Script assembly signing & NuGet package creation in Azure Pipelines (#2862)

Windows x86/x64 builds now parallelized
Windows assemblies now signed
NuGet package created
NuGet package signed
NuGet package published to NuGet.org

scripts/authorization.json

@@ -1,15 +0,0 @@
-{
-  "Version": "1.0.0",
-  "AuthenticationType": "AAD_CERT",
-  "ClientId": "1c614a83-2dbe-4d3c-853b-effaefd4fb20",
-  "AuthCert": {
-    "SubjectName": "1c614a83-2dbe-4d3c-853b-effaefd4fb20.microsoft.com",
-    "StoreLocation": "LocalMachine",
-    "StoreName": "My"
-  },
-  "RequestSigningCert": {
-    "SubjectName": "1c614a83-2dbe-4d3c-853b-effaefd4fb20",
-    "StoreLocation": "LocalMachine",
-    "StoreName": "My"
-  }
-}

scripts/build-win-signed.yml

@@ -0,0 +1,81 @@
+parameters:
+  ReleaseVersion: ''
+  BuildArchitecture: ''
+
+jobs:
+- job: WindowsBuild${{parameters.BuildArchitecture}}
+  displayName: "Windows build (${{parameters.BuildArchitecture}})"
+  pool:
+    vmImage: "vs2017-win2016"
+  steps:
+  - powershell: write-host $(System.TeamProjectId)
+    displayName: 'System.TeamProjectId'
+  - powershell: write-host $(System.DefinitionId)
+    displayName: 'System.DefinitionId'
+  - powershell: write-host $(Build.BuildId)
+    displayName: 'Build.BuildId'
+  - task: CmdLine@2
+    displayName: Build
+    inputs:
+      script:
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{parameters.BuildArchitecture}} &&
+        python scripts\mk_win_dist.py
+            --${{parameters.BuildArchitecture}}-only
+            --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
+  - task: EsrpCodeSigning@1
+    displayName: Sign
+    inputs:
+      ConnectedServiceName: 'z3-esrp-signing'
+      FolderPath: 'dist/z3-${{parameters.ReleaseVersion}}-${{parameters.BuildArchitecture}}-win/bin'
+      Pattern: 'Microsoft.Z3.dll,libz3.dll,libz3java.dll,z3.exe'
+      signConfigType: 'inlineSignParams'
+      inlineOperation: |
+        [
+          {
+            "keyCode": "CP-230012",
+            "operationSetCode": "SigntoolSign",
+            "parameters": [
+              {
+                "parameterName": "OpusName",
+                "parameterValue": "Microsoft"
+              },
+              {
+                "parameterName": "OpusInfo",
+                "parameterValue": "http://www.microsoft.com"
+              },
+              {
+                "parameterName": "PageHash",
+                "parameterValue": "/NPH"
+              },
+              {
+                "parameterName": "FileDigest",
+                "parameterValue": "/fd sha256"
+              },
+              {
+                "parameterName": "TimeStamp",
+                "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+              }
+            ],
+            "toolName": "signtool.exe",
+            "toolVersion": "6.2.9304.0"
+          }
+        ]
+      SessionTimeout: '60'
+      MaxConcurrency: '50'
+      MaxRetryAttempts: '5'
+  - task: DeleteFiles@1
+    displayName: Cleanup
+    inputs:
+      SourceFolder: 'dist/z3-${{parameters.ReleaseVersion}}-${{parameters.BuildArchitecture}}-win/bin'
+      Contents: 'CodeSignSummary*'
+  - task: ArchiveFiles@2
+    displayName: Zip
+    inputs:
+      rootFolderOrFile: 'dist/z3-${{parameters.ReleaseVersion}}-${{parameters.BuildArchitecture}}-win'
+      includeRootFolder: true
+      archiveType: 'zip'
+      archiveFile: '$(Build.ArtifactStagingDirectory)/z3-${{parameters.ReleaseVersion}}-${{parameters.BuildArchitecture}}-win.zip'
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Build.ArtifactStagingDirectory)/z3-${{parameters.ReleaseVersion}}-${{parameters.BuildArchitecture}}-win.zip'
+      artifactName: 'WindowsBuild-${{parameters.BuildArchitecture}}'

scripts/mk_nuget_release.py

@@ -125,55 +125,6 @@ Linux Dependencies:
 def create_nuget_package():
     subprocess.call(["nuget", "pack"], cwd="out")
 
-nuget_sign_input = """
-{
-  "Version": "1.0.0",
-  "SignBatches"
-  :
-  [
-   {
-    "SourceLocationType": "UNC",
-    "SourceRootDirectory": "%s",
-    "DestinationLocationType": "UNC",
-    "DestinationRootDirectory": "%s",
-    "SignRequestFiles": [
-     {
-      "CustomerCorrelationId": "42fc9577-af9e-4ac9-995d-1788d8721d17",
-      "SourceLocation": "Microsoft.Z3.x64.%s.nupkg",
-      "DestinationLocation": "Microsoft.Z3.x64.%s.nupkg"
-     }
-    ],
-    "SigningInfo": {
-     "Operations": [
-      {
-       "KeyCode" : "CP-401405",
-       "OperationCode" : "NuGetSign",
-       "Parameters" : {},
-       "ToolName" : "sign",
-       "ToolVersion" : "1.0"
-      },
-      {
-       "KeyCode" : "CP-401405",
-       "OperationCode" : "NuGetVerify",
-       "Parameters" : {},
-       "ToolName" : "sign",
-       "ToolVersion" : "1.0"
-      }
-     ]
-    }
-   }
-  ]
-}"""
-
-def sign_nuget_package():
-    package_name = "Microsoft.Z3.x64.%s.nupkg" % release_version
-    input_file = "out/nuget_sign_input.json"
-    output_path = os.path.abspath("out").replace("\\","\\\\") 
-    with open(input_file, 'w') as f:
-        f.write(nuget_sign_input % (output_path, output_path, release_version, release_version))
-    r = subprocess.call(["EsrpClient.exe", "sign", "-a", "authorization.json", "-p", "policy.json", "-i", input_file, "-o", "out\\diagnostics.json"], shell=True, stderr=subprocess.STDOUT)
-    print(r)
-    
 def main():
     mk_dir("packages")
     download_installs()
@@ -181,7 +132,5 @@ def main():
     mk_targets()
     create_nuget_spec()
     create_nuget_package()
-    sign_nuget_package()
-
 
 main()

scripts/mk_nuget_task.py

@@ -64,11 +64,19 @@ def unpack(packages):
                     zip_ref.extract("%s/bin/%s" % (package_dir, b), "tmp")
                     shutil.move("tmp/%s/bin/%s" % (package_dir, b), "out/lib/netstandard1.4/%s" % b)
 
-def mk_targets():
+def mk_targets(source_root):
     mk_dir("out/build")
-    shutil.copy("../src/api/dotnet/Microsoft.Z3.targets.in", "out/build/Microsoft.Z3.targets")
+    shutil.copy("{}/src/api/dotnet/Microsoft.Z3.targets.in".format(source_root), "out/build/Microsoft.Z3.x64.targets")
+
+def mk_icon(source_root):
+    mk_dir("out/content")
+    shutil.copy("{}/resources/icon.jpg".format(source_root), "out/content/icon.jpg")
+
+def mk_license(source_root):
+    mk_dir("out/content")
+    shutil.copy("{}/LICENSE.txt".format(source_root), "out/content/LICENSE.txt")
     
-def create_nuget_spec(release_version, release_commit):
+def create_nuget_spec(version, repo, branch, commit):
     contents = """<?xml version="1.0" encoding="utf-8"?>
 <package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
     <metadata>
@@ -83,78 +91,34 @@ Linux Dependencies:
         </description>
         <copyright>&#169; Microsoft Corporation. All rights reserved.</copyright>
         <tags>smt constraint solver theorem prover</tags>
-        <iconUrl>https://raw.githubusercontent.com/Z3Prover/z3/{1}/resources/icon.jpg</iconUrl>
+        <icon>content/icon.jpg</icon>
         <projectUrl>https://github.com/Z3Prover/z3</projectUrl>
-        <licenseUrl>https://raw.githubusercontent.com/Z3Prover/z3/{1}/LICENSE.txt</licenseUrl>
-        <repository type="git" url="https://github.com/Z3Prover/z3.git" branch="master" commit="{1}" />
+        <license type="file">content/LICENSE.txt</license>
+        <repository type="git" url="{1}" branch="{2}" commit="{3}" />
         <requireLicenseAcceptance>true</requireLicenseAcceptance>
         <language>en</language>
+        <dependencies>
+            <group targetFramework=".NETStandard1.4" />
+        </dependencies>
     </metadata>
-</package>""".format(release_version, release_commit)
+</package>""".format(version, repo, branch, commit)
     print(contents)
     with open("out/Microsoft.Z3.x64.nuspec", 'w') as f:
         f.write(contents)
         
-nuget_sign_input = """
-{
-  "Version": "1.0.0",
-  "SignBatches"
-  :
-  [
-   {
-    "SourceLocationType": "UNC",
-    "SourceRootDirectory": "%s",
-    "DestinationLocationType": "UNC",
-    "DestinationRootDirectory": "%s",
-    "SignRequestFiles": [
-     {
-      "CustomerCorrelationId": "42fc9577-af9e-4ac9-995d-1788d8721d17",
-      "SourceLocation": "Microsoft.Z3.x64.%s.nupkg",
-      "DestinationLocation": "Microsoft.Z3.x64.%s.nupkg"
-     }
-    ],
-    "SigningInfo": {
-     "Operations": [
-      {
-       "KeyCode" : "CP-401405",
-       "OperationCode" : "NuGetSign",
-       "Parameters" : {},
-       "ToolName" : "sign",
-       "ToolVersion" : "1.0"
-      },
-      {
-       "KeyCode" : "CP-401405",
-       "OperationCode" : "NuGetVerify",
-       "Parameters" : {},
-       "ToolName" : "sign",
-       "ToolVersion" : "1.0"
-      }
-     ]
-    }
-   }
-  ]
-}"""
-
-def create_sign_input(release_version):
-    package_name = "Microsoft.Z3.x64.%s.nupkg" % release_version
-    input_file = "out/nuget_sign_input.json"
-    output_path = os.path.abspath("out").replace("\\","\\\\") 
-    with open(input_file, 'w') as f:
-        f.write(nuget_sign_input % (output_path, output_path, release_version, release_version))
-    
-    
 def main():
     packages = sys.argv[1]
-    release_version = sys.argv[2]
-    release_commit = sys.argv[3]
+    version = sys.argv[2]
+    repo = sys.argv[3]
+    branch = sys.argv[4]
+    commit = sys.argv[5]
+    source_root = sys.argv[6]
     print(packages)
-    mk_dir(packages)    
+    mk_dir(packages)
     unpack(packages)
-    mk_targets()
-    create_nuget_spec(release_version, release_commit)
-    create_sign_input(release_version)
-#    create_nuget_package()
-#    sign_nuget_package(release_version)
-
+    mk_targets(source_root)
+    mk_icon(source_root)
+    mk_license(source_root)
+    create_nuget_spec(version, repo, branch, commit)
 
 main()

scripts/mk_util.py

@@ -91,7 +91,6 @@ VS_PROJ = False
 TRACE = False
 PYTHON_ENABLED=False
 DOTNET_CORE_ENABLED=False
-ESRP_SIGN=False
 DOTNET_KEY_FILE=getenv("Z3_DOTNET_KEY_FILE", None)
 JAVA_ENABLED=False
 ML_ENABLED=False
@@ -679,14 +678,14 @@ def display_help(exit_code):
 # Parse configuration option for mk_make script
 def parse_options():
     global VERBOSE, DEBUG_MODE, IS_WINDOWS, VS_X64, ONLY_MAKEFILES, SHOW_CPPS, VS_PROJ, TRACE, VS_PAR, VS_PAR_NUM
-    global DOTNET_CORE_ENABLED, DOTNET_KEY_FILE, JAVA_ENABLED, ML_ENABLED, JS_ENABLED, STATIC_LIB, STATIC_BIN, PREFIX, GMP, PYTHON_PACKAGE_DIR, GPROF, GIT_HASH, GIT_DESCRIBE, PYTHON_INSTALL_ENABLED, PYTHON_ENABLED, ESRP_SIGN
+    global DOTNET_CORE_ENABLED, DOTNET_KEY_FILE, JAVA_ENABLED, ML_ENABLED, JS_ENABLED, STATIC_LIB, STATIC_BIN, PREFIX, GMP, PYTHON_PACKAGE_DIR, GPROF, GIT_HASH, GIT_DESCRIBE, PYTHON_INSTALL_ENABLED, PYTHON_ENABLED
     global LINUX_X64, SLOW_OPTIMIZE, LOG_SYNC, SINGLE_THREADED
     global GUARD_CF, ALWAYS_DYNAMIC_BASE
     try:
         options, remainder = getopt.gnu_getopt(sys.argv[1:],
                                                'b:df:sxhmcvtnp:gj',
                                                ['build=', 'debug', 'silent', 'x64', 'help', 'makefiles', 'showcpp', 'vsproj', 'guardcf',
-                                                'trace', 'dotnet', 'dotnet-key=', 'esrp', 'staticlib', 'prefix=', 'gmp', 'java', 'parallel=', 'gprof', 'js',
+                                                'trace', 'dotnet', 'dotnet-key=', 'staticlib', 'prefix=', 'gmp', 'java', 'parallel=', 'gprof', 'js',
                                                 'githash=', 'git-describe', 'x86', 'ml', 'optimize', 'pypkgdir=', 'python', 'staticbin', 'log-sync', 'single-threaded'])
     except:
         print("ERROR: Invalid command line option")
@@ -722,8 +721,6 @@ def parse_options():
             DOTNET_CORE_ENABLED = True
         elif opt in ('--dotnet-key'):
             DOTNET_KEY_FILE = arg
-        elif opt in ('--esrp'):
-            ESRP_SIGN = True
         elif opt in ('--staticlib'):
             STATIC_LIB = True
         elif opt in ('--staticbin'):
@@ -1697,79 +1694,9 @@ class DotNetDLLComponent(Component):
         dotnetCmdLine.extend(['-o', path])
             
         MakeRuleCmd.write_cmd(out, ' '.join(dotnetCmdLine))
-        self.sign_esrp(out)
         out.write('\n')        
         out.write('%s: %s\n\n' % (self.name, dllfile))
 
-    def sign_esrp(self, out):
-        global ESRP_SIGNx
-        print("esrp-sign", ESRP_SIGN)
-        if not ESRP_SIGN:
-            return
-        
-        import uuid
-        guid = str(uuid.uuid4())
-        path = os.path.abspath(BUILD_DIR).replace("\\","\\\\")        
-        assemblySignStr = """
-{
-  "Version": "1.0.0",
-  "SignBatches"
-  :
-  [
-   {
-    "SourceLocationType": "UNC",
-    "SourceRootDirectory": "%s",
-    "DestinationLocationType": "UNC",
-    "DestinationRootDirectory": "c:\\\\ESRP\\\\output",
-    "SignRequestFiles": [
-     {
-      "CustomerCorrelationId": "%s",
-      "SourceLocation": "libz3.dll",
-      "DestinationLocation": "libz3.dll"
-     },
-     {
-      "CustomerCorrelationId": "%s",
-      "SourceLocation": "Microsoft.Z3.dll",
-      "DestinationLocation": "Microsoft.Z3.dll"
-     }
-    ],
-    "SigningInfo": {
-     "Operations": [
-      {
-       "KeyCode" : "CP-230012",
-       "OperationCode" : "SigntoolSign",
-       "Parameters" : {
-        "OpusName": "Microsoft",
-        "OpusInfo": "http://www.microsoft.com",
-        "FileDigest": "/fd \\"SHA256\\"",
-        "PageHash": "/NPH",
-        "TimeStamp": "/tr \\"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\\" /td sha256"
-       },
-       "ToolName" : "sign",
-       "ToolVersion" : "1.0"
-      },
-      {
-       "KeyCode" : "CP-230012",
-       "OperationCode" : "SigntoolVerify",
-       "Parameters" : {},
-       "ToolName" : "sign",
-       "ToolVersion" : "1.0"
-      }
-     ]
-    }
-   }
-  ]
-}       """ % (path, guid, guid)
-        assemblySign = os.path.join(os.path.abspath(BUILD_DIR), 'dotnet', 'assembly-sign-input.json')
-        with open(assemblySign, 'w') as ous:
-            ous.write(assemblySignStr)
-        outputFile = os.path.join(os.path.abspath(BUILD_DIR), 'dotnet', "esrp-out.json")
-        esrpCmdLine = ["esrpclient.exe", "sign", "-a", "C:\\esrp\\config\\authorization.json", "-p", "C:\\esrp\\config\\policy.json", "-i", assemblySign, "-o", outputFile]
-        MakeRuleCmd.write_cmd(out, ' '.join(esrpCmdLine))
-        MakeRuleCmd.write_cmd(out, "move /Y C:\\esrp\\output\\libz3.dll .")
-        MakeRuleCmd.write_cmd(out, "move /Y C:\\esrp\\output\\Microsoft.Z3.dll .")
-
-
     def main_component(self):
         return is_dotnet_core_enabled()
     

scripts/mk_win_dist.cmd

@@ -1,9 +0,0 @@
-
-call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
-
-python scripts\mk_win_dist.py --x64-only --dotnet-key=$(Build.SourcesDirectory/resources/z3.snk
-
-call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars32.bat"
-
-python scripts\mk_win_dist.py --x86-only --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
-

scripts/mk_win_dist.py

@@ -25,9 +25,9 @@ VERBOSE=True
 DIST_DIR='dist'
 FORCE_MK=False
 DOTNET_CORE_ENABLED=True
-ESRP_SIGN=False
 DOTNET_KEY_FILE=None
 JAVA_ENABLED=True
+ZIP_BUILD_OUTPUTS=False
 GIT_HASH=False
 PYTHON_ENABLED=True
 X86ONLY=False
@@ -63,10 +63,10 @@ def display_help():
     print("  -b <sudir>, --build=<subdir>  subdirectory where x86 and x64 Z3 versions will be built (default: build-dist).")
     print("  -f, --force                   force script to regenerate Makefiles.")
     print("  --nodotnet                    do not include .NET bindings in the binary distribution files.")
-    print("  --dotnet-key=<file>           sign the .NET assembly with the private key in <file>.")
-    print("  --esrp                        sign with esrp.")
+    print("  --dotnet-key=<file>           strongname sign the .NET assembly with the private key in <file>.")
     print("  --nojava                      do not include Java bindings in the binary distribution files.")
     print("  --nopython                    do not include Python bindings in the binary distribution files.")
+    print("  --zip                         package build outputs in zip file.")
     print("  --githash                     include git hash in the Zip file.")
     print("  --x86-only                    x86 dist only.")
     print("  --x64-only                    x64 dist only.")
@@ -74,7 +74,7 @@ def display_help():
 
 # Parse configuration option for mk_make script
 def parse_options():
-    global FORCE_MK, JAVA_ENABLED, GIT_HASH, DOTNET_CORE_ENABLED, DOTNET_KEY_FILE, PYTHON_ENABLED, X86ONLY, X64ONLY, ESRP_SIGN
+    global FORCE_MK, JAVA_ENABLED, ZIP_BUILD_OUTPUTS, GIT_HASH, DOTNET_CORE_ENABLED, DOTNET_KEY_FILE, PYTHON_ENABLED, X86ONLY, X64ONLY
     path = BUILD_DIR
     options, remainder = getopt.gnu_getopt(sys.argv[1:], 'b:hsf', ['build=',
                                                                    'help',
@@ -83,7 +83,7 @@ def parse_options():
                                                                    'nojava',
                                                                    'nodotnet',
                                                                    'dotnet-key=',
-                                                                   'esrp',
+                                                                   'zip',
                                                                    'githash',
                                                                    'nopython',
                                                                    'x86-only',
@@ -107,10 +107,10 @@ def parse_options():
             PYTHON_ENABLED = False
         elif opt == '--dotnet-key':
             DOTNET_KEY_FILE = arg
-        elif opt == '--esrp':
-            ESRP_SIGN = True
         elif opt == '--nojava':
             JAVA_ENABLED = False
+        elif opt == '--zip':
+            ZIP_BUILD_OUTPUTS = True
         elif opt == '--githash':
             GIT_HASH = True
         elif opt == '--x86-only' and not X64ONLY:
@@ -138,8 +138,6 @@ def mk_build_dir(path, x64):
             opts.append('--java')
         if x64:
             opts.append('-x')
-        if ESRP_SIGN:
-            opts.append('--esrp')
         if GIT_HASH:
             opts.append('--githash=%s' % mk_util.git_hash())
             opts.append('--git-describe')
@@ -208,7 +206,6 @@ def get_z3_name(x64):
         return 'z3-%s.%s.%s-%s-win' % (major, minor, build, platform)
 
 def mk_dist_dir(x64):
-    global ESRP_SIGN
     if x64:
         platform = "x64"
         build_path = BUILD_X64_DIR
@@ -217,7 +214,6 @@ def mk_dist_dir(x64):
         build_path = BUILD_X86_DIR
     dist_path = os.path.join(DIST_DIR, get_z3_name(x64))
     mk_dir(dist_path)
-    mk_util.ESRP_SIGN = ESRP_SIGN
     mk_util.DOTNET_CORE_ENABLED = True
     mk_util.DOTNET_KEY_FILE = DOTNET_KEY_FILE
     mk_util.JAVA_ENABLED = JAVA_ENABLED
@@ -320,7 +316,8 @@ def main():
         mk_dist_dir(False)
         cp_license(False)
         cp_vs_runtime(False)
-        mk_zip(False)
+        if ZIP_BUILD_OUTPUTS:
+            mk_zip(False)
     elif X64ONLY:
         mk_build_dir(BUILD_X64_DIR, True)
         mk_z3(True)
@@ -328,7 +325,8 @@ def main():
         mk_dist_dir(True)
         cp_license(True)
         cp_vs_runtime(True)
-        mk_zip(True)
+        if ZIP_BUILD_OUTPUTS:
+            mk_zip(True)
     else:
         mk_build_dirs()
         mk_z3s()
@@ -336,7 +334,8 @@ def main():
         mk_dist_dirs()
         cp_licenses()
         cp_vs_runtimes()
-        mk_zips()
+        if ZIP_BUILD_OUTPUTS:
+            mk_zips()
 
 main()
 

scripts/nightly.yaml

@@ -48,31 +48,67 @@ jobs:
       artifactName: 'Manylinux'
       targetPath: $(Build.ArtifactStagingDirectory)
 
-- job: Windows
-  displayName: "Windows build"
+- job: Windows32
+  displayName: "Windows 32-bit build"
   pool:
     vmImage: "vs2017-win2016"
   steps:
-  - script: scripts\mk_win_dist.cmd
-#  - script: git clone https://github.com/z3prover/z3test z3test
-#  - script: python z3test/scripts/test_benchmarks.py build-dist\z3.exe z3test/regressions/smt2    
-  - script: xcopy dist\*.zip $(Build.ArtifactStagingDirectory)\* /y
-  - task: PublishPipelineArtifact@0
+  - task: CmdLine@2
+    inputs:
+      script:
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars32.bat" &&
+        python scripts\mk_win_dist.py
+            --x86-only
+            --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
+            --zip
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: dist
+      contents: '*.zip'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+  - task: PublishPipelineArtifact@1
     inputs:
-      artifactName: 'Windows'
       targetPath: $(Build.ArtifactStagingDirectory)
+      artifactName: 'Windows32'
 
+- job: Windows64
+  displayName: "Windows 64-bit build"
+  pool:
+    vmImage: "vs2017-win2016"
+  steps:
+  - task: CmdLine@2
+    inputs:
+      script:
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat" &&
+        python scripts\mk_win_dist.py
+            --x64-only
+            --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
+            --zip
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: dist
+      contents: '*.zip'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+      artifactName: 'Windows64'
 
 - job: NuGet
   displayName: "Create Nuget Package"
   dependsOn:
   - Mac
   - Ubuntu
-  - Windows
+  - Windows32
+  - Windows64
   steps:
   - task: DownloadPipelineArtifact@0
     inputs:
-     artifactName: 'Windows'
+     artifactName: 'Windows32'
+     targetPath: tmp
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Windows64'
      targetPath: tmp
   - task: DownloadPipelineArtifact@0
     inputs:
@@ -108,14 +144,19 @@ jobs:
   displayName: "Python packaging"
   dependsOn:
   - Manylinux
-  - Windows
+  - Windows32
+  - Windows64
   - Mac
   pool:
     vmImage: "ubuntu-16.04"
   steps:
   - task: DownloadPipelineArtifact@0
     inputs:
-     artifactName: 'Windows'
+     artifactName: 'Windows32'
+     targetPath: $(Agent.TempDirectory)
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Windows64'
      targetPath: $(Agent.TempDirectory)
   - task: DownloadPipelineArtifact@0
     inputs:
@@ -146,12 +187,17 @@ jobs:
   dependsOn: 
   - Mac
   - Ubuntu
-  - Windows
+  - Windows32
+  - Windows64
   - Python
   steps:
   - task: DownloadPipelineArtifact@0
     inputs:
-     artifactName: 'Windows'
+     artifactName: 'Windows32'
+     targetPath: tmp
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Windows64'
      targetPath: tmp
   - task: DownloadPipelineArtifact@0
     inputs:

scripts/release.yml

@@ -1,141 +1,321 @@
-jobs:
+# Release pipeline (must be triggered manually)
+# * Builds for all platforms (with code signing)
+# * Creates packages for Python and NuGet
+# * Uploads build products to stores (GitHub, NuGet, PyPI)
 
-- job: Mac
-  displayName: "Mac Build"
-  pool:
-    vmImage: "macOS-10.14"
-  steps:
-  - script: python scripts/mk_unix_dist.py --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
-  - script: git clone https://github.com/z3prover/z3test z3test
-  - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
-  - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: 'Mac'
-      targetPath: $(Build.ArtifactStagingDirectory)
+trigger: none
 
-- job: Ubuntu
-  displayName: "Ubuntu build"
-  pool:
-    vmImage: "ubuntu-16.04"
-  steps:
-  - script: python scripts/mk_unix_dist.py --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
-  - script: git clone https://github.com/z3prover/z3test z3test
-  - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
-  - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: 'Ubuntu'
-      targetPath: $(Build.ArtifactStagingDirectory)
+variables:
+  ReleaseVersion: '4.8.8'
 
-- job: Manylinux
-  displayName: "Manylinux build"
-  pool:
-    vmImage: "ubuntu-16.04"
-  container: "rhelmot/manylinux1_x86_64:latest"
-  variables:
-    python: "/opt/python/cp35-cp35m/bin/python"
-  steps:
-  - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava
-  - script: git clone https://github.com/z3prover/z3test z3test
-  - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
-  - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: 'Manylinux'
-      targetPath: $(Build.ArtifactStagingDirectory)
+stages:
 
-- job: Windows
-  displayName: "Windows build"
-  pool:
-    vmImage: "vs2017-win2016"
-  steps:
-  - script: scripts\mk_win_dist.cmd
-  - script: xcopy dist\*.zip $(Build.ArtifactStagingDirectory)\* /y
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: 'Windows'
-      targetPath: $(Build.ArtifactStagingDirectory)
+# Builds Z3 on various platforms
+- stage: Build
+  jobs:
 
+  - job: MacBuild
+    displayName: "macOS Build"
+    pool:
+      vmImage: "macOS-10.14"
+    steps:
+    - task: PythonScript@0
+      displayName: Build
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: scripts/mk_unix_dist.py
+        arguments: --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
+    - script: git clone https://github.com/z3prover/z3test z3test
+      displayName: 'Clone z3test'
+    - task: PythonScript@0
+      displayName: Test
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: z3test/scripts/test_benchmarks.py
+        arguments: build-dist/z3 z3test/regressions/smt2
+    - task: CopyFiles@2
+      inputs:
+        sourceFolder: dist
+        contents: '*.zip'
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'macOSBuild'
+        targetPath: $(Build.ArtifactStagingDirectory)
 
-- job: Python
-  displayName: "Python packaging"
-  dependsOn:
-  - Manylinux
-  - Windows
-  pool:
-    vmImage: "ubuntu-16.04"
-  steps:
-  - task: DownloadPipelineArtifact@0
-    inputs:
-     artifactName: 'Windows'
-     targetPath: $(Agent.TempDirectory)
-  - task: DownloadPipelineArtifact@0
-    inputs:
-     artifactName: 'Manylinux'
-     targetPath: $(Agent.TempDirectory)
-  - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*centos*.zip
-  - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
-  - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
-  - script: python -m pip install --user -U setuptools wheel
-  - script: cd src/api/python; python setup.py sdist
-  # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
-  - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
-  - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
-  - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: 'Python packages'
-      targetPath: src/api/python/dist
+  - job: UbuntuBuild
+    displayName: "Ubuntu build"
+    pool:
+      vmImage: "ubuntu-16.04"
+    steps:
+    - task: PythonScript@0
+      displayName: Build
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: scripts/mk_unix_dist.py
+        arguments: --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
+    - script: git clone https://github.com/z3prover/z3test z3test
+      displayName: 'Clone z3test'
+    - task: PythonScript@0
+      displayName: Test
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: z3test/scripts/test_benchmarks.py
+        arguments: build-dist/z3 z3test/regressions/smt2
+    - task: CopyFiles@2
+      inputs:
+        sourceFolder: dist
+        contents: '*.zip'
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'UbuntuBuild'
+        targetPath: $(Build.ArtifactStagingDirectory)
 
-- job: Deploy
-  displayName: "Deploy into GitHub and PyPI"
-  dependsOn:
-  - Mac
-  - Ubuntu
-  - Windows
-  - Python
-  steps:
-  - task: DownloadPipelineArtifact@0
-    inputs:
-     artifactName: 'Windows'
-     targetPath: tmp
-  - task: DownloadPipelineArtifact@0
-    inputs:
-     artifactName: 'Mac'
-     targetPath: tmp
-  - task: DownloadPipelineArtifact@0
-    inputs:
-     artifactName: 'Ubuntu'
-     targetPath: tmp
-# TBD: build NuGet package
-# TBD: this script should build a specific pre-specified tag
-  - task: GitHubRelease@0
-    inputs:
-      gitHubConnection: Z3GitHub
-      repositoryName: 'Z3Prover/z3'
-      action: 'create'
-      target: '$(Build.SourceVersion)'
-      tagSource: 'manual'
-      tag: 'z3-4.8.7'
-      title: 'z3-4.8.7'
-      releaseNotesSource: 'input'
-      releaseNotes: '4.8.7 release'
-      assets: 'tmp/*'
-      isDraft: true
-      isPreRelease: true
-  - task: DownloadPipelineArtifact@0
-    inputs:
-      artifactName: 'Python packages'
-      targetPath: dist
-  - task: DownloadSecureFile@1
-    name: pypirc
-    inputs:
-      secureFile: 'pypirc'
-  - script: pip install --upgrade pip
-  - script: python -m pip install --user -U setuptools importlib_metadata wheel twine 
-# Uncomment on release:
-  - script: python -m twine upload --config-file $(pypirc.secureFilePath) -r $(pypiReleaseServer) dist/*
+  - job: ManyLinuxBuild
+    displayName: "ManyLinux build"
+    pool:
+      vmImage: "ubuntu-16.04"
+    container: "rhelmot/manylinux1_x86_64:latest"
+    variables:
+      python: "/opt/python/cp35-cp35m/bin/python"
+    steps:
+    - task: PythonScript@0
+      displayName: Build
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: scripts/mk_unix_dist.py
+        arguments: --nodotnet --nojava
+        pythonInterpreter: $(python)
+    - script: git clone https://github.com/z3prover/z3test z3test
+      displayName: 'Clone z3test'
+    - task: PythonScript@0
+      displayName: Test
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: z3test/scripts/test_benchmarks.py
+        arguments: build-dist/z3 z3test/regressions/smt2
+        pythonInterpreter: $(python)
+    - task: CopyFiles@2
+      inputs:
+        sourceFolder: dist
+        contents: '*.zip'
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'ManyLinuxBuild'
+        targetPath: $(Build.ArtifactStagingDirectory)
+  
+  - template: build-win-signed.yml
+    parameters:
+      ReleaseVersion: $(ReleaseVersion)
+      BuildArchitecture: 'x64'
 
+  - template: build-win-signed.yml
+    parameters:
+      ReleaseVersion: $(ReleaseVersion)
+      BuildArchitecture: 'x86'
 
-# TBD: run regression tests on generated binaries.
+# Creates Z3 packages in various formats
+- stage: Package
+  jobs:
+
+  - job: NuGetPackage
+    displayName: "NuGet packaging"
+    pool:
+      vmImage: "windows-latest"
+    steps:
+    - powershell: write-host $(System.DefinitionId)
+      displayName: 'System.DefinitionId'
+    - powershell: write-host $(Build.BuildId)
+      displayName: 'Build.BuildId'
+    - powershell: write-host $(System.TeamProjectId)
+      displayName: 'System.TeamProjectId'
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download Win64 Build'
+      inputs:
+        artifact: 'WindowsBuild-x64'
+        path: $(Agent.TempDirectory)\package
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download Ubuntu Build'
+      inputs:
+        artifact: 'UbuntuBuild'
+        path: $(Agent.TempDirectory)\package
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download macOS Build'
+      inputs:
+        artifact: 'macOSBuild'
+        path: $(Agent.TempDirectory)\package
+    - task: PythonScript@0
+      displayName: 'Python: assemble files'
+      inputs:
+        scriptSource: 'filepath'
+        scriptPath: scripts\mk_nuget_task.py
+        workingDirectory: $(Agent.TempDirectory)\package
+        arguments:
+          $(Agent.TempDirectory)\package
+          $(ReleaseVersion)
+          $(Build.Repository.Uri)
+          $(Build.SourceBranchName)
+          $(Build.SourceVersion)
+          $(Build.SourcesDirectory)
+    - task: NuGetToolInstaller@0
+      inputs:
+        versionSpec: 5.x
+        checkLatest: false
+    - task: NugetCommand@2
+      displayName: 'NuGet Pack'
+      inputs:
+        command: pack
+        packagesToPack: $(Agent.TempDirectory)\package\out\Microsoft.Z3.x64.nuspec
+        basePath: $(Agent.TempDirectory)\package\out
+        packDestination: $(Build.ArtifactStagingDirectory)
+        verbosityPack: detailed
+    - task: EsrpCodeSigning@1
+      displayName: 'Sign Package'
+      inputs:
+        ConnectedServiceName: 'z3-esrp-signing'
+        FolderPath: $(Build.ArtifactStagingDirectory)
+        Pattern: Microsoft.Z3.x64.$(ReleaseVersion).nupkg
+        signConfigType: 'inlineSignParams'
+        inlineOperation: |
+          [
+            {
+              "KeyCode" : "CP-401405",
+              "OperationCode" : "NuGetSign",
+              "Parameters" : {},
+              "ToolName" : "sign",
+              "ToolVersion" : "1.0"
+            },
+            {
+              "KeyCode" : "CP-401405",
+              "OperationCode" : "NuGetVerify",
+              "Parameters" : {},
+              "ToolName" : "sign",
+              "ToolVersion" : "1.0"
+            }
+          ]
+        SessionTimeout: '60'
+        MaxConcurrency: '50'
+        MaxRetryAttempts: '5'
+    - task: PublishPipelineArtifact@1
+      inputs:
+        targetPath: $(Build.ArtifactStagingDirectory)\Microsoft.Z3.x64.$(ReleaseVersion).nupkg
+        artifactName: 'NuGetPackage'
+
+  - job: PythonPackage
+    displayName: "Python packaging"
+    pool:
+      vmImage: "ubuntu-16.04"
+    steps:
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download macOS Build'
+      inputs:
+        artifact: 'macOSBuild'
+        path: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download ManyLinux Build'
+      inputs:
+        artifact: 'ManyLinuxBuild'
+        path: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download Win32 Build'
+      inputs:
+        artifact: 'WindowsBuild-x86'
+        path: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download Win64 Build'
+      inputs:
+        artifact: 'WindowsBuild-x64'
+        path: $(Agent.TempDirectory)
+    - script: cd $(Agent.TempDirectory); mkdir osx-bin; cd osx-bin; unzip ../*osx*.zip
+    - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*centos*.zip
+    - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
+    - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
+    - script: python -m pip install --user -U setuptools wheel
+    - script: cd src/api/python; python setup.py sdist
+    # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/osx-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'PythonPackage'
+        targetPath: src/api/python/dist
+
+# Uploads Z3 packages to various package stores
+- stage: Publish
+  jobs:
+
+  - job: GitHubPublish
+    displayName: "Publish to GitHub"
+    pool:
+      vmImage: "windows-latest"
+    steps:
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download macOS Build'
+      inputs:
+        artifact: 'macOSBuild'
+        path: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download Win32 Build'
+      inputs:
+        artifact: 'WindowsBuild-x86'
+        path: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download Win64 Build'
+      inputs:
+        artifact: 'WindowsBuild-x64'
+        path: $(Agent.TempDirectory)
+    - task: GitHubRelease@0
+      inputs:
+        gitHubConnection: Z3GitHub
+        repositoryName: $(Build.Repository.Name)
+        action: 'create'
+        target: '$(Build.SourceVersion)'
+        tagSource: 'manual'
+        tag: 'z3-$(ReleaseVersion)'
+        title: 'z3-$(ReleaseVersion)'
+        releaseNotesSource: 'input'
+        releaseNotes: '$(ReleaseVersion) release'
+        assets: '$(Agent.TempDirectory)/*.zip'
+        isDraft: true
+        isPreRelease: false
+
+  - job: NuGetPublish
+    displayName: "Publish to NuGet.org"
+    steps:
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download NuGet Package'
+      inputs:
+        artifact: 'NuGetPackage'
+        path: $(Agent.TempDirectory)
+    - task: NuGetAuthenticate@0
+      inputs:
+        nuGetServiceConnections: Z3Nuget
+#    - task: NuGetCommand@2
+#      inputs:
+#        command: push
+#        nuGetFeedType: external
+#        feedsToUse: select
+#        includeNuGetOrg: true
+#        packagesToPush: $(Agent.TempDirectory)
+
+  - job: PyPIPublish
+    displayName: "Publish to PyPI"
+    pool:
+      vmImage: "ubuntu-16.04"
+    steps:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: 'PythonPackage'
+        path: dist
+    - task: DownloadSecureFile@1
+      name: pypirc
+      inputs:
+        secureFile: 'pypirc'
+    - script: pip install --upgrade pip
+    - script: python -m pip install --user -U setuptools importlib_metadata wheel twine 
+    # Uncomment on release:
+    #- script: python -m twine upload --config-file $(pypirc.secureFilePath) -r $(pypiReleaseServer) dist/*