z3/scripts/nightly.yaml

variables:
  Major: '4'
  Minor: '12'
  Patch: '1'
  AssemblyVersion: $(Major).$(Minor).$(Patch).$(Build.BuildId)
  NightlyVersion: $(AssemblyVersion)-$(Build.DefinitionName)

stages:
- stage: Build
  jobs:
  - job: Mac
    displayName: "Mac Build"
    pool:
      vmImage: "macOS-11"
    steps:
    - script: python scripts/mk_unix_dist.py --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
    - script: git clone https://github.com/z3prover/z3test z3test
    - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2    
    - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
    - task: PublishPipelineArtifact@1
      inputs:
        artifactName: 'Mac'
        targetPath: $(Build.ArtifactStagingDirectory)

  - job: MacArm64
    displayName: "Mac ARM64 Build"
    pool:
      vmImage: "macOS-11"
    steps:
    - script: python scripts/mk_unix_dist.py --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk --arch=arm64 --os=osx-11.0
    - script: git clone https://github.com/z3prover/z3test z3test
    - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
    - task: PublishPipelineArtifact@1
      inputs:
        artifactName: 'MacArm64'
        targetPath: $(Build.ArtifactStagingDirectory)

  - job: Ubuntu
    displayName: "Ubuntu build"
    pool:
      vmImage: "ubuntu-latest"
    steps:
    - script: python scripts/mk_unix_dist.py --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
    - script: git clone https://github.com/z3prover/z3test z3test
    - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2    
    - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: 'Ubuntu'
        targetPath: $(Build.ArtifactStagingDirectory)

  - job: UbuntuDoc
    displayName: "Ubuntu Doc build"
    pool:
      vmImage: "ubuntu-latest"
    steps:
    - script: sudo apt-get install ocaml opam libgmp-dev
    - script: opam init -y
    - script: eval `opam config env`; opam install zarith ocamlfind -y
    - script: eval `opam config env`; python scripts/mk_make.py --ml
    - script: sudo apt-get install doxygen
    - script: sudo apt-get install graphviz
    - script: |
        set -e
        cd build
        eval `opam config env` 
        make -j3
        make -j3 examples
        make -j3 test-z3
        cd ..        
    - script: |
        set -e
        eval `opam config env` 
        cd doc
        python mk_api_doc.py --mld --z3py-package-path=../build/python/z3
        python mk_params_doc.py 
        mkdir api/html/ml
        ocamldoc -html -d api/html/ml -sort -hide Z3 -I $( ocamlfind query zarith ) -I ../build/api/ml ../build/api/ml/z3enums.mli ../build/api/ml/z3.mli
        cd ..        
    - script: zip -r z3doc.zip doc/api
    - script: cp z3doc.zip $(Build.ArtifactStagingDirectory)/.
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: 'UbuntuDoc'
        targetPath: $(Build.ArtifactStagingDirectory)

  - job: ManyLinuxBuild
    variables:
      python: "/opt/python/cp37-cp37m/bin/python"
      name: ManyLinux
    displayName: "ManyLinux build"
    pool:
      vmImage: "ubuntu-latest"
    container: "quay.io/pypa/manylinux2014_x86_64:latest"
    steps:
    - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava
    - script: git clone https://github.com/z3prover/z3test z3test
    - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
    - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: '$(name)Build'
        targetPath: $(Build.ArtifactStagingDirectory)

#  - job: MuslLinuxBuild
#    condition: eq(0,1)
#    variables:
#      python: "/opt/python/cp310-cp310/bin/python"
#      name: MuslLinux
#    displayName: "MuslLinux build"
#    pool:
#      vmImage: "ubuntu-latest"
#    container: "quay.io/pypa/musllinux_1_1_x86_64:latest"
#    steps:
#    - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava
#    - script: git clone https://github.com/z3prover/z3test z3test
#    - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
#    - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/
#    - task: PublishPipelineArtifact@0
#      inputs:
#        artifactName: '$(name)Build'
#        targetPath: $(Build.ArtifactStagingDirectory)

  - job: Windows32
    displayName: "Windows 32-bit build"
    pool:
      vmImage: "windows-latest"
    steps:
    - task: CmdLine@2
      inputs:
        script:
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86 &
          python scripts\mk_win_dist.py
            --assembly-version=$(AssemblyVersion)
            --x86-only
            --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
            --zip
    - task: CopyFiles@2
      inputs:
        sourceFolder: dist
        contents: '*.zip'
        targetFolder: $(Build.ArtifactStagingDirectory)
    - task: PublishPipelineArtifact@1
      inputs:
        targetPath: $(Build.ArtifactStagingDirectory)
        artifactName: 'Windows32'
    - task: CopyFiles@2
      displayName: 'Collect Symbols'
      inputs:
        sourceFolder: dist
        contents: '**/*.pdb'
        targetFolder: '$(Build.ArtifactStagingDirectory)/symbols'
    # Publish symbol archive to match nuget package
    # Index your source code and publish symbols to a file share or Azure Artifacts symbol server
    - task: PublishSymbols@2
      inputs:
        symbolsFolder: '$(Build.ArtifactStagingDirectory)/symbols'
        searchPattern: '**/*.pdb' 
        indexSources: false # Github not supported
        publishSymbols: true
        symbolServerType: TeamServices
        detailedLog: true

  - job: Windows64
    displayName: "Windows 64-bit build"
    pool:
      vmImage: "windows-latest"
    steps:
    - task: CmdLine@2
      inputs:
        script:
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 &
          python scripts\mk_win_dist.py
            --assembly-version=$(AssemblyVersion)
            --x64-only
            --dotnet-key=$(Build.SourcesDirectory)/resources/z3.snk
            --zip
    - task: CopyFiles@2
      inputs:
        sourceFolder: dist
        contents: '*.zip'
        targetFolder: $(Build.ArtifactStagingDirectory)
    - task: PublishPipelineArtifact@1
      inputs:
        targetPath: $(Build.ArtifactStagingDirectory)
        artifactName: 'Windows64'
    - task: CopyFiles@2
      displayName: 'Collect Symbols'
      inputs:
        sourceFolder: dist
        contents: '**/*.pdb'
        targetFolder: '$(Build.ArtifactStagingDirectory)/symbols'
    # Publish symbol archive to match nuget package
    # Index your source code and publish symbols to a file share or Azure Artifacts symbol server
    - task: PublishSymbols@2
      inputs:
        symbolsFolder: '$(Build.ArtifactStagingDirectory)/symbols'
        searchPattern: '**/*.pdb' 
        indexSources: false # Github not supported
        publishSymbols: true
        symbolServerType: TeamServices
        detailedLog: true

- stage: Package
  jobs:
  - job: NuGet64
    displayName: "NuGet 64 packaging"
    pool:
      vmImage: "windows-latest"
    steps:
    - powershell: write-host $(System.DefinitionId)
      displayName: 'System.DefinitionId'
    - powershell: write-host $(Build.BuildId)
      displayName: 'Build.BuildId'
    - powershell: write-host $(System.TeamProjectId)
      displayName: 'System.TeamProjectId'
    - task: DownloadPipelineArtifact@2
      displayName: 'Download Win64 Build'
      inputs:
        artifact: 'Windows64'
        path: $(Agent.TempDirectory)\package
    - task: DownloadPipelineArtifact@2
      displayName: 'Download Ubuntu Build'
      inputs:
        artifact: 'Ubuntu'
        path: $(Agent.TempDirectory)\package
    - task: DownloadPipelineArtifact@2
      displayName: 'Download macOS Build'
      inputs:
        artifact: 'Mac'
        path: $(Agent.TempDirectory)\package
    - task: DownloadPipelineArtifact@2
      displayName: 'Download macOS Arm64 Build'
      inputs:
        artifact: 'MacArm64'
        path: $(Agent.TempDirectory)\package	
    - task: NuGetToolInstaller@0
      inputs:
        versionSpec: 5.x
        checkLatest: false        
    - task: PythonScript@0
      displayName: 'Python: assemble files'
      inputs:
        scriptSource: 'filepath'
        scriptPath: scripts\mk_nuget_task.py
        workingDirectory: $(Agent.TempDirectory)\package
        arguments:
          $(Agent.TempDirectory)\package
          $(NightlyVersion)
          $(Build.Repository.Uri)
          $(Build.SourceBranchName)
          $(Build.SourceVersion)
          $(Build.SourcesDirectory)
          symbols
    - task: NugetCommand@2
      displayName: 'NuGet Pack Symbols'
      inputs:
        command: custom
        versioningScheme: byPrereleaseNumber
        majorVersion: $(Major)
        minorVersion: $(Minor)
        patchVersion: $(Patch)
        arguments: 'pack $(Agent.TempDirectory)\package\out\Microsoft.Z3.sym.nuspec -Version $(NightlyVersion) -OutputDirectory $(Build.ArtifactStagingDirectory) -Verbosity detailed -Symbols -SymbolPackageFormat snupkg -BasePath $(Agent.TempDirectory)\package\out' 
    - task: EsrpCodeSigning@2
      continueOnError: true
      displayName: 'Sign Package'
      inputs:
        ConnectedServiceName: 'z3-esrp-signing-2'
        FolderPath: $(Build.ArtifactStagingDirectory)
        Pattern: Microsoft.Z3.$(NightlyVersion).nupkg
        signConfigType: 'inlineSignParams'
        inlineOperation: |
          [
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetSign",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            },
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetVerify",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            }
          ]          
        SessionTimeout: '60'
        MaxConcurrency: '50'
        MaxRetryAttempts: '5'
    - task: EsrpCodeSigning@2
      continueOnError: true 
      displayName: 'Sign Symbol Package'
      inputs:
        ConnectedServiceName: 'z3-esrp-signing-2'
        FolderPath: $(Build.ArtifactStagingDirectory)
        Pattern: Microsoft.Z3.$(NightlyVersion).snupkg
        signConfigType: 'inlineSignParams'
        inlineOperation: |
          [
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetSign",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            },
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetVerify",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            }
          ]          
        SessionTimeout: '60'
        MaxConcurrency: '50'
        MaxRetryAttempts: '5'        
    - task: PublishPipelineArtifact@1
      inputs:
        targetPath: $(Build.ArtifactStagingDirectory)
        artifactName: 'NuGet'

  - job: NuGet32
    displayName: "NuGet 32 packaging"
    pool:
      vmImage: "windows-latest"
    steps:
    - powershell: write-host $(System.DefinitionId)
      displayName: 'System.DefinitionId'
    - powershell: write-host $(Build.BuildId)
      displayName: 'Build.BuildId'
    - powershell: write-host $(System.TeamProjectId)
      displayName: 'System.TeamProjectId'
    - task: DownloadPipelineArtifact@2
      displayName: 'Download Win32 Build'
      inputs:
        artifact: 'Windows32'
        path: $(Agent.TempDirectory)\package
    - task: NuGetToolInstaller@0
      inputs:
        versionSpec: 5.x
        checkLatest: false        
    - task: PythonScript@0
      displayName: 'Python: assemble files'
      inputs:
        scriptSource: 'filepath'
        scriptPath: scripts\mk_nuget_task.py
        workingDirectory: $(Agent.TempDirectory)\package
        arguments:
          $(Agent.TempDirectory)\package
          $(NightlyVersion)
          $(Build.Repository.Uri)
          $(Build.SourceBranchName)
          $(Build.SourceVersion)
          $(Build.SourcesDirectory)
          symbols
          x86
    - task: NugetCommand@2
      displayName: 'NuGet Pack Symbols'
      inputs:
        command: custom
        versioningScheme: byPrereleaseNumber
        majorVersion: $(Major)
        minorVersion: $(Minor)
        patchVersion: $(Patch)
        arguments: 'pack $(Agent.TempDirectory)\package\out\Microsoft.Z3.x86.sym.nuspec -Version $(NightlyVersion) -OutputDirectory $(Build.ArtifactStagingDirectory) -Verbosity detailed -Symbols -SymbolPackageFormat snupkg -BasePath $(Agent.TempDirectory)\package\out' 
    - task: EsrpCodeSigning@2
      continueOnError: true 
      displayName: 'Sign Package'
      inputs:
        ConnectedServiceName: 'z3-esrp-signing-2'
        FolderPath: $(Build.ArtifactStagingDirectory)
        Pattern: Microsoft.Z3.x86.$(NightlyVersion).nupkg
        signConfigType: 'inlineSignParams'
        inlineOperation: |
          [
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetSign",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            },
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetVerify",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            }
          ]          
        SessionTimeout: '60'
        MaxConcurrency: '50'
        MaxRetryAttempts: '5'
    - task: EsrpCodeSigning@2
      continueOnError: true 
      displayName: 'Sign Symbol Package'
      inputs:
        ConnectedServiceName: 'z3-esrp-signing-2'
        FolderPath: $(Build.ArtifactStagingDirectory)
        Pattern: Microsoft.Z3.x86.$(NightlyVersion).snupkg
        signConfigType: 'inlineSignParams'
        inlineOperation: |
          [
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetSign",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            },
            {
              "KeyCode" : "CP-401405",
              "OperationCode" : "NuGetVerify",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
            }
          ]          
        SessionTimeout: '60'
        MaxConcurrency: '50'
        MaxRetryAttempts: '5'        
    - task: PublishPipelineArtifact@1
      inputs:
        targetPath: $(Build.ArtifactStagingDirectory)
        artifactName: 'NuGet32'


  - job: Python
    displayName: "Python packaging"
    pool:
      vmImage: "ubuntu-latest"
    steps:
    - task: DownloadPipelineArtifact@2
      inputs:
        artifactName: 'Windows32'
        targetPath: $(Agent.TempDirectory)
    - task: DownloadPipelineArtifact@2
      inputs:
        artifactName: 'Windows64'
        targetPath: $(Agent.TempDirectory)
    - task: DownloadPipelineArtifact@2
      inputs:
        artifactName: 'ManyLinuxBuild'
        targetPath: $(Agent.TempDirectory)
#    - task: DownloadPipelineArtifact@2
#      displayName: 'Download MuslLinux Build'
#      inputs:
#        artifact: 'MuslLinuxBuild'
#        path: $(Agent.TempDirectory)	
    - task: DownloadPipelineArtifact@2
      inputs:
        artifactName: 'Mac'
        targetPath: $(Agent.TempDirectory)
    - task: DownloadPipelineArtifact@2
      inputs:
        artifactName: 'MacArm64'
        targetPath: $(Agent.TempDirectory)
    - script: cd $(Agent.TempDirectory); mkdir osx-x64-bin; cd osx-x64-bin; unzip ../*x64-osx*.zip
    - script: cd $(Agent.TempDirectory); mkdir osx-arm64-bin; cd osx-arm64-bin; unzip ../*arm64-osx*.zip
    - script: cd $(Agent.TempDirectory); mkdir libc-bin; cd libc-bin; unzip ../*glibc*.zip
#    - script: cd $(Agent.TempDirectory); mkdir musl-bin; cd musl-bin; unzip ../*-linux.zip
    - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
    - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
    - script: python3 -m pip install --user -U setuptools wheel
    - script: cd src/api/python; python3 setup.py sdist
    # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
    - script: cd src/api/python; echo $(Agent.TempDirectory)/libc-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
#   - script: cd src/api/python; echo $(Agent.TempDirectory)/musl-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
    - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
    - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
    - script: cd src/api/python; echo $(Agent.TempDirectory)/osx-x64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
    - script: cd src/api/python; echo $(Agent.TempDirectory)/osx-arm64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
    - task: PublishPipelineArtifact@0
      inputs:
        artifactName: 'Python packages'
        targetPath: src/api/python/dist


- stage: Deployment
  jobs:
  - job: Deploy
    displayName: "Deploy into GitHub"
    continueOnError: true
    pool:
      vmImage: "ubuntu-latest"
    steps:
    - task: DownloadPipelineArtifact@2
      displayName: "Download windows32"
      inputs:
        artifactName: 'Windows32'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download windows64"
      inputs:
        artifactName: 'Windows64'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download Mac"
      inputs:
        artifactName: 'Mac'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download MacArm64"
      inputs:
        artifactName: 'MacArm64'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download Ubuntu"
      inputs:
        artifactName: 'Ubuntu'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download Doc"
      inputs:
        artifactName: 'UbuntuDoc'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download Python"
      inputs:
        artifactName: 'Python packages'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download NuGet"
      inputs:
        artifactName: 'NuGet'
        targetPath: tmp
    - task: DownloadPipelineArtifact@2
      displayName: "Download NuGet32"
      inputs:
        artifactName: 'NuGet32'
        targetPath: tmp
    - task: GitHubRelease@0
      continueOnError: true 
      inputs:
        gitHubConnection: Z3Prover
        repositoryName: 'Z3Prover/z3'
        action: 'delete'
#     target: '$(Build.SourceVersion)'
        tagSource: 'manual'
        tag: 'Nightly'
    - task: GitHubRelease@0
      continueOnError: true 
      inputs:
        gitHubConnection: Z3Prover
        repositoryName: 'Z3Prover/z3'
        action: 'create'
#     target: '$(Build.SourceVersion)'
        tagSource: 'manual'
        tag: 'Nightly'
        title: 'Nightly'
        releaseNotesSource: 'input'
        releaseNotes: 'nightly build'
        assets: 'tmp/*'
        assetUploadMode: 'replace'
        isDraft: false
        isPreRelease: true

- stage: NugetPublishNightly
  jobs:
   # Publish to nightly feed on Azure
  - job: NuGetPublishNightly
    displayName: "Push nuget packages to Azure Feed"
    steps:
    - task: NuGetAuthenticate@0
      displayName: 'NuGet Authenticate'
    - task: NuGetToolInstaller@0
      inputs:
        versionSpec: 5.x
        checkLatest: false
    - task: DownloadPipelineArtifact@2
      displayName: 'Download NuGet x86 Package'
      inputs:
        artifact: 'NuGet32'
        path: $(Agent.TempDirectory)/x86
    - task: DownloadPipelineArtifact@2
      displayName: 'Download NuGet x64 Package'
      inputs:
        artifact: 'NuGet'
        path: $(Agent.TempDirectory)/x64
    - task: NuGetCommand@2
      displayName: 'NuGet Nightly x64 push'
      inputs:
        command: push
        publishVstsFeed: 'Z3Build/Z3-Nightly-Builds'
        packagesToPush: $(Agent.TempDirectory)/x64/*.nupkg
        allowPackageConflicts: true
    - task: NuGetCommand@2
      displayName: 'NuGet Nightly x86 push'
      inputs:
        command: push
        publishVstsFeed: 'Z3Build/Z3-Nightly-Builds'
        packagesToPush: $(Agent.TempDirectory)/x86/*.nupkg
        allowPackageConflicts: true
    
       
# TBD: run regression tests on generated binaries.