From fcb55257beede8e5882f20e04de5537341b90acb Mon Sep 17 00:00:00 2001
From: 0152la <al2510@imperial.ac.uk>
Date: Mon, 2 Aug 2021 20:01:41 +0100
Subject: [PATCH] Improve coverage CI script (#5451)
* Due to the long duration of the CI execution, execute it at a set time
daily (currently 11 UTC / 4 PDT)
* Use `Ninja` to build instead of Makefile, due to better compilation
time
* Execute all the available Z3 tests and examples: `test-z3 -a`,
`z3test` regression suites (`smt2`, `smt2-debug`, and `smt2-extra`),
`z3test` coverage tests, and the 4 provided examples
* Upload `gcovr` report as an artifact associated with the CI run
TODOs:
* Fix `gcovr` emitting an empty report
* Potentially take the artifact and upload it somewhere accessible
Co-authored-by: Andrei Lascu <andrei.lascu10@imperial.ac.uk>
---
.github/workflows/coverage.yml | 82 ++++++++++++++++++++++++++++------
1 file changed, 69 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 85639bd95..e5b19bebd 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -1,34 +1,90 @@
name: Code Coverage
on:
- push:
- branches: [ master ]
- pull_request:
- branches: [ master ]
+ #push:
+ #branches: [ master ]
+ schedule:
+ - cron: "0 11 * * *"
env:
BUILD_TYPE: Debug
+ CMAKE_GENERATOR: Ninja
jobs:
build:
runs-on: ubuntu-latest
+ env:
+ CC: clang
+ CXX: clang++
+
steps:
- uses: actions/checkout@v2
- - name: Configure CMake
- run: CXXFLAGS=--coverage LDFLAGS=-lgcov CC=clang CXX=clang++ cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_INSTALL_PREFIX=./install
+ - name: Setup
+ run: |
+ sudo apt-get remove -y --purge man-db
+ sudo apt-get update -y
+ sudo apt-get install -y gcovr ninja-build
- - name: Build
+ ## Building
+ - name: Configure CMake Z3
+ run: CFLAGS=="--coverage" CXXFLAGS="--coverage" LDFLAGS="-lgcov" cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_INSTALL_PREFIX=./install
+
+ - name: Build Z3
# Build your program with the given configuration
run: cmake --build ${{github.workspace}}/build --target install --config ${{env.BUILD_TYPE}}
+ - name: Build test-z3
+ run: cmake --build ${{github.workspace}}/build --target test-z3 --config ${{env.BUILD_TYPE}}
+
+ - name: Build examples
+ run: |
+ cmake --build ${{github.workspace}}/build --target c_example
+ cmake --build ${{github.workspace}}/build --target cpp_example
+ cmake --build ${{github.workspace}}/build --target z3_tptp5
+ cmake --build ${{github.workspace}}/build --target c_maxsat_example
+
- name: Clone z3test
run: git clone https://github.com/z3prover/z3test z3test
-
+
+ ## Testing
+ - name: Run test-z3
+ run: |
+ cd ${{github.workspace}}/build
+ ./test-z3 -a
+ cd -
+
+ - name: Run examples
+ run: |
+ ${{github.workspace}}/build/examples/c_example_build_dir/c_example
+ ${{github.workspace}}/build/examples/cpp_example_build_dir/cpp_example
+ ${{github.workspace}}/build/examples/tptp_build_dir/z3_tptp5 --help
+ ${{github.workspace}}/build/examples/c_maxsat_example_build_dir/c_maxsat_example ${{github.workspace}}/examples/maxsat/ex.smt
+
- name: Run regressions
- run: python z3test/scripts/test_benchmarks.py build/z3 z3test/regressions/smt2
-
- - name: Run coverage
- run: python z3test/scripts/test_coverage_tests.py ./install z3test/coverage/cpp
-
+ run: |
+ python z3test/scripts/test_benchmarks.py build/z3 z3test/regressions/smt2
+ python z3test/scripts/test_benchmarks.py build/z3 z3test/regressions/smt2-debug
+ python z3test/scripts/test_benchmarks.py build/z3 z3test/regressions/smt2-extra
+
+ - name: Run coverage tests
+ run: python z3test/scripts/test_coverage_tests.py ./install z3test/coverage/cpp
+
+ ## Artifact
+ - name: Gather coverage
+ run: |
+ cd ${{github.workspace}}
+ gcovr
+ #gcovr --html -o coverage.html .
+ cd -
+
+ - name: Get date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+ - uses: actions/upload-artifact@v2
+ with:
+ name: coverage-${{steps.date.outputs.date}}
+ path: coverage.html
+ retention-days: 10