diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 60fe481e7..c9dc5fc22 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -42,7 +42,7 @@ runs: if: runner.os == 'Linux' && inputs.get-build-deps == 'true' uses: awalsh128/cache-apt-pkgs-action@v1.6.0 with: - packages: bison clang flex libffi-dev libfl-dev libreadline-dev pkg-config tcl-dev zlib1g-dev + packages: bison clang flex libffi-dev libfl-dev libreadline-dev pkg-config tcl-dev zlib1g-dev libgtest-dev version: ${{ inputs.runs-on }}-buildys - name: Linux docs dependencies @@ -52,15 +52,6 @@ runs: packages: graphviz xdot version: ${{ inputs.runs-on }}-docsys - # if updating test dependencies, make sure to update - # docs/source/yosys_internals/extending_yosys/test_suites.rst to match. - - name: Linux test dependencies - if: runner.os == 'Linux' && inputs.get-test-deps == 'true' - uses: awalsh128/cache-apt-pkgs-action@v1.6.0 - with: - packages: libgtest-dev - version: ${{ inputs.runs-on }}-testys - - name: Install macOS Dependencies if: runner.os == 'macOS' shell: bash diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index ab6eb3148..ebdeb15d8 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -72,6 +72,7 @@ jobs: cd build make -f ../Makefile config-$CC make -f ../Makefile -j$procs + make -f ../Makefile unit-test -j$procs - name: Log yosys-config output run: | @@ -81,7 +82,7 @@ jobs: shell: bash run: | cd build - tar -cvf ../build.tar share/ yosys yosys-* + tar -cvf ../build.tar share/ yosys yosys-* libyosys.so - name: Store build artifact uses: actions/upload-artifact@v4 @@ -131,7 +132,7 @@ jobs: - name: Run tests shell: bash run: | - make -j$procs test TARGETS= EXTRA_TARGETS= CONFIG=$CC + make -j$procs vanilla-test TARGETS= EXTRA_TARGETS= CONFIG=$CC - name: Report errors if: ${{ failure() }} diff --git a/.github/workflows/test-sanitizers.yml b/.github/workflows/test-sanitizers.yml index 11a339cd3..7650470c3 100644 --- a/.github/workflows/test-sanitizers.yml +++ b/.github/workflows/test-sanitizers.yml @@ -65,7 +65,7 @@ jobs: - name: Run tests shell: bash run: | - make -j$procs test TARGETS= EXTRA_TARGETS= + make -j$procs vanilla-test TARGETS= EXTRA_TARGETS= - name: Report errors if: ${{ failure() }} @@ -73,7 +73,3 @@ jobs: run: | find tests/**/*.err -print -exec cat {} \; - - name: Run unit tests - shell: bash - run: | - make -j$procs unit-test ENABLE_LIBYOSYS=1 diff --git a/.github/workflows/test-verific.yml b/.github/workflows/test-verific.yml index adc6f59d8..cd2545cc8 100644 --- a/.github/workflows/test-verific.yml +++ b/.github/workflows/test-verific.yml @@ -68,7 +68,7 @@ jobs: - name: Run Yosys tests run: | - make -j$procs test + make -j$procs vanilla-test - name: Run Verific specific Yosys tests run: | @@ -80,11 +80,6 @@ jobs: run: | make -C sby run_ci - - name: Run unit tests - shell: bash - run: | - make -j$procs unit-test ENABLE_LTO=1 ENABLE_LIBYOSYS=1 - test-pyosys: needs: pre-job if: ${{ needs.pre-job.outputs.should_skip != 'true' && github.repository == 'YosysHQ/Yosys' }} diff --git a/Makefile b/Makefile index 0f76cdbef..28ee48ef9 100644 --- a/Makefile +++ b/Makefile @@ -979,9 +979,11 @@ makefile-tests/%: %/run-test.mk $(TARGETS) $(EXTRA_TARGETS) $(MAKE) -C $* -f run-test.mk +@echo "...passed tests in $*" -test: makefile-tests abcopt-tests seed-tests +test: vanilla-test unit-test + +vanilla-test: makefile-tests abcopt-tests seed-tests @echo "" - @echo " Passed \"make test\"." + @echo " Passed \"make vanilla-test\"." ifeq ($(ENABLE_VERIFIC),1) ifeq ($(YOSYS_NOVERIFIC),1) @echo " Ran tests without verific support due to YOSYS_NOVERIFIC=1." @@ -1013,11 +1015,11 @@ ystests: $(TARGETS) $(EXTRA_TARGETS) # Unit test unit-test: libyosys.so - @$(MAKE) -C $(UNITESTPATH) CXX="$(CXX)" CC="$(CC)" CPPFLAGS="$(CPPFLAGS)" \ + @$(MAKE) -f $(UNITESTPATH)/Makefile CXX="$(CXX)" CC="$(CC)" CPPFLAGS="$(CPPFLAGS)" \ CXXFLAGS="$(CXXFLAGS)" LINKFLAGS="$(LINKFLAGS)" LIBS="$(LIBS)" ROOTPATH="$(CURDIR)" clean-unit-test: - @$(MAKE) -C $(UNITESTPATH) clean + @$(MAKE) -f $(UNITESTPATH)/Makefile clean install-dev: $(PROGRAM_PREFIX)yosys-config share $(INSTALL_SUDO) mkdir -p $(DESTDIR)$(BINDIR) diff --git a/README.md b/README.md index 3b2f41768..df65a6a10 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ To build Yosys simply type 'make' in this directory. $ sudo make install Tests are located in the tests subdirectory and can be executed using the test -target. Note that you need gawk as well as a recent version of iverilog (i.e. -build from git). Then, execute tests via: +target. Note that you need gawk, a recent version of iverilog, and gtest. +Execute tests via: $ make test diff --git a/docs/source/yosys_internals/extending_yosys/test_suites.rst b/docs/source/yosys_internals/extending_yosys/test_suites.rst index 81a79e77f..d3422b23a 100644 --- a/docs/source/yosys_internals/extending_yosys/test_suites.rst +++ b/docs/source/yosys_internals/extending_yosys/test_suites.rst @@ -8,7 +8,44 @@ Running the included test suite The Yosys source comes with a test suite to avoid regressions and keep everything working as expected. Tests can be run by calling ``make test`` from -the root Yosys directory. +the root Yosys directory. By default, this runs vanilla and unit tests. + +Vanilla tests +~~~~~~~~~~~~~ + +These make up the majority of our testing coverage. +They can be run with ``make vanilla-test`` and are based on calls to +make subcommands (``make makefile-tests``) and shell scripts +(``make seed-tests`` and ``make abcopt-tests``). Both use ``run-test.sh`` +files, but make-based tests only call ``tests/gen-tests-makefile.sh`` +to generate a makefile appropriate for the given directory, so only +afterwards when make is invoked do the tests actually run. + +Usually their structure looks something like this: +you write a .ys file that gets automatically run, +which runs a frontend like ``read_verilog`` or ``read_rtlil`` with +a relative path or a heredoc, then runs some commands including the command +under test, and then uses :doc:`/using_yosys/more_scripting/selections` +with ``-assert-count``. Usually it's unnecessary to "register" the test anywhere +as if it's being added to an existing directory, depending +on how the ``run-test.sh`` in that directory works. + +Unit tests +~~~~~~~~~~ + +Running the unit tests requires the following additional packages: + +.. tab:: Ubuntu + + .. code:: console + + sudo apt-get install libgtest-dev + +.. tab:: macOS + + No additional requirements. + +Unit tests can be run with ``make unit-test``. Functional tests ~~~~~~~~~~~~~~~~ @@ -41,23 +78,6 @@ instructions `_. Then, set the :makevar:`ENABLE_FUNCTIONAL_TESTS` make variable when calling ``make test`` and the functional tests will be run as well. -Unit tests -~~~~~~~~~~ - -Running the unit tests requires the following additional packages: - -.. tab:: Ubuntu - - .. code:: console - - sudo apt-get install libgtest-dev - -.. tab:: macOS - - No additional requirements. - -Unit tests can be run with ``make unit-test``. - Docs tests ~~~~~~~~~~ diff --git a/tests/unit/Makefile b/tests/unit/Makefile index b275d7f41..e8f76cba9 100644 --- a/tests/unit/Makefile +++ b/tests/unit/Makefile @@ -18,10 +18,11 @@ endif EXTRAFLAGS := -lyosys -pthread -OBJTEST := objtest -BINTEST := bintest +MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +OBJTEST := $(MAKEFILE_DIR)objtest +BINTEST := $(MAKEFILE_DIR)bintest -ALLTESTFILE := $(shell find . -name '*Test.cc' | sed 's|^\./||' | tr '\n' ' ') +ALLTESTFILE := $(shell cd $(MAKEFILE_DIR) && find . -name '*Test.cc' | sed 's|^\./||' | tr '\n' ' ') TESTDIRS := $(sort $(dir $(ALLTESTFILE))) TESTS := $(addprefix $(BINTEST)/, $(basename $(ALLTESTFILE:%Test.cc=%Test.o))) @@ -34,7 +35,7 @@ $(BINTEST)/%: $(OBJTEST)/%.o | prepare $(CXX) -L$(ROOTPATH) $(RPATH) $(LINKFLAGS) -o $@ $^ $(LIBS) \ $(GTEST_LDFLAGS) $(EXTRAFLAGS) -$(OBJTEST)/%.o: $(basename $(subst $(OBJTEST),.,%)).cc | prepare +$(OBJTEST)/%.o: $(MAKEFILE_DIR)/%.cc | prepare $(CXX) -o $@ -c -I$(ROOTPATH) $(CPPFLAGS) $(CXXFLAGS) $(GTEST_CXXFLAGS) $^ .PHONY: prepare run-tests clean