Merge pull request #5676 from YosysHQ/emil/unit-test-by-default

Run unit tests on make test

.github/actions/setup-build-env/action.yml

@@ -42,7 +42,7 @@ runs:
       if: runner.os == 'Linux' && inputs.get-build-deps == 'true'
       uses: awalsh128/cache-apt-pkgs-action@v1.6.0
       with:
-        packages: bison clang flex libffi-dev libfl-dev libreadline-dev pkg-config tcl-dev zlib1g-dev
+        packages: bison clang flex libffi-dev libfl-dev libreadline-dev pkg-config tcl-dev zlib1g-dev libgtest-dev
         version: ${{ inputs.runs-on }}-buildys
 
     - name: Linux docs dependencies
@@ -52,15 +52,6 @@ runs:
         packages: graphviz xdot
         version: ${{ inputs.runs-on }}-docsys
 
-    # if updating test dependencies, make sure to update
-    # docs/source/yosys_internals/extending_yosys/test_suites.rst to match.
-    - name: Linux test dependencies
-      if: runner.os == 'Linux' && inputs.get-test-deps == 'true'
-      uses: awalsh128/cache-apt-pkgs-action@v1.6.0
-      with:
-        packages: libgtest-dev
-        version: ${{ inputs.runs-on }}-testys
-
     - name: Install macOS Dependencies
       if: runner.os == 'macOS'
       shell: bash

.github/workflows/test-build.yml

@@ -72,6 +72,7 @@ jobs:
           cd build
           make -f ../Makefile config-$CC
           make -f ../Makefile -j$procs
+          make -f ../Makefile unit-test -j$procs
 
       - name: Log yosys-config output
         run: |
@@ -81,7 +82,7 @@ jobs:
         shell: bash
         run: |
           cd build
-          tar -cvf ../build.tar share/ yosys yosys-*
+          tar -cvf ../build.tar share/ yosys yosys-* libyosys.so
 
       - name: Store build artifact
         uses: actions/upload-artifact@v4
@@ -131,7 +132,7 @@ jobs:
       - name: Run tests
         shell: bash
         run: |
-          make -j$procs test TARGETS= EXTRA_TARGETS= CONFIG=$CC
+          make -j$procs vanilla-test TARGETS= EXTRA_TARGETS= CONFIG=$CC
 
       - name: Report errors
         if: ${{ failure() }}

.github/workflows/test-sanitizers.yml

@@ -65,7 +65,7 @@ jobs:
       - name: Run tests
         shell: bash
         run: |
-          make -j$procs test TARGETS= EXTRA_TARGETS=
+          make -j$procs vanilla-test TARGETS= EXTRA_TARGETS=
 
       - name: Report errors
         if: ${{ failure() }}
@@ -73,7 +73,3 @@ jobs:
         run: |
           find tests/**/*.err -print -exec cat {} \;
 
-      - name: Run unit tests
-        shell: bash
-        run: |
-          make -j$procs unit-test ENABLE_LIBYOSYS=1

.github/workflows/test-verific.yml

@@ -68,7 +68,7 @@ jobs:
 
       - name: Run Yosys tests
         run: |
-          make -j$procs test
+          make -j$procs vanilla-test
 
       - name: Run Verific specific Yosys tests
         run: |
@@ -80,11 +80,6 @@ jobs:
         run: |
           make -C sby run_ci
 
-      - name: Run unit tests
-        shell: bash
-        run: |
-          make -j$procs unit-test ENABLE_LTO=1 ENABLE_LIBYOSYS=1
-
   test-pyosys:
     needs: pre-job
     if: ${{ needs.pre-job.outputs.should_skip != 'true' && github.repository == 'YosysHQ/Yosys' }}

Makefile

@@ -979,9 +979,11 @@ makefile-tests/%: %/run-test.mk $(TARGETS) $(EXTRA_TARGETS)
 	$(MAKE) -C $* -f run-test.mk
 	+@echo "...passed tests in $*"
 
-test: makefile-tests abcopt-tests seed-tests
+test: vanilla-test unit-test
+
+vanilla-test: makefile-tests abcopt-tests seed-tests
 	@echo ""
-	@echo "  Passed \"make test\"."
+	@echo "  Passed \"make vanilla-test\"."
 ifeq ($(ENABLE_VERIFIC),1)
 ifeq ($(YOSYS_NOVERIFIC),1)
 	@echo "  Ran tests without verific support due to YOSYS_NOVERIFIC=1."
@@ -1013,11 +1015,11 @@ ystests: $(TARGETS) $(EXTRA_TARGETS)
 
 # Unit test
 unit-test: libyosys.so
-	@$(MAKE) -C $(UNITESTPATH) CXX="$(CXX)" CC="$(CC)" CPPFLAGS="$(CPPFLAGS)" \
+	@$(MAKE) -f $(UNITESTPATH)/Makefile CXX="$(CXX)" CC="$(CC)" CPPFLAGS="$(CPPFLAGS)" \
 		CXXFLAGS="$(CXXFLAGS)" LINKFLAGS="$(LINKFLAGS)" LIBS="$(LIBS)" ROOTPATH="$(CURDIR)"
 
 clean-unit-test:
-	@$(MAKE) -C $(UNITESTPATH) clean
+	@$(MAKE) -f $(UNITESTPATH)/Makefile clean
 
 install-dev: $(PROGRAM_PREFIX)yosys-config share
 	$(INSTALL_SUDO) mkdir -p $(DESTDIR)$(BINDIR)

README.md

@@ -114,8 +114,8 @@ To build Yosys simply type 'make' in this directory.
 	$ sudo make install
 
 Tests are located in the tests subdirectory and can be executed using the test
-target. Note that you need gawk as well as a recent version of iverilog (i.e.
-build from git). Then, execute tests via:
+target. Note that you need gawk, a recent version of iverilog, and gtest.
+Execute tests via:
 
 	$ make test
 

docs/source/yosys_internals/extending_yosys/test_suites.rst

@@ -8,7 +8,44 @@ Running the included test suite
 
 The Yosys source comes with a test suite to avoid regressions and keep
 everything working as expected.  Tests can be run by calling ``make test`` from
-the root Yosys directory.
+the root Yosys directory. By default, this runs vanilla and unit tests.
+
+Vanilla tests
+~~~~~~~~~~~~~
+
+These make up the majority of our testing coverage.
+They can be run with ``make vanilla-test`` and are based on calls to
+make subcommands (``make makefile-tests``) and shell scripts
+(``make seed-tests`` and ``make abcopt-tests``). Both use ``run-test.sh``
+files, but make-based tests only call ``tests/gen-tests-makefile.sh``
+to generate a makefile appropriate for the given directory, so only
+afterwards when make is invoked do the tests actually run.
+
+Usually their structure looks something like this:
+you write a .ys file that gets automatically run,
+which runs a frontend like ``read_verilog`` or ``read_rtlil`` with
+a relative path or a heredoc, then runs some commands including the command
+under test, and then uses :doc:`/using_yosys/more_scripting/selections`
+with ``-assert-count``. Usually it's unnecessary to "register" the test anywhere
+as if it's being added to an existing directory, depending
+on how the ``run-test.sh`` in that directory works.
+
+Unit tests
+~~~~~~~~~~
+
+Running the unit tests requires the following additional packages:
+
+.. tab:: Ubuntu
+
+   .. code:: console
+
+      sudo apt-get install libgtest-dev
+
+.. tab:: macOS
+
+   No additional requirements.
+
+Unit tests can be run with ``make unit-test``.
 
 Functional tests
 ~~~~~~~~~~~~~~~~
@@ -41,23 +78,6 @@ instructions <https://github.com/Z3Prover/z3>`_.
 Then, set the :makevar:`ENABLE_FUNCTIONAL_TESTS` make variable when calling
 ``make test`` and the functional tests will be run as well.
 
-Unit tests
-~~~~~~~~~~
-
-Running the unit tests requires the following additional packages:
-
-.. tab:: Ubuntu
-
-   .. code:: console
-
-      sudo apt-get install libgtest-dev
-
-.. tab:: macOS
-
-   No additional requirements.
-
-Unit tests can be run with ``make unit-test``. 
-
 Docs tests
 ~~~~~~~~~~
 

tests/unit/Makefile

@@ -18,10 +18,11 @@ endif
 
 EXTRAFLAGS := -lyosys -pthread
 
-OBJTEST := objtest
-BINTEST := bintest
+MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+OBJTEST := $(MAKEFILE_DIR)objtest
+BINTEST := $(MAKEFILE_DIR)bintest
 
-ALLTESTFILE := $(shell find . -name '*Test.cc' | sed 's|^\./||' | tr '\n' ' ')
+ALLTESTFILE := $(shell cd $(MAKEFILE_DIR) && find . -name '*Test.cc' | sed 's|^\./||' | tr '\n' ' ')
 TESTDIRS := $(sort $(dir $(ALLTESTFILE)))
 TESTS := $(addprefix $(BINTEST)/, $(basename $(ALLTESTFILE:%Test.cc=%Test.o)))
 
@@ -34,7 +35,7 @@ $(BINTEST)/%: $(OBJTEST)/%.o | prepare
 	$(CXX) -L$(ROOTPATH) $(RPATH) $(LINKFLAGS) -o $@ $^ $(LIBS) \
 		$(GTEST_LDFLAGS) $(EXTRAFLAGS)
 
-$(OBJTEST)/%.o: $(basename $(subst $(OBJTEST),.,%)).cc | prepare
+$(OBJTEST)/%.o: $(MAKEFILE_DIR)/%.cc | prepare
 	$(CXX) -o $@ -c -I$(ROOTPATH) $(CPPFLAGS) $(CXXFLAGS) $(GTEST_CXXFLAGS) $^
 
 .PHONY: prepare run-tests clean