Merge pull request #5959 from YosysHQ/improve_test

Improve test
2026-06-17 06:16:22 +00:00 · 2026-06-16 08:46:11 +00:00 · 2026-06-16 08:46:11 +00:00 · e2903c4a5c
commit e2903c4a5c
parent 211fa48ce5 3af45e7d04
5 changed files with 52 additions and 80 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -39,6 +39,7 @@ set(CMAKE_CXX_SCAN_FOR_MODULES NO)
 set(YOSYS_COMPILER_LAUNCHER "" CACHE STRING "Compiler launcher (ccache, sccache)")
 option(YOSYS_ENABLE_COVERAGE "Enable code coverage" OFF)
 option(YOSYS_ENABLE_PROFILING "Enable instruction profiling" OFF)
+option(YOSYS_ENABLE_FUNCTIONAL_TESTS "Enable running functional tests" OFF)

 set(YOSYS_PROGRAM_PREFIX "" CACHE STRING "Name prefix for programs, libraries, and data")
 set(YOSYS_COMPONENTS "everything" CACHE STRING "List of components to build (use pass names)")
@ -534,8 +535,19 @@ if (NOT YOSYS_BUILD_PYTHON_ONLY)

 	add_custom_target(test-vanilla
 		COMMAND make vanilla-test ${makefile_vars}
+			ENABLE_FUNCTIONAL_TESTS=$<IF:$<BOOL:${YOSYS_ENABLE_FUNCTIONAL_TESTS}>,1,0>
 		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests
 		DEPENDS ${makefile_depends}
+		USES_TERMINAL
+		JOB_SERVER_AWARE TRUE
+	)
+
+	add_custom_target(test-functional
+		COMMAND make functional ${makefile_vars}
+			ENABLE_FUNCTIONAL_TESTS=$<IF:$<BOOL:${YOSYS_ENABLE_FUNCTIONAL_TESTS}>,1,0>
+		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests
+		DEPENDS ${makefile_depends}
+		USES_TERMINAL
 		JOB_SERVER_AWARE TRUE
 	)

--- a/docs/source/yosys_internals/extending_yosys/test_suites.rst
+++ b/docs/source/yosys_internals/extending_yosys/test_suites.rst
@ -16,6 +16,20 @@ tests.
   cmake -B build .
   cmake --build build --target test --parallel $(nproc)

+.. warning::
+
+   There are limitations when using `Ninja` as generator, so we suggest using
+   `Unix Makefiles` to make running tests in parallel possible. However, it is
+   possible to use it directly by running:
+
+.. code:: console
+
+   cd tests
+   make -j9
+
+Please note that in this case default build directory is `build` but can be
+overwritten by providing `BUILD_DIR` variable.
+
 Vanilla tests
 ~~~~~~~~~~~~~

@ -76,15 +90,19 @@ If you don't have one of the :ref:`getting_started/installation:CAD suite(s)`
 installed, you should also install Z3 `following their
 instructions <https://github.com/Z3Prover/z3>`_.

-.. TODO:: CMAKE_TODO
+Functional tests are disabled by default, to enable them use next code snippet
+and run tests as usual:

-   How does this work under CMake?  Is it only via ``make -C tests
-   ENABLE_FUNCTIONAL_TESTS=1`` and then manually setting ``BUILD_DIR`` and
-   ``PROGRAM_PREFIX``?  And possibly also setting ``YOSYS`` et al if there is a
-   ``.exe``.  Previous instructions:
+.. code:: console

-   Then, set the :makevar:`ENABLE_FUNCTIONAL_TESTS` make variable when calling
-   ``make test`` and the functional tests will be run as well.
+   cmake -B build . -DYOSYS_ENABLE_FUNCTIONAL_TESTS=ON
+   cmake --build build --target test --parallel $(nproc)
+
+Or run just functional tests with:
+
+.. code:: console
+
+   cmake --build build --target test-functional

 Docs tests
 ~~~~~~~~~~
@ -164,6 +182,9 @@ compiler versions.  For up to date information, including OS versions, refer to
   test for ``kernel/celledges.cc``, you will need to create a file like this:
   ``tests/unit/kernel/celledgesTest.cc``;
   * Implement your unit test
+   * Add unit test to file list in `CMakeLists.txt`
+   In case unit tests are added to new directory, note that you need also to
+   create new `CmakeList.txt` file and add ``yosys_gtest(dir-name unit-test.cc)```

   Run unit tests
   ~~~~~~~~~~~~~~
@ -172,10 +193,5 @@ compiler versions.  For up to date information, including OS versions, refer to

   .. code-block:: console

-      make unit-test
+      cmake --build build --target test-unit

-   If you want to remove all unit test files, type:
-
-   .. code-block:: console
-
-      make clean-unit-test
--- a/tests/common.mk
+++ b/tests/common.mk
@ -28,6 +28,12 @@ export YOSYS_MAX_THREADS
 export LLVM_PROFILE_FILE
 export LLVM_PROFILE_FILE_BUFFER_SIZE=0

+ifeq ($(or $(V),$(VERBOSE)),1)
+  QUIET :=
+else
+  QUIET := >/dev/null 2>&1
+endif
+
 all:

 ifndef OVERRIDE_MAIN
@ -38,7 +44,7 @@ endif
 define run_test
 	@set -e; \
 	rc=0; \
-	( set -e; $(2) ) >/dev/null 2>&1 || rc=$$?; \
+	( set -e; $(2) ) $(QUIET) || rc=$$?; \
 	if [ $$rc -eq 0 ]; then \
 		echo "PASS $1"; \
 		echo PASS > $1.result; \
--- a/tests/gen_tests_makefile.py
+++ b/tests/gen_tests_makefile.py
@ -26,27 +26,27 @@ def generate_target(name, command, deps = None):
    print(f"\t@$(call run_test,{target}, $({target}_cmd))")

 def generate_ys_test(ys_file, yosys_args="", commands=""):
-    cmd = f'$(YOSYS) -ql {ys_file}.err {yosys_args} {ys_file} && mv {ys_file}.err {ys_file}.log'
+    cmd = f'$(YOSYS) -l {ys_file}.err {yosys_args} {ys_file} && mv {ys_file}.err {ys_file}.log'
    if commands:
        cmd += f"; \\\n{commands}"
    generate_target(ys_file, cmd)

 def generate_tcl_test(tcl_file, yosys_args="", commands=""):
-    cmd = f'$(YOSYS) -ql {tcl_file}.err {yosys_args} {tcl_file} && mv {tcl_file}.err {tcl_file}.log'
+    cmd = f'$(YOSYS) -l {tcl_file}.err {yosys_args} {tcl_file} && mv {tcl_file}.err {tcl_file}.log'
    if commands:
        cmd += f"; \\\n{commands}"
    generate_target(tcl_file, cmd)

 def generate_sv_check(sv_file, yosys_args="", yosys_cmds=""):
    yosys_cmd = f'read -sv {sv_file}; {yosys_cmds}'
-    cmd = f'$(YOSYS) -ql {sv_file}.err -p "{yosys_cmd}" {yosys_args} && mv {sv_file}.err {sv_file}.log'
+    cmd = f'$(YOSYS) -l {sv_file}.err -p "{yosys_cmd}" {yosys_args} && mv {sv_file}.err {sv_file}.log'
    generate_target(sv_file, cmd)

 def generate_sv_test(sv_file, yosys_args="", commands=""):
    base = os.path.splitext(sv_file)[0]
    if not os.path.exists(base + ".ys"):
        yosys_cmd = '-p "prep -top top; async2sync; sat -enable_undef -verify -prove-asserts"'
-        cmd = f'$(YOSYS) -ql {sv_file}.err {yosys_cmd} {yosys_args} {sv_file} && mv {sv_file}.err {sv_file}.log'
+        cmd = f'$(YOSYS) -l {sv_file}.err {yosys_cmd} {yosys_args} {sv_file} && mv {sv_file}.err {sv_file}.log'
        if commands:
            cmd += f"; \\\n{commands}"
        generate_target(sv_file, cmd)
--- a/tests/unit/Makefile
+++ b/tests/unit/Makefile
@ -1,62 +0,0 @@
-UNAME_S := $(shell uname -s)
-
-# GoogleTest flags
-GTEST_PREFIX := $(shell brew --prefix googletest 2>/dev/null)
-ifeq ($(GTEST_PREFIX),)
-  GTEST_CXXFLAGS :=
-  GTEST_LDFLAGS := -lgtest -lgmock -lgtest_main
-else
-  GTEST_CXXFLAGS := -I$(GTEST_PREFIX)/include
-  GTEST_LDFLAGS := -L$(GTEST_PREFIX)/lib -lgtest -lgmock -lgtest_main
-endif
-
-ifeq ($(UNAME_S),Darwin)
-  RPATH = -Wl,-rpath,$(ROOTPATH)
-else
-  RPATH = -Wl,-rpath=$(ROOTPATH)
-endif
-
-EXTRAFLAGS := -lyosys -pthread
-
-MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-OBJTEST := $(MAKEFILE_DIR)objtest
-BINTEST := $(MAKEFILE_DIR)bintest
-
-ALLTESTFILE := $(shell cd $(MAKEFILE_DIR) && find . -name '*Test.cc' | sed 's|^\./||' | tr '\n' ' ')
-TESTDIRS := $(sort $(dir $(ALLTESTFILE)))
-TESTS := $(addprefix $(BINTEST)/, $(basename $(ALLTESTFILE:%Test.cc=%Test.o)))
-
-# Prevent make from removing our .o files
-.SECONDARY:
-
-all: prepare $(TESTS) run-tests
-
-$(BINTEST)/%: $(OBJTEST)/%.o | prepare
-	$(CXX) -L$(ROOTPATH) $(RPATH) $(LINKFLAGS) -o $@ $^ $(LIBS) \
-		$(GTEST_LDFLAGS) $(EXTRAFLAGS)
-
-$(OBJTEST)/%.o: $(MAKEFILE_DIR)/%.cc | prepare
-	$(CXX) -o $@ -c -I$(ROOTPATH) $(CPPFLAGS) $(CXXFLAGS) $(GTEST_CXXFLAGS) $^
-
-.PHONY: prepare run-tests clean
-
-run-tests: $(TESTS)
-ifeq ($(UNAME_S),Darwin)
-	@for t in $^; do \
-		echo "Running $$t"; \
-		DYLD_LIBRARY_PATH=$(ROOTPATH) $$t || exit 1; \
-	done
-else
-	@for t in $^; do \
-		echo "Running $$t"; \
-		$$t || exit 1; \
-	done
-endif
-
-prepare:
-	mkdir -p $(addprefix $(BINTEST)/,$(TESTDIRS))
-	mkdir -p $(addprefix $(OBJTEST)/,$(TESTDIRS))
-
-clean:
-	rm -rf $(OBJTEST)
-	rm -rf $(BINTEST)