CI: Update Testing Options (#625)

* Always enable testing * Consistent naming of CMake testing options * Add warnings/errors for some testing option combinations * Add documentation for testing options * Move testing README into testing docs * Cleanup using default testing options * Enable running test in containers without enabling development tests * Simplify setting pre and post executable run commands/options in sundials_add_test * Remove redundant Jenkins tests * Remove precision dependent diff option in unit tests
LLNL · Dec 19, 2024 · ab2f4ad · ab2f4ad
1 parent 6a615a9
commit ab2f4ad
Show file tree

Hide file tree

Showing 40 changed files with 1,129 additions and 949 deletions.
diff --git a/.github/workflows/ubuntu-clang-latest.yml b/.github/workflows/ubuntu-clang-latest.yml
@@ -44,8 +44,8 @@ jobs:
         -D SUNDIALS_LOGGING_LEVEL=${{matrix.logging_level}} \
         -D ENABLE_ALL_WARNINGS=ON \
         -D ENABLE_WARNINGS_AS_ERRORS=ON \
-        -D SUNDIALS_TEST_DEVTESTS=ON \
-        -D SUNDIALS_TEST_UNITTESTS=ON \
+        -D SUNDIALS_TEST_ENABLE_DEV_TESTS=ON \
+        -D SUNDIALS_TEST_ENABLE_UNIT_TESTS=ON \
         -D SUNDIALS_TEST_ENABLE_GTEST=OFF \
         -D SUNDIALS_TEST_FLOAT_PRECISION=4 \
         -D SUNDIALS_TEST_INTEGER_PRECISION=10

diff --git a/.github/workflows/windows-latest-intel.yml b/.github/workflows/windows-latest-intel.yml
@@ -30,7 +30,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Configure CMake (Static)
-      run: cmake -G "Ninja" -B ${{github.workspace}}/build_static -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_COMPILER=icx-cl -DCMAKE_CXX_COMPILER=icx-cl -DCMAKE_Fortran_COMPILER=ifx -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_FORTRAN_MODULE_INTERFACE=ON -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_TEST_UNITTESTS=OFF -DEXAMPLES_ENABLE_CXX=ON
+      run: cmake -G "Ninja" -B ${{github.workspace}}/build_static -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_COMPILER=icx-cl -DCMAKE_CXX_COMPILER=icx-cl -DCMAKE_Fortran_COMPILER=ifx -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_FORTRAN_MODULE_INTERFACE=ON -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_TEST_ENABLE_UNIT_TESTS=OFF -DEXAMPLES_ENABLE_CXX=ON
 
     - name: Build (Static)
       run: cmake --build ${{github.workspace}}/build_static --verbose
@@ -40,7 +40,7 @@ jobs:
       run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure
 
     - name: Configure CMake (Shared)
-      run: cmake -G "Ninja" -B ${{github.workspace}}/build_shared -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_COMPILER=icx-cl -DCMAKE_CXX_COMPILER=icx-cl -DCMAKE_Fortran_COMPILER=ifx -DBUILD_STATIC_LIBS=OFF -DBUILD_SHARED_LIBS=ON -DBUILD_FORTRAN_MODULE_INTERFACE=ON -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_TEST_UNITTESTS=OFF -DEXAMPLES_ENABLE_CXX=ON
+      run: cmake -G "Ninja" -B ${{github.workspace}}/build_shared -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_COMPILER=icx-cl -DCMAKE_CXX_COMPILER=icx-cl -DCMAKE_Fortran_COMPILER=ifx -DBUILD_STATIC_LIBS=OFF -DBUILD_SHARED_LIBS=ON -DBUILD_FORTRAN_MODULE_INTERFACE=ON -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_TEST_ENABLE_UNIT_TESTS=OFF -DEXAMPLES_ENABLE_CXX=ON
 
     - name: Build (Shared)
       run: cmake --build ${{github.workspace}}/build_shared --verbose

diff --git a/.github/workflows/windows-latest-mingw.yml b/.github/workflows/windows-latest-mingw.yml
@@ -58,7 +58,7 @@ jobs:
           -DBUILD_SHARED_LIBS=OFF \
           -DSUNDIALS_BUILD_WITH_PROFILING=ON \
           -DSUNDIALS_LOGGING_LEVEL=2 \
-          -DSUNDIALS_TEST_UNITTESTS=OFF \
+          -DSUNDIALS_TEST_ENABLE_UNIT_TESTS=OFF \
           -DEXAMPLES_ENABLE_CXX=ON \
           -DENABLE_KLU=ON
 
@@ -85,7 +85,7 @@ jobs:
           -DBUILD_STATIC_LIBS=OFF \
           -DSUNDIALS_BUILD_WITH_PROFILING=ON \
           -DSUNDIALS_LOGGING_LEVEL=2 \
-          -DSUNDIALS_TEST_UNITTESTS=OFF \
+          -DSUNDIALS_TEST_ENABLE_UNIT_TESTS=OFF \
           -DEXAMPLES_ENABLE_CXX=ON \
           -DENABLE_KLU=ON
 

diff --git a/.github/workflows/windows-latest.yml b/.github/workflows/windows-latest.yml
@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Configure CMake
-      run: cmake -G "Visual Studio 17 2022" -B ${{github.workspace}}/build -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_TEST_UNITTESTS=ON -DEXAMPLES_ENABLE_CXX=ON
+      run: cmake -G "Visual Studio 17 2022" -B ${{github.workspace}}/build -DSUNDIALS_BUILD_WITH_PROFILING=ON -DSUNDIALS_TEST_ENABLE_UNIT_TESTS=ON -DEXAMPLES_ENABLE_CXX=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --verbose

diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh
@@ -19,15 +19,15 @@ echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 export CUDA_LAUNCH_BLOCKING=1
 
 if [[ -d ${build_dir} ]]
-then 
+then
 
     # configure for multinode
     $cmake_exe \
         -C "${hostconfig_path}" \
         -DSUNDIALS_BENCHMARK_NUM_CPUS=${nresc} \
         -DSUNDIALS_BENCHMARK_NUM_GPUS=${nresg} \
         "${project_dir}"
-    
+
     cd ${build_dir}
     make benchmark
 
@@ -37,4 +37,3 @@ echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo "~~~~~ CLEAN UP"
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 make clean
-
diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh
@@ -178,7 +178,8 @@ then
             -DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \
             -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \
             -DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \
-            -DSUNDIALS_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \
+            -DSUNDIALS_TEST_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \
+            -DSUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \
             "${project_dir}"
 
     else

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -199,6 +199,12 @@ include(SundialsBuildOptionsPost)
 
 include(SundialsSetupConfig)
 
+# ===============================================================
+# Setup testing
+# ===============================================================
+
+include(SundialsSetupTesting)
+
 # ===============================================================
 # Add src and tests; optionally add examples and unit tests.
 # ===============================================================
@@ -208,17 +214,17 @@ add_subdirectory(src)
 
 # Add selected examples to the build
 if(_BUILD_EXAMPLES)
-  include(SundialsSetupTesting)
   add_subdirectory(examples)
 endif()
 
 # Add benchmarks to the build
 if(BUILD_BENCHMARKS)
+  include(SundialsSetupBenchmarks)
   add_subdirectory(benchmarks)
 endif()
 
-# Add unit tests to the build if they are enabled
-if(SUNDIALS_TEST_UNITTESTS)
+# Add unit tests to the build
+if(SUNDIALS_TEST_ENABLE_UNIT_TESTS)
   add_subdirectory(test/unit_tests)
 endif()
 

diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake
@@ -311,26 +311,85 @@ endif()
 # Options for SUNDIALS testing
 # ---------------------------------------------------------------
 
+sundials_option(SUNDIALS_TEST_ENABLE_DEV_TESTS BOOL "Include development tests"
+                OFF ADVANCED)
+
+sundials_option(SUNDIALS_TEST_ENABLE_UNIT_TESTS BOOL "Include unit tests" OFF
+                ADVANCED)
+
+if(SUNDIALS_TEST_ENABLE_UNIT_TESTS)
+  set(_default_gtest ON)
+else()
+  set(_default_gtest OFF)
+endif()
+
+sundials_option(SUNDIALS_TEST_ENABLE_GTEST BOOL "Include GTest unit tests"
+                ${_default_gtest} ADVANCED)
+
+if(SUNDIALS_TEST_ENABLE_GTEST AND NOT SUNDIALS_TEST_ENABLE_UNIT_TESTS)
+  message(
+    FATAL_ERROR "Unit tests with Google test are enabled but unit tests are OFF"
+  )
+endif()
+
+if(SUNDIALS_TEST_ENABLE_UNIT_TESTS AND NOT SUNDIALS_TEST_ENABLE_GTEST)
+  message(
+    WARNING
+      "Unit tests are enabled but unit tests with Google test are OFF. Some "
+      "unit test will not be run.")
+endif()
+
+if(SUNDIALS_TEST_ENABLE_DEV_TESTS OR SUNDIALS_TEST_ENABLE_UNIT_TESTS)
+  set(_default_diff_output ON)
+else()
+  set(_default_diff_output OFF)
+endif()
+
+sundials_option(
+  SUNDIALS_TEST_ENABLE_DIFF_OUTPUT BOOL
+  "Compare test output with saved answer files" ${_default_diff_output}
+  ADVANCED)
+
+if((SUNDIALS_TEST_ENABLE_DEV_TESTS OR SUNDIALS_TEST_ENABLE_UNIT_TESTS)
+   AND NOT SUNDIALS_TEST_ENABLE_DIFF_OUTPUT)
+  message(
+    WARNING "Development or unit tests are enabled but output comparison is OFF"
+  )
+endif()
+
 sundials_option(
   SUNDIALS_TEST_FLOAT_PRECISION STRING
-  "Precision for floating point comparisons (number of digits)" "-1" ADVANCED)
+  "Precision for floating point comparisons (number of digits)" "4" ADVANCED)
 
 sundials_option(
   SUNDIALS_TEST_INTEGER_PRECISION STRING
-  "Precision for integer comparisons (percent difference)" "-1" ADVANCED)
+  "Precision for integer comparisons (percent difference)" "10" ADVANCED)
+
+sundials_option(
+  SUNDIALS_TEST_OUTPUT_DIR PATH "Location to write test output files"
+  "${PROJECT_BINARY_DIR}/Testing/output" ADVANCED)
 
-sundials_option(SUNDIALS_TEST_OUTPUT_DIR PATH
-                "Location to write testing output files" "" ADVANCED)
+sundials_option(SUNDIALS_TEST_ANSWER_DIR PATH "Location of test answer files"
+                "" ADVANCED)
 
-sundials_option(SUNDIALS_TEST_ANSWER_DIR PATH
-                "Location of testing answer files" "" ADVANCED)
+if(SUNDIALS_TEST_ENABLE_DIFF_OUTPUT AND NOT SUNDIALS_TEST_ANSWER_DIR)
+  message(
+    WARNING
+      "Test output comparison is enabled but an answer directory was not "
+      "supplied. Using the default answer files may produce erroneous test "
+      "failures due to hardware or round-off differences.")
+endif()
 
-sundials_option(SUNDIALS_TEST_PROFILE BOOL
-                "Use Caliper to profile SUNDIALS tests" OFF ADVANCED)
+sundials_option(SUNDIALS_TEST_ENABLE_PROFILING BOOL "Profile tests" OFF
+                ADVANCED)
 
 sundials_option(
-  SUNDIALS_TEST_NODIFF BOOL
-  "Disable output comparison in the regression test suite" OFF ADVANCED)
+  SUNDIALS_TEST_CALIPER_OUTPUT_DIR PATH "Location to write test Caliper files"
+  "${PROJECT_BINARY_DIR}/Testing/caliper" ADVANCED)
+
+# ---------------------------------------------------------------
+# Options for SUNDIALS testing with containers
+# ---------------------------------------------------------------
 
 sundials_option(SUNDIALS_TEST_CONTAINER_EXE PATH "Path to docker or podman" ""
                 ADVANCED)
@@ -344,29 +403,31 @@ sundials_option(
   SUNDIALS_TEST_CONTAINER_MNT STRING
   "Path to project root inside the container" "/sundials" ADVANCED)
 
-# Include development examples in regression tests
-sundials_option(SUNDIALS_TEST_DEVTESTS BOOL
-                "Include development tests in make test" OFF ADVANCED)
-
-# Include unit tests in regression tests
-sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test"
-                OFF ADVANCED)
-
-# Include googletest unit tests in regression tests
-sundials_option(SUNDIALS_TEST_ENABLE_GTEST BOOL "Disable GTest unit tests" ON
-                ADVANCED)
+# ---------------------------------------------------------------
+# Options for SUNDIALS development
+# ---------------------------------------------------------------
 
 sundials_option(SUNDIALS_DEV_IWYU BOOL "Enable include-what-you-use" OFF
                 ADVANCED)
 
 sundials_option(SUNDIALS_DEV_CLANG_TIDY BOOL "Enable clang-tidy" OFF ADVANCED)
 
+# ---------------------------------------------------------------
+# Options for SUNDIALS benchmarks
+# ---------------------------------------------------------------
+
 sundials_option(
   SUNDIALS_SCHEDULER_COMMAND STRING
   "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED)
 
-sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR PATH
-                "Location to write caliper output files" "" ADVANCED)
+sundials_option(
+  SUNDIALS_BENCHMARK_OUTPUT_DIR PATH "Location to write benchmark output files"
+  "${PROJECT_BINARY_DIR}/Benchmarking/output" ADVANCED)
+
+sundials_option(
+  SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR PATH
+  "Location to write benchmark caliper files"
+  "${PROJECT_BINARY_DIR}/Benchmarking/caliper" ADVANCED)
 
 sundials_option(SUNDIALS_BENCHMARK_NUM_CPUS STRING
                 "Number of CPU cores to run benchmarks with" "40" ADVANCED)

diff --git a/cmake/SundialsDeprecated.cmake b/cmake/SundialsDeprecated.cmake
@@ -189,3 +189,64 @@ if(DEFINED CUDA_ARCH)
       CACHE STRING "CUDA Architectures" FORCE)
   unset(CUDA_ARCH)
 endif()
+
+#
+# Deprecated Testing Options
+#
+
+if(SUNDIALS_TEST_PROFILE)
+  message(DEPRECATION "The CMake option SUNDIALS_TEST_PROFILE is deprecated. "
+                      "Use SUNDIALS_TEST_ENABLE_PROFILING instead.")
+  set(SUNDIALS_TEST_ENABLE_PROFILING
+      ${SUNDIALS_TEST_PROFILE}
+      CACHE BOOL "Profile tests" FORCE)
+  unset(SUNDIALS_TEST_PROFILE)
+endif()
+
+if(SUNDIALS_TEST_NODIFF)
+  message(DEPRECATION "The CMake option SUNDIALS_TEST_NODIFF is deprecated. "
+                      "Use SUNDIALS_TEST_ENABLE_DIFF_OUTPUT instead.")
+  if(SUNDIALS_TEST_NODIFF)
+    set(_new_value OFF)
+  else()
+    set(_new_value ON)
+  endif()
+  set(SUNDIALS_TEST_ENABLE_DIFF_OUTPUT
+      ${_new_value}
+      CACHE BOOL "Compare test output with saved answer files" FORCE)
+  unset(SUNDIALS_TEST_NODIFF)
+  unset(_new_value)
+endif()
+
+if(SUNDIALS_TEST_DEVTESTS)
+  message(DEPRECATION "The CMake option SUNDIALS_TEST_DEVTESTS is deprecated. "
+                      "Use SUNDIALS_TEST_ENABLE_DEV_TESTS instead.")
+  set(SUNDIALS_TEST_ENABLE_DEV_TESTS
+      ${SUNDIALS_TEST_DEVTESTS}
+      CACHE BOOL "Include development tests" FORCE)
+  unset(SUNDIALS_TEST_DEVTESTS)
+endif()
+
+if(SUNDIALS_TEST_UNITTESTS)
+  message(DEPRECATION "The CMake option SUNDIALS_TEST_UNITTESTS is deprecated. "
+                      "Use SUNDIALS_TEST_ENABLE_UNIT_TESTS instead.")
+  set(SUNDIALS_TEST_ENABLE_UNIT_TESTS
+      ${SUNDIALS_TEST_UNITTESTS}
+      CACHE BOOL "Include units tests" FORCE)
+  unset(SUNDIALS_TEST_UNITTESTS)
+endif()
+
+if(SUNDIALS_CALIPER_OUTPUT_DIR)
+  message(
+    DEPRECATION
+      "The CMake option SUNDIALS_CALIPER_OUTPUT_DIR is deprecated. "
+      "Use SUNDIALS_TEST_CALIPER_OUTPUT_DIR and SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR instead."
+  )
+  set(SUNDIALS_TEST_CALIPER_OUTPUT_DIR
+      ${SUNDIALS_TEST_CALIPER_OUTPUT_DIR}
+      CACHE PATH "Location to write test caliper files" FORCE)
+  set(SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR
+      ${SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR}
+      CACHE PATH "Location to write benchmark caliper files" FORCE)
+  unset(SUNDIALS_CALIPER_OUTPUT_DIR)
+endif()
diff --git a/cmake/SundialsSetupBenchmarks.cmake b/cmake/SundialsSetupBenchmarks.cmake
@@ -0,0 +1,55 @@
+# ---------------------------------------------------------------
+# Programmer(s): David J. Gardner @ LLNL
+# ---------------------------------------------------------------
+# SUNDIALS Copyright Start
+# Copyright (c) 2002-2024, Lawrence Livermore National Security
+# and Southern Methodist University.
+# All rights reserved.
+#
+# See the top-level LICENSE and NOTICE files for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# SUNDIALS Copyright End
+# ---------------------------------------------------------------
+# Enable SUNDIALS Benchmarks
+# ---------------------------------------------------------------
+
+#
+# Check if the test runner is needed
+#
+if(NOT TESTRUNNER)
+  # Python is needed to use the test runner
+  find_package(Python3 REQUIRED)
+  # Look for the testRunner script in the test directory
+  find_program(
+    TESTRUNNER testRunner
+    PATHS test
+    NO_DEFAULT_PATH REQUIRED)
+  message(STATUS "Found testRunner: ${TESTRUNNER}")
+  set(TESTRUNNER
+      ${TESTRUNNER}
+      CACHE INTERNAL "")
+endif()
+
+# Create the benchmark output directory
+if(NOT EXISTS ${SUNDIALS_BENCHMARK_OUTPUT_DIR})
+  file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_OUTPUT_DIR})
+endif()
+message(STATUS "Benchmark output directory: ${SUNDIALS_BENCHMARK_OUTPUT_DIR}")
+
+if(ENABLE_CALIPER)
+  message(STATUS "Enabled benchmark profiling with Caliper")
+  if(NOT EXISTS ${SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR})
+    file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR})
+  endif()
+  message(
+    STATUS
+      "Benchmark Caliper output directory: ${SUNDIALS_BENCHMARK_CALIPER_OUTPUT_DIR}"
+  )
+endif()
+
+#
+# Create `make benchmark`
+#
+add_custom_target(benchmark ${CMAKE_COMMAND} -E cmake_echo_color --cyan
+                            "All benchmarks complete.")
diff --git a/cmake/SundialsSetupCompilers.cmake b/cmake/SundialsSetupCompilers.cmake
@@ -421,7 +421,7 @@ endif()
 # ===============================================================
 
 if(BUILD_BENCHMARKS
-   OR SUNDIALS_TEST_UNITTESTS
+   OR SUNDIALS_TEST_ENABLE_UNIT_TESTS
    OR EXAMPLES_ENABLE_CXX
    OR ENABLE_CUDA
    OR ENABLE_HIP