Merge Pull Request #12609 from trilinos/Trilinos/master_merge_2023121…

…5_175959

Automatically Merged using Trilinos Master Merge AutoTester
PR Title: b'Trilinos Master Merge PR Generator: Auto PR created to promote from master_merge_20231215_175959 branch to master'
PR Author: trilinos-autotester

cmake/SimpleTesting/cmake/ctest-cdash-setup.cmake

@@ -36,6 +36,13 @@ endif()
 list(APPEND CTEST_CONFIGURE_COMMAND_ARGS
     "-C \"${package_enables_file}\""
     "-G \"${CTEST_CMAKE_GENERATOR}\""
+)
+
+if(EXTRA_CONFIGURE_ARGS)
+    list(APPEND CTEST_CONFIGURE_COMMAND_ARGS ${EXTRA_CONFIGURE_ARGS})
+endif()
+
+list(APPEND CTEST_CONFIGURE_COMMAND_ARGS
     "${CTEST_SOURCE_DIRECTORY}"
 )
 

packages/PyTrilinos2/CMakeLists.txt

@@ -379,7 +379,7 @@ ADD_SUBDIRECTORY( src )
 file(COPY ${PyTrilinos2PyFiles} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/PyTrilinos2/.)
 #file(COPY ${PyTrilinos2PyFilesSo} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/PyTrilinos2/.)
 
-SET(PyTrilinos2_PYTHONPATH "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}")
+SET(PyTrilinos2_PYTHONPATH "${CMAKE_CURRENT_BINARY_DIR}:$ENV{PYTHONPATH}")
 
 TRIBITS_ADD_TEST_DIRECTORIES(test)
 

packages/PyTrilinos2/cmake/PyTrilinos2MakeTest.cmake

@@ -52,10 +52,8 @@ MACRO(PyTrilinos2_MAKE_MPI_TEST TEST_NAME)
     NAME ${TEST_NAME}
     ARGS "${TEST_NAME}.py"
     PASS_REGULAR_EXPRESSION "OK"
+    ENVIRONMENT "PYTHONPATH=${PyTrilinos2_PYTHONPATH}"
     ${ARGN}
     )
 
-  tribits_set_tests_properties(PyTrilinos2_${TEST_NAME}_MPI_4
-    PROPERTIES ENVIRONMENT "${PYTHON_LIBRARY_PREFIX}:${PyTrilinos2_PYTHONPATH}")
-
 ENDMACRO(PyTrilinos2_MAKE_MPI_TEST TEST_NAME)

packages/amesos2/src/Amesos2_Superludist_TypeMap.hpp

@@ -69,10 +69,13 @@
 
 #include "Amesos2_TypeMap.hpp"
 
+#ifdef KOKKOS_ENABLE_CUDA
+  #include <cublas_v2.h>
+  #include <cuda_runtime_api.h>
+#endif
 
-namespace SLUD {
 
-extern "C" {
+namespace SLUD {
 
 #if SUPERLU_DIST_MAJOR_VERSION > 4
 // SuperLU_Dist before major version 5 does not contain the config file
@@ -108,9 +111,6 @@ extern "C" {
   }
 #endif  // HAVE_TEUCHOS_COMPLEX
 
-
-} // end extern "C"
-
 // multiplication of SLUD types
 template <typename slu_scalar_t, typename slu_mag_t>
 struct slu_dist_mult {};

packages/amesos2/test/adapters/Tpetra_CrsMatrix_Adapter_UnitTests.cpp

@@ -118,9 +118,6 @@ namespace {
 
   // Where to look for input files
   string filedir;
-
-
-
   TEUCHOS_STATIC_SETUP()
   {
     Teuchos::CommandLineProcessor &clp = Teuchos::UnitTestRepository::getCLP();
@@ -202,7 +199,7 @@ namespace {
     }
     eye->fillComplete();
     // Create using non-member function
-    RCP<ADAPT> adapter  = Amesos2::createMatrixAdapter<MAT>(eye);
+    //RCP<ADAPT> adapter  = Amesos2::createMatrixAdapter<MAT>(eye);
 
     // The following should all pass at compile time
     TEST_ASSERT( (std::is_same_v<Scalar,typename ADAPT::scalar_t>) );
@@ -435,9 +432,6 @@ namespace {
     RCP<const Comm<int> > comm = Tpetra::getDefaultComm();
     const size_t numprocs = comm->getSize();
     const size_t rank     = comm->getRank();
-    // create a Map for our matrix
-    global_size_t nrows = 6;
-    RCP<const Map<LO,GO,Node> > map = createUniformContigMap<LO,GO>(nrows,comm);
 
     /* We will be using the following matrix for this test (amesos2_test_mat0[_complex].mtx):
      *

packages/amesos2/test/solvers/SolverFactory.cpp

@@ -106,9 +106,7 @@ namespace {
     typedef Tpetra::Map<LO,GO,NT> map_type;
     //typedef Teuchos::ScalarTraits<SC> STS; // unused
 
-    Teuchos::OSTab tab0 (out);
     out << "Create test matrix with " << gblNumRows << " row(s)" << endl;
-    Teuchos::OSTab tab1 (out);
 
     TEUCHOS_TEST_FOR_EXCEPTION
       ( gblNumRows == 0, std::invalid_argument, "gblNumRows = 0");
@@ -208,9 +206,7 @@ namespace {
     typedef typename MV::mag_type mag_type;
     typedef Teuchos::ScalarTraits<mag_type> STM;
 
-    Teuchos::OSTab tab0 (out);
     out << "Test solver \"" << solverName << "\" from Amesos2 package" << endl;
-    Teuchos::OSTab tab1 (out);
 
     // NDE: Beginning changes towards passing parameter list to shylu basker
     // for controlling various parameters per test, matrix, etc.

packages/compadre/src/Compadre_Operators.hpp

@@ -8,7 +8,7 @@
 namespace Compadre {
 
     //! Available target functionals
-    enum TargetOperation {
+    enum TargetOperation : int {
         //! Point evaluation of a scalar
         ScalarPointEvaluation,
         //! Point evaluation of a vector (reconstructs entire vector at once, requiring a 
@@ -92,7 +92,7 @@ namespace Compadre {
     }
 
     //! Space in which to reconstruct polynomial
-    enum ReconstructionSpace {
+    enum ReconstructionSpace : int {
         //! Scalar polynomial basis centered at the target site and scaled by sum of basis powers 
         //! e.g. \f$(x-x_t)^2*(y-y_t)*(z-z_t)^3/factorial(2+1+3)\f$ would be a member of 3rd order in 3D, where 
         //! \f$(x_t,y_t,z_t)\f$ is the coordinate of the target site in 3D coordinates.
@@ -122,7 +122,7 @@ namespace Compadre {
     }
 
     //! Describes the SamplingFunction relationship to targets, neighbors
-    enum SamplingTransformType {
+    enum SamplingTransformType : int {
         Identity,           ///< No action performed on data before GMLS target operation
         SameForAll,         ///< Each neighbor for each target all apply the same transform
         DifferentEachTarget,   ///< Each target applies a different data transform, but the same to each neighbor
@@ -206,15 +206,15 @@ namespace Compadre {
         CellIntegralSample = make_sampling_functional(0,0,false,false,(int)DifferentEachNeighbor);
 
     //! Dense solver type
-    enum DenseSolverType {
+    enum DenseSolverType : int {
         //! QR+Pivoting factorization performed on P*sqrt(w) matrix
         QR, 
         //! LU factorization performed on P^T*W*P matrix
         LU, 
     };
 
     //! Problem type, that optionally can handle manifolds
-    enum ProblemType {
+    enum ProblemType : int {
         //! Standard GMLS problem type
         STANDARD, 
         //! Solve GMLS problem on a manifold (will use QR or SVD to solve the resultant GMLS 
@@ -223,15 +223,15 @@ namespace Compadre {
     };
 
     //! Constraint type
-    enum ConstraintType {
+    enum ConstraintType : int {
         //! No constraint
         NO_CONSTRAINT,
         //! Neumann Gradient Scalar Type
         NEUMANN_GRAD_SCALAR,
     };
 
     //! Available weighting kernel function types
-    enum WeightingFunctionType {
+    enum WeightingFunctionType : int {
         Power,
         Gaussian,
         CubicSpline,
@@ -241,7 +241,7 @@ namespace Compadre {
 
     //! Coordinate type for input and output format of vector data on manifold problems.
     //! Anything without a manifold is always Ambient.
-    enum CoordinatesType {
+    enum CoordinatesType : int {
         Ambient, ///< a 2D manifold in 3D in ambient coordinates would have 3 components for a vector
         Local,   ///< a 2D manifold in 3D in local coordinates would have 2 components for a vector
     };

packages/compadre/src/Compadre_Quadrature.hpp

@@ -8,7 +8,7 @@
 namespace Compadre {
 
 // ENUM for quadrature types
-enum QuadratureType {INVALID, LINE, TRI, QUAD, TET, HEX};
+enum QuadratureType : int {INVALID, LINE, TRI, QUAD, TET, HEX};
 
 //!  Quadrature
 /*!

packages/framework/ini-files/config-specs.ini

@@ -1863,12 +1863,12 @@ use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 
 use COMMON
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                            STRING       : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                            STRING       : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                         BOOL         : ON
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D          BOOL         : ON
 opt-set-cmake-var KokkosKernels_blas_serial_MPI_1_DISABLE                BOOL         : ON
 opt-set-cmake-var ROL_example_PDE-OPT_helmholtz_example_02_MPI_1_DISABLE BOOL         : ON
-opt-set-cmake-var CMAKE_CXX_FLAGS                                        STRING FORCE : ${CMAKE_CXX_FLAGS|CMAKE} -Wall -Wno-clobbered -Wno-vla -Wno-pragmas -Wno-unknown-pragmas -Wno-unused-local-typedefs -Wno-literal-suffix -Wno-deprecated-declarations -Wno-misleading-indentation -Wno-int-in-bool-context -Wno-maybe-uninitialized -Wno-nonnull-compare -Wno-address -Wno-inline -DTRILINOS_HIDE_DEPRECATED_HEADER_WARNINGS
+opt-set-cmake-var CMAKE_CXX_FLAGS                                        STRING FORCE : -Wall -Wno-clobbered -Wno-vla -Wno-pragmas -Wno-unknown-pragmas -Wno-unused-local-typedefs -Wno-literal-suffix -Wno-deprecated-declarations -Wno-misleading-indentation -Wno-int-in-bool-context -Wno-maybe-uninitialized -Wno-nonnull-compare -Wno-address -Wno-inline -DTRILINOS_HIDE_DEPRECATED_HEADER_WARNINGS
 
 # Test failures as of 11-28-22
 opt-set-cmake-var ROL_example_PDE-OPT_navier-stokes_example_01_MPI_4_DISABLE BOOL : ON
@@ -1956,7 +1956,7 @@ use USE-DEPRECATED|YES
 
 use COMMON
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Tpetra_INST_SERIAL                            BOOL FORCE : ON
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL       : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL       : OFF
@@ -1982,7 +1982,7 @@ use USE-DEPRECATED|NO
 
 use COMMON
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Tpetra_INST_SERIAL                            BOOL FORCE : ON
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL       : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL       : OFF
@@ -2010,7 +2010,7 @@ use USE-DEPRECATED|YES
 use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 use COMMON
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING       : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING       : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL         : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL         : OFF
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D BOOL         : ON
@@ -2042,7 +2042,7 @@ use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 
 use COMMON
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D BOOL       : ON
 
 use RHEL7_TEST_DISABLES|CLANG
@@ -2090,7 +2090,7 @@ use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D BOOL       : ON
 
 use RHEL7_TEST_DISABLES|CLANG
@@ -2164,7 +2164,7 @@ use USE-DEPRECATED|YES
 use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 
 opt-set-cmake-var TPL_Netcdf_LIBRARIES STRING FORCE : -L${NETCDF_C_LIB|ENV}/lib;${NETCDF_C_LIB|ENV}/libnetcdf.so;${PARALLEL_NETCDF_LIB|ENV}/libpnetcdf.a
 opt-set-cmake-var TPL_HDF5_LIBRARIES STRING FORCE : ${HDF5_LIB|ENV}/libhdf5_hl.so;${HDF5_LIB|ENV}/libhdf5.a;${ZLIB_LIB|ENV}/libz.a;-ldl
@@ -2646,7 +2646,7 @@ use USE-MPI|YES
 use USE-PT|NO
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                            STRING       : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                            STRING       : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                         BOOL         : ON
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D          BOOL         : ON
 opt-set-cmake-var KokkosKernels_blas_serial_MPI_1_DISABLE                BOOL         : ON
@@ -2684,7 +2684,7 @@ use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                            STRING       : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                            STRING       : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                         BOOL         : ON
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D          BOOL         : ON
 opt-set-cmake-var KokkosKernels_blas_serial_MPI_1_DISABLE                BOOL         : ON
@@ -2756,7 +2756,7 @@ use PACKAGE-ENABLES|PR
 
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Tpetra_INST_SERIAL                            BOOL FORCE : ON
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL       : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL       : OFF
@@ -2784,7 +2784,7 @@ use PACKAGE-ENABLES|PR
 
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING     : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Tpetra_INST_SERIAL                            BOOL FORCE : ON
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL       : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL       : OFF
@@ -2813,7 +2813,7 @@ use USE-DEPRECATED|YES
 use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 
 use COMMON_SPACK_TPLS
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING       : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                   STRING       : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                BOOL         : ON
 opt-set-cmake-var CMAKE_CXX_EXTENSIONS                          BOOL         : OFF
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D BOOL         : ON
@@ -2849,7 +2849,7 @@ use PACKAGE-ENABLES|NO-PACKAGE-ENABLES
 
 use COMMON_SPACK_TPLS
 
-opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                                STRING       : --bind-to;none
+opt-set-cmake-var MPI_EXEC_PRE_NUMPROCS_FLAGS                                STRING       : --bind-to;none --mca btl vader,self
 opt-set-cmake-var Trilinos_ENABLE_COMPLEX_DOUBLE                             BOOL         : ON
 opt-set-cmake-var Teko_DISABLE_LSCSTABALIZED_TPETRA_ALPAH_INV_D              BOOL         : ON
 opt-set-cmake-var KokkosKernels_blas_serial_MPI_1_DISABLE                    BOOL         : ON

packages/framework/pr_tools/PullRequestLinuxDriver.sh

@@ -107,7 +107,7 @@ sig_merge_old=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullReques
 
 if [[ ${on_kokkos_develop} == "1" ]]; then
     message_std "PRDriver> --kokkos-develop is set - setting kokkos and kokkos-kernels packages to current develop"
-    "$SCRIPTPATH"/SetKokkosDevelop.sh
+    "${SCRIPTPATH}"/SetKokkosDevelop.sh
 else
     print_banner "Merge Source into Target"
     message_std "PRDriver> " "TRILINOS_SOURCE_SHA: ${TRILINOS_SOURCE_SHA:?}"
@@ -212,6 +212,11 @@ test_cmd_options=(
     --ctest-drop-site=${TRILINOS_CTEST_DROP_SITE:?}
 )
 
+if [[ ${on_kokkos_develop} == "1" ]]
+then
+    test_cmd_options+=( "--extra-configure-args=\"-DKokkos_SOURCE_DIR_OVERRIDE:string=kokkos;-DKokkosKernels_SOURCE_DIR_OVERRIDE:string=kokkos-kernels\" ")
+fi
+
 if [[ ${GENCONFIG_BUILD_NAME} == *"gnu"* ]]
 then
     test_cmd_options+=( "--use-explicit-cachefile ")

packages/framework/pr_tools/PullRequestLinuxDriverTest.py

@@ -238,6 +238,12 @@ def parse_args():
                           default=False,
                           help="Enable dry-run mode. Script will run but not execute the build steps. Default = %(default)s")
 
+    optional.add_argument("--extra-configure-args",
+                          dest="extra_configure_args",
+                          action="store",
+                          default="",
+                          help="Extra arguments that will be passed to CMake for configuring Trilinos.")
+
     arguments = parser.parse_args()
 
     # Type conversions
@@ -273,6 +279,7 @@ def parse_args():
     print("| - [O] req-mem-per-core            : {req_mem_per_core}".format(**vars(arguments)))
     print("| - [O] test-mode                   : {test_mode}".format(**vars(arguments)))
     print("| - [O] workspace-dir               : {workspace_dir}".format(**vars(arguments)))
+    print("| - [O] extra_configure_args        : {extra_configure_args}".format(**vars(arguments)))
     #print("| - [O] : {}".format(**vars(arguments)))
     print("+" + "="*78 + "+")
 

packages/framework/pr_tools/SetKokkosDevelop.sh

@@ -2,19 +2,21 @@
 SCRIPTFILE=$(realpath ${WORKSPACE:?}/Trilinos/packages/framework/pr_tools/SetKokkosDevelop.sh)
 SCRIPTPATH=$(dirname $SCRIPTFILE)
 source ${SCRIPTPATH:?}/common.bash
-PACKAGESPATH=$(realpath ${WORKSPACE:?}/Trilinos/packages)
+DIR_CONTAINING_TRILINOS=$(realpath ${WORKSPACE:?})
+TRILINOS_SRC=${DIR_CONTAINING_TRILINOS}/Trilinos
 
 # Ensures git is loaded properly
 if ! command -v git &> /dev/null; then
     message_std "SetKokkosDevelop> ERROR: git is not available"
     exit 1
 fi
 
-cd $PACKAGESPATH
-rm -rf kokkos kokkos-kernels
-git clone --depth=1 --single-branch --branch=develop --shallow-submodules https://github.com/kokkos/kokkos.git
-git clone --depth=1 --single-branch --branch=develop --shallow-submodules https://github.com/kokkos/kokkos-kernels.git
+cd $DIR_CONTAINING_TRILINOS
+git clone --depth=1 --single-branch --branch=develop --shallow-submodules https://github.com/kokkos/kokkos.git kokkos
+git clone --depth=1 --single-branch --branch=develop --shallow-submodules https://github.com/kokkos/kokkos-kernels.git kokkos-kernels
 message_std "SetKokkosDevelop> INFO: updated kokkos and kokkos-kernels packages with current develop"
 
-# Returns to previous path from before running this script
-cd -
+ln -s "${DIR_CONTAINING_TRILINOS}/kokkos" "$TRILINOS_SRC/kokkos"
+ln -s "${DIR_CONTAINING_TRILINOS}/kokkos-kernels" "$TRILINOS_SRC/kokkos-kernels"
+
+cd -