Bugfix: CMake FindMAGMA (#356)

Always append hipblas and hipsparse CMake targets with HIP backend as these libraries are not reliably included in the pkgconfig file. Replace cublas and cusparse with the CMake targets because the library path in the pkgconfig file is not reliable. Specifically, the path is wrong on systems using the NVIDIA HPC SDK. --------- Co-authored-by: David Gardner <[email protected]>
LLNL · Oct 27, 2023 · b84b330 · b84b330
1 parent 747da2e
commit b84b330
Show file tree

Hide file tree

Showing 17 changed files with 193 additions and 54 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,6 +27,9 @@ Fixed scaling bug in `SUNMatScaleAddI_Sparse` for non-square matrices.
 Fixed missing soversions in some `SUNLinearSolver` and `SUNNonlinearSolver`
 CMake targets.
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Added the fourth order ERK method `ARKODE_SOFRONIOU_SPALETTA_5_3_4`.
 
 ## Changes to SUNDIALS in release 6.6.1

diff --git a/cmake/tpl/FindMAGMA.cmake b/cmake/tpl/FindMAGMA.cmake
@@ -66,24 +66,42 @@ if(MAGMA_LIBRARY AND MAGMA_INCLUDE_DIR)
     list(SUBLIST _libraries_list 1 -1 _libraries_list) # remove 'Libs:' part
 
     set(_interface_libraires )
+
+    if(SUNDIALS_MAGMA_BACKENDS MATCHES "HIP")
+      if(NOT TARGET roc::hipblas)
+        find_package(hipblas REQUIRED)
+      endif()
+      if(NOT TARGET roc::hipsparse)
+        find_package(hipsparse REQUIRED)
+      endif()
+      # MAGMA does not reliably include these in the pkgconfig file
+      list(APPEND _interface_libraires "roc::hipblas;roc::hipsparse")
+    endif()
+
+    if(SUNDIALS_MAGMA_BACKENDS MATCHES "CUDA")
+      if (NOT TARGET CUDA::cudart)
+        find_package(CUDAToolkit REQUIRED)
+      endif() 
+    endif()
+
     foreach(lib ${_libraries_list})
       if(NOT (lib STREQUAL "-lmagma" OR lib STREQUAL "-lmagma_sparse"
             OR lib STREQUAL "-L\${libdir}" OR lib STREQUAL "") )
-
-        # Remove -l only from the beginning of the string
-        string(REPLACE "^-l" "" lib ${lib})
-        list(APPEND _interface_libraires ${lib})
-
-        # Check if we need to find roc::hipblas or roc::hipsparse
-        if(SUNDIALS_MAGMA_BACKENDS MATCHES "HIP")
-          if((lib STREQUAL "roc::hipblas") AND (NOT TARGET roc::hipblas))
-            find_package(hipblas REQUIRED)
+
+        # Check if we need to find cusparse or cublas
+        if(SUNDIALS_MAGMA_BACKENDS MATCHES "CUDA")
+          # Replace cublas, cusparse with the CMake targets because the library path in
+          # the magma pkgconfig is not reliable. Sepcifically, the path is wrong on systems
+          # like Perlmutter where the NVIDIA HPC SDK is used.
+          if(lib STREQUAL "-lcublas")
+            set(lib CUDA::cublas)
           endif()
-          if((lib STREQUAL "roc::hipsparse") AND (NOT TARGET roc::hipsparse))
-            find_package(hipsparse REQUIRED)
+          if(lib STREQUAL "-lcusparse")
+            set(lib CUDA::cusparse)
           endif()
         endif()
-
+
+        list(APPEND _interface_libraires ${lib})
       endif()
     endforeach()
 

diff --git a/doc/arkode/guide/source/Introduction.rst b/doc/arkode/guide/source/Introduction.rst
@@ -133,6 +133,9 @@ Changes from previous versions
 Changes in vX.X.X
 -----------------
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Fixed a regression introduced by the stop time bug fix in v6.6.1 where ARKODE
 steppers would return at the stop time rather than the requested output time if
 the stop time was reached in the same step in which the output time was passed.

diff --git a/doc/cvode/guide/source/Introduction.rst b/doc/cvode/guide/source/Introduction.rst
@@ -114,6 +114,9 @@ Changes from previous versions
 Changes in vX.X.X
 -----------------
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Fixed a regression introduced by the stop time bug fix in v6.6.1 where CVODE
 would return at the stop time rather than the requested output time if the stop
 time was reached in the same step in which the output time was passed.

diff --git a/doc/cvodes/guide/source/Introduction.rst b/doc/cvodes/guide/source/Introduction.rst
@@ -114,6 +114,9 @@ Changes from previous versions
 Changes in vX.X.X
 -----------------
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Fixed a regression introduced by the stop time bug fix in v6.6.1 where CVODES
 would return at the stop time rather than the requested output time if the stop
 time was reached in the same step in which the output time was passed.

diff --git a/doc/ida/guide/source/Introduction.rst b/doc/ida/guide/source/Introduction.rst
@@ -75,6 +75,9 @@ Changes from previous versions
 Changes in vX.X.X
 -----------------
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Fixed a regression introduced by the stop time bug fix in v6.6.1 where IDA would
 return at the stop time rather than the requested output time if the stop time
 was reached in the same step in which the output time was passed.

diff --git a/doc/idas/guide/source/Introduction.rst b/doc/idas/guide/source/Introduction.rst
@@ -89,6 +89,9 @@ Changes from previous versions
 Changes in vX.X.X
 -----------------
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Fixed a regression introduced by the stop time bug fix in v6.6.1 where IDAS
 would return at the stop time rather than the requested output time if the stop
 time was reached in the same step in which the output time was passed.

diff --git a/doc/kinsol/guide/source/Introduction.rst b/doc/kinsol/guide/source/Introduction.rst
@@ -91,6 +91,9 @@ Changes from previous versions
 Changes in vX.X.X
 -----------------
 
+Fixed the build system support for MAGMA when using a NVIDIA HPC SDK installation of CUDA
+and fixed the targets used for rocBLAS and rocSPARSE.
+
 Improved computational complexity of ``SUNMatScaleAddI_Sparse`` from ``O(M*N)`` to
 ``O(NNZ)``.
 

diff --git a/scripts/shared b/scripts/shared
@@ -246,6 +246,8 @@ $tar $tarfile $distrobase/src/sundials/sundials_band.c
 $tar $tarfile $distrobase/src/sundials/sundials_context_impl.h
 $tar $tarfile $distrobase/src/sundials/sundials_context.c
 $tar $tarfile $distrobase/src/sundials/sundials_cuda.h
+$tar $tarfile $distrobase/src/sundials/sundials_cusolver.h
+$tar $tarfile $distrobase/src/sundials/sundials_cusparse.h
 $tar $tarfile $distrobase/src/sundials/sundials_cuda_kernels.cuh
 $tar $tarfile $distrobase/src/sundials/sundials_debug.h
 $tar $tarfile $distrobase/src/sundials/sundials_dense.c

diff --git a/src/sundials/sundials_cuda.h b/src/sundials/sundials_cuda.h
@@ -21,8 +21,6 @@
 #include <stdio.h>
 
 #include <cuda_runtime.h>
-#include <cusolverSp.h>
-#include <cusparse.h>
 
 #include <sundials/sundials_types.h>
 
@@ -38,8 +36,6 @@ extern "C" {
  * ---------------------------------------------------------------------------*/
 
 #define SUNDIALS_CUDA_VERIFY(cuerr) SUNDIALS_CUDA_Assert(cuerr, __FILE__, __LINE__)
-#define SUNDIALS_CUSPARSE_VERIFY(cuerr) SUNDIALS_CUSPARSE_Assert(cuerr, __FILE__, __LINE__)
-#define SUNDIALS_CUSOLVER_VERIFY(cuerr) SUNDIALS_CUSOLVER_Assert(cuerr, __FILE__, __LINE__)
 
 #define SUNDIALS_KERNEL_NAME(...) __VA_ARGS__
 #ifndef SUNDIALS_DEBUG_CUDA_LASTERROR
@@ -75,42 +71,9 @@ inline booleantype SUNDIALS_CUDA_Assert(cudaError_t cuerr, const char *file, int
   return SUNTRUE; /* Assert OK */
 }
 
-inline booleantype SUNDIALS_CUSPARSE_Assert(cusparseStatus_t status, const char *file, int line)
-{
-  if (status != CUSPARSE_STATUS_SUCCESS)
-  {
-#ifdef SUNDIALS_DEBUG
-    fprintf(stderr,
-            "ERROR in cuSPARSE runtime operation: cusparseStatus_t = %d %s:%d\n",
-            status, file, line);
-#ifdef SUNDIALS_DEBUG_ASSERT
-    assert(false);
-#endif
-#endif
-    return SUNFALSE; /*  Assert failed */
-  }
-  return SUNTRUE; /* Assert OK */
-}
-
-inline booleantype SUNDIALS_CUSOLVER_Assert(cusolverStatus_t status, const char *file, int line)
-{
-  if (status != CUSOLVER_STATUS_SUCCESS)
-  {
-#ifdef SUNDIALS_DEBUG
-    fprintf(stderr,
-            "ERROR in cuSOLVER runtime operation: cusolverStatus_t = %d %s:%d\n",
-            status, file, line);
-#ifdef SUNDIALS_DEBUG_ASSERT
-    assert(false);
-#endif
-#endif
-    return SUNFALSE; /*  Assert failed */
-  }
-  return SUNTRUE; /* Assert OK */
-}
 
 #ifdef __cplusplus  /* wrapper to enable C++ usage */
 }
 #endif
 
-#endif /* _SUNDIALS_CUDA_H */
+#endif /* _SUNDIALS_CUDA_H */
diff --git a/src/sundials/sundials_cusolver.h b/src/sundials/sundials_cusolver.h
@@ -0,0 +1,68 @@
+/*
+ * -----------------------------------------------------------------
+ * Programmer(s): Cody J. Balos @ LLNL
+ * -----------------------------------------------------------------
+ * SUNDIALS Copyright Start
+ * Copyright (c) 2002-2023, Lawrence Livermore National Security
+ * and Southern Methodist University.
+ * All rights reserved.
+ *
+ * See the top-level LICENSE and NOTICE files for details.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SUNDIALS Copyright End
+ * -----------------------------------------------------------------
+ * This header files defines internal utility functions and macros
+ * for working with CUDA.
+ * -----------------------------------------------------------------
+ */
+
+#include <assert.h>
+#include <stdio.h>
+
+#include <cuda_runtime.h>
+#include <cusolverDn.h>
+#include <cusolverSp.h>
+
+#include <sundials/sundials_types.h>
+
+#ifndef _SUNDIALS_CUSOLVER_H
+#define _SUNDIALS_CUSOLVER_H
+
+#ifdef __cplusplus  /* wrapper to enable C++ usage */
+extern "C" {
+#endif
+
+/* ---------------------------------------------------------------------------
+ * Utility macros
+ * ---------------------------------------------------------------------------*/
+
+#define SUNDIALS_CUSOLVER_VERIFY(cuerr) SUNDIALS_CUSOLVER_Assert(cuerr, __FILE__, __LINE__)
+
+
+/* ---------------------------------------------------------------------------
+ * Utility functions
+ * ---------------------------------------------------------------------------*/
+
+inline booleantype SUNDIALS_CUSOLVER_Assert(cusolverStatus_t status, const char *file, int line)
+{
+  if (status != CUSOLVER_STATUS_SUCCESS)
+  {
+#ifdef SUNDIALS_DEBUG
+    fprintf(stderr,
+            "ERROR in cuSOLVER runtime operation: cusolverStatus_t = %d %s:%d\n",
+            status, file, line);
+#ifdef SUNDIALS_DEBUG_ASSERT
+    assert(false);
+#endif
+#endif
+    return SUNFALSE; /*  Assert failed */
+  }
+  return SUNTRUE; /* Assert OK */
+}
+
+#ifdef __cplusplus  /* wrapper to enable C++ usage */
+}
+#endif
+
+#endif /* _SUNDIALS_CUSOLVER_H */
diff --git a/src/sundials/sundials_cusparse.h b/src/sundials/sundials_cusparse.h
@@ -0,0 +1,66 @@
+/*
+ * -----------------------------------------------------------------
+ * Programmer(s): Cody J. Balos @ LLNL
+ * -----------------------------------------------------------------
+ * SUNDIALS Copyright Start
+ * Copyright (c) 2002-2023, Lawrence Livermore National Security
+ * and Southern Methodist University.
+ * All rights reserved.
+ *
+ * See the top-level LICENSE and NOTICE files for details.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ * SUNDIALS Copyright End
+ * -----------------------------------------------------------------
+ * This header files defines internal utility functions and macros
+ * for working with CUDA.
+ * -----------------------------------------------------------------
+ */
+
+#include <assert.h>
+#include <stdio.h>
+
+#include <cuda_runtime.h>
+#include <cusparse.h>
+
+#include <sundials/sundials_types.h>
+
+#ifndef _SUNDIALS_CUSPARSE_H
+#define _SUNDIALS_CUSPARSE_H
+
+#ifdef __cplusplus  /* wrapper to enable C++ usage */
+extern "C" {
+#endif
+
+/* ---------------------------------------------------------------------------
+ * Utility macros
+ * ---------------------------------------------------------------------------*/
+
+#define SUNDIALS_CUSPARSE_VERIFY(cuerr) SUNDIALS_CUSPARSE_Assert(cuerr, __FILE__, __LINE__)
+
+/* ---------------------------------------------------------------------------
+ * Utility functions
+ * ---------------------------------------------------------------------------*/
+
+inline booleantype SUNDIALS_CUSPARSE_Assert(cusparseStatus_t status, const char *file, int line)
+{
+  if (status != CUSPARSE_STATUS_SUCCESS)
+  {
+#ifdef SUNDIALS_DEBUG
+    fprintf(stderr,
+            "ERROR in cuSPARSE runtime operation: cusparseStatus_t = %d %s:%d\n",
+            status, file, line);
+#ifdef SUNDIALS_DEBUG_ASSERT
+    assert(false);
+#endif
+#endif
+    return SUNFALSE; /*  Assert failed */
+  }
+  return SUNTRUE; /* Assert OK */
+}
+
+#ifdef __cplusplus  /* wrapper to enable C++ usage */
+}
+#endif
+
+#endif /* _SUNDIALS_CUSPARSE_H */
diff --git a/src/sunlinsol/cusolversp/sunlinsol_cusolversp_batchqr.cu b/src/sunlinsol/cusolversp/sunlinsol_cusolversp_batchqr.cu
@@ -23,6 +23,7 @@
 #include <sunlinsol/sunlinsol_cusolversp_batchqr.h>
 
 #include "sundials_cuda.h"
+#include "sundials_cusolver.h"
 #include "sundials_debug.h"
 
 #define ZERO RCONST(0.0)

diff --git a/src/sunlinsol/magmadense/CMakeLists.txt b/src/sunlinsol/magmadense/CMakeLists.txt
@@ -19,7 +19,7 @@ if(SUNDIALS_MAGMA_BACKENDS MATCHES "CUDA")
   set(_libs_needed sundials_sunmatrixmagmadense sundials_nveccuda)
 elseif(SUNDIALS_MAGMA_BACKENDS MATCHES "HIP")
   set_source_files_properties(sunlinsol_magmadense.cpp PROPERTIES LANGUAGE CXX)
-  set(_libs_needed sundials_sunmatrixmagmadense sundials_nvechip)
+  set(_libs_needed sundials_sunmatrixmagmadense sundials_nvechip hip::device)
 endif()
 
 # Add the sunlinsol_magmadense library

diff --git a/src/sunmatrix/cusparse/CMakeLists.txt b/src/sunmatrix/cusparse/CMakeLists.txt
@@ -28,8 +28,7 @@ sundials_add_library(sundials_sunmatrixcusparse
     sundials_generic_obj
     sundials_sunmemcuda_obj
   LINK_LIBRARIES
-    PUBLIC CUDA::cusparse
-    PRIVATE CUDA::cusolver
+    PUBLIC CUDA::cusparse CUDA::cusolver
   OUTPUT_NAME
     sundials_sunmatrixcusparse
   VERSION

diff --git a/src/sunmatrix/cusparse/sunmatrix_cusparse.cu b/src/sunmatrix/cusparse/sunmatrix_cusparse.cu
@@ -24,6 +24,7 @@
 #include <sunmatrix/sunmatrix_cusparse.h>
 
 #include "sundials_cuda.h"
+#include "sundials_cusparse.h"
 #include "sundials_debug.h"
 #include "cusparse_kernels.cuh"
 

diff --git a/src/sunmatrix/magmadense/CMakeLists.txt b/src/sunmatrix/magmadense/CMakeLists.txt
@@ -16,7 +16,7 @@ install(CODE "MESSAGE(\"\nInstall SUNMATRIX_MAGMADENSE with ${SUNDIALS_MAGMA_BAC
 
 if(SUNDIALS_MAGMA_BACKENDS MATCHES "CUDA")
   set_source_files_properties(sunmatrix_magmadense.cpp PROPERTIES LANGUAGE CUDA)
-  set(_libs_needed sundials_nveccuda ${CUDA_CUBLAS_LIBRARIES})
+  set(_libs_needed sundials_nveccuda)
 elseif(SUNDIALS_MAGMA_BACKENDS MATCHES "HIP")
   set_source_files_properties(sunmatrix_magmadense.cpp PROPERTIES LANGUAGE CXX)
   set(_libs_needed sundials_nvechip hip::device)