WIP: Feature/probinso/hipcc support

SetupBLT.cmake

@@ -187,7 +187,9 @@ if (NOT BLT_LOADED)
     # File extension lists used to filter sources based on languages for code checks
     # and filtering Fortran sources out of cuda/hip source lists
     # Note: this filtering is case-insensitive
-    set(BLT_C_FILE_EXTS ".cpp" ".hpp" ".cxx" ".hxx" ".c" ".h" ".cc" ".hh" ".inl" ".cu" ".cuh"
+    set(BLT_C_FILE_EXTS ".c" ".h"
+               CACHE STRING "List of known file extensions used for C/CXX sources")
+    set(BLT_CXX_FILE_EXTS ".cpp" ".hpp" ".cxx" ".hxx"  ".cc" ".hh" ".inl" ".cu" ".cuh"
                CACHE STRING "List of known file extensions used for C/CXX sources")
     set(BLT_Fortran_FILE_EXTS ".f" ".f90"
                CACHE STRING "List of known file extensions used for Fortran sources")

cmake/BLTMacros.cmake

@@ -654,6 +654,13 @@ macro(blt_add_library)
                 DEPENDS_ON   ${arg_DEPENDS_ON}
                 LIBRARY_TYPE ${_lib_type})
         endif()
+        if (ENABLE_HIP)
+           blt_setup_hip_target(
+                NAME         ${arg_NAME}
+                SOURCES      ${arg_SOURCES}
+                DEPENDS_ON   ${arg_DEPENDS_ON}
+                LIBRARY_TYPE ${_lib_type})
+        endif()
     else()
         #
         #  Header-only library support
@@ -732,7 +739,6 @@ macro(blt_add_library)
         # the white-list of properties that are allowed
         blt_clean_target(TARGET ${arg_NAME})
     endif()
-
 endmacro(blt_add_library)
 
 
@@ -776,6 +782,12 @@ macro(blt_add_executable)
             SOURCES      ${arg_SOURCES}
             DEPENDS_ON   ${arg_DEPENDS_ON})
     endif()
+    if (ENABLE_HIP)
+       blt_setup_hip_target(
+            NAME         ${arg_NAME}
+            SOURCES      ${arg_SOURCES}
+            DEPENDS_ON   ${arg_DEPENDS_ON})
+    endif()
 
     # CMake wants to load with C++ if any of the libraries are C++.
     # Force to load with Fortran if the first file is Fortran.

cmake/BLTPrivateMacros.cmake

@@ -190,7 +190,7 @@ macro(blt_find_executable)
     string(TOUPPER ${arg_NAME} _ucname)
 
     message(STATUS "${arg_NAME} support is ${ENABLE_${_ucname}}")
-    if (ENABLE_${_ucname})
+    if (${ENABLE_${_ucname}})
         set(_exes ${arg_NAME})
         if (DEFINED arg_EXECUTABLES)
             set(_exes ${arg_EXECUTABLES})
@@ -491,6 +491,87 @@ macro(blt_setup_target)
 
 endmacro(blt_setup_target)
 
+##------------------------------------------------------------------------------
+## blt_setup_hip_target(NAME <name of target>
+##                       SOURCES <list of sources>
+##                       DEPENDS_ON <list of dependencies>
+##                       LIBRARY_TYPE <STATIC, SHARED, OBJECT, or blank for executables>)
+##------------------------------------------------------------------------------
+macro(blt_setup_hip_target)
+
+    set(options)
+    set(singleValueArgs NAME LIBRARY_TYPE)
+    set(multiValueArgs SOURCES DEPENDS_ON)
+
+    # Parse the arguments
+    cmake_parse_arguments(arg "${options}" "${singleValueArgs}"
+                            "${multiValueArgs}" ${ARGN} )
+
+    # Check arguments
+    if ( NOT DEFINED arg_NAME )
+        message( FATAL_ERROR "Must provide a NAME argument to the 'blt_setup_hip_target' macro")
+    endif()
+
+    if ( NOT DEFINED arg_SOURCES )
+        message( FATAL_ERROR "Must provide SOURCES to the 'blt_setup_hip_target' macro")
+    endif()
+
+    # Determine if hip or hip_runtime are in DEPENDS_ON
+    list(FIND arg_DEPENDS_ON "blt::hip" _hip_index)
+    set(_depends_on_hip FALSE)
+    if(${_hip_index} GREATER -1)
+        set(_depends_on_hip TRUE)
+    endif()
+    list(FIND arg_DEPENDS_ON "blt::hip_runtime" _hip_runtime_index)
+    set(_depends_on_hip_runtime FALSE)
+    if(${_hip_runtime_index} GREATER -1)
+        set(_depends_on_hip_runtime TRUE)
+    endif()
+    message("DEPENDS ON HIP ${_depends_on_hip} DEPENDS_ON_HIP_RUNTIME ${_depends_on_hip_runtime}\n\t ${arg_DEPENDS_ON}")
+    if (${_depends_on_hip_runtime} OR ${_depends_on_hip})
+        if (HIP_LINK_WITH_HIPCC)
+            set_target_properties( ${arg_NAME} PROPERTIES LINKER_LANGUAGE HIP)
+            # This will be propagated up to executable targets that depend on this
+            # library, which will need the HIP linker
+            set_target_properties( ${arg_NAME} PROPERTIES INTERFACE_BLT_LINKER_LANGUAGE_OVERRIDE HIP)
+        endif()
+    endif()
+
+    if (${_depends_on_hip})
+        # if hip is in depends_on, flag each file's language as HIP
+        # instead of leaving it up to CMake to decide
+        # Note: we don't do this when depending on just 'hip_runtime'
+        set(_hip_sources)
+        set(_non_hip_sources)
+        blt_split_source_list_by_language(SOURCES      ${arg_SOURCES}
+                                          CXX_LIST       _hip_sources
+                                          C_LIST       _non_hip_sources
+                                          Fortran_LIST _non_hip_sources)
+        message("ALL_SOURCES: ${arg_SOURCES}\n\t HIP SOURCES ${_hip_sources}")
+        set_source_files_properties( ${_hip_sources} PROPERTIES
+                                     LANGUAGE HIP)
+
+#        if (HIP_SEPARABLE_COMPILATION)
+#            set_source_files_properties( ${_hip_sources} PROPERTIES
+#                                         HIP_SEPARABLE_COMPILATION ON)
+#            set_target_properties( ${arg_NAME} PROPERTIES
+#                                   HIP_SEPARABLE_COMPILATION ON)
+#        endif()
+
+        if (DEFINED arg_LIBRARY_TYPE)
+            if (${arg_LIBRARY_TYPE} STREQUAL "static")
+                set_target_properties( ${arg_NAME} PROPERTIES
+                                       CMAKE_HIP_CREATE_STATIC_LIBRARY ON)
+            else()
+                set_target_properties( ${arg_NAME} PROPERTIES
+                                       CMAKE_HIP_CREATE_STATIC_LIBRARY OFF)
+            endif()
+        endif()
+
+    endif()
+#    message(FATAL_ERROR "stopping")
+endmacro(blt_setup_hip_target)
+
 
 ##------------------------------------------------------------------------------
 ## blt_setup_cuda_target(NAME <name of target>
@@ -546,6 +627,7 @@ macro(blt_setup_cuda_target)
         set(_non_cuda_sources)
         blt_split_source_list_by_language(SOURCES      ${arg_SOURCES}
                                           C_LIST       _cuda_sources
+                                          CXX_LIST     _cuda_sources
                                           Fortran_LIST _non_cuda_sources)
 
         set_source_files_properties( ${_cuda_sources} PROPERTIES
@@ -622,6 +704,7 @@ endmacro(blt_make_file_ext_regex)
 ##------------------------------------------------------------------------------
 ## blt_split_source_list_by_language( SOURCES <sources>
 ##                                    C_LIST <list name>
+##                                    CXX_LIST <list name>
 ##                                    Fortran_LIST <list name>
 ##                                    Python_LIST <list name>
 ##                                    CMAKE_LIST <list name>)
@@ -637,7 +720,7 @@ endmacro(blt_make_file_ext_regex)
 macro(blt_split_source_list_by_language)
 
     set(options)
-    set(singleValueArgs C_LIST Fortran_LIST Python_LIST CMAKE_LIST)
+    set(singleValueArgs C_LIST CXX_LIST Fortran_LIST Python_LIST CMAKE_LIST)
     set(multiValueArgs SOURCES)
 
     # Parse the arguments
@@ -653,6 +736,9 @@ macro(blt_split_source_list_by_language)
     set(BLT_C_FILE_REGEX)
     blt_make_file_ext_regex(EXTENSIONS   ${BLT_C_FILE_EXTS}
                             OUTPUT_REGEX BLT_C_FILE_REGEX)
+    set(BLT_CXX_FILE_REGEX)
+    blt_make_file_ext_regex(EXTENSIONS   ${BLT_CXX_FILE_EXTS}
+                            OUTPUT_REGEX BLT_CXX_FILE_REGEX)
     set(BLT_Fortran_FILE_REGEX)
     blt_make_file_ext_regex(EXTENSIONS   ${BLT_Fortran_FILE_EXTS}
                             OUTPUT_REGEX BLT_Fortran_FILE_REGEX)
@@ -686,6 +772,10 @@ macro(blt_split_source_list_by_language)
             if (DEFINED arg_C_LIST)
                 list(APPEND ${arg_C_LIST} "${_file}")
             endif()
+        elseif("${_lower_file}" MATCHES "${BLT_CXX_FILE_REGEX}")
+            if (DEFINED arg_CXX_LIST)
+                list(APPEND ${arg_CXX_LIST} "${_file}")
+            endif()
         elseif("${_lower_file}" MATCHES "${BLT_Fortran_FILE_REGEX}")
             if (DEFINED arg_Fortran_LIST)
                 list(APPEND ${arg_Fortran_LIST} "${_file}")

cmake/thirdparty/SetupHIP.cmake

@@ -22,6 +22,7 @@ if (NOT ROCM_PATH)
         /opt/rocm)
 endif()
 
+
 # Update CMAKE_PREFIX_PATH to make sure all the configs that hip depends on are
 # found.
 set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${ROCM_PATH}")
@@ -31,6 +32,9 @@ find_package(hip REQUIRED CONFIG PATHS ${HIP_PATH} ${ROCM_PATH})
 message(STATUS "ROCM path:        ${ROCM_PATH}")
 message(STATUS "HIP version:      ${hip_VERSION}")
 
+if ( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.21.0" )
+   enable_language(HIP)
+endif()
 # AMDGPU_TARGETS should be defined in the hip-config.cmake that gets "included" via find_package(hip)
 # This file is also what hardcodes the --offload-arch flags we're removing here
 if(DEFINED AMDGPU_TARGETS)
@@ -54,9 +58,47 @@ if(DEFINED AMDGPU_TARGETS)
             list(REMOVE_ITEM _hip_link_libs ${_flag})
         endif()
     endforeach()
+     if (${BLT_BUILD_HIP_WITH_HIPCC})
+        list(REMOVE_ITEM _hip_compile_options "$<$<COMPILE_LANGUAGE:CXX>:SHELL:-mllvm")
+        list(REMOVE_ITEM _hip_compile_options "-amdgpu-early-inline-all=true")
+        list(REMOVE_ITEM _hip_compile_options "-mllvm")
+        list(REMOVE_ITEM _hip_compile_options "-amdgpu-function-calls=false>")
+        get_target_property(_hip_lang_compile_options hip-lang::device INTERFACE_COMPILE_OPTIONS)
+        set(_flag "-mllvm-amdgpu-early-inline-all=true-mllvm-amdgpu-function-calls=false")
+        set(_generator_compile_flag "$<$<COMPILE_LANGUAGE:HIP>:SHELL:${_flag}>")
+        list(REMOVE_ITEM _hip_lang_compile_options "$<$<COMPILE_LANGUAGE:HIP>:SHELL:-mllvm")
+        list(REMOVE_ITEM _hip_lang_compile_options "-amdgpu-early-inline-all=true")
+        list(REMOVE_ITEM _hip_lang_compile_options "-mllvm")
+        list(REMOVE_ITEM _hip_lang_compile_options "-amdgpu-function-calls=false>")
+        set_property(TARGET hip-lang::device PROPERTY INTERFACE_COMPILE_OPTIONS ${_hip_lang_compile_options})
+     endif()
 
+#   when we are using hipcc instead of amdclang, the hip cmake stuff fails to find the clang runtime link libraries
+#   and leaves CLANGRT_BUILTINS-NOTFOUND in the interface. We remove it here.
+#   Note sometimes the entry is the last entry of a generator expression, ending in '>', so we use transform to
+#   preserve the '>'
+    if (${BLT_BUILD_HIP_WITH_HIPCC})
+       get_target_property(_hip_lang_link_libs hip-lang::device INTERFACE_LINK_LIBRARIES)
+       get_target_property(_hip_host_link_libs hip::host INTERFACE_LINK_LIBRARIES)
+       list(TRANSFORM _hip_lang_link_libs REPLACE "CLANGRT_BUILTINS-NOTFOUND" "")
+       list(TRANSFORM _hip_host_link_libs REPLACE "CLANGRT_BUILTINS-NOTFOUND" "")
+       set_property(TARGET hip::host PROPERTY INTERFACE_LINK_LIBRARIES ${_hip_host_link_libs})
+       set_property(TARGET hip-lang::device PROPERTY  INTERFACE_LINK_LIBRARIES ${_hip_lang_link_libs})
+       # also transform preexisting _hip_link_libs before set occurs after this conditional block
+       list(TRANSFORM _hip_link_libs REPLACE "CLANGRT_BUILTINS-NOTFOUND" "")
+    endif()
+
     set_property(TARGET hip::device PROPERTY INTERFACE_COMPILE_OPTIONS ${_hip_compile_options})
     set_property(TARGET hip::device PROPERTY INTERFACE_LINK_LIBRARIES ${_hip_link_libs})
+    if (${BLT_BUILD_HIP_WITH_HIPCC})
+#   remove interface include directories - these are bugged in ROCM 5.2.1
+       set_property(TARGET hip-lang::device PROPERTY INTERFACE_INCLUDE_DIRECTORIES "")
+       set_property(TARGET hip::device PROPERTY INTERFACE_INCLUDE_DIRECTORIES "")
+       set_property(TARGET hip::host PROPERTY INTERFACE_INCLUDE_DIRECTORIES "")
+       set_property(TARGET hip-lang::device PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "")
+       set_property(TARGET hip::device PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "")
+       set_property(TARGET hip::host PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "")
+    endif()
 
     if(DEFINED CMAKE_HIP_ARCHITECTURES)
         set(AMDGPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "" FORCE)
@@ -69,10 +111,16 @@ if (${BLT_EXPORT_THIRDPARTY})
     set(_blt_hip_is_global Off)
 endif ()
 
+if (${BLT_BUILD_HIP_WITH_HIPCC})
+blt_import_library(NAME       blt_hip
+                   EXPORTABLE ${BLT_EXPORT_THIRDPARTY}
+                   GLOBAL ${_blt_hip_is_global})
+else()
 blt_import_library(NAME       blt_hip
                    COMPILE_FLAGS "--rocm-path=${ROCM_PATH}"
                    EXPORTABLE ${BLT_EXPORT_THIRDPARTY}
                    GLOBAL ${_blt_hip_is_global})
+endif()
 
 # Hard-copy inheritable properties instead of depending on hip::device so that we can export all required
 # information in our target blt_hip