From 310fbe52a6d560d7b2fc7f9c3dde0ce078ac1c43 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Thu, 24 Oct 2024 14:36:08 -0400 Subject: [PATCH 1/7] updates to build on Gaea-C5 and C6 --- ...ea.intel.lua => gfsutils_gaeac5.intel.lua} | 4 +-- modulefiles/gfsutils_gaeac6.intel.lua | 32 +++++++++++++++++++ ush/detect_machine.sh | 16 +++++++--- 3 files changed, 45 insertions(+), 7 deletions(-) rename modulefiles/{gfsutils_gaea.intel.lua => gfsutils_gaeac5.intel.lua} (94%) create mode 100644 modulefiles/gfsutils_gaeac6.intel.lua diff --git a/modulefiles/gfsutils_gaea.intel.lua b/modulefiles/gfsutils_gaeac5.intel.lua similarity index 94% rename from modulefiles/gfsutils_gaea.intel.lua rename to modulefiles/gfsutils_gaeac5.intel.lua index 0d7f9df0..9b5ade06 100644 --- a/modulefiles/gfsutils_gaea.intel.lua +++ b/modulefiles/gfsutils_gaeac5.intel.lua @@ -8,10 +8,10 @@ whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]= prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) -stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" diff --git a/modulefiles/gfsutils_gaeac6.intel.lua b/modulefiles/gfsutils_gaeac6.intel.lua new file mode 100644 index 00000000..9b5ade06 --- /dev/null +++ b/modulefiles/gfsutils_gaeac6.intel.lua @@ -0,0 +1,32 @@ +help([[ + This module loads libraries required for building and running GFS UTILS + on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. +]]) + +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===]) + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" +load(pathJoin("stack-python", stack_python_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) + +load("gfsutils_common") +load("nccmp/1.9.0.1") + +unload("darshan-runtime") +unload("cray-libsci") + +setenv("CC","cc") +setenv("CXX","CC") +setenv("FC","ftn") +setenv("CMAKE_Platform","gaea.intel") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index b049a604..a2867769 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -21,8 +21,11 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 + + gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 + gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 @@ -81,9 +84,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 ]]; then + # We are on GAEAC5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 ]]; then + # We are on GAEAC6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 From b4250167317627f075a1fea28c9b258ad03e9a27 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 24 Oct 2024 19:01:58 +0000 Subject: [PATCH 2/7] updates for container --- modulefiles/gfsutils_container.intel.lua | 17 +++++++++++++++++ modulefiles/gfsutils_generic.lua | 22 ++++++++++++++++++++++ ush/detect_machine.sh | 5 ++++- ush/module-setup.sh | 7 +++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 modulefiles/gfsutils_container.intel.lua create mode 100644 modulefiles/gfsutils_generic.lua diff --git a/modulefiles/gfsutils_container.intel.lua b/modulefiles/gfsutils_container.intel.lua new file mode 100644 index 00000000..441ee10f --- /dev/null +++ b/modulefiles/gfsutils_container.intel.lua @@ -0,0 +1,17 @@ +help([[ +Build environment for GFS utilities in a container +]]) + +prepend_path("MODULEPATH", "/opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.12.1" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gfsutils_generic") + +whatis("Description: GFS utilities environment in container with Intel Compilers") diff --git a/modulefiles/gfsutils_generic.lua b/modulefiles/gfsutils_generic.lua new file mode 100644 index 00000000..822a156e --- /dev/null +++ b/modulefiles/gfsutils_generic.lua @@ -0,0 +1,22 @@ +help([[ +Load common modules to build GFS utilities on all machines +]]) + +load("libpng") + +load("netcdf-c") +load("netcdf-fortran") + +load("bufr") +load("bacio") +load("w3emc") +load("sp") +load("ip") +load("sigio") +load("sfcio") +load("nemsio") +load("wrf-io") +load("ncio") +load("g2") +load("landsfcutil") +load("wgrib2") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index a2867769..4167d617 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -64,7 +64,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then fi # Try searching based on paths since hostname may not match on compute nodes -if [[ -d /lfs/h3 ]]; then +if [[ -d /opt/spack-stack ]]; then + # We are in a container + MACHINE_ID=container +elif [[ -d /lfs/h3 ]]; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then diff --git a/ush/module-setup.sh b/ush/module-setup.sh index a4635808..14904aee 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -52,6 +52,13 @@ elif [[ $MACHINE_ID = cheyenne* ]] ; then fi module purge +elif [[ $MACHINE_ID = container ]] ; then + # We are in a container + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /usr/lmod/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = stampede* ]] ; then # We are on TACC Stampede if ( ! eval module help > /dev/null 2>&1 ) ; then From 768d409f8fcf1ac1d6457f647a1bb90cdfae8600 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 24 Oct 2024 19:18:04 +0000 Subject: [PATCH 3/7] added option for building without w3emc in tave and vint --- CMakeLists.txt | 1 + src/tave.fd/CMakeLists.txt | 7 ++++++- src/vint.fd/CMakeLists.txt | 11 ++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a213736..2493be0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ endif() # User options. option(OPENMP "use OpenMP threading" ON) +option(LINK_W3EMC "Link w3emc library" ON) # Find packages. find_package(MPI REQUIRED) diff --git a/src/tave.fd/CMakeLists.txt b/src/tave.fd/CMakeLists.txt index 70905a0b..1d4cd1d4 100644 --- a/src/tave.fd/CMakeLists.txt +++ b/src/tave.fd/CMakeLists.txt @@ -10,8 +10,13 @@ endif() set(exe_name tave.x) add_executable(${exe_name} ${fortran_src}) -target_link_libraries(${exe_name} PRIVATE bacio::bacio_4 +if(LINK_W3EMC) + target_link_libraries(${exe_name} PRIVATE bacio::bacio_4 w3emc::w3emc_d g2::g2_d) +else() + target_link_libraries(${exe_name} PRIVATE bacio::bacio_4 + g2::g2_d) +endif() install(TARGETS ${exe_name} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/vint.fd/CMakeLists.txt b/src/vint.fd/CMakeLists.txt index 118faab2..a9d21699 100644 --- a/src/vint.fd/CMakeLists.txt +++ b/src/vint.fd/CMakeLists.txt @@ -10,8 +10,13 @@ endif() set(exe_name vint.x) add_executable(${exe_name} ${fortran_src}) -target_link_libraries(${exe_name} PRIVATE bacio::bacio_4 - w3emc::w3emc_d - g2::g2_d) +if(LINK_W3EMC) + target_link_libraries(${exe_name} PRIVATE bacio::bacio_4 + w3emc::w3emc_d + g2::g2_d) +else() + target_link_libraries(${exe_name} PRIVATE bacio::bacio_4 + g2::g2_d) +endif() install(TARGETS ${exe_name} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) From 06153e7bf61b8d490676f52e7049df03791aefc5 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Thu, 24 Oct 2024 15:18:47 -0400 Subject: [PATCH 4/7] update mod file descriptions --- modulefiles/gfsutils_gaeac5.intel.lua | 4 ++-- modulefiles/gfsutils_gaeac6.intel.lua | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modulefiles/gfsutils_gaeac5.intel.lua b/modulefiles/gfsutils_gaeac5.intel.lua index 9b5ade06..9502dcee 100644 --- a/modulefiles/gfsutils_gaeac5.intel.lua +++ b/modulefiles/gfsutils_gaeac5.intel.lua @@ -3,7 +3,7 @@ help([[ on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. ]]) -whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===]) +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5]===]) prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") @@ -29,4 +29,4 @@ unload("cray-libsci") setenv("CC","cc") setenv("CXX","CC") setenv("FC","ftn") -setenv("CMAKE_Platform","gaea.intel") +setenv("CMAKE_Platform","gaeac5.intel") diff --git a/modulefiles/gfsutils_gaeac6.intel.lua b/modulefiles/gfsutils_gaeac6.intel.lua index 9b5ade06..e27a285c 100644 --- a/modulefiles/gfsutils_gaeac6.intel.lua +++ b/modulefiles/gfsutils_gaeac6.intel.lua @@ -1,9 +1,9 @@ help([[ This module loads libraries required for building and running GFS UTILS - on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. + on the NOAA RDHPC machine Gaea C6 using Intel-2023.1.0. ]]) -whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===]) +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6 ]===]) prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") @@ -29,4 +29,4 @@ unload("cray-libsci") setenv("CC","cc") setenv("CXX","CC") setenv("FC","ftn") -setenv("CMAKE_Platform","gaea.intel") +setenv("CMAKE_Platform","gaeac6.intel") From 1d1dc63bbc5469fb545f4f9374f7affdb56dad6e Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Mon, 4 Nov 2024 17:40:56 -0500 Subject: [PATCH 5/7] point to C6 install of spack-stack --- modulefiles/gfsutils_gaeac6.intel.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/gfsutils_gaeac6.intel.lua b/modulefiles/gfsutils_gaeac6.intel.lua index e27a285c..24b6372b 100644 --- a/modulefiles/gfsutils_gaeac6.intel.lua +++ b/modulefiles/gfsutils_gaeac6.intel.lua @@ -5,13 +5,13 @@ help([[ whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6 ]===]) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) -stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" From fc46a1e12ce19336ddd66b3c9ca7049aae81f3af Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Wed, 13 Nov 2024 18:56:20 +0000 Subject: [PATCH 6/7] switched container modules to use environment variables for version numbers --- modulefiles/gfsutils_container.intel.lua | 2 +- modulefiles/gfsutils_generic.lua | 22 ---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 modulefiles/gfsutils_generic.lua diff --git a/modulefiles/gfsutils_container.intel.lua b/modulefiles/gfsutils_container.intel.lua index 441ee10f..cddef43a 100644 --- a/modulefiles/gfsutils_container.intel.lua +++ b/modulefiles/gfsutils_container.intel.lua @@ -12,6 +12,6 @@ load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) load(pathJoin("cmake", cmake_ver)) -load("gfsutils_generic") +load("gfsutils_common") whatis("Description: GFS utilities environment in container with Intel Compilers") diff --git a/modulefiles/gfsutils_generic.lua b/modulefiles/gfsutils_generic.lua deleted file mode 100644 index 822a156e..00000000 --- a/modulefiles/gfsutils_generic.lua +++ /dev/null @@ -1,22 +0,0 @@ -help([[ -Load common modules to build GFS utilities on all machines -]]) - -load("libpng") - -load("netcdf-c") -load("netcdf-fortran") - -load("bufr") -load("bacio") -load("w3emc") -load("sp") -load("ip") -load("sigio") -load("sfcio") -load("nemsio") -load("wrf-io") -load("ncio") -load("g2") -load("landsfcutil") -load("wgrib2") From 22ae6c315e14390e1214fd058e26a6de2d2066ba Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO Date: Wed, 13 Nov 2024 14:29:07 -0500 Subject: [PATCH 7/7] Update C5 C6 modulefiles --- modulefiles/gfsutils_gaeac5.intel.lua | 9 ++------- modulefiles/gfsutils_gaeac6.intel.lua | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/modulefiles/gfsutils_gaeac5.intel.lua b/modulefiles/gfsutils_gaeac5.intel.lua index 9502dcee..7844b57e 100644 --- a/modulefiles/gfsutils_gaeac5.intel.lua +++ b/modulefiles/gfsutils_gaeac5.intel.lua @@ -1,11 +1,10 @@ help([[ This module loads libraries required for building and running GFS UTILS - on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. + on the NOAA RDHPC machine Gaea C5 using Intel-2023.2.0. ]]) -whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5]===]) +whatis([===[Loads libraries needed for building the GFS utilities on Gaea C5]===]) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" @@ -14,16 +13,12 @@ load(pathJoin("stack-intel", stack_intel_ver)) stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) -stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" -load(pathJoin("stack-python", stack_python_ver)) - cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) load("gfsutils_common") load("nccmp/1.9.0.1") -unload("darshan-runtime") unload("cray-libsci") setenv("CC","cc") diff --git a/modulefiles/gfsutils_gaeac6.intel.lua b/modulefiles/gfsutils_gaeac6.intel.lua index 24b6372b..619caa99 100644 --- a/modulefiles/gfsutils_gaeac6.intel.lua +++ b/modulefiles/gfsutils_gaeac6.intel.lua @@ -1,11 +1,10 @@ help([[ This module loads libraries required for building and running GFS UTILS - on the NOAA RDHPC machine Gaea C6 using Intel-2023.1.0. + on the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0. ]]) -whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6 ]===]) +whatis([===[Loads libraries needed for building the GFS utilities on Gaea C6 ]===]) -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" @@ -14,16 +13,12 @@ load(pathJoin("stack-intel", stack_intel_ver)) stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) -stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" -load(pathJoin("stack-python", stack_python_ver)) - cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) load("gfsutils_common") load("nccmp/1.9.0.1") -unload("darshan-runtime") unload("cray-libsci") setenv("CC","cc")