Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/fhd_heffte' into fhd_heffte
Browse files Browse the repository at this point in the history
  • Loading branch information
isriva committed Oct 31, 2024
2 parents de7ad32 + 5dddc8d commit 41d1da3
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 26 deletions.
6 changes: 1 addition & 5 deletions exec/compressible_stag/GNUmakefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ ifeq ($(USE_HEFFTE_FFTW),TRUE)
else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
HEFFTE_HOME ?= ../../../heffte-org/build_aware/
else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
HEFFTE_HOME ?= ../../../heffte/
HEFFTE_HOME ?= ../../../heffte-org/build_noaware/
endif

include $(AMREX_HOME)/Tools/GNUMake/Make.defs
Expand Down Expand Up @@ -71,10 +71,6 @@ ifeq ($(USE_HEFFTE_FFTW),TRUE)
LIBRARIES += -L$(FFTW_DIR) -lfftw3_mpi -lfftw3 -lfftw3f
else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
DEFINES += -DHEFFTE_CUFFT
VPATH_LOCATIONS += $(HEFFTE_HOME)/include
INCLUDE_LOCATIONS += $(HEFFTE_HOME)/include
LIBRARY_LOCATIONS += $(HEFFTE_HOME)/lib
LIBRARIES += -lheffte
else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
DEFINES += -DHEFFTE_ROCFFT
endif
Expand Down
27 changes: 27 additions & 0 deletions exec/compressible_stag/SPECTRAL_FILTER/build_frontier.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/bash

## load necessary modules
module load craype-accel-amd-gfx90a
module load amd-mixed
#module load rocm/5.2.0 # waiting for 5.6 for next bump
module load cray-mpich/8.1.23
module load cce/15.0.0 # must be loaded after rocm

# GPU-aware MPI
export MPICH_GPU_SUPPORT_ENABLED=1

# optimize CUDA compilation for MI250X
export AMREX_AMD_ARCH=gfx90a

# compiler environment hints
##export CC=$(which hipcc)
##export CXX=$(which hipcc)
##export FC=$(which ftn)
##export CFLAGS="-I${ROCM_PATH}/include"
##export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
##export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"
export LDFLAGS="-L${MPICH_DIR}/lib -lmpi ${CRAY_XPMEM_POST_LINK_OPTS} -lxpmem ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}"
export CXXFLAGS="-I${MPICH_DIR}/include"
export HIPFLAGS="--amdgpu-target=gfx90a"

make -j10 USE_HIP=TRUE USE_HEFFTE_ROCFFT=TRUE USE_ASSERTION=TRUE
28 changes: 7 additions & 21 deletions exec/compressible_stag/TURB_PDFS/build_frontier.sh
Original file line number Diff line number Diff line change
@@ -1,27 +1,13 @@
#!/usr/bin/bash

## load necessary modules
module load craype-accel-amd-gfx90a
module load amd-mixed
#module load rocm/5.2.0 # waiting for 5.6 for next bump
module load cray-mpich/8.1.23
module load cce/15.0.0 # must be loaded after rocm

# GPU-aware MPI
export MPICH_GPU_SUPPORT_ENABLED=1

# optimize CUDA compilation for MI250X
export AMREX_AMD_ARCH=gfx90a
module load PrgEnv-cray
module load cray-mpich
module load cce

# compiler environment hints
##export CC=$(which hipcc)
##export CXX=$(which hipcc)
##export FC=$(which ftn)
##export CFLAGS="-I${ROCM_PATH}/include"
##export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
##export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"
export LDFLAGS="-L${MPICH_DIR}/lib -lmpi ${CRAY_XPMEM_POST_LINK_OPTS} -lxpmem ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}"
export CXXFLAGS="-I${MPICH_DIR}/include"
export HIPFLAGS="--amdgpu-target=gfx90a"
export CC=$(which craycc)
export CXX=$(which craycc)
export FC=$(which crayftn)

make -j10 USE_CUDA=FALSE USE_HIP=TRUE USE_ASSERTION=TRUE
make -j10 USE_CUDA=FALSE USE_HIP=FALSE USE_ASSERTION=TRUE
24 changes: 24 additions & 0 deletions exec/compressible_stag/build_frontier_101324.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/bash

## load necessary modules
module load craype-accel-amd-gfx90a
module load rocm
module load cray-mpich
module load cce # must be loaded after rocm
module load cray-fftw

# GPU-aware MPI
export MPICH_GPU_SUPPORT_ENABLED=1

# optimize CUDA compilation for MI250X
export AMREX_AMD_ARCH=gfx90a

# compiler environment hints
export CC=$(which hipcc)
export CXX=$(which hipcc)
export FC=$(which ftn)
export CFLAGS="-I${ROCM_PATH}/include"
export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"

make -j8 USE_HIP=TRUE DO_TURB=TRUE MAX_SPEC=2 USE_HEFFTE_ROCFFT=FALSE USE_ASSERTION=TRUE

0 comments on commit 41d1da3

Please sign in to comment.