Skip to content

Commit

Permalink
more compilation mods to get heffte working on frontier
Browse files Browse the repository at this point in the history
  • Loading branch information
Ishan Srivastava committed Oct 11, 2023
1 parent 48bca93 commit b625dc4
Show file tree
Hide file tree
Showing 5 changed files with 433 additions and 225 deletions.
34 changes: 15 additions & 19 deletions exec/compressible_stag/GNUmakefile
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
# AMREX_HOME defines the directory in which we will find all the AMReX code.
# If you set AMREX_HOME as an environment variable, this line will be ignored
AMREX_HOME ?= ../../../amrex/
HEFFTE_HOME ?= ../../../heffte/

DEBUG = FALSE
USE_MPI = TRUE
USE_OMP = FALSE
USE_CUDA = TRUE
USE_CUDA = FALSE
USE_HIP = FALSE
COMP = gnu
DIM = 3
TINY_PROFILE = FALSE
MAX_SPEC = 8

USE_PARTICLES = FALSE
DO_TURB = FALSE
USE_HEFFTE = FALSE
DO_TURB = FALSE
USE_HEFFTE_FFTW = FALSE
USE_HEFFTE_CUFFT = FALSE
USE_HEFFTE_ROCFFT = FALSE

include $(AMREX_HOME)/Tools/GNUMake/Make.defs

Expand All @@ -41,31 +44,20 @@ include ../../src_rng/Make.package
VPATH_LOCATIONS += ../../src_rng/
INCLUDE_LOCATIONS += ../../src_rng/


include ../../src_common/Make.package
VPATH_LOCATIONS += ../../src_common/
INCLUDE_LOCATIONS += ../../src_common/

include $(HEFFTE_HOME)/src/Make.package

include $(AMREX_HOME)/Src/Base/Make.package

include ../../src_analysis/Make.package
VPATH_LOCATIONS += ../../src_analysis/
INCLUDE_LOCATIONS += ../../src_analysis/

include $(AMREX_HOME)/Tools/GNUMake/Make.rules

ifeq ($(USE_HEFFTE),TRUE)
ifeq ($(USE_CUDA),TRUE)
HEFFTE_HOME ?= ../../../heffte/build_gpu
else
HEFFTE_HOME ?= ../../../heffte/build
endif
libraries += -lheffte
VPATH_LOCATIONS += $(HEFFTE_HOME)/include
INCLUDE_LOCATIONS += $(HEFFTE_HOME)/include
LIBRARY_LOCATIONS += $(HEFFTE_HOME)/lib
endif

ifeq ($(USE_CUDA),TRUE)
LIBRARIES += -lcufft
else ifeq ($(USE_HIP),TRUE)
Expand All @@ -81,8 +73,12 @@ ifeq ($(DO_TURB), TRUE)
DEFINES += -DTURB
endif

ifeq ($(USE_HEFFTE), TRUE)
DEFINES += -DHEFFTE
ifeq ($(USE_HEFFTE_FFTW),TRUE)
DEFINES += -DHEFFTE_FFTW
else ifeq ($(USE_HEFFTE_CUFFT),TRUE)
DEFINES += -DHEFFTE_CUFFT
else ifeq ($(USE_HEFFTE_ROCFFT),TRUE)
DEFINES += -DHEFFTE_ROCFFT
endif

MAXSPECIES := $(strip $(MAX_SPEC))
Expand Down
15 changes: 7 additions & 8 deletions exec/compressible_stag/build_frontier.sh
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#!/usr/bin/bash

## load necessary modules
module load cmake/3.23.2
module load craype-accel-amd-gfx90a
module load rocm/5.2.0 # waiting for 5.6 for next bump
module load cray-mpich
Expand All @@ -14,11 +13,11 @@ export MPICH_GPU_SUPPORT_ENABLED=1
export AMREX_AMD_ARCH=gfx90a

# compiler environment hints
export CC=$(which hipcc)
export CXX=$(which hipcc)
export FC=$(which ftn)
export CFLAGS="-I${ROCM_PATH}/include"
export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"
##export CC=$(which hipcc)
##export CXX=$(which hipcc)
##export FC=$(which ftn)
##export CFLAGS="-I${ROCM_PATH}/include"
##export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
##export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa"

make -j18 USE_CUDA=FALSE USE_HIP=TRUE DO_TURB=TRUE MAX_SPEC=2
make -j10 USE_CUDA=FALSE USE_HIP=TRUE DO_TURB=TRUE MAX_SPEC=2 USE_HEFFTE_ROCFFT=TRUE
61 changes: 37 additions & 24 deletions src_analysis/TurbSpectra.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
#include <AMReX_Vector.H>
#include <AMReX_VisMF.H>

// These are for heFFTe / FFTW / cuFFT / rocFFT

#if defined(HEFFTE)
// HEFFTE
#if defined(HEFFTE_FFTW) || defined(HEFFTE_CUFFT) || defined(HEFFTE_ROCFFT)
#include <heffte.h>
#endif

// non-HEFFTE
#ifdef AMREX_USE_CUDA
#include <cufft.h>
#elif AMREX_USE_HIP
Expand All @@ -34,56 +36,67 @@

using namespace amrex;

#if !defined(HEFFTE_FFTW) && !defined(HEFFTE_CUFFT) && !defined(HEFFTE_ROCFFT)
#ifdef AMREX_USE_CUDA
std::string cufftError (const cufftResult& err);
#endif
#ifdef AMREX_USE_HIP
std::string rocfftError (const rocfft_status err);
void rocfft_status (std::string const& name, rocfft_status status);
void Assert_rocfft_status (std::string const& name, rocfft_status status);
#endif
#endif

#if defined(HEFFTE)
void IntegrateKScalar(const BaseFab<GpuComplex<Real> >& spectral_field,
const std::string& name, const Real& scaling,
const Box& c_local_box,
const Real& sqrtnpts,
const int& step,
const int& comp);
void IntegrateKVelocity(const BaseFab<GpuComplex<Real> >& spectral_field,
const std::string& name, const Real& scaling,
const Box& c_local_box,
const int& step);
#else
#if defined(HEFFTE_FFTW) || defined(HEFFTE_CUFFT) || defined(HEFFTE_ROCFFT)
void IntegrateKScalarHeffte(const BaseFab<GpuComplex<Real> >& spectral_field,
const std::string& name, const Real& scaling,
const Box& c_local_box,
const Real& sqrtnpts,
const int& step);
void IntegrateKVelocityHeffte(const BaseFab<GpuComplex<Real> >& spectral_fieldx,
const BaseFab<GpuComplex<Real> >& spectral_fieldy,
const BaseFab<GpuComplex<Real> >& spectral_fieldz,
const std::string& name, const Real& scaling,
const Box& c_local_box,
const int& step);
void TurbSpectrumScalarHeffte(const MultiFab& variables,
const amrex::Geometry& geom,
const int& step,
const amrex::Vector<amrex::Real>& var_scaling,
const amrex::Vector< std::string >& var_names);
void TurbSpectrumVelDecompHeffte(const MultiFab& vel,
MultiFab& vel_decomp,
const amrex::Geometry& geom,
const int& step,
const amrex::Real& var_scaling,
const amrex::Vector< std::string >& var_names);
#endif

#if !defined(HEFFTE_FFTW) && !defined(HEFFTE_CUFFT) && !defined(HEFFTE_ROCFFT)
void IntegrateKScalar(const Vector<std::unique_ptr<BaseFab<GpuComplex<Real> > > >& spectral_field,
const MultiFab& variables_onegrid,
const std::string& name,
const Real& scaling,
const Real& sqrtnpts,
const int& step,
const int& comp);
const int& step);
void IntegrateKVelocity(const Vector<std::unique_ptr<BaseFab<GpuComplex<Real> > > >& spectral_fieldx,
const Vector<std::unique_ptr<BaseFab<GpuComplex<Real> > > >& spectral_fieldy,
const Vector<std::unique_ptr<BaseFab<GpuComplex<Real> > > >& spectral_fieldz,
const MultiFab& vel_onegrid,
const std::string& name,
const Real& scaling,
const int& step);
void InverseFFTVel(Vector<std::unique_ptr<BaseFab<GpuComplex<Real> > > >& spectral_field,
MultiFab& vel_decomp_onegrid, const IntVect& fft_size);
#endif

void TurbSpectrumScalar(const MultiFab& variables,
const amrex::Geometry& geom,
const int& step,
const amrex::Vector<amrex::Real>& var_scaling,
const amrex::Vector< std::string >& var_names);

void TurbSpectrumVelDecomp(const MultiFab& vel,
MultiFab& vel_decomp,
const amrex::Geometry& geom,
const int& step,
const amrex::Real& var_scaling,
const amrex::Vector< std::string >& var_names);


void InverseFFTVel(Vector<std::unique_ptr<BaseFab<GpuComplex<Real> > > >& spectral_field,
MultiFab& vel_decomp_onegrid, const IntVect& fft_size);
#endif
#endif
Loading

0 comments on commit b625dc4

Please sign in to comment.