diff --git a/Docs/compile-notes/README.md b/Docs/compile-notes/README.md new file mode 100644 index 00000000..cacc9bf6 --- /dev/null +++ b/Docs/compile-notes/README.md @@ -0,0 +1,3 @@ +# Compile-Guides + +Please go to [lwJi/ETK-Compile-Guides](https://github.com/lwJi/ETK-Compile-Guides) for more informaiton. diff --git a/Docs/compile-notes/frontera/README.md b/Docs/compile-notes/frontera/README.md deleted file mode 100644 index 9eb923c4..00000000 --- a/Docs/compile-notes/frontera/README.md +++ /dev/null @@ -1,144 +0,0 @@ -# Install CarpetX with Spack (Frontera) - -* Use interactive session - - - Compile CPU version: `idev -m 120` - - - Compile GPU version: `idev -p rtx-dev -m 120` - -* Download CarpetX and SpacetimeX - - ```bash - curl -kLO https://raw.githubusercontent.com/gridaphobe/CRL/master/GetComponents - chmod a+x GetComponents - ./GetComponents --root Cactus --parallel --no-shallow https://raw.githubusercontent.com/lwJi/SpacetimeX/main/Docs/thornlist/spacetimex.th - ``` - - -## The Short Way - -### Intel-Oneapi version (`oneapi@2023.1.0`) - -* Load `intel/23.1.0` - - - `module load intel/23.1.0` - -* Install SpacetimeX - - ```bash - cd Cactus - gmake SpacetimeX-oneapi options=repos/SpacetimeX/Docs/compile-notes/frontera/configs/config-frontera-oneapi-impi.cfg - cp repos/SpacetimeX/Docs/thornlist/spacetimex.th configs/SpacetimeX-oneapi/ThornList - gmake -j24 SpacetimeX-oneapi - ``` - -### CUDA version (`cuda@11.8.0`) - -* Load `intel/19.1.1` - - - `module reset` or `module load intel/19.1.1` - -* Install SpacetimeX - - ```bash - . /work2/08708/liwei/frontera/SpackSource/spack/share/spack/setup-env.sh - spack load gcc@11.2.0 - spack load cuda@11.8.0 - - cd Cactus - gmake SpacetimeX-cuda options=repos/SpacetimeX/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg - cp repos/SpacetimeX/Docs/thornlist/spacetimex.th configs/SpacetimeX-cuda/ThornList - gmake -j16 SpacetimeX-cuda - ``` - - -## The Long Way - -### Intel-Oneapi version (`oneapi@2023.1.0`) - -* Load `intel/23.1.0` - - - `module load intel/23.1.0` - -* Download spack - - - `git clone -c feature.manyFiles=true https://github.com/spack/spack.git` - - - `git checkout releases/v0.20` - - - `. share/spack/setup-env.sh` - - - `spack compiler find` - -* Create a dir where you want put `view` in (say `/work2/.../username/frontera/SpackView/oneapi`) - -* Replace the last line of `oneapi23.1.0-impi21.9.0/spack.yaml` with your own dir (say `/work2/.../username/frontera/SpackView/oneapi/view`) - -* Replace the dir `/work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view` (with say `/work2/.../username/frontera/SpackView/oneapi/view`) in `configs/config-frontera-oneapi-impi.cfg` - -* Install other required packages - - ```bash - env TMPDIR=$WORK/tmp spack --env-dir ./oneapi23.1.0-impi21.9.0 compiler find - env TMPDIR=$WORK/tmp spack --env-dir ./oneapi23.1.0-impi21.9.0 concretize --force - env TMPDIR=$WORK/tmp spack --env-dir ./oneapi23.1.0-impi21.9.0 install --fail-fast - ``` - -* Install SpacetimeX - - ```bash - cd Cactus - gmake SpacetimeX-oneapi options=config-frontera-oneapi-impi.cfg - cp repos/SpacetimeX/Docs/thornlist/spacetimex.th configs/SpacetimeX-oneapi/ThornList - gmake -j24 SpacetimeX-oneapi - ``` - - -### Cuda version (`cuda@11.8.0` with `intel@19.1.1`) - -* Load `intel/19.1.1` - - - `module reset` or `module load intel/19.1.1` - -* Download spack - - - `git clone -c feature.manyFiles=true https://github.com/spack/spack.git` - - - `git checkout relesases/v0.21` - - - `. share/spack/setup-env.sh` - - - `spack compiler find` - -* Install gcc@11.2.0 - -```bash -spack install gcc@11.2.0 %gcc@4.8.5 -spack compiler add ... # ... is the last line of previous command -``` - -* Create a dir where you want put `view` in (say `/work2/.../username/frontera/SpackView/cuda`) - -* Replace the last line of `gcc11.2.0-cuda11.8.0-impi19.0.9/spack_yaml` with your dir (say `/work2/.../username/frontera/SpackView/cuda/view`) - -* Replace the dir `/work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view` (with say `/work2/.../username/frontera/SpackView/cuda/view`) -in `config-frontera-gcc-cuda-impi.cfg` - -* Install other required packages - -```bash -env TMPDIR=$WORK/tmp spack --env-dir ./gcc11.2.0-cuda11.8.0-impi19.0.9 compiler find view-cuda-compilers -env TMPDIR=$WORK/tmp spack --env-dir ./gcc11.2.0-cuda11.8.0-impi19.0.9 concretize --force -env TMPDIR=$WORK/tmp spack --env-dir ./gcc11.2.0-cuda11.8.0-impi19.0.9 install --fail-fast -``` - -* Install SpacetimeX - -```bash -spack load gcc@11.2.0 -spack load cuda@11.8.0 -cd Cactus -gmake SpacetimeX-cuda options=config-frontera-gcc-cuda-impi.cfg -cp repos/SpacetimeX/Docs/thornlist/spacetimex.th configs/SpacetimeX-cuda/ThornList -gmake -j16 SpacetimeX-cuda -``` diff --git a/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg b/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg deleted file mode 100644 index f422a40e..00000000 --- a/Docs/compile-notes/frontera/configs/config-frontera-gcc-cuda-impi.cfg +++ /dev/null @@ -1,102 +0,0 @@ -# Configuration for an frontera installation: cuda - -# Whenever this version string changes, the application is configured -# and rebuilt from scratch -VERSION = frontera-2023-09-30 - -CPP = cpp -CC = gcc -CXX = nvcc -x cu -FPP = cpp -FC = gfortran -F90 = gfortran -LD = nvcc -lcurand - -CPPFLAGS = -DSIMD_CPU -CFLAGS = -pipe -g -std=gnu11 -CXXFLAGS = -pipe -g0 -std=c++17 --compiler-options -std=gnu++17 --expt-relaxed-constexpr --extended-lambda --gpu-architecture sm_70 --forward-unknown-to-host-compiler --Werror ext-lambda-captures-this --relocatable-device-code=true --objdir-as-tempdir -FPPFLAGS = -traditional -F90FLAGS = -pipe -g -fcray-pointer -ffixed-line-length-none -LIBS = nvToolsExt gfortran - -#CUCC = nvcc -#CUCCFLAGS = -std=c++14 -I/opt/apps/cuda/11.3/include #-L/opt/apps/cuda/11.3/lib64 -#LDFLAGS = -rdynamic -#LDFLAGS = -Wl,-rpath,/opt/apps/cuda/11.3/lib64 -Wl,-rpath,/work2/08708/liwei/frontera/SpackView/cuda/view/lib -Wl,-rpath,/work2/08708/liwei/frontera/SpackView/cuda/view/lib64 -#LIBDIRS = /opt/apps/cuda/11.3/lib64 - -C_LINE_DIRECTIVES = yes -F_LINE_DIRECTIVES = yes - -INTEGER_PRECISION = 4 -REAL_PRECISION = 8 - -# CUDA does not support these -DISABLE_INT16 = yes -DISABLE_REAL16 = yes - -DEBUG = no -CPP_DEBUG_FLAGS = -DCARPET_DEBUG -C_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -FPP_DEBUG_FLAGS = -DCARPET_DEBUG -F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv - -OPTIMISE = yes -C_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations -CXX_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations -F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations - -OPENMP = yes -CPP_OPENMP_FLAGS = -fopenmp -FPP_OPENMP_FLAGS = -D_OPENMP - -WARN = yes -CPP_WARN_FLAGS = -Wall -C_WARN_FLAGS = -CXX_WARN_FLAGS = -FPP_WARN_FLAGS = -Wall -F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising - -VECTORISE = no - -MPI_DIR = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi -MPI_LIB_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release -MPI_INC_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include -MPI_LIBS = mpi - -GSL_DIR = /opt/apps/intel19/gsl/2.6 -HWLOC_DIR = /opt/apps/hwloc/1.11.12 - -BLAS_DIR = NO_BUILD -BLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -LAPACK_DIR = NO_BUILD -LAPACK_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -OPENBLAS_DIR = NO_BUILD -OPENBLAS_LIBS = # /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 - -PTHREADS_DIR = NO_BUILD - -FFTW3_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -HDF5_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -HDF5_ENABLE_CXX = yes -HDF5_ENABLE_FORTRAN = yes -HDF5_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/include -HDF5_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib -HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5 -ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib64 -ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 -AMREX_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/include -NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib -NSIMD_ARCH = AVX2 -OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -SSHT_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -SILO_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -#SILO_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/include -#SILO_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view/lib -YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view -ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view diff --git a/Docs/compile-notes/frontera/configs/config-frontera-gcc-impi.cfg b/Docs/compile-notes/frontera/configs/config-frontera-gcc-impi.cfg deleted file mode 100644 index b2ea3be6..00000000 --- a/Docs/compile-notes/frontera/configs/config-frontera-gcc-impi.cfg +++ /dev/null @@ -1,84 +0,0 @@ -# Configuration for an frontera installation: gcc - -# Whenever this version string changes, the application is configured -# and rebuilt from scratch -VERSION = frontera-2023-09-30 - -CPP = cpp -CC = gcc -CXX = g++ -FPP = cpp -F77 = gfortran -F90 = gfortran - -CFLAGS = -pipe -g -std=gnu11 -CXXFLAGS = -pipe -g -std=gnu++17 -FPPFLAGS = -traditional -F90FLAGS = -pipe -g -fcray-pointer -ffixed-line-length-none -#LDFLAGS = -Wl,-rpath=/work2/08708/liwei/frontera/SpackView/gcc/view/lib -#CPPFLAGS = -DMPICH_IGNORE_CXX_SEEK - -C_LINE_DIRECTIVES = yes -F_LINE_DIRECTIVES = yes - -INTEGER_PRECISION = 4 -REAL_PRECISION = 8 - -DEBUG = no -CPP_DEBUG_FLAGS = -DCARPET_DEBUG -C_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -FPP_DEBUG_FLAGS = -DCARPET_DEBUG -F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv - -OPTIMISE = yes -C_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations -CXX_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations -F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans -funsafe-math-optimizations - -OPENMP = yes -CPP_OPENMP_FLAGS = -fopenmp -FPP_OPENMP_FLAGS = -D_OPENMP - -WARN = yes -CPP_WARN_FLAGS = -Wall -C_WARN_FLAGS = -CXX_WARN_FLAGS = -FPP_WARN_FLAGS = -Wall -F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising - -MPI_DIR = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64 -MPI_LIB_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release -MPI_INC_DIRS = /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include -MPI_LIBS = mpi - -GSL_DIR = /opt/apps/intel19/gsl/2.6 -FFTW3_DIR = /opt/apps/intel19/impi19_0/fftw3/3.3.10 -HWLOC_DIR = /opt/apps/hwloc/1.11.12 - -BLAS_DIR = /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -BLAS_LIBS = mkl_intel_lp64 mkl_core mkl_gnu_thread -LAPACK_DIR = /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -LAPACK_LIBS = mkl_intel_lp64 mkl_core mkl_gnu_thread -OPENBLAS_DIR = /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64 -OPENBLAS_LIBS = mkl_intel_lp64 mkl_core mkl_gnu_thread - -PTHREADS_DIR = NO_BUILD - -HDF5_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5 -ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view/lib64 -ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 -AMREX_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view/include -NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view/lib -NSIMD_ARCH = AVX512_SKYLAKE -NSIMD_OPTIONS = #AVX2 FMA -OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -SSHT_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -SILO_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view -ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view diff --git a/Docs/compile-notes/frontera/configs/config-frontera-oneapi-impi.cfg b/Docs/compile-notes/frontera/configs/config-frontera-oneapi-impi.cfg deleted file mode 100644 index 81bc56bd..00000000 --- a/Docs/compile-notes/frontera/configs/config-frontera-oneapi-impi.cfg +++ /dev/null @@ -1,92 +0,0 @@ -# Configuration for an frontera installation: oneapi - -# Whenever this version string changes, the application is configured -# and rebuilt from scratch -VERSION = frontera-2023-09-30 - -CPP = cpp -CC = icx -CXX = icpx -FPP = cpp -F77 = ifx -F90 = ifx - -CPPFLAGS = -g -D_XOPEN_SOURCE -D_XOPEN_SOURCE_EXTENDED -DSIMD_CPU -FPPFLAGS = -g -traditional -CFLAGS = -g -pipe -std=c11 -CXXFLAGS = -g -pipe -std=c++17 -F77FLAGS = -g -traceback -debug all -xCORE-AVX512 -align -pad -safe-cray-ptr -F90FLAGS = -g -traceback -debug all -xCORE-AVX512 -align -pad -safe-cray-ptr - -#LDFLAGS = -Wl,-rpath=/work2/08708/liwei/frontera/SpackView/oneapi/view/lib -#CPPFLAGS = -DMPICH_IGNORE_CXX_SEEK - -C_LINE_DIRECTIVES = yes -F_LINE_DIRECTIVES = yes - -INTEGER_PRECISION = 4 -REAL_PRECISION = 8 - -DEBUG = no -CPP_DEBUG_FLAGS = -DCARPET_DEBUG -C_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -CXX_DEBUG_FLAGS = -fbounds-check -fsanitize=undefined -fstack-protector-all -ftrapv -FPP_DEBUG_FLAGS = -DCARPET_DEBUG -F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -fsanitize=undefined -fstack-protector-all -ftrapv - -OPTIMISE = yes -C_OPTIMISE_FLAGS = -Ofast -CXX_OPTIMISE_FLAGS = -Ofast -F90_OPTIMISE_FLAGS = -Ofast - -OPENMP = yes -CPP_OPENMP_FLAGS = -fopenmp -FPP_OPENMP_FLAGS = -fopenmp -C_OPENMP_FLAGS = -fopenmp -CXX_OPENMP_FLAGS = -fopenmp -F77_OPENMP_FLAGS = -fopenmp -F90_OPENMP_FLAGS = -fopenmp - -WARN = yes -CPP_WARN_FLAGS = -Wall -C_WARN_FLAGS = -Wno-implicit-function-declaration -CXX_WARN_FLAGS = -FPP_WARN_FLAGS = -F90_WARN_FLAGS = - -MPI_DIR = /opt/intel/oneapi/mpi/2021.9.0 -MPI_LIB_DIRS = /opt/intel/oneapi/mpi/2021.9.0/lib -MPI_INC_DIRS = /opt/intel/oneapi/mpi/2021.9.0/include -MPI_LIBS = mpi - -HWLOC_DIR = /opt/apps/hwloc/1.11.12 - -BLAS_DIR = NO_BUILD -BLAS_LIBS = -mkl -LAPACK_DIR = NO_BUILD -LAPACK_LIBS = -mkl -OPENBLAS_DIR = NO_BUILD -OPENBLAS_LIBS = -mkl - -PTHREADS_DIR = NO_BUILD - -GSL_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -ADIOS2_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -ADIOS2_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view/lib64 -ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 -OPENPMD_API_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -OPENPMD_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -FFTW3_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -HDF5_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5 -SILO_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -BOOST_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -AMREX_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -NSIMD_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -NSIMD_INC_DIRS = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view/include -NSIMD_LIB_DIRS = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view/lib -NSIMD_ARCH = AVX512_SKYLAKE -NSIMD_OPTIONS = #AVX2 FMA -SSHT_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -YAML_CPP_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view -ZLIB_DIR = /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view diff --git a/Docs/compile-notes/frontera/gcc11.2.0-cuda11.8.0-impi19.0.9/spack.yaml b/Docs/compile-notes/frontera/gcc11.2.0-cuda11.8.0-impi19.0.9/spack.yaml deleted file mode 100644 index cc0c0a91..00000000 --- a/Docs/compile-notes/frontera/gcc11.2.0-cuda11.8.0-impi19.0.9/spack.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# source $WORK/SpackSource/spack/share/spack/setup-env.sh -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-cuda11.8.0-impi19.0.9 compiler find view-cuda-compilers && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-cuda11.8.0-impi19.0.9 concretize --force && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-cuda11.8.0-impi19.0.9 install --fail-fast -spack: - definitions: - - compilers: - - gcc @11.2.0 #arch=linux-centos7-broadwell - - packages: - - adios2 ~python - - amrex dimensions=3 +cuda cuda_arch=70 ~fortran +hdf5 +openmp +particles - - cuda @11.8.0 +allow-unsupported-compilers - - fftw +mpi +openmp - - hwloc @1.11.12 - - petsc @3.17 +cuda +fftw +hwloc +openmp - - nsimd @3.0.1 - - openpmd-api - - ssht - - hdf5 @1.10.9 +cxx +fortran +hl +mpi +threadsafe - - silo@4.10.2 +hdf5 ~fortran ~pic ~shared - - yaml-cpp - - zlib - # - gsl - # - openblas - packages: - petsc: - buildable: false - externals: - - spec: petsc@3.17 - prefix: /home1/apps/intel19/impi19_0/petsc/3.17 - hwloc: - buildable: false - externals: - - spec: hwloc@1.11.12 - prefix: /opt/apps/hwloc/1.11.12 - #cmake: - # buildable: false - # externals: - # - spec: cmake@3.24.2 - # prefix: /opt/apps/cmake/3.24.2 - intel-mkl: - buildable: false - externals: - - spec: intel-mkl@19.1.1 - prefix: /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl - #fftw: - # buildable: false - # externals: - # - spec: fftw@3.3.10 - # prefix: /opt/apps/intel19/impi19_0/fftw3/3.3.10 - #hdf5: - # buildable: false - # externals: - # - spec: hdf5@1.10.4 - # prefix: /opt/apps/intel19/hdf5/1.10.4/x86_64 - libfabric: - buildable: false - externals: - - spec: libfabric@19.0.9 - prefix: /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib - intel-mpi: - buildable: false - externals: - - spec: intel-mpi@19.0.9 - prefix: /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi - mpi: - buildable: false - specs: - - matrix: - - [$%compilers] - - [$packages] - concretizer: - reuse: false - unify: true - view: /work2/08708/liwei/frontera/SpackView/gcc11.2.0-cuda11.8.0-impi19.0.9/view diff --git a/Docs/compile-notes/frontera/gcc11.2.0-impi19.0.9/spack.yaml b/Docs/compile-notes/frontera/gcc11.2.0-impi19.0.9/spack.yaml deleted file mode 100644 index 776b8785..00000000 --- a/Docs/compile-notes/frontera/gcc11.2.0-impi19.0.9/spack.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# source $WORK/SpackSource/spack/share/spack/setup-env.sh -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-impi19.0.9 compiler find && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-impi19.0.9 concretize --force && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/gcc11.2.0-impi19.0.9 install --fail-fast -spack: - definitions: - - compilers: - - gcc @11.2.0 #arch=linux-centos7-broadwell - - packages: - - adios2 - - amrex ~fortran +hdf5 +openmp +particles - - fftw +mpi +openmp - - hwloc @1.11.12 - - nsimd @3.0.1 - - openpmd-api @0.15.1 - - ssht - - hdf5 @1.10.9 +cxx +fortran +hl +mpi +threadsafe - - silo@4.10.2 +hdf5 - - yaml-cpp - - zlib - # - petsc @3.17 +fftw +hwloc +openmp - # - gsl - # - openblas - packages: - #petsc: - # buildable: false - # externals: - # - spec: petsc@3.17 - # prefix: /home1/apps/intel19/impi19_0/petsc/3.17 - hwloc: - buildable: false - externals: - - spec: hwloc@1.11.12 - prefix: /opt/apps/hwloc/1.11.12 - #cmake: - # buildable: false - # externals: - # - spec: cmake@3.24.2 - # prefix: /opt/apps/cmake/3.24.2 - intel-mkl: - buildable: false - externals: - - spec: intel-mkl@19.1.1 - prefix: /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl - fftw: - buildable: false - externals: - - spec: fftw@3.3.10 - prefix: /opt/apps/intel19/impi19_0/fftw3/3.3.10 - libfabric: - buildable: false - externals: - - spec: libfabric@19.0.9 - prefix: /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib - intel-mpi: - buildable: false - externals: - - spec: intel-mpi@19.0.9 - prefix: /opt/intel/compilers_and_libraries_2020.4.304/linux/mpi - mpi: - buildable: false - specs: - - matrix: - - [$%compilers] - - [$packages] - concretizer: - reuse: false - unify: true - view: /work2/08708/liwei/frontera/SpackView/gcc11.2.0-impi19.0.9/view diff --git a/Docs/compile-notes/frontera/oneapi23.1.0-impi21.9.0/spack.yaml b/Docs/compile-notes/frontera/oneapi23.1.0-impi21.9.0/spack.yaml deleted file mode 100644 index 68b93c92..00000000 --- a/Docs/compile-notes/frontera/oneapi23.1.0-impi21.9.0/spack.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# source $WORK/SpackSource/spack/share/spack/setup-env.sh -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/oneapi23.1.0-impi21.9.0 compiler find && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/oneapi23.1.0-impi21.9.0 concretize --force && -# env TMPDIR=$WORK/tmp spack --env-dir $WORK/Tools/spack-files/frontera/github/oneapi23.1.0-impi21.9.0 install --fail-fast -spack: - definitions: - - compilers: - - oneapi@2023.1.0 - - packages: - - gmake@4.3 - - adios2 - - amrex dimensions=3 ~fortran +hdf5 +openmp +particles - - boost cxxstd=17 +filesystem +mpi +system - - fftw +mpi +openmp - - hwloc @1.11.12 - - nsimd - - openpmd-api @0.15.1 - - ssht - - hdf5 +cxx +fortran +hl +mpi +threadsafe - - silo +hdf5 - - yaml-cpp - - zlib - - gsl - packages: - hwloc: - buildable: false - externals: - - spec: hwloc@1.11.12 - prefix: /opt/apps/hwloc/1.11.12 - intel-mkl: - buildable: false - externals: - - spec: intel-mkl@23.1.0 - prefix: /opt/intel/oneapi/mkl/2023.1.0 - libfabric: - buildable: false - externals: - - spec: libfabric@2021.9.0 - prefix: /opt/intel/oneapi/mpi/2021.9.0/libfabric/lib - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.9.0 - prefix: /opt/intel/oneapi - mpi: - buildable: false - specs: - - matrix: - - [$%compilers] - - [$packages] - concretizer: - reuse: false - unify: true - view: /work2/08708/liwei/frontera/SpackView/oneapi23.1.0-impi21.9.0/view diff --git a/Docs/compile-notes/frontera/submitscript/sub-cpu b/Docs/compile-notes/frontera/submitscript/sub-cpu deleted file mode 100644 index 2c0b2160..00000000 --- a/Docs/compile-notes/frontera/submitscript/sub-cpu +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -#SBATCH -J qc0_X_N4 -#SBATCH -p development # Queue (partition) name -#SBATCH -N 4 # Total # of nodes 40 -#SBATCH -n 8 # Total # of mpi tasks 280 -#SBATCH -o myjob.o%j # Name of stdout output file -#SBATCH -e myjob.e%j # Name of stderr error file -#SBATCH -t 00:30:00 # Run time (hh:mm:ss) -#SBATCH -A PHY20010 # Project/Allocation name (req'd if you have more than 1) - -##SBATCH --mail-type=all # Send email at begin and end of job -##SBATCH --mail-user=...@rit.edu - -# Any other commands must follow all #SBATCH directives... -export OMP_NUM_THREADS=2 -#export FI_PROVIDER=sockets -#export I_MPI_DEBUG=4 - -module load intel/23.1.0 -ml - -source /work2/08708/liwei/frontera/spack/share/spack/setup-env.sh -spack load gcc@11.2.0 - -# Launch MPI code... -#mpiexec -np $ranks -ppn $ppn ./cactus_Spherical SphericalNR_B2_off_centered.par -ibrun ./cactus_SpacetimeX-oneapi qc0.par diff --git a/Docs/compile-notes/frontera/submitscript/sub-gpu b/Docs/compile-notes/frontera/submitscript/sub-gpu deleted file mode 100644 index 3cf96dd5..00000000 --- a/Docs/compile-notes/frontera/submitscript/sub-gpu +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -#SBATCH -J qc0_X_N4 -#SBATCH -p rtx-dev # Queue (partition) name -#SBATCH -N 1 # Total # of nodes 40 -#SBATCH -n 4 # Total # of mpi tasks 280 -#SBATCH -o myjob.o%j # Name of stdout output file -#SBATCH -e myjob.e%j # Name of stderr error file -#SBATCH -t 02:00:00 # Run time (hh:mm:ss) -#SBATCH -A PHY20010 # Project/Allocation name (req'd if you have more than 1) - -##SBATCH --ntasks-per-node=4 -##SBATCH --gpu-bind=map_gpu:0,1,2,3 - -##SBATCH --mail-type=all # Send email at begin and end of job -##SBATCH --mail-user=@rit.edu - -# Any other commands must follow all #SBATCH directives... -#export OMP_NUM_THREADS=1 - -module reset -ml - -source /work2/08708/liwei/frontera/SpackSource/spack/share/spack/setup-env.sh -spack load gcc@11.2.0 -spack load cuda@11.8.0 - -ibrun ./cactus_SpacetimeX-cuda qc0.par diff --git a/Docs/compile-notes/frontier/Load-Module-CarpetX-cce15.sh b/Docs/compile-notes/frontier/Load-Module-CarpetX-cce15.sh deleted file mode 100755 index c0f01cda..00000000 --- a/Docs/compile-notes/frontier/Load-Module-CarpetX-cce15.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -module load craype-accel-amd-gfx90a -module load rocm/5.3.0 - -export MPICH_GPU_SUPPORT_ENABLED=1 -export PE_MPICH_GTL_DIR_amd_gfx90a="-L/opt/cray/pe/mpich/8.1.23/gtl/lib" -export PE_MPICH_GTL_LIBS_amd_gfx90a="-lmpi_gtl_hsa" - -module load adios2/2.8.3 -module load amrex/22.11 -module load boost/1.79.0-cxx17 -module load cray-fftw/3.3.10.3 -module load cray-hdf5-parallel/1.12.2.1 -module load gsl/2.7.1 -module load hwloc/2.5.0 -module load libjpeg-turbo/2.1.0 -module load openblas/0.3.17 -module load openpmd-api/0.14.4 -module load zlib/1.2.11 diff --git a/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17-amd.sh b/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17-amd.sh deleted file mode 100755 index 5b5b8481..00000000 --- a/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17-amd.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -module purge -module load PrgEnv-cray/8.5.0 -module load cce/17.0.0 -module load cpe/23.12 -module load amd-mixed/6.0.0 -module load craype-accel-amd-gfx90a -module load craype-x86-trento -module load openpmd-api -module load boost -module load cray-fftw -module load cray-hdf5-parallel -module load gsl -module load hwloc -module load openblas -module load zlib -module load adios2 -module load libjpeg-turbo -module unload darshan-runtime - - -################################################################################ -# old -################################################################################ - -#module purge -#module load DefApps/default -#module load libfabric/1.15.2.0 -#module load craype-x86-trento -#module load craype-network-ofi -#module load cray-pmi/6.1.8 -#module load craype-accel-amd-gfx90a -#module load cce/15.0.0 -#module load craype/2.7.19 -#module load cray-dsmml/0.2.2 -#module load cray-mpich/8.1.23 -#module load cray-libsci/22.12.1.1 -# -#module load perftools-base/22.12.0 -#module load xpmem/2.6.2-2.5_2.22__gd067c3f.shasta diff --git a/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17-cray.sh b/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17-cray.sh deleted file mode 100755 index db6385c7..00000000 --- a/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17-cray.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -module purge -module load PrgEnv-cray/8.5.0 -module load cce/17.0.0 -module load cpe/23.12 -module load rocm/6.0.0 -module load craype-accel-amd-gfx90a -module load craype-x86-trento -module load openpmd-api -module load boost -module load cray-fftw -module load cray-hdf5-parallel -module load gsl -module load hwloc -module load openblas -module load zlib -module load adios2 -module load libjpeg-turbo -module unload darshan-runtime - - -################################################################################ -# old -################################################################################ - -#module purge -#module load DefApps/default -#module load libfabric/1.15.2.0 -#module load craype-x86-trento -#module load craype-network-ofi -#module load cray-pmi/6.1.8 -#module load craype-accel-amd-gfx90a -#module load cce/15.0.0 -#module load craype/2.7.19 -#module load cray-dsmml/0.2.2 -#module load cray-mpich/8.1.23 -#module load cray-libsci/22.12.1.1 -# -#module load perftools-base/22.12.0 -#module load xpmem/2.6.2-2.5_2.22__gd067c3f.shasta diff --git a/Docs/compile-notes/frontier/README.md b/Docs/compile-notes/frontier/README.md deleted file mode 100644 index 0eb577cb..00000000 --- a/Docs/compile-notes/frontier/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Compile CarpetX (Frontier) - -* Download CarpetX and SpacetimeX - - ```bash - curl -kLO https://raw.githubusercontent.com/gridaphobe/CRL/master/GetComponents - chmod a+x GetComponents - ./GetComponents --root Cactus --parallel --no-shallow https://raw.githubusercontent.com/lwJi/SpacetimeX/main/Docs/thornlist/spacetimex.th - ``` - -## The Short Way - - -### cce-17.0.0 - -* Load modules - -```bash -source repos/SpacetimeX/Docs/compile-notes/frontier/Load-Module-CarpetX-cce17.sh -``` - -* Install AsterX - -```bash -cd Cactus -gmake AsterX options=repos/SpacetimeX/Docs/compile-notes/frontier/configs/frontier-cce17.cfg -cp repos/SpacetimeX/Docs/thornlist/asterx-frontier.th configs/AsterX/ThornList -gmake -j24 AsterX -``` - - -### cce-15.0.0 - -* Load modules - -```bash -source repos/SpacetimeX/Docs/compile-notes/frontier/Load-Module-CarpetX-cce15.sh -``` - -* Install AsterX - -```bash -cd Cactus -gmake AsterX options=simfactory/mdb/optionlists/frontier.cfg -cp repos/SpacetimeX/Docs/thornlist/asterx-frontier.th configs/AsterX/ThornList -gmake -j24 AsterX -``` - - - -## The Long Way - -### Compile AMReX - -* Clone `amrex` - -```bash -git clone https://github.com/AMReX-Codes/amrex.git -``` - -* Install `amrex` to `$HOME/local/amrex-24.06` - -```bash -cd amrex -mkdir build -cd build - -source repos/SpacetimeX/Docs/compile-notes/frontier/amrex/Load-Module-AMReX.sh -source repos/SpacetimeX/Docs/compile-notes/frontier/amrex/Export-AMReX.sh -source repos/SpacetimeX/Docs/compile-notes/frontier/amrex/Compile-AMReX.sh - -make -j24 install -``` diff --git a/Docs/compile-notes/frontier/amrex/amd/Compile-AMReX.sh b/Docs/compile-notes/frontier/amrex/amd/Compile-AMReX.sh deleted file mode 100755 index 2b8c68e4..00000000 --- a/Docs/compile-notes/frontier/amrex/amd/Compile-AMReX.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=${HOME}/local/amrex-24.06-amd \ - -DCMAKE_PREFIX_PATH='/opt/rocm-6.0.0/lib/cmake/AMDDeviceLibs;/opt/rocm-6.0.0/lib/cmake/amd_comgr;/opt/rocm-6.0.0/lib/cmake/hip;/opt/rocm-6.0.0/lib/cmake/hiprand;/opt/rocm-6.0.0/lib/cmake/hsa-runtime64;/opt/rocm-6.0.0/lib/cmake/rocprim;/opt/rocm/lib/cmake/rocrand' \ - -DAMReX_GPU_BACKEND=HIP \ - -DAMReX_AMD_ARCH=gfx90a \ - -DAMReX_FORTRAN=OFF \ - -DAMReX_FORTRAN_INTERFACES=OFF \ - -DAMReX_OMP=OFF \ - -DAMReX_PARTICLES=ON \ - -DAMReX_PRECISION=DOUBLE \ - .. - # -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/amdclang++ \ - # -DCMAKE_C_COMPILER=${ROCM_PATH}/bin/amdclang \ diff --git a/Docs/compile-notes/frontier/amrex/amd/Export-AMReX.sh b/Docs/compile-notes/frontier/amrex/amd/Export-AMReX.sh deleted file mode 100755 index cd9f9e99..00000000 --- a/Docs/compile-notes/frontier/amrex/amd/Export-AMReX.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -export MPICH_GPU_SUPPORT_ENABLED=1 -export AMREX_AMD_ARCH=gfx90a -export CC=$(which hipcc) -export CXX=$(which hipcc) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" diff --git a/Docs/compile-notes/frontier/amrex/amd/Load-Module-AMReX.sh b/Docs/compile-notes/frontier/amrex/amd/Load-Module-AMReX.sh deleted file mode 100755 index 1af19ce1..00000000 --- a/Docs/compile-notes/frontier/amrex/amd/Load-Module-AMReX.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -module purge -module load PrgEnv-cray/8.5.0 -module load cce/17.0.0 -module load cpe/23.12 -module load amd-mixed/6.0.0 -module load craype-accel-amd-gfx90a -module load craype-x86-trento -module unload darshan-runtime diff --git a/Docs/compile-notes/frontier/amrex/cray/Compile-AMReX.sh b/Docs/compile-notes/frontier/amrex/cray/Compile-AMReX.sh deleted file mode 100755 index 87ed34e3..00000000 --- a/Docs/compile-notes/frontier/amrex/cray/Compile-AMReX.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=${HOME}/local/amrex-24.06-cray \ - -DCMAKE_PREFIX_PATH='/opt/rocm-6.0.0/lib/cmake/AMDDeviceLibs;/opt/rocm-6.0.0/lib/cmake/amd_comgr;/opt/rocm-6.0.0/lib/cmake/hip;/opt/rocm-6.0.0/lib/cmake/hiprand;/opt/rocm-6.0.0/lib/cmake/hsa-runtime64;/opt/rocm-6.0.0/lib/cmake/rocprim;/opt/rocm/lib/cmake/rocrand' \ - -DAMReX_GPU_BACKEND=HIP \ - -DAMReX_AMD_ARCH=gfx90a \ - -DAMReX_FORTRAN=OFF \ - -DAMReX_FORTRAN_INTERFACES=OFF \ - -DAMReX_OMP=OFF \ - -DAMReX_PARTICLES=ON \ - -DAMReX_PRECISION=DOUBLE \ - .. - # -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/amdclang++ \ - # -DCMAKE_C_COMPILER=${ROCM_PATH}/bin/amdclang \ diff --git a/Docs/compile-notes/frontier/amrex/cray/Export-AMReX.sh b/Docs/compile-notes/frontier/amrex/cray/Export-AMReX.sh deleted file mode 100755 index 2a62cf8c..00000000 --- a/Docs/compile-notes/frontier/amrex/cray/Export-AMReX.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -export MPICH_GPU_SUPPORT_ENABLED=1 -export AMREX_AMD_ARCH=gfx90a -export CXX=$(which CC) -export CC=$(which cc) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" diff --git a/Docs/compile-notes/frontier/amrex/cray/Load-Module-AMReX.sh b/Docs/compile-notes/frontier/amrex/cray/Load-Module-AMReX.sh deleted file mode 100755 index 8f947ea2..00000000 --- a/Docs/compile-notes/frontier/amrex/cray/Load-Module-AMReX.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -module purge -module load PrgEnv-cray/8.5.0 -module load cce/17.0.0 -module load cpe/23.12 -module load rocm/6.0.0 -module load craype-accel-amd-gfx90a -module load craype-x86-trento -module unload darshan-runtime - -################################################################################ -# from AMReX-Astro -################################################################################ -#module load PrgEnv-gnu -#module load cray-mpich/8.1.27 -#module load craype-accel-amd-gfx90a -#module load amd-mixed/6.0.0 -#module unload darshan-runtime - -################################################################################ -# from vass -################################################################################ -#module load cce/17.0.0 -#module load cpe/23.12 -#module load rocm/6.0.0 -#module load craype-accel-amd-gfx90a -#module load cray-hdf5-parallel -#module load cray-fftw -#module load boost -#module load adios2 -#module load hsi -#module load gsl -#module load lfs-wrapper -#module load cray-dsmml/0.2.2 - - -################################################################################ -# from simfactory -################################################################################ -#module load craype-accel-amd-gfx90a -#module load rocm/5.3.0 -# -#export MPICH_GPU_SUPPORT_ENABLED=1 -#export PE_MPICH_GTL_DIR_amd_gfx90a="-L/opt/cray/pe/mpich/8.1.23/gtl/lib" -#export PE_MPICH_GTL_LIBS_amd_gfx90a="-lmpi_gtl_hsa" -# -#module load adios2/2.8.3 -#module load amrex/22.11 -#module load boost/1.79.0-cxx17 -#module load cray-fftw/3.3.10.3 -#module load cray-hdf5-parallel/1.12.2.1 -#module load gsl/2.7.1 -#module load hwloc/2.5.0 -#module load libjpeg-turbo/2.1.0 -#module load openblas/0.3.17 -#module load openpmd-api/0.14.4 -#module load zlib/1.2.11 - - -################################################################################ -# Old module list -################################################################################ - -#module load craype-x86-trento -#module load libfabric/1.15.2.0 -#module load craype-network-ofi -#module load perftools-base/22.12.0 -#module load xpmem/2.6.2-2.5_2.22__gd067c3f.shasta -#module load cray-pmi/6.1.8 -#module load DefApps/default -#module load ccache/4.5.1 -#module load cmake/3.23.2 -#module load craype-accel-amd-gfx90a -##module load rocm/5.2.0 -#module load rocm/5.3.0 -#module load craype/2.7.19 -#module load cray-dsmml/0.2.2 -#module load cray-libsci/22.12.1.1 -#module load PrgEnv-gnu/8.3.3 diff --git a/Docs/compile-notes/frontier/configs/frontier-amd.cfg b/Docs/compile-notes/frontier/configs/frontier-amd.cfg deleted file mode 100644 index 80a2ca98..00000000 --- a/Docs/compile-notes/frontier/configs/frontier-amd.cfg +++ /dev/null @@ -1,77 +0,0 @@ -# Option list for the Einstein Toolkit - -# The "weird" options here should probably be made the default in the -# ET instead of being set here. - -# Whenever this version string changes, the application is configured -# and rebuilt from scratch -VERSION = frontier-2024-06-15 - -CPP = cpp -FPP = cpp -CC = hipcc --offload-arch=gfx90a -CXX = hipcc --offload-arch=gfx90a -F90 = amdflang -LD = hipcc --offload-arch=gfx90a -lhiprand - -CPPFLAGS = -DSIMD_CPU -FPPFLAGS = -traditional -CFLAGS = -g -std=gnu11 -CXXFLAGS = -g -std=c++17 -D__cpp_lib_filesystem=1L -F90FLAGS = -g -LDFLAGS = -fgpu-rdc --hip-link - -SYS_INC_DIRS = ${ROCM_PATH}/include -LIBDIRS = ${ROCM_PATH}/lib -LIBS = stdc++fs amdhip64 pthread - -C_LINE_DIRECTIVES = yes -F_LINE_DIRECTIVES = yes - -DISABLE_REAL16 = yes - -DEBUG = no -CPP_DEBUG_FLAGS = -DCARPET_DEBUG -C_DEBUG_FLAGS = -CXX_DEBUG_FLAGS = -FPP_DEBUG_FLAGS = -DCARPET_DEBUG -F90_DEBUG_FLAGS = - -OPTIMISE = yes -C_OPTIMISE_FLAGS = -O3 -CXX_OPTIMISE_FLAGS = -O3 -F90_OPTIMISE_FLAGS = -O3 - -OPENMP = no - -WARN = yes -CPP_WARN_FLAGS = -Wall -C_WARN_FLAGS = -CXX_WARN_FLAGS = -FPP_WARN_FLAGS = -Wall -F90_WARN_FLAGS = - -VECTORISE = no - -ADIOS2_DIR = ${OLCF_ADIOS2_ROOT} -ADIOS2_INC_DIRS = ${OLCF_ADIOS2_ROOT}/include -ADIOS2_LIB_DIRS = ${OLCF_ADIOS2_ROOT}/lib64 -ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 - -AMREX_DIR = /ccs/home/liwei/local/amrex-24.06-amd - -FFTW3_DIR = ${FFTW_ROOT} -HDF5_DIR = ${HDF5_ROOT} -BOOST_DIR = ${OLCF_BOOST_ROOT} -GSL_DIR = ${OLCF_GSL_ROOT} -HWLOC_DIR = ${OLCF_HWLOC_ROOT} -LIBJPEG_DIR = ${OLCF_LIBJPEG_TURBO_ROOT} -MPI_DIR = ${CRAY_MPICH_DIR} -MPI_LIB_DIRS = ${CRAY_MPICH_DIR}/lib ${CRAY_MPICH_ROOTDIR}/gtl/lib -MPI_LIBS = mpi mpi_gtl_hsa -OPENBLAS_DIR = ${OLCF_OPENBLAS_ROOT} -PTHREADS_DIR = NO_BUILD -SILO_DIR = BUILD -SSHT_DIR = BUILD -YAML_CPP_DIR = BUILD -ZLIB_DIR = ${OLCF_ZLIB_ROOT} diff --git a/Docs/compile-notes/frontier/configs/frontier-cray.cfg b/Docs/compile-notes/frontier/configs/frontier-cray.cfg deleted file mode 100644 index 7e55b569..00000000 --- a/Docs/compile-notes/frontier/configs/frontier-cray.cfg +++ /dev/null @@ -1,86 +0,0 @@ -# Option list for the Einstein Toolkit - -# The "weird" options here should probably be made the default in the -# ET instead of being set here. - -# Whenever this version string changes, the application is configured -# and rebuilt from scratch -VERSION = frontier-2024-06-13 - -EXTERNAL_CC = cc -EXTERNAL_CXX = CC -EXTERNAL_F90 = amdflang -EXTERNAL_LD = CC - -EXTERNAL_CFLAGS = -g -O3 -EXTERNAL_CXXFLAGS = -g -O3 -EXTERNAL_F90FLAGS = -g -O3 -EXTERNAL_LDFLAGS = -g -O3 - -CPP = cpp -CC = cc -CXX = hipcc --offload-arch=gfx90a -FPP = cpp -F90 = amdflang -LD = hipcc --offload-arch=gfx90a -lhiprand - -CPPFLAGS = -DSIMD_CPU -CFLAGS = -g -std=gnu11 -CXXFLAGS = -g -std=c++17 -D__cpp_lib_filesystem=1L -FPPFLAGS = -traditional -F90FLAGS = -g -LDFLAGS = -fgpu-rdc --hip-link - -LIBDIRS = /opt/cray/pe/cce/17.0.0/cce/x86_64/lib ${OLCF_ROCM_ROOT}/lib -LIBS = stdc++fs f u amdhip64 pthread - -C_LINE_DIRECTIVES = yes -F_LINE_DIRECTIVES = yes - -DISABLE_REAL16 = yes - -DEBUG = no -CPP_DEBUG_FLAGS = -DCARPET_DEBUG -C_DEBUG_FLAGS = -CXX_DEBUG_FLAGS = -FPP_DEBUG_FLAGS = -DCARPET_DEBUG -F90_DEBUG_FLAGS = - -OPTIMISE = yes -C_OPTIMISE_FLAGS = -O3 -CXX_OPTIMISE_FLAGS = -O3 -F90_OPTIMISE_FLAGS = -O3 - -OPENMP = no - -WARN = yes -CPP_WARN_FLAGS = -Wall -C_WARN_FLAGS = -CXX_WARN_FLAGS = -FPP_WARN_FLAGS = -Wall -F90_WARN_FLAGS = - -VECTORISE = no - -ADIOS2_DIR = ${OLCF_ADIOS2_ROOT} -ADIOS2_INC_DIRS = ${OLCF_ADIOS2_ROOT}/include -ADIOS2_LIB_DIRS = ${OLCF_ADIOS2_ROOT}/lib64 -ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11 - -AMREX_DIR = /ccs/home/liwei/local/amrex-24.06 - -FFTW3_DIR = ${FFTW_ROOT} -HDF5_DIR = ${HDF5_ROOT} -BOOST_DIR = ${OLCF_BOOST_ROOT} -GSL_DIR = ${OLCF_GSL_ROOT} -HWLOC_DIR = ${OLCF_HWLOC_ROOT} -LIBJPEG_DIR = ${OLCF_LIBJPEG_TURBO_ROOT} -MPI_DIR = ${CRAY_MPICH_DIR} -MPI_LIB_DIRS = ${CRAY_MPICH_DIR}/lib ${CRAY_MPICH_ROOTDIR}/gtl/lib -MPI_LIBS = mpi mpi_gtl_hsa -OPENBLAS_DIR = ${OLCF_OPENBLAS_ROOT} -PTHREADS_DIR = NO_BUILD -SILO_DIR = BUILD -SSHT_DIR = BUILD -YAML_CPP_DIR = BUILD -ZLIB_DIR = ${OLCF_ZLIB_ROOT}