#2465 Bring to master

stfc · Jan 11, 2024 · f2361fe · f2361fe
2 parents 35a7805 + 8fa5800
commit f2361fe
Show file tree

Hide file tree

Showing 235 changed files with 12,142 additions and 9,225 deletions.
diff --git a/.github/workflows/compilation.yml b/.github/workflows/compilation.yml
@@ -35,7 +35,7 @@
 # Modified by R. W. Ford, STFC Daresbury Lab
 # Modified by A. J. Voysey, Met Office
 # Modified by J. Henrichs, Bureau of Meteorology
-# Modified by N. M. Nobre, STFC Daresbury Lab
+# Modified by N. Nobre, STFC Daresbury Lab
 
 # This workflow will use a self-hosted runner to perform the more expensive
 # compilation tests that are not run on GHA systems. At the moment it:
@@ -51,10 +51,14 @@ on:
   push
 
 env:
-  PYTHON_VERSION: 3.11.4
+  CUDA_VERSION: 12.3.1
   GFORTRAN_VERSION: 13.2.0
-  NVFORTRAN_VERSION: 23.7
-  CUDA_VERSION: 12.2
+  HDF5_VERSION: 1.14.3
+  NETCDF_C_VERSION: 4.9.2
+  NETCDF_FORTRAN_VERSION: 4.6.1
+  NVFORTRAN_VERSION: 23.11
+  OPENMPI_VERSION: 5.0.0
+  PYTHON_VERSION: 3.12.1
 
 jobs:
   run_if_on_mirror:
@@ -80,56 +84,49 @@ jobs:
         python -m venv .runner_venv
         . .runner_venv/bin/activate
         python -m pip install --upgrade pip
-        # If you wish to install the version of fparser pointed to by the
-        # submodule instead of the released version (from PyPI) then
-        # uncomment the following line:
         pip install external/fparser
         pip install .[test,psydata]
     - name: Unit tests with compilation - gfortran
       run: |
-        module load python/${PYTHON_VERSION}
         . .runner_venv/bin/activate
-        module load gcc/${GFORTRAN_VERSION} openmpi netcdf_fortran
+        module load gcc/${GFORTRAN_VERSION} openmpi/${OPENMPI_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
         pytest -n 4 --f90=gfortran --f90flags="-ffree-line-length-none" --compile --compileopencl src/psyclone/tests
-        module rm netcdf_fortran gcc
     - name: Unit tests with compilation - nvfortran
       run: |
-        module load python/${PYTHON_VERSION}
         . .runner_venv/bin/activate
-        module load nvidia-hpcsdk/${NVFORTRAN_VERSION} netcdf_fortran
-        # We have to tell nvfortran where to find the OpenCL library.
-        pytest -n 4 --f90=nvfortran --f90flags="-L/apps/packages/compilers/nvidia-hpcsdk/Linux_x86_64/${NVFORTRAN_VERSION}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib" --compile --compileopencl src/psyclone/tests
-        module rm netcdf_fortran nvidia-hpcsdk
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        pytest -n 4 --f90=nvfortran --compile --compileopencl src/psyclone/tests
     - name: Examples with compilation - gfortran
       run: |
-        module load python/${PYTHON_VERSION}
         . .runner_venv/bin/activate
-        module load gcc/${GFORTRAN_VERSION} openmpi netcdf_fortran
-        # Although we're using gfortran, we link with the OpenCL lib that comes
-        # with CUDA.
         make -C examples allclean
-        F90=gfortran F90FLAGS="-ffree-line-length-none -L/apps/packages/compilers/nvidia-hpcsdk/Linux_x86_64/${NVFORTRAN_VERSION}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib" make -C examples compile
+        # Although we're using gfortran, we link with CUDA's OpenCL library
+        module load gcc/${GFORTRAN_VERSION} openmpi/${OPENMPI_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        module load cuda/${CUDA_VERSION}
+        F90=gfortran make -C examples compile
     - name: Tutorials with compilation - gfortran
       run: |
-        module load python/${PYTHON_VERSION}
         . .runner_venv/bin/activate
         make -C tutorial/practicals allclean
-        module load gcc/${GFORTRAN_VERSION} openmpi netcdf_fortran
+        module load gcc/${GFORTRAN_VERSION} openmpi/${OPENMPI_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
         make -C tutorial/practicals compile
     - name: Examples with compilation - nvfortran
       run: |
-        module load python/${PYTHON_VERSION}
         . .runner_venv/bin/activate
         make -C examples allclean
-        module load nvidia-hpcsdk/${NVFORTRAN_VERSION} netcdf_fortran
-        # We have to tell nvfortran where to find the OpenCL library.
-        F90=nvfortran F90FLAGS="-acc -Minfo=all -L/apps/packages/compilers/nvidia-hpcsdk/Linux_x86_64/${NVFORTRAN_VERSION}/cuda/${CUDA_VERSION}/targets/x86_64-linux/lib" make -C examples compile
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        F90=nvfortran F90FLAGS="-acc -Minfo=all" make -C examples compile
     - name: Tutorials with compilation - nvfortran
       run: |
-        module load python/${PYTHON_VERSION}
         . .runner_venv/bin/activate
         make -C tutorial/practicals allclean
-        module load nvidia-hpcsdk/${NVFORTRAN_VERSION} netcdf_fortran
-        # TODO #2251. Cannot build the LFRic practicals with 23.5/7 of nvfortran
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        # TODO #2251. Cannot build the LFRic practicals with nvfortran 23.5-23.9
         # because the compilation of nan_test.f90 gives an ICE.
         F90=nvfortran F90FLAGS="-acc -Minfo=all" make -C tutorial/practicals/nemo compile
diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml
@@ -47,7 +47,7 @@ jobs:
     runs-on: self-hosted
     env:
       LFRIC_REV: 45376
-      PYTHON_VERSION: 3.11.4
+      PYTHON_VERSION: 3.12.1
 
     steps:
     - uses: actions/checkout@v3
@@ -69,9 +69,6 @@ jobs:
         python -m venv .runner_venv
         . .runner_venv/bin/activate
         python -m pip install --upgrade pip
-        # If you wish to install the version of fparser pointed to by the
-        # submodule instead of the released version (from PyPI) then
-        # uncomment the following line:
         pip install external/fparser
         pip install .[test]
         pip install jinja2

diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml
@@ -46,11 +46,13 @@ jobs:
     if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
     runs-on: self-hosted
     env:
-        PERL5LIB: /home/aporter/perl5/lib/perl5
-        PERL_LOCAL_LIB_ROOT: /home/aporter/perl5
-        PERL_MB_OPT: "--install_base \"/home/aporter/perl5\""
-        PERL_MM_OPT: "INSTALL_BASE=/home/aporter/perl5"
-        PYTHON_VERSION: 3.11.4
+      HDF5_VERSION: 1.14.3
+      NETCDF_C_VERSION: 4.9.2
+      NETCDF_FORTRAN_VERSION: 4.6.1
+      NVFORTRAN_VERSION: 23.7
+      ONEAPI_VERSION: 2024.0.0
+      PERL_VERSION: 5.38.2
+      PYTHON_VERSION: 3.12.1
 
     steps:
     - uses: actions/checkout@v3
@@ -71,38 +73,37 @@ jobs:
         python -m venv .runner_venv
         . .runner_venv/bin/activate
         python -m pip install --upgrade pip
-        # If you wish to install the version of fparser pointed to by the
-        # submodule instead of the released version (from PyPI) then
-        # uncomment the following line:
         pip install external/fparser
         pip install .[test]
-        # Add Perl to the PATH
-        echo "/home/aporter/perl5/bin" >> $GITHUB_PATH
         # Compile nvidia profiling tools
-        module load nvidia-hpcsdk
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
         cd lib/profiling/nvidia/
         F90=nvfortran make
 
     # PSyclone passthrough for MetOffice NEMO
     - name: NEMO MetOffice Passthrough
       run: |
-        source .runner_venv/bin/activate
+        . .runner_venv/bin/activate
         export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
         export NEMO_DIR=${HOME}/NEMO
-        cd examples/nemo/scripts
+        cd $PSYCLONE_NEMO_DIR
         make -j 4 passthrough
-        module load nvidia-hpcsdk netcdf_fortran
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        module load perl/${PERL_VERSION}
         make -j 4 compile-passthrough
 
     # PSyclone, compile and run MetOffice NEMO with OpenMP for GPUs
     - name: NEMO MetOffice OpenMP for GPU
       run: |
-        source .runner_venv/bin/activate
+        . .runner_venv/bin/activate
         export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
         export NEMO_DIR=${HOME}/NEMO
-        cd examples/nemo/scripts
+        cd $PSYCLONE_NEMO_DIR
         make -j 4 openmp_gpu
-        module load nvidia-hpcsdk netcdf_fortran
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        module load perl/${PERL_VERSION}
         make -j 4 compile-openmp_gpu
         export NV_ACC_POOL_THRESHOLD=75
         make run-openmp_gpu |& tee output.txt
@@ -125,12 +126,14 @@ jobs:
     # PSyclone, compile and run MetOffice NEMO with OpenACC kernels for GPUs
     - name: NEMO MetOffice OpenACC kernels for GPU
       run: |
-        source .runner_venv/bin/activate
+        . .runner_venv/bin/activate
         export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
         export NEMO_DIR=${HOME}/NEMO
-        cd examples/nemo/scripts
+        cd $PSYCLONE_NEMO_DIR
         make -j 4 openacc_kernels
-        module load nvidia-hpcsdk netcdf_fortran
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        module load perl/${PERL_VERSION}
         COMPILER_ARCH=linux_nvidia_acc_gpu make -j 4 compile-openacc_kernels
         export NV_ACC_POOL_THRESHOLD=75
         make run-openacc_kernels |& tee output.txt
@@ -151,23 +154,56 @@ jobs:
         gist_id: a4049a0fc0a0a11651a5ce6a04d76160
         file_path: ../../../../store_results/performance_history_openacc_kernels_gpu
 
-    # PSyclone, compile and run ECMWF NEMO with OpenMP for CPUs
+    # PSyclone, compile and run MetOffice NEMO with OpenACC loops for GPUs
+    - name: NEMO MetOffice OpenACC loops for GPU
+      run: |
+        . .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export NEMO_DIR=${HOME}/NEMO
+        cd $PSYCLONE_NEMO_DIR
+        make -j 4 openacc_loops
+        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        module load perl/${PERL_VERSION}
+        COMPILER_ARCH=linux_nvidia_acc_gpu make -j 4 compile-openacc_loops
+        export NV_ACC_POOL_THRESHOLD=75
+        make run-openacc_loops |& tee output.txt
+        # Check the output is as expected for the first 6 digits
+        tail -n 1 output.txt | grep -q " it :      10"
+        tail -n 1 output.txt | grep -q "|ssh|_max:  0.259483"
+        tail -n 1 output.txt | grep -q "|U|_max:  0.458515"
+        tail -n 1 output.txt | grep -q "S_min:  0.482686"
+        tail -n 1 output.txt | grep -q "S_max:  0.407622"
+        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1)
+        echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME >> ${HOME}/store_results/performance_history_openacc_loops_gpu
+        # Upload the results into a Github Gist
+
+    - name: Upload NEMO OpenACC loops GPU results
+      uses: exuanbo/actions-deploy-gist@v1
+      with:
+        token: ${{ secrets.GIST_TOKEN }}
+        gist_id: a4049a0fc0a0a11651a5ce6a04d76160
+        file_path: ../../../../store_results/performance_history_openacc_loops_gpu
+
+    # PSyclone, compile and run ECMWF NEMO with OpenMP for CPUs. This uses
+    # the Intel compiler and does not disable MPI. Therefore we have to ensure
+    # that we provide the path to the header files for Intel MPI.
     - name: NEMO ECMWF OpenMP for CPU
       run: |
-        source .runner_venv/bin/activate
+        . .runner_venv/bin/activate
+        module load oneapi/${ONEAPI_VERSION}
+        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
+        module load perl/${PERL_VERSION}
         export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
         export NEMO_DIR=${HOME}/NEMOGCM_V40
         export COMPILER_ARCH=linux_intel
         export ADD_KEYS="IEEE_IS_NAN=ieee_is_nan key_nosignedzero"
         export DEL_KEYS="key_iomput"
         export MODEL_DIR=/archive/ssiso/ecmwf_eORCA1_GO8/
         export NAMELISTS_DIR=${NEMO_DIR}/testscripts_V40/output/openmp_outer_V40_eORCA1_GO8_Z75_20170906_cray_dp_1x1/
-        cd examples/nemo/scripts
+        export MPI_INC_DIR=${I_MPI_ROOT}/include
+        cd $PSYCLONE_NEMO_DIR
         make -j 4 openmp_cpu
-        module load intel/oneapi compiler mpi
-        export LD_LIBRARY_PATH=/home/ssiso/ecmwf_nemo/ESiWACE2/scripts/dev/netcdf-c-4.9.0/lib/:$LD_LIBRARY_PATH
-        export LD_LIBRARY_PATH=/home/ssiso/ecmwf_nemo/ESiWACE2/scripts/dev/netcdf-fortran-4.5.4/lib:$LD_LIBRARY_PATH
-        export LD_LIBRARY_PATH=/home/ssiso/ecmwf_nemo/ESiWACE2/scripts/dev/hdf5-1.12.2/lib/:$LD_LIBRARY_PATH
         make -j 4 compile-openmp_cpu
         export OMP_NUM_THREADS=4
         make run-openmp_cpu |& tee output.txt

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -35,7 +35,7 @@
 # Modified by R. W. Ford, STFC Daresbury Lab
 # Modified by A. J. Voysey, Met Office
 # Modified by J. Henrichs, Bureau of Meteorology
-# Modified by N. M. Nobre, STFC Daresbury Lab
+# Modified by N. Nobre, STFC Daresbury Lab
 
 # This workflow will install Python dependencies, run tests and lint with a
 # variety of Python versions.
@@ -106,7 +106,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.11]
+        python-version: [3.7, 3.8, 3.12]
     steps:
     - uses: actions/checkout@v3
       with:
@@ -130,9 +130,6 @@ jobs:
         # Some of the examples use Jupyter. We need to keep IPython to
         # below 8.0 as we still run the test suite with Python 3.7.
         pip install "ipython>=7.0,<8.0" jupyter
-        # If you wish to install the version of fparser pointed to by the
-        # submodule instead of the released version (from PyPI) then
-        # uncomment the following line:
         pip install external/fparser
         pip install .[test]
     - name: Lint with flake8