Update testing strategy (fortran-lang#51)

* separate testing and reporting mode via dynamic test matrix
* test known supported combinations on push and pull request
* test all combos and report compatibility on schedule and manual dispatch
* add long_compat.csv to version control (easier dynamic matrix generation)
* update compatibility files and readme to reflect recently added nvidia support
* move test programs and compatibility files and scripts to subdirectories
* introduce requirements.txt file for python dependencies
* run scheduled testing weekly not monthly
* update .gitignore

Co-authored-by: w-bonelli <[email protected]>

.github/actions/test-cc/action.yml

@@ -30,6 +30,7 @@ runs:
         [[ "$ccv" == ${{ inputs.version }}* ]] && (echo "found ${{ env.CC }} version: $ccv") || (echo "unexpected ${{ env.CC }} version: $ccv"; exit 1)
     
     - name: Test compile (bash)
+      working-directory: test
       shell: bash
       run: |
         ${{ env.CC }} -o hw hw.c

.github/actions/test-cxx/action.yml

@@ -30,6 +30,7 @@ runs:
         [[ "$cxxv" == ${{ matrix.toolchain.version }}* ]] && (echo "found ${{ env.CXX }} version: $cxxv") || (echo "unexpected ${{ env.CXX }} version: $cxxv"; exit 1)
     
     - name: Test compile (bash)
+      working-directory: test
       shell: bash
       run: |
         ${{ env.CXX }} -o hw hw.cpp

.github/actions/test-fc/action.yml

@@ -30,6 +30,7 @@ runs:
         [[ "$fcv" == ${{ inputs.version }}* ]] && (echo "found ${{ env.FC }} version: $fcv") || (echo "unexpected ${{ env.FC }} version: $fcv"; exit 1)
 
     - name: Test compile (bash)
+      working-directory: test
       shell: bash
       run: |
         # macos-13/gfortran 7-9 compatibility workaround
@@ -42,13 +43,15 @@ runs:
             args="-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"
           fi
         fi
-
+        
+        # hello world program
         ${{ env.FC }} $args -o hw hw.f90
         output=$(./hw '2>&1')
-        [[ "$output" == *"hello world"* ]] && echo "$output" || (echo "Unexpected Fortran program output: $output"; exit 1)
+        [[ "$output" == *"hello world"* ]] && echo "$output" || (echo "Unexpected Fortran program 'hw' output: $output"; exit 1)
         rm hw
 
     - name: Test compile Fortran (pwsh)
+      working-directory: test
       if: ${{ (success() || failure()) && runner.os == 'Windows' }}
       shell: pwsh
       run: |
@@ -63,6 +66,7 @@ runs:
         rm hw.exe
 
     - name: Test compile Fortran (powershell)
+      working-directory: test
       if: ${{ (success() || failure()) && runner.os == 'Windows' }}
       shell: powershell
       run: |
@@ -77,6 +81,7 @@ runs:
         rm hw.exe
 
     - name: Test compile Fortran (cmd)
+      working-directory: test
       if: ${{ (success() || failure()) && runner.os == 'Windows' }}
       shell: cmd
       run: |

.github/compat/compat.csv

@@ -0,0 +1,10 @@
+compiler,gcc,gcc,gcc,gcc,gcc,gcc,gcc,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel,intel,intel,intel,intel,intel,intel,intel,intel,intel,intel,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc
+version,10,11,12,13,7,8,9,2021.1,2021.10,2021.2,2021.3,2021.4,2021.5,2021.6,2021.7.1,2021.7,2021.8,2021.9,2021.1.2,2021.1.2,2021.1,2021.2,2021.4,2022.0,2022.1,2022.2.1,2022.2,2023.0,2023.1,2023.2,20.11,21.11,22.1,22.11,23.11,23.3,23.5,23.7,23.9
+runner,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+macos-11,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,,,,,,,,,,,,,,,,,,,,,
+macos-12,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,,,,,,,,,,,,,,,,,,,,,
+macos-13,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,,,,,,,,,,,,,,,,,,,,,
+ubuntu-20.04,✓,✓,,✓,✓,✓,✓,✓,✓,✓,,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓
+ubuntu-22.04,✓,✓,✓,✓,,,✓,✓,✓,✓,,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓,✓
+windows-2019,✓,✓,✓,✓,,✓,✓,,✓,,,,,✓,,✓,✓,✓,,,,,,,✓,,✓,✓,✓,✓,,,,,,,,,
+windows-2022,✓,✓,✓,✓,,✓,✓,,✓,,,,,✓,,✓,✓,✓,,,,,,,✓,,✓,✓,✓,✓,,,,,,,,,

.github/compat/long_compat.csv

@@ -0,0 +1,209 @@
+runner,compiler,version,support
+macos-11,gcc,10,✓
+macos-11,gcc,11,✓
+macos-11,gcc,12,✓
+macos-11,gcc,13,✓
+macos-11,gcc,6,
+macos-11,gcc,7,✓
+macos-11,gcc,8,✓
+macos-11,gcc,9,✓
+macos-11,intel-classic,2021.1.2,
+macos-11,intel-classic,2021.1,✓
+macos-11,intel-classic,2021.10,✓
+macos-11,intel-classic,2021.2,✓
+macos-11,intel-classic,2021.3,✓
+macos-11,intel-classic,2021.4,✓
+macos-11,intel-classic,2021.5,✓
+macos-11,intel-classic,2021.6,✓
+macos-11,intel-classic,2021.7.1,✓
+macos-11,intel-classic,2021.7,✓
+macos-11,intel-classic,2021.8,✓
+macos-11,intel-classic,2021.9,✓
+macos-12,gcc,10,✓
+macos-12,gcc,11,✓
+macos-12,gcc,12,✓
+macos-12,gcc,13,✓
+macos-12,gcc,6,
+macos-12,gcc,7,✓
+macos-12,gcc,8,✓
+macos-12,gcc,9,✓
+macos-12,intel-classic,2021.1.2,
+macos-12,intel-classic,2021.1,✓
+macos-12,intel-classic,2021.10,✓
+macos-12,intel-classic,2021.2,✓
+macos-12,intel-classic,2021.3,✓
+macos-12,intel-classic,2021.4,✓
+macos-12,intel-classic,2021.5,✓
+macos-12,intel-classic,2021.6,✓
+macos-12,intel-classic,2021.7.1,✓
+macos-12,intel-classic,2021.7,✓
+macos-12,intel-classic,2021.8,✓
+macos-12,intel-classic,2021.9,✓
+macos-13,gcc,10,✓
+macos-13,gcc,11,✓
+macos-13,gcc,12,✓
+macos-13,gcc,13,✓
+macos-13,gcc,6,
+macos-13,gcc,7,✓
+macos-13,gcc,8,✓
+macos-13,gcc,9,✓
+macos-13,intel-classic,2021.1.2,
+macos-13,intel-classic,2021.1,✓
+macos-13,intel-classic,2021.10,✓
+macos-13,intel-classic,2021.2,✓
+macos-13,intel-classic,2021.3,✓
+macos-13,intel-classic,2021.4,✓
+macos-13,intel-classic,2021.5,✓
+macos-13,intel-classic,2021.6,✓
+macos-13,intel-classic,2021.7.1,✓
+macos-13,intel-classic,2021.7,✓
+macos-13,intel-classic,2021.8,✓
+macos-13,intel-classic,2021.9,✓
+ubuntu-20.04,gcc,10,✓
+ubuntu-20.04,gcc,11,✓
+ubuntu-20.04,gcc,12,
+ubuntu-20.04,gcc,13,✓
+ubuntu-20.04,gcc,6,
+ubuntu-20.04,gcc,7,✓
+ubuntu-20.04,gcc,8,✓
+ubuntu-20.04,gcc,9,✓
+ubuntu-20.04,intel-classic,2021.1.2,✓
+ubuntu-20.04,intel-classic,2021.1,✓
+ubuntu-20.04,intel-classic,2021.10,✓
+ubuntu-20.04,intel-classic,2021.2,✓
+ubuntu-20.04,intel-classic,2021.3,
+ubuntu-20.04,intel-classic,2021.4,✓
+ubuntu-20.04,intel-classic,2021.5,✓
+ubuntu-20.04,intel-classic,2021.6,✓
+ubuntu-20.04,intel-classic,2021.7.1,✓
+ubuntu-20.04,intel-classic,2021.7,✓
+ubuntu-20.04,intel-classic,2021.8,✓
+ubuntu-20.04,intel-classic,2021.9,✓
+ubuntu-20.04,intel,2021.1.2,✓
+ubuntu-20.04,intel,2021.1,✓
+ubuntu-20.04,intel,2021.2,✓
+ubuntu-20.04,intel,2021.4,✓
+ubuntu-20.04,intel,2022.0,✓
+ubuntu-20.04,intel,2022.1,✓
+ubuntu-20.04,intel,2022.2.1,✓
+ubuntu-20.04,intel,2022.2,✓
+ubuntu-20.04,intel,2023.0,✓
+ubuntu-20.04,intel,2023.1,✓
+ubuntu-20.04,intel,2023.2,✓
+ubuntu-20.04,nvidia-hpc,20.11,✓
+ubuntu-20.04,nvidia-hpc,20.7,
+ubuntu-20.04,nvidia-hpc,20.9,
+ubuntu-20.04,nvidia-hpc,21.1,
+ubuntu-20.04,nvidia-hpc,21.11,✓
+ubuntu-20.04,nvidia-hpc,22.1,✓
+ubuntu-20.04,nvidia-hpc,22.11,✓
+ubuntu-20.04,nvidia-hpc,23.11,✓
+ubuntu-20.04,nvidia-hpc,23.3,✓
+ubuntu-20.04,nvidia-hpc,23.5,✓
+ubuntu-20.04,nvidia-hpc,23.7,✓
+ubuntu-20.04,nvidia-hpc,23.9,✓
+ubuntu-22.04,gcc,10,✓
+ubuntu-22.04,gcc,11,✓
+ubuntu-22.04,gcc,12,✓
+ubuntu-22.04,gcc,13,✓
+ubuntu-22.04,gcc,6,
+ubuntu-22.04,gcc,7,
+ubuntu-22.04,gcc,8,
+ubuntu-22.04,gcc,9,✓
+ubuntu-22.04,intel-classic,2021.1.2,✓
+ubuntu-22.04,intel-classic,2021.1,✓
+ubuntu-22.04,intel-classic,2021.10,✓
+ubuntu-22.04,intel-classic,2021.2,✓
+ubuntu-22.04,intel-classic,2021.3,
+ubuntu-22.04,intel-classic,2021.4,✓
+ubuntu-22.04,intel-classic,2021.5,✓
+ubuntu-22.04,intel-classic,2021.6,✓
+ubuntu-22.04,intel-classic,2021.7.1,✓
+ubuntu-22.04,intel-classic,2021.7,✓
+ubuntu-22.04,intel-classic,2021.8,✓
+ubuntu-22.04,intel-classic,2021.9,✓
+ubuntu-22.04,intel,2021.1.2,✓
+ubuntu-22.04,intel,2021.1,✓
+ubuntu-22.04,intel,2021.2,✓
+ubuntu-22.04,intel,2021.4,✓
+ubuntu-22.04,intel,2022.0,✓
+ubuntu-22.04,intel,2022.1,✓
+ubuntu-22.04,intel,2022.2.1,✓
+ubuntu-22.04,intel,2022.2,✓
+ubuntu-22.04,intel,2023.0,✓
+ubuntu-22.04,intel,2023.1,✓
+ubuntu-22.04,intel,2023.2,✓
+ubuntu-22.04,nvidia-hpc,20.11,✓
+ubuntu-22.04,nvidia-hpc,20.7,
+ubuntu-22.04,nvidia-hpc,20.9,
+ubuntu-22.04,nvidia-hpc,21.1,
+ubuntu-22.04,nvidia-hpc,21.11,✓
+ubuntu-22.04,nvidia-hpc,22.1,✓
+ubuntu-22.04,nvidia-hpc,22.11,✓
+ubuntu-22.04,nvidia-hpc,23.11,✓
+ubuntu-22.04,nvidia-hpc,23.3,✓
+ubuntu-22.04,nvidia-hpc,23.5,✓
+ubuntu-22.04,nvidia-hpc,23.7,✓
+ubuntu-22.04,nvidia-hpc,23.9,✓
+windows-2019,gcc,10,✓
+windows-2019,gcc,11,✓
+windows-2019,gcc,12,✓
+windows-2019,gcc,13,✓
+windows-2019,gcc,6,
+windows-2019,gcc,7,
+windows-2019,gcc,8,✓
+windows-2019,gcc,9,✓
+windows-2019,intel-classic,2021.1.2,
+windows-2019,intel-classic,2021.1,
+windows-2019,intel-classic,2021.10,✓
+windows-2019,intel-classic,2021.2,
+windows-2019,intel-classic,2021.3,
+windows-2019,intel-classic,2021.4,
+windows-2019,intel-classic,2021.5,
+windows-2019,intel-classic,2021.6,✓
+windows-2019,intel-classic,2021.7.1,
+windows-2019,intel-classic,2021.7,✓
+windows-2019,intel-classic,2021.8,✓
+windows-2019,intel-classic,2021.9,✓
+windows-2019,intel,2021.1.2,
+windows-2019,intel,2021.1,
+windows-2019,intel,2021.2,
+windows-2019,intel,2021.4,
+windows-2019,intel,2022.0,
+windows-2019,intel,2022.1,✓
+windows-2019,intel,2022.2.1,
+windows-2019,intel,2022.2,✓
+windows-2019,intel,2023.0,✓
+windows-2019,intel,2023.1,✓
+windows-2019,intel,2023.2,✓
+windows-2022,gcc,10,✓
+windows-2022,gcc,11,✓
+windows-2022,gcc,12,✓
+windows-2022,gcc,13,✓
+windows-2022,gcc,6,
+windows-2022,gcc,7,
+windows-2022,gcc,8,✓
+windows-2022,gcc,9,✓
+windows-2022,intel-classic,2021.1.2,
+windows-2022,intel-classic,2021.1,
+windows-2022,intel-classic,2021.10,✓
+windows-2022,intel-classic,2021.2,
+windows-2022,intel-classic,2021.3,
+windows-2022,intel-classic,2021.4,
+windows-2022,intel-classic,2021.5,
+windows-2022,intel-classic,2021.6,✓
+windows-2022,intel-classic,2021.7.1,
+windows-2022,intel-classic,2021.7,✓
+windows-2022,intel-classic,2021.8,✓
+windows-2022,intel-classic,2021.9,✓
+windows-2022,intel,2021.1.2,
+windows-2022,intel,2021.1,
+windows-2022,intel,2021.2,
+windows-2022,intel,2021.4,
+windows-2022,intel,2022.0,
+windows-2022,intel,2022.1,✓
+windows-2022,intel,2022.2.1,
+windows-2022,intel,2022.2,✓
+windows-2022,intel,2023.0,✓
+windows-2022,intel,2023.1,✓
+windows-2022,intel,2023.2,✓

.github/compat/matrix.yml

@@ -0,0 +1,83 @@
+os:
+  - ubuntu-22.04
+  - ubuntu-20.04
+  - macos-13
+  - macos-12
+  - macos-11
+  - windows-2022
+  - windows-2019
+toolchain:
+  - {compiler: gcc, version: 13}
+  - {compiler: gcc, version: 12}
+  - {compiler: gcc, version: 11}
+  - {compiler: gcc, version: 10}
+  - {compiler: gcc, version:  9}
+  - {compiler: gcc, version:  8}
+  - {compiler: gcc, version:  7}
+  - {compiler: gcc, version:  6}
+  - {compiler: intel, version: '2023.2'}
+  - {compiler: intel, version: '2023.1'}
+  - {compiler: intel, version: '2023.0'}
+  - {compiler: intel, version: '2022.2.1'}
+  - {compiler: intel, version: '2022.2'}
+  - {compiler: intel, version: '2022.1'}
+  - {compiler: intel, version: '2022.0'}
+  - {compiler: intel, version: '2021.4'}
+  - {compiler: intel, version: '2021.2'}
+  - {compiler: intel, version: '2021.1.2'}
+  - {compiler: intel, version: '2021.1'}
+  - {compiler: intel-classic, version: '2021.10'}
+  - {compiler: intel-classic, version: '2021.9'}
+  - {compiler: intel-classic, version: '2021.8'}
+  - {compiler: intel-classic, version: '2021.7.1'}
+  - {compiler: intel-classic, version: '2021.7'}
+  - {compiler: intel-classic, version: '2021.6'}
+  - {compiler: intel-classic, version: '2021.5'}
+  - {compiler: intel-classic, version: '2021.4'}
+  - {compiler: intel-classic, version: '2021.3'}
+  - {compiler: intel-classic, version: '2021.2'}
+  - {compiler: intel-classic, version: '2021.1.2'}
+  - {compiler: intel-classic, version: '2021.1'}
+  - {compiler: nvidia-hpc, version: '23.11'}
+  - {compiler: nvidia-hpc, version: '23.9'}
+  - {compiler: nvidia-hpc, version: '23.7'}
+  - {compiler: nvidia-hpc, version: '23.5'}
+  - {compiler: nvidia-hpc, version: '23.3'}
+  - {compiler: nvidia-hpc, version: '23.1'}
+  - {compiler: nvidia-hpc, version: '22.11'}
+  - {compiler: nvidia-hpc, version: '22.9'}
+  - {compiler: nvidia-hpc, version: '22.7'}
+  - {compiler: nvidia-hpc, version: '22.5'}
+  - {compiler: nvidia-hpc, version: '22.3'}
+  - {compiler: nvidia-hpc, version: '22.2'}
+  - {compiler: nvidia-hpc, version: '22.1'}
+  - {compiler: nvidia-hpc, version: '21.11'}
+  - {compiler: nvidia-hpc, version: '21.9'}
+  - {compiler: nvidia-hpc, version: '21.7'}
+  - {compiler: nvidia-hpc, version: '21.5'}
+  - {compiler: nvidia-hpc, version: '21.3'}
+  - {compiler: nvidia-hpc, version: '21.2'}
+  - {compiler: nvidia-hpc, version: '21.1'}
+  - {compiler: nvidia-hpc, version: '20.11'}
+  - {compiler: nvidia-hpc, version: '20.9'}
+  - {compiler: nvidia-hpc, version: '20.7'}
+exclude:
+  # ifx not available for mac
+  - os: macos-13
+    toolchain: {compiler: intel}
+  - os: macos-12
+    toolchain: {compiler: intel}
+  - os: macos-11
+    toolchain: {compiler: intel}
+  # nvidia-hpc not available for mac
+  - os: macos-13
+    toolchain: {compiler: nvidia-hpc}
+  - os: macos-12
+    toolchain: {compiler: nvidia-hpc}
+  - os: macos-11
+    toolchain: {compiler: nvidia-hpc}
+  # nvidia-hpc not available for windows
+  - os: windows-2022
+    toolchain: {compiler: nvidia-hpc}
+  - os: windows-2019
+    toolchain: {compiler: nvidia-hpc}

.github/compat/matrix_json_from_csv.py

@@ -0,0 +1,24 @@
+import csv
+import json
+import sys
+from pathlib import Path
+
+csv_path = Path(sys.argv[1])  # path to CSV file
+jsn_path = Path(sys.argv[2])  # path to JSON file
+
+include = []
+
+with open(csv_path, "r") as csv_file:
+    reader = csv.DictReader(csv_file)
+    for row in reader:
+        if not any(row["support"].strip()):
+            continue
+        include.append(
+            {
+                "os": row["runner"],
+                "toolchain": {"compiler": row["compiler"], "version": row["version"]},
+            }
+        )
+
+with open(jsn_path, "w") as jsn_file:
+    json.dump({"include": include}, jsn_file)

update_compat_table.py → .github/compat/update_compat_table.py

@@ -19,6 +19,6 @@
         r"<!\-\- compat starts \-\->.*<!\-\- compat ends \-\->",
         re.DOTALL,
     )
-    ct = '<!-- compat starts -->{}<!-- compat ends -->'.format('\n{}\n'.format(table))
+    ct = "<!-- compat starts -->{}<!-- compat ends -->".format("\n{}\n".format(table))
     readme = update_path.open().read()
     update_path.open("w").write(r.sub(ct, readme))