Merge pull request #12895 from trilinos/master_merge_20240405_175846

Trilinos Master Merge PR Generator: Auto PR created to promote from master_merge_20240405_175846 branch to master

.github/workflows/PR-gcc-openmpi.yml → .github/workflows/AT2.yml

@@ -1,11 +1,13 @@
-name: PR-gcc-openmpi
+name: AT2-EXPERIMENTAL
 
 on:
   pull_request:
     types:
-      - labeled
       - opened
       - synchronize
+  pull_request_review:
+    types: 
+      - submitted
     branches:
     - master
     - develop
@@ -15,14 +17,9 @@ permissions:
   contents: read
 
 jobs:
-  trilogpu02-gcc:
-    permissions:
-      actions: write  # for styfle/cancel-workflow-action to cancel/stop running workflows
-      contents: read  # for actions/checkout to fetch code
-    runs-on: [self-hosted, trilogpu02]
-    container:
-      image: registry-ex.sandia.gov/trilinos-project/trilinos-containers/rhel8/trilinos-pr-env:gcc-8.5.0
-      options: --hostname trilinos-container-gcc-8
+  gcc10-openmpi416-EXPERIMENTAL:
+    runs-on: [self-hosted, gcc-10.3.0_openmpi-4.1.6]
+    if: ${{ github.event.action == 'synchronize' || github.event.action == 'opened' || github.event.review.state == 'APPROVED' }}
     steps:
       - name: env
         env:
@@ -62,7 +59,7 @@ jobs:
       - name: Generate PR cmake fragment
         working-directory: /home/Trilinos/build
         run: |
-          bash -l -c "source ${GITHUB_WORKSPACE}/packages/framework/GenConfig/gen-config.sh rhel8_gcc-openmpi_debug_shared_no-kokkos-arch_no-asan_complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables --cmake-fragment PR-configuration.cmake ${GITHUB_WORKSPACE}"
+          bash -l -c "source ${GITHUB_WORKSPACE}/packages/framework/GenConfig/gen-config.sh rhel8_gcc-openmpi_debug_shared_no-kokkos-arch_no-asan_complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables --force --cmake-fragment PR-configuration.cmake ${GITHUB_WORKSPACE}"
       - name: Generate enable packages cmake fragment
         working-directory: /home/Trilinos/build
         run: |
@@ -71,14 +68,9 @@ jobs:
         working-directory: /home/Trilinos/build
         run: |
           bash -l -c "cmake -C PR-configuration.cmake -C package-enables.cmake ${GITHUB_WORKSPACE}"
-      - name: CMakeCache.txt
-        if: always()
-        working-directory: /home/Trilinos/build
-        run: |
-          bash -l -c "cat CMakeCache.txt"
       - name: build trilinos
         working-directory: /home/Trilinos/build
-        run: bash -l -c "ninja"
+        run: bash -l -c "ninja -j36"
       - name: ctest
         working-directory: /home/Trilinos/build
         run: bash -l -c "ctest -j36"

.github/workflows/dependency-review.yml

@@ -24,4 +24,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@733dd5d4a5203f238c33806593ec0f5fc5343d8c # v4.2.4
+        uses: actions/dependency-review-action@5bbc3ba658137598168acb2ab73b21c432dd411b # v4.2.5

.github/workflows/spack.yml

@@ -0,0 +1,30 @@
+name: spack
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  pull_request_review:
+    types: 
+      - submitted
+  workflow_dispatch:
+
+jobs:
+  gcc10-openmpi416:
+    runs-on: [self-hosted, gcc-10.3.0_openmpi-4.1.6]
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - name: Clone Trilinos
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Spack build
+        shell: bash -l {0}
+        run: |
+          spack develop --no-clone --path $GITHUB_WORKSPACE trilinos@develop
+          spack add trilinos@develop
+          spack concretize -f
+          spack install --cdash-upload-url=https://sems-cdash-son.sandia.gov/cdash/submit.php?project=Trilinos -j16

.github/workflows/title_to_mention.yml

@@ -31,3 +31,10 @@ jobs:
         issue-number: ${{ github.event.issue.number }}  
         body: |
           Automatic mention of the @trilinos/ifpack2 team
+    - name: Mention ROL
+      uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+      if: (contains(github.event.action, 'labeled') && contains(github.event.label.name, 'ROL')) || (contains(github.event.action, 'opened') && contains(github.event.issue.title, 'ROL'))
+      with:
+        issue-number: ${{ github.event.issue.number }}
+        body: |
+          Automatic mention of @aj463

commonTools/framework/ProjectCiFileChangeLogic.py

@@ -85,6 +85,12 @@ def isGlobalBuildFileRequiringGlobalRebuild(self, modifiedFileFullPath):
       # Changes under packages/framework/ likely impact the GenConfig PR build
       # configurations and therefore to be safe, everything needs to be tested.
       return True
+    elif lenPathArray >= 2 and modifiedFileFullPathArray[0] == '.github' and \
+      modifiedFileFullPathArray[1] == 'workflows' \
+      :
+      # Changes under .github/workflows/ impact CI-type runs on GitHub Actions
+      # and therefore to be safe, everything needs to be tested.
+      return True
     # Any other files not already covered above should *not* trigger a global
     # build
     return False

commonTools/framework/ProjectCiFileChangeLogic_UnitTests.py

@@ -116,6 +116,9 @@ def test_packages_something(self):
   def test_packages_framework(self):
     self.check('packages/framework/something', True)
 
+  def test_dotgithub_workflows(self):
+    self.check('.github/workflows/something', True)
+
 
 if __name__ == '__main__':
   unittest.main()

packages/framework/ini-files/config-specs.ini

@@ -2826,6 +2826,7 @@ opt-set-cmake-var ROL_example_PDE-OPT_navier-stokes_example_01_MPI_4_DISABLE
 opt-set-cmake-var Pliris_vector_random_MPI_3_DISABLE                             BOOL : ON
 opt-set-cmake-var Pliris_vector_random_MPI_4_DISABLE                             BOOL : ON
 
+opt-set-cmake-var Zoltan_ch_7944_parmetis_parallel_DISABLE                       BOOL : ON
 opt-set-cmake-var Zoltan_ch_simple_parmetis_parallel_DISABLE                     BOOL : ON
 opt-set-cmake-var Belos_bl_gmres_complex_hb_3_MPI_4_DISABLE                      BOOL : ON
 opt-set-cmake-var Belos_hybrid_gmres_complex_hb_0_MPI_4_DISABLE                  BOOL : ON

packages/ifpack2/example/BlockTriDi.cpp

@@ -18,7 +18,7 @@ namespace { // (anonymous)
 
 // Values of command-line arguments.
 struct CmdLineArgs {
-  CmdLineArgs ():blockSize(-1),numIters(10),numRepeats(1),tol(1e-12),nx(172),ny(-1),nz(-1),mx(1),my(1),mz(1),sublinesPerLine(1),sublinesPerLineSchur(1),useStackedTimer(false),overlapCommAndComp(false){}
+  CmdLineArgs ():blockSize(-1),numIters(10),numRepeats(1),tol(1e-12),nx(172),ny(-1),nz(-1),mx(1),my(1),mz(1),sublinesPerLine(1),sublinesPerLineSchur(1),useStackedTimer(false),usePointMatrix(false),overlapCommAndComp(false){}
 
   std::string mapFilename;
   std::string matrixFilename;
@@ -37,6 +37,7 @@ struct CmdLineArgs {
   int sublinesPerLine;
   int sublinesPerLineSchur;
   bool useStackedTimer;
+  bool usePointMatrix;
   bool overlapCommAndComp;
   std::string problemName;
   std::string matrixType;
@@ -68,6 +69,8 @@ getCmdLineArgs (CmdLineArgs& args, int argc, char* argv[])
   cmdp.setOption ("sublinesPerLine", &args.sublinesPerLine, "If using inline meshing, number of sublines per mesh x line. If set to -1 the block Jacobi algorithm is used.");
   cmdp.setOption ("withStackedTimer", "withoutStackedTimer", &args.useStackedTimer,
       "Whether to run with a StackedTimer and print the timer tree at the end (and try to output Watchr report)");
+  cmdp.setOption ("withPointMatrix", "withoutPointMatrix", &args.usePointMatrix,
+      "Whether to run with a point matrix");
   cmdp.setOption ("withOverlapCommAndComp", "withoutOverlapCommAndComp", &args.overlapCommAndComp,
 		  "Whether to run with overlapCommAndComp)");
   cmdp.setOption("problemName", &args.problemName, "Human-readable problem name for Watchr plot");
@@ -279,7 +282,6 @@ main (int argc, char* argv[])
   using std::cerr;
   using std::endl;
   typedef Tpetra::CrsMatrix<> crs_matrix_type;
-  typedef Tpetra::BlockCrsMatrix<> block_crs_matrix_type;
   typedef Tpetra::Map<> map_type;
   typedef Tpetra::MultiVector<> MV;
   typedef Tpetra::RowMatrix<> row_matrix_type;
@@ -358,11 +360,15 @@ main (int argc, char* argv[])
 
   // Read sparse matrix A from Matrix Market file.
   RCP<crs_matrix_type> A;
-  RCP<block_crs_matrix_type> Ablock;
+  RCP<row_matrix_type> Ablock;
   RCP<MV> B,X;
   RCP<IV> line_info;
 #if defined(HAVE_IFPACK2_XPETRA)
   if(args.matrixFilename == "") {
+    if (args.usePointMatrix) {
+      std::string msg = "usePointMatrix with inline matrix is not yet implemented";
+      throw std::runtime_error(msg);      
+    }
     // matrix
     Teuchos::ParameterList plist;
     if(args.matrixType == "") {
@@ -575,7 +581,10 @@ main (int argc, char* argv[])
 
   {
     Teuchos::TimeMonitor precSetupTimeMon (*precSetupTime);
-    precond = rcp(new BTDC(Ablock,parts,args.sublinesPerLineSchur,args.overlapCommAndComp));
+    if(args.usePointMatrix)
+      precond = rcp(new BTDC(A,parts,args.sublinesPerLineSchur,args.overlapCommAndComp, false, args.blockSize));
+    else
+      precond = rcp(new BTDC(Ablock,parts,args.sublinesPerLineSchur,args.overlapCommAndComp));
 
     if(rank0) std::cout<<"Initializing preconditioner..."<<std::endl;
     precond->initialize ();

packages/ifpack2/src/Ifpack2_BlockRelaxation_decl.hpp

@@ -139,6 +139,8 @@ class BlockRelaxation :
   typedef Tpetra::Vector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> vector_type;
   typedef Teuchos::ScalarTraits<scalar_type> STS;
   typedef Teuchos::ScalarTraits<magnitude_type> STM;
+  typedef Tpetra::CrsMatrix<scalar_type, local_ordinal_type,
+                                 global_ordinal_type, node_type> crs_matrix_type;
   typedef Tpetra::BlockCrsMatrix<scalar_type, local_ordinal_type,
                                  global_ordinal_type, node_type> block_crs_matrix_type;
   typedef Tpetra::Map<local_ordinal_type, global_ordinal_type, node_type> map_type;

packages/ifpack2/src/Ifpack2_BlockRelaxation_def.hpp

@@ -53,6 +53,7 @@
 #include "Ifpack2_LocalFilter.hpp"
 #include "Ifpack2_Parameters.hpp"
 #include "Teuchos_TimeMonitor.hpp"
+#include "Tpetra_BlockCrsMatrix_Helpers_decl.hpp"
 
 namespace Ifpack2 {
 
@@ -176,6 +177,8 @@ getValidParameters () const
   validParams->set("partitioner: coordinates",dummy);
   validParams->set("timer for apply", true);
   validParams->set("partitioner: subparts per part", 1);
+  validParams->set("partitioner: block size", -1);
+  validParams->set("partitioner: print level", false);
 
   return validParams;
 }
@@ -629,11 +632,16 @@ initialize ()
     Teuchos::RCP<const block_crs_matrix_type> A_bcrs =
       Teuchos::rcp_dynamic_cast<const block_crs_matrix_type> (A_);
     hasBlockCrsMatrix_ = !A_bcrs.is_null();
-    if (A_bcrs.is_null ()) {
-      hasBlockCrsMatrix_ = false;
-    }
-    else {
+
+    if(!hasBlockCrsMatrix_ && List_.isParameter("relaxation: container") && List_.get<std::string>("relaxation: container") == "BlockTriDi" ) {
+      TEUCHOS_FUNC_TIME_MONITOR("Ifpack2::BlockRelaxation::initialize::convertToBlockCrsMatrix");
+      int block_size = List_.get<int>("partitioner: block size");
+      TEUCHOS_TEST_FOR_EXCEPT_MSG
+        (block_size == -1, "A pointwise matrix and block_size = -1 were given as inputs.");
+      A_bcrs = Tpetra::convertToBlockCrsMatrix(*Teuchos::rcp_dynamic_cast<const crs_matrix_type>(A_), block_size);
+      A_ = A_bcrs;
       hasBlockCrsMatrix_ = true;
+      Kokkos::DefaultExecutionSpace().fence();
     }
 
     NumLocalRows_      = A_->getLocalNumRows ();

packages/ifpack2/src/Ifpack2_BlockTriDiContainer_decl.hpp

@@ -174,6 +174,8 @@ namespace Ifpack2 {
     //typedef Tpetra::MultiVector<local_scalar_type, local_ordinal_type, global_ordinal_type, node_type> local_mv_type;
     //typedef typename Kokkos::View<local_scalar_type**, Kokkos::HostSpace> HostViewLocal;
 
+    typedef Tpetra::CrsMatrix
+    <scalar_type,local_ordinal_type,global_ordinal_type,node_type> crs_matrix_type;
     typedef Tpetra::BlockCrsMatrix
     <scalar_type,local_ordinal_type,global_ordinal_type,node_type> block_crs_matrix_type;
 
@@ -231,7 +233,9 @@ namespace Ifpack2 {
     BlockTriDiContainer (const Teuchos::RCP<const row_matrix_type>& matrix,
                          const Teuchos::Array<Teuchos::Array<local_ordinal_type> >& partitions,
                          const int n_subparts_per_part = 1,
-                         bool overlapCommAndComp = false, bool useSequentialMethod = false);
+                         bool overlapCommAndComp = false, 
+                         bool useSequentialMethod = false,
+                         const int block_size = -1);
 
     //! Destructor (declared virtual for memory safety of derived classes).
     ~BlockTriDiContainer () override;
@@ -398,7 +402,8 @@ namespace Ifpack2 {
     void initInternal (const Teuchos::RCP<const row_matrix_type>& matrix,
                        const Teuchos::RCP<const import_type> &importer,
                        const bool overlapCommAndComp,
-                       const bool useSeqMethod);
+                       const bool useSeqMethod,
+                       const int block_size = -1);
 
     void clearInternal();
   };

packages/ifpack2/src/Ifpack2_BlockTriDiContainer_def.hpp

@@ -47,6 +47,7 @@
 
 #include <Tpetra_Distributor.hpp>
 #include <Tpetra_BlockMultiVector.hpp>
+#include <Tpetra_BlockCrsMatrix_Helpers.hpp>
 
 #include <Kokkos_ArithTraits.hpp>
 #include <KokkosBatched_Util.hpp>
@@ -81,7 +82,8 @@ namespace Ifpack2 {
   ::initInternal (const Teuchos::RCP<const row_matrix_type>& matrix,
                   const Teuchos::RCP<const import_type>& importer,
                   const bool overlapCommAndComp,
-                  const bool useSeqMethod) 
+                  const bool useSeqMethod,
+                  const int block_size) 
   {
     IFPACK2_BLOCKHELPER_TIMER("BlockTriDiContainer::initInternal");
 
@@ -98,8 +100,15 @@ namespace Ifpack2 {
     {
       IFPACK2_BLOCKHELPER_TIMER("BlockTriDiContainer::setA");
       impl_->A = Teuchos::rcp_dynamic_cast<const block_crs_matrix_type>(matrix);
-      TEUCHOS_TEST_FOR_EXCEPT_MSG
-        (impl_->A.is_null(), "BlockTriDiContainer currently supports Tpetra::BlockCrsMatrix only.");
+      if (impl_->A.is_null()) {
+        TEUCHOS_TEST_FOR_EXCEPT_MSG
+          (block_size == -1, "A pointwise matrix and block_size = -1 were given as inputs.");
+        {
+          IFPACK2_BLOCKHELPER_TIMER("BlockTriDiContainer::setA::convertToBlockCrsMatrix");
+          impl_->A = Tpetra::convertToBlockCrsMatrix(*Teuchos::rcp_dynamic_cast<const crs_matrix_type>(matrix), block_size);
+          IFPACK2_BLOCKHELPER_TIMER_FENCE(typename BlockHelperDetails::ImplType<MatrixType>::execution_space)
+        }
+      }
       IFPACK2_BLOCKHELPER_TIMER_FENCE(typename BlockHelperDetails::ImplType<MatrixType>::execution_space)
     }
 
@@ -197,11 +206,12 @@ namespace Ifpack2 {
                        const Teuchos::Array<Teuchos::Array<local_ordinal_type> >& partitions,
                        const int n_subparts_per_part,
                        const bool overlapCommAndComp, 
-                       const bool useSeqMethod)
+                       const bool useSeqMethod,
+                       const int block_size)
     : Container<MatrixType>(matrix, partitions, false), partitions_(partitions)
   {
     IFPACK2_BLOCKHELPER_TIMER("BlockTriDiContainer::BlockTriDiContainer");
-    initInternal(matrix, Teuchos::null, overlapCommAndComp, useSeqMethod);
+    initInternal(matrix, Teuchos::null, overlapCommAndComp, useSeqMethod, block_size);
     n_subparts_per_part_ = n_subparts_per_part;
     IFPACK2_BLOCKHELPER_TIMER_FENCE(typename BlockHelperDetails::ImplType<MatrixType>::execution_space)
   }