From 616908697b85cc05d9e153ccc51cf458241b4fb4 Mon Sep 17 00:00:00 2001 From: StepSecurity Bot Date: Mon, 22 Apr 2024 15:39:23 +0000 Subject: [PATCH 01/14] [StepSecurity] ci: Harden GitHub Actions Signed-off-by: StepSecurity Bot --- .github/workflows/AT2.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/AT2.yml b/.github/workflows/AT2.yml index 4bef4385ccf5..63b88c266843 100644 --- a/.github/workflows/AT2.yml +++ b/.github/workflows/AT2.yml @@ -32,7 +32,7 @@ jobs: bash -l -c "module list" printenv PATH - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.12.1 + uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 with: access_token: ${{ github.token }} - name: make dirs @@ -41,7 +41,7 @@ jobs: mkdir -p /home/Trilinos/src/Trilinos mkdir -p /home/Trilinos/build - name: Clone trilinos - uses: actions/checkout@v4.1.2 + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - name: Repo status @@ -90,7 +90,7 @@ jobs: bash -l -c "module list" printenv PATH - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.12.1 + uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1 with: access_token: ${{ github.token }} - name: make dirs @@ -99,7 +99,7 @@ jobs: mkdir -p /home/Trilinos/src/Trilinos mkdir -p /home/Trilinos/build - name: Clone trilinos - uses: actions/checkout@v4.1.2 + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - name: Repo status From c536c170629f23939fbdad8806348ed582670147 Mon Sep 17 00:00:00 2001 From: Alan Williams Date: Mon, 22 Apr 2024 10:53:05 -0600 Subject: [PATCH 02/14] STK: Snapshot 04-22-24 10:53 from Sierra 5.19.2-926-gd0a856d5 --- packages/stk/CHANGELOG.md | 4 ++- .../cmake_install_test/build_stk_user_facing | 4 +++ .../stk/stk_search/stk_search/CMakeLists.txt | 31 ++++++++++++------- packages/stk/stk_util/stk_util/Version.hpp | 2 +- .../stk_util/registry/ProductRegistry.cpp | 2 +- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/stk/CHANGELOG.md b/packages/stk/CHANGELOG.md index 74bfbf5f467a..12ce8f91eb44 100644 --- a/packages/stk/CHANGELOG.md +++ b/packages/stk/CHANGELOG.md @@ -1,5 +1,7 @@ # CHANGELOG +5.19.2 (STK_VERSION 5190201) 4/22/2024 + Fixed broken stk_search install targets 5.19.2 (STK_VERSION 5190200) 4/17/2024 -Initial creation of STK_VERSION macro + Initial creation of STK_VERSION macro diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing index dca1fe7944d2..0b77cf8ae7b5 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing @@ -3,6 +3,10 @@ exe() { stdbuf -o0 -e0 echo "% $@" ; eval "$@" ; + if [ $? -ne 0 ] ; then + echo "'$@' failed."; + exit 1; + fi } sierra_proj=${SIERRA_PROJ:-${PWD}} diff --git a/packages/stk/stk_search/stk_search/CMakeLists.txt b/packages/stk/stk_search/stk_search/CMakeLists.txt index dbb061f773a8..45b6b7b9d45b 100644 --- a/packages/stk/stk_search/stk_search/CMakeLists.txt +++ b/packages/stk/stk_search/stk_search/CMakeLists.txt @@ -34,22 +34,27 @@ SET(HEADERS "") SET(SOURCES "") -FILE(GLOB HEADERS *.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/kdtree/*.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/morton_lbvh/*.hpp -) -FILE(GLOB SOURCES *.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/kdtree/*.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/morton_lbvh/*.cpp -) +FILE(GLOB HEADERS_KDTREE + ${CMAKE_CURRENT_SOURCE_DIR}/kdtree/*.hpp) +FILE(GLOB HEADERS_MORTON + ${CMAKE_CURRENT_SOURCE_DIR}/morton_lbvh/*.hpp) + +FILE(GLOB HEADERS *.hpp) + +FILE(GLOB SOURCES_KDTREE + ${CMAKE_CURRENT_SOURCE_DIR}/kdtree/*.cpp) +FILE(GLOB SOURCES_MORTON + ${CMAKE_CURRENT_SOURCE_DIR}/morton_lbvh/*.cpp) + +FILE(GLOB SOURCES *.cpp) if(HAVE_STK_Trilinos) TRIBITS_ADD_LIBRARY(stk_search - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} + NOINSTALLHEADERS ${HEADERS} ${HEADERS_KDTREE} ${HEADERS_MORTON} + SOURCES ${SOURCES} ${SOURCES_KDTREE} ${SOURCES_MORTON} ) else() - add_library(stk_search ${SOURCES}) + add_library(stk_search ${SOURCES} ${SOURCES_KDTREE} ${SOURCES_MORTON}) target_link_libraries(stk_search PUBLIC stk_math) target_link_libraries(stk_search PUBLIC stk_util_parallel) @@ -69,6 +74,10 @@ target_include_directories(stk_search PUBLIC INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_search) +INSTALL(FILES ${HEADERS_KDTREE} DESTINATION + ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_search/kdtree) +INSTALL(FILES ${HEADERS_MORTON} DESTINATION + ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_search/morton_lbvh) if(NOT HAVE_STK_Trilinos) INSTALL(TARGETS stk_search DESTINATION ${STK_INSTALL_LIBDIR}) diff --git a/packages/stk/stk_util/stk_util/Version.hpp b/packages/stk/stk_util/stk_util/Version.hpp index 49275003a6fe..3336576bbbf8 100644 --- a/packages/stk/stk_util/stk_util/Version.hpp +++ b/packages/stk/stk_util/stk_util/Version.hpp @@ -44,7 +44,7 @@ //See the file CHANGELOG.md for a listing that shows the //correspondence between version numbers and API changes. -#define STK_VERSION 5190200 +#define STK_VERSION 5190201 namespace stk diff --git a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp index 8516c153bbcc..1ec227291a39 100644 --- a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp +++ b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp @@ -42,7 +42,7 @@ //In Sierra, STK_VERSION_STRING is provided on the compile line by bake. //For Trilinos stk snapshots, the following macro definition gets populated with //the real version string by the trilinos_snapshot.sh script. -#define STK_VERSION_STRING "5.19.2-901-ge8f5b533" +#define STK_VERSION_STRING "5.19.2-926-gd0a856d5" #endif namespace stk { From 728fa6153e0ce31a38c84e7f3e1fb5a1e0274892 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 22:52:56 +0000 Subject: [PATCH 03/14] Bump actions/upload-artifact from 4.3.1 to 4.3.3 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.3.1 to 4.3.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/5d5d22a31266ced268874388b861e4b58bb5c2f3...65462800fd760344b1a7b4382951275a0abb4808) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/clang_format.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clang_format.yml b/.github/workflows/clang_format.yml index 0447591161fe..36f174695a9d 100644 --- a/.github/workflows/clang_format.yml +++ b/.github/workflows/clang_format.yml @@ -20,7 +20,7 @@ jobs: - run: git diff HEAD > format_patch.txt - run: if [ "$(cat format_patch.txt)" == "" ] ; then rm format_patch.txt ; else cat format_patch.txt; fi - - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 id: upload-artf if: ${{ hashFiles('format_patch.txt') != '' }} with: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index ffdb8bc48497..15189c9c8721 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -58,7 +58,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: SARIF file path: results.sarif From 05cd750fd085ec7411e67ad3cecf11270fbd2b61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 22:53:05 +0000 Subject: [PATCH 04/14] Bump github/codeql-action from 3.24.10 to 3.25.2 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.10 to 3.25.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3.24.10...8f596b4ae3cb3c588a5c46780b86dd53fef16c52) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/scorecards.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a27bd2ec983d..9d921949f490 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -62,7 +62,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0 + uses: github/codeql-action/init@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -85,6 +85,6 @@ jobs: make -j 2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0 + uses: github/codeql-action/analyze@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index ffdb8bc48497..605acb2c5487 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -66,6 +66,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@4355270be187e1b672a7a1c7c7bae5afdc1ab94a # v3.24.10 + uses: github/codeql-action/upload-sarif@8f596b4ae3cb3c588a5c46780b86dd53fef16c52 # v3.25.2 with: sarif_file: results.sarif From 72d6506278af9c122b5e981d22cdc90c18fc6b10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 22:53:12 +0000 Subject: [PATCH 05/14] Bump actions/checkout from 4.1.1 to 4.1.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.1.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.1...1d96c772d19495a3b5c517cd2bc0cb401ea0529f) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/AT2.yml | 4 ++-- .github/workflows/clang_format.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/detect-git-lfs.yml | 2 +- .github/workflows/detect-mpi-comm-world.yml | 2 +- .github/workflows/scorecards.yml | 2 +- .github/workflows/spack.yml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/AT2.yml b/.github/workflows/AT2.yml index 63b88c266843..60d5783aaf0c 100644 --- a/.github/workflows/AT2.yml +++ b/.github/workflows/AT2.yml @@ -41,7 +41,7 @@ jobs: mkdir -p /home/Trilinos/src/Trilinos mkdir -p /home/Trilinos/build - name: Clone trilinos - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 with: fetch-depth: 0 - name: Repo status @@ -99,7 +99,7 @@ jobs: mkdir -p /home/Trilinos/src/Trilinos mkdir -p /home/Trilinos/build - name: Clone trilinos - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 with: fetch-depth: 0 - name: Repo status diff --git a/.github/workflows/clang_format.yml b/.github/workflows/clang_format.yml index 0447591161fe..e22b6a1096c3 100644 --- a/.github/workflows/clang_format.yml +++ b/.github/workflows/clang_format.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 - uses: DoozyX/clang-format-lint-action@11b773b1598aa4ae3b32f023701bca5201c3817d # v0.17 with: source: './packages/muelu ./packages/tempus ./packages/teko ./packages/xpetra' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a27bd2ec983d..44049d50c1c4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,7 +58,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 97323a31371c..a5729061f210 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -22,6 +22,6 @@ jobs: egress-policy: audit - name: 'Checkout Repository' - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 - name: 'Dependency Review' uses: actions/dependency-review-action@5bbc3ba658137598168acb2ab73b21c432dd411b # v4.2.5 diff --git a/.github/workflows/detect-git-lfs.yml b/.github/workflows/detect-git-lfs.yml index f8498ce6fd6f..484ac13824f1 100644 --- a/.github/workflows/detect-git-lfs.yml +++ b/.github/workflows/detect-git-lfs.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 with: fetch-depth: 0 diff --git a/.github/workflows/detect-mpi-comm-world.yml b/.github/workflows/detect-mpi-comm-world.yml index e4e0b5269637..82ea4178dd5c 100644 --- a/.github/workflows/detect-mpi-comm-world.yml +++ b/.github/workflows/detect-mpi-comm-world.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 with: fetch-depth: 0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index ffdb8bc48497..c3982538e6bb 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -31,7 +31,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 with: persist-credentials: false diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index fa8a97a66091..a240e42eac09 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -24,7 +24,7 @@ jobs: with: access_token: ${{ github.token }} - name: Clone Trilinos - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 with: fetch-depth: 1 - name: Spack build From c0752000fa1333f3728f35f84d89b49a54145493 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 19 Apr 2024 14:28:27 -0600 Subject: [PATCH 06/14] NOX: add observer to control how often the preconditioner is updated --- .../NOX_Observer_ReusePreconditioner.cpp | 152 ++++++++++++++++++ .../NOX_Observer_ReusePreconditioner.hpp | 85 ++++++++++ ...OX_Observer_ReusePreconditionerFactory.cpp | 35 ++++ ...OX_Observer_ReusePreconditionerFactory.hpp | 27 ++++ packages/nox/src-thyra/NOX_Thyra_Group.C | 24 ++- packages/nox/src-thyra/NOX_Thyra_Group.H | 11 ++ packages/nox/src/CMakeLists.txt | 4 + packages/nox/test/tpetra/Tpetra_1DFEM.cpp | 129 +++++++++++++++ 8 files changed, 465 insertions(+), 2 deletions(-) create mode 100644 packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp create mode 100644 packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.hpp create mode 100644 packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.cpp create mode 100644 packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.hpp diff --git a/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp b/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp new file mode 100644 index 000000000000..baff05432bb3 --- /dev/null +++ b/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp @@ -0,0 +1,152 @@ +#include "NOX_Observer_ReusePreconditioner.hpp" +#include "NOX_Solver_Generic.H" +#include "NOX_SolverStats.hpp" +#include "NOX_Thyra_Group.H" +#include "Thyra_PreconditionerFactoryBase.hpp" +#include "Thyra_PreconditionerFactoryHelpers.hpp" + +NOX::ObserverReusePreconditioner::ObserverReusePreconditioner() + : update_at_start_of_solve_(false), + update_after_n_iterations_(false), + num_iterations_for_update_(5), + update_if_stalled_(false), + max_linear_iterations_for_stall_(-1), + max_count_for_stall_(-1), + stall_count_(0), + iterations_since_last_update_(0) +{} + +void +NOX::ObserverReusePreconditioner:: +setOperatorsAndFactory(const Teuchos::RCP<::Thyra::PreconditionerBase>& precOperator, + const Teuchos::RCP<::Thyra::PreconditionerFactoryBase>& precFactory) +{ + precOperator_ = precOperator; + precFactory_ = precFactory; +} + +void +NOX::ObserverReusePreconditioner::updateAtStartOfSolve() +{ + update_at_start_of_solve_ = true; +} + +void +NOX::ObserverReusePreconditioner:: +updateAfterNIterations(const int num_iterations_for_update) +{ + update_after_n_iterations_ = true; + num_iterations_for_update_ = num_iterations_for_update; +} + +void +NOX::ObserverReusePreconditioner:: +updateOnLinearSolverStall(const int max_linear_iterations, + const int max_count) +{ + update_if_stalled_ = true; + max_linear_iterations_for_stall_ = max_linear_iterations; + max_count_for_stall_ = max_count; +} + +void NOX::ObserverReusePreconditioner::runPreSolve(const NOX::Solver::Generic& solver) +{ + // First time through, grab the operator and tell the group that the + // user will control preconditioning. + if (precOperator_.is_null()) { + const auto& const_group = solver.getSolutionGroup(); + auto& group = const_cast(const_group); + auto& thyra_group = dynamic_cast(group); + + auto lows_factory = thyra_group.getLinearOpWithSolveFactory(); + TEUCHOS_ASSERT(nonnull(lows_factory)); + + // Creating the preconditioner objects follows the logic of the + // NOX::Thyra::Group constructors for consistency. + precFactory_ = thyra_group.getPreconditionerFactory(); + if (precFactory_.is_null()) { + precFactory_ = lows_factory->getPreconditionerFactory(); + } + + precOperator_ = thyra_group.getNonconstPreconditioner(); + if (precOperator_.is_null()) { + if (nonnull(precFactory_)) { + precOperator_ = precFactory_->createPrec(); + } else if (thyra_group.getModel()->createOutArgs().supports( ::Thyra::ModelEvaluatorBase::OUT_ARG_W_prec)) { + // precOperator_ = thyra_group.getModel()->create_W_prec(); + } + } + + thyra_group.takeControlOfPreconditionerUpdates(precOperator_); + + TEUCHOS_TEST_FOR_EXCEPTION(precFactory_.is_null(),std::runtime_error,"ERROR: the ReusePreconditioner observer is registered but there is no preconditioner available!"); + TEUCHOS_TEST_FOR_EXCEPTION(precOperator_.is_null(),std::runtime_error,"ERROR: the ReusePreconditioner observer is registered but there is no preconditioner available!"); + + // First time through, create the preconditioner + this->updatePreconditioner(thyra_group); + } +} + +void NOX::ObserverReusePreconditioner::runPreIterate(const NOX::Solver::Generic& solver) +{ + TEUCHOS_ASSERT(this->isInitialized()); + + bool update_the_preconditioner = false; + + if (update_at_start_of_solve_ && (solver.getNumIterations() == 0) ) { + update_the_preconditioner = true; + } + + if (update_after_n_iterations_) { + if (solver.getNumIterations() % num_iterations_for_update_ == 0) { + update_the_preconditioner = true; + } + } + + if (update_if_stalled_) { + const int num_linear_iterations = solver.getSolverStatistics()->linearSolve.lastLinearSolve_NumIterations; + const bool is_converged = solver.getSolverStatistics()->linearSolve.lastLinearSolve_Converged; + + if (num_linear_iterations >= max_linear_iterations_for_stall_) { + ++stall_count_; + } else { // reset count on a successful linear solve + stall_count_ = 0; + } + + if ( (!is_converged) || (stall_count_ >= max_count_for_stall_) ) + update_the_preconditioner = true; + } + + if (update_the_preconditioner) { + const auto& const_group = solver.getSolutionGroup(); + auto& group = const_cast(const_group); + auto& thyraGroup = dynamic_cast(group); + this->updatePreconditioner(thyraGroup); + } +} + +void NOX::ObserverReusePreconditioner::runPostIterate(const NOX::Solver::Generic& solver) +{++iterations_since_last_update_;} + +void NOX::ObserverReusePreconditioner::updatePreconditioner(NOX::Thyra::Group& group) +{ + TEUCHOS_ASSERT(this->isInitialized()); + + if (!group.isJacobian()) + group.computeJacobian(); + + auto jacOperator = group.getScaledJacobianOperator(); + + ::Thyra::initializePrec(*precFactory_,jacOperator,precOperator_.ptr()); + + group.unscaleJacobianOperator(); + + stall_count_ = 0; + iterations_since_last_update_ = 0; +} + +bool NOX::ObserverReusePreconditioner::isInitialized() const +{ + return ( (update_at_start_of_solve_ || update_after_n_iterations_ || update_if_stalled_) && + nonnull(precOperator_) && nonnull(precFactory_) ); +} diff --git a/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.hpp b/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.hpp new file mode 100644 index 000000000000..edc4260c9e3d --- /dev/null +++ b/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.hpp @@ -0,0 +1,85 @@ +#ifndef NOX_OBSERVER_REUSE_PRECONDITIONER_HPP +#define NOX_OBSERVER_REUSE_PRECONDITIONER_HPP + +#include "NOX_Config.h" +#include "NOX_Observer.hpp" +#include "Teuchos_RCP.hpp" + +namespace Thyra { + template class PreconditionerBase; + template class PreconditionerFactoryBase; +} + +namespace NOX { + + namespace Thyra { + class Group; + } + + /** \brief Observer that controls when to update the preconditioner for the Thyra interface. + + The preconditioner can be updated at the start of each new + nonlinear solve and/or when convergence stalls out. + */ + class ObserverReusePreconditioner : public NOX::Observer { + public: + using Scalar = double; + + ObserverReusePreconditioner(); + + /// Setup the preconditioner objects used to update the preconditioner + void setOperatorsAndFactory(const Teuchos::RCP<::Thyra::PreconditionerBase>& precOperator, + const Teuchos::RCP<::Thyra::PreconditionerFactoryBase>& precFactory); + + /** \brief Enables updating the preconditioner at the start of each nonlinear solve */ + void updateAtStartOfSolve(); + + /** \brief Enables updating of the preconditioner after a set number of nonlinear iteraitons. + + @param [in] num_iterations_for_update (int) Updates the preconditioner after this number of nonlinear iterations. + */ + void updateAfterNIterations(const int num_iterations_for_update); + + /** \brief Enables updating of preconditioner if the observer detects a stall or failure in the linear solver. + + This algorithm tries to assess a stalled computation due to + reusing the preconditioner. It will always recompute for a + failed linear solve. It will also recompute if the last + max_count number of iterations each had linear solves that + took more iterations than max_linear_iterations. + + @param [in] max_linear_iterations (int) Declare a stalled iteraiton if the number of linear solver iterations is above this value + @param [in] max_count (int) Recompute the preconditioner after this many stalled iterations + */ + void updateOnLinearSolverStall(const int max_linear_iterations, + const int max_count); + + // Derived from NOX::Observer + void runPreSolve(const NOX::Solver::Generic& solver) override; + void runPreIterate(const NOX::Solver::Generic& solver) override; + void runPostIterate(const NOX::Solver::Generic& solver) override; + + private: + void updatePreconditioner(NOX::Thyra::Group& group); + bool isInitialized() const; + + private: + bool update_at_start_of_solve_; + + bool update_after_n_iterations_; + int num_iterations_for_update_; + + bool update_if_stalled_; + int max_count_for_stall_; + int max_linear_iterations_for_stall_; + int stall_count_; + + int iterations_since_last_update_; + + Teuchos::RCP<::Thyra::PreconditionerBase> precOperator_; + Teuchos::RCP<::Thyra::PreconditionerFactoryBase> precFactory_; + }; + +} + +#endif diff --git a/packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.cpp b/packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.cpp new file mode 100644 index 000000000000..57a59e3a7cd6 --- /dev/null +++ b/packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.cpp @@ -0,0 +1,35 @@ +#include "NOX_Observer_ReusePreconditionerFactory.hpp" +#include "NOX_Observer_ReusePreconditioner.hpp" +#include "Teuchos_ParameterList.hpp" + +Teuchos::RCP +NOX::createReusePreconditionerObserver(Teuchos::ParameterList& pl) +{ + Teuchos::ParameterList validParams; + validParams.set("Update prec at start of nonlinear solve",true); + validParams.set("Update prec after this many nonlinear iterations",0); + validParams.set("Update prec after this many stalled linear solves",0); + validParams.set("Max linear iterations for stall",50); + + pl.validateParametersAndSetDefaults(validParams); + + bool update_at_start = pl.get("Update prec at start of nonlinear solve"); + int update_after_n_iters = pl.get("Update prec after this many nonlinear iterations"); + int update_after_n_stalls = pl.get("Update prec after this many stalled linear solves"); + int max_iters_for_stall = pl.get("Max linear iterations for stall"); + + auto observer = Teuchos::rcp(new NOX::ObserverReusePreconditioner); + + if (update_at_start) + observer->updateAtStartOfSolve(); + + // Zero or negative value disables + if (update_after_n_iters > 0) + observer->updateAfterNIterations(update_after_n_iters); + + // Zero or negative value disables + if (update_after_n_stalls > 0) + observer->updateOnLinearSolverStall(max_iters_for_stall,update_after_n_stalls); + + return observer; +} diff --git a/packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.hpp b/packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.hpp new file mode 100644 index 000000000000..02adb9191f41 --- /dev/null +++ b/packages/nox/src-thyra/NOX_Observer_ReusePreconditionerFactory.hpp @@ -0,0 +1,27 @@ +#ifndef NOX_OBSERVER_REUSE_PRECONDITIONER_FACTORY_HPP +#define NOX_OBSERVER_REUSE_PRECONDITIONER_FACTORY_HPP + +#include "NOX_Config.h" +#include "NOX_Observer_ReusePreconditioner.hpp" +#include "Teuchos_RCP.hpp" + +namespace Teuchos { + class ParameterList; +} + +namespace NOX { + + /** Non-member function to create an observer that will reuse the preconditioner across multiple linear solves and time steps. + + When to update the preconditioner is controlled by the ParameterList arguments below. + + @param "Update prec at start of nonlinear solve" (bool) Update preconditioner at the start of each nonlinear solve. Defaults to true. + @param "Update prec after this many nonlinear iterations" (int) Update preconditioner after this many nonlinear iterations. Setting to 0 disables. Defaults to 0 (disabled). + @param "Update prec after this many stalled linear solves" (int) Update prec after this many stalled linear solves. Setting to 0 disables. Defaults to 0 (disabled). + @param "Max linear iterations for stall" (int) Maximum number of linear iterations that triggers a nonlinear iteration to be declared stalled. Defaults to 50. + */ + Teuchos::RCP createReusePreconditionerObserver(Teuchos::ParameterList& pl); + +} + +#endif diff --git a/packages/nox/src-thyra/NOX_Thyra_Group.C b/packages/nox/src-thyra/NOX_Thyra_Group.C index 002ade4ff463..ad64269ae28d 100644 --- a/packages/nox/src-thyra/NOX_Thyra_Group.C +++ b/packages/nox/src-thyra/NOX_Thyra_Group.C @@ -1000,7 +1000,7 @@ void NOX::Thyra::Group::updateLOWS() const prec_, shared_jacobian_->getObject(this).ptr()); } - else if (nonnull(prec_factory_)) { + else if (nonnull(prec_factory_) && updatePreconditioner_) { // Automatically update using the user supplied prec factory prec_factory_->initializePrec(losb_, prec_.get()); @@ -1009,7 +1009,7 @@ void NOX::Thyra::Group::updateLOWS() const prec_, shared_jacobian_->getObject(this).ptr()); } - else if ( nonnull(prec_) && (model_->createOutArgs().supports( ::Thyra::ModelEvaluatorBase::OUT_ARG_W_prec)) ) { + else if ( nonnull(prec_) && (model_->createOutArgs().supports( ::Thyra::ModelEvaluatorBase::OUT_ARG_W_prec)) && updatePreconditioner_) { // Automatically update using the ModelEvaluator auto in_args = model_->createInArgs(); auto out_args = model_->createOutArgs(); @@ -1143,3 +1143,23 @@ void NOX::Thyra::Group::unsetBasePoint() bool NOX::Thyra::Group::usingBasePoint() const { return use_base_point_; } + +Teuchos::RCP> +NOX::Thyra::Group::getLinearOpWithSolveFactory() const +{ return lows_factory_; } + +Teuchos::RCP<::Thyra::PreconditionerFactoryBase> +NOX::Thyra::Group::getPreconditionerFactory() const +{ return prec_factory_; } + +void +NOX::Thyra::Group:: +takeControlOfPreconditionerUpdates(const Teuchos::RCP< ::Thyra::PreconditionerBase>& prec) +{ + prec_ = prec; + prec_factory_ = Teuchos::null; + updatePreconditioner_ = false; + + // Unset factory so solver doesn't automatically update preconditioner + Teuchos::rcp_const_cast<::Thyra::LinearOpWithSolveFactoryBase>(lows_factory_)->unsetPreconditionerFactory(nullptr,nullptr); +} diff --git a/packages/nox/src-thyra/NOX_Thyra_Group.H b/packages/nox/src-thyra/NOX_Thyra_Group.H index d1f2e7026d77..383ae73ef2c4 100644 --- a/packages/nox/src-thyra/NOX_Thyra_Group.H +++ b/packages/nox/src-thyra/NOX_Thyra_Group.H @@ -365,6 +365,17 @@ namespace NOX { /// Returns true if a base point has been set. bool usingBasePoint() const; + /// Returns the thyra linear solver factory. + Teuchos::RCP> + getLinearOpWithSolveFactory() const; + + /// Returns the thyra preconditioner factory. + Teuchos::RCP<::Thyra::PreconditionerFactoryBase> + getPreconditionerFactory() const; + + /// User will take control of updating an already registered preconditioner. Typically used with NOX::ObserverReusePreconditioner observer. + void takeControlOfPreconditionerUpdates(const Teuchos::RCP< ::Thyra::PreconditionerBase>& prec); + protected: //! resets the isValid flags to false diff --git a/packages/nox/src/CMakeLists.txt b/packages/nox/src/CMakeLists.txt index 440fb359bf8d..a57c2b566107 100644 --- a/packages/nox/src/CMakeLists.txt +++ b/packages/nox/src/CMakeLists.txt @@ -239,6 +239,8 @@ IF(NOX_ENABLE_ABSTRACT_IMPLEMENTATION_THYRA) ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_MatrixFree_ModelEvaluatorDecorator.hpp ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_LineSearch_SafeguardedDirection.hpp ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_TOpEleWiseMinSwap.hpp + ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_Observer_ReusePreconditioner.hpp + ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_Observer_ReusePreconditionerFactory.hpp ) APPEND_SET(SOURCES ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_Thyra_Vector.C @@ -249,6 +251,8 @@ IF(NOX_ENABLE_ABSTRACT_IMPLEMENTATION_THYRA) ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_MeritFunction_Weighted.cpp ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_Solver_PseudoTransient.cpp ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_LineSearch_SafeguardedDirection.cpp + ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_Observer_ReusePreconditioner.cpp + ${PACKAGE_SOURCE_DIR}/src-thyra/NOX_Observer_ReusePreconditionerFactory.cpp ) ENDIF() diff --git a/packages/nox/test/tpetra/Tpetra_1DFEM.cpp b/packages/nox/test/tpetra/Tpetra_1DFEM.cpp index 35f8e6e3bffa..59fb498570a9 100644 --- a/packages/nox/test/tpetra/Tpetra_1DFEM.cpp +++ b/packages/nox/test/tpetra/Tpetra_1DFEM.cpp @@ -68,6 +68,12 @@ #include "NOX_Thyra_MatrixFreeJacobianOperator.hpp" #include "NOX_MatrixFree_ModelEvaluatorDecorator.hpp" +#include "NOX_Observer_ReusePreconditioner.hpp" +#include "NOX_Observer_ReusePreconditionerFactory.hpp" +#include "NOX_Observer_Print.hpp" +#include "NOX_Observer_Vector.hpp" +#include "Thyra_VectorBase.hpp" +#include "Thyra_VectorStdOps.hpp" TEUCHOS_UNIT_TEST(NOX_Tpetra_1DFEM, AnalyticJacobian_NoPrec) { @@ -499,3 +505,126 @@ TEUCHOS_UNIT_TEST(NOX_Tpetra_1DFEM, JFNK_UserPrec) Teuchos::TimeMonitor::summarize(); } +TEUCHOS_UNIT_TEST(NOX_Tpetra_1DFEM, AnalyticJacobian_UserUpdatedIfpack2Prec) +{ + Teuchos::TimeMonitor::zeroOutTimers(); + + Teuchos::RCP > comm = Tpetra::getDefaultComm(); + + auto utils = Teuchos::rcp(new NOX::Utils(0,comm->getRank(),0,8)); + utils->out() << "\n\n"; + + // Get default Tpetra template types + typedef Tpetra::Vector<>::scalar_type Scalar; + typedef Tpetra::Vector<>::local_ordinal_type LO; + typedef Tpetra::Vector<>::global_ordinal_type GO; + typedef Tpetra::Vector<>::node_type Node; + + // Create the model evaluator object + Scalar x00 = 0.0; + Scalar x01 = 1.0; + const Tpetra::global_size_t numGlobalElements = 100; + Teuchos::RCP > model = + evaluatorTpetra1DFEM(comm, numGlobalElements, x00, x01); + + // Set the source term high to make the problem more difficult + ::Thyra::put_scalar(10.0,Teuchos::rcp_const_cast<::Thyra::VectorBase>(model->getNominalValues().get_p(2)).ptr()); + + Stratimikos::DefaultLinearSolverBuilder builder; + + Teuchos::RCP p = Teuchos::parameterList(); + p->set("Linear Solver Type", "Belos"); + Teuchos::ParameterList& belosList = p->sublist("Linear Solver Types").sublist("Belos"); + belosList.set("Solver Type", "Pseudo Block GMRES"); + belosList.sublist("Solver Types").sublist("Pseudo Block GMRES").set("Maximum Iterations", 200); + belosList.sublist("Solver Types").sublist("Pseudo Block GMRES").set("Num Blocks", 200); + belosList.sublist("Solver Types").sublist("Pseudo Block GMRES").set("Verbosity", 0x7f); + belosList.sublist("Solver Types").sublist("Pseudo Block GMRES").set("Output Frequency", 100); + belosList.sublist("VerboseObject").set("Verbosity Level", "none"); + p->set("Preconditioner Type", "Ifpack2"); + Teuchos::ParameterList& ifpackList = p->sublist("Preconditioner Types").sublist("Ifpack2"); + ifpackList.set("Prec Type", "ILUT"); + ifpackList.set("Overlap", 3); + ifpackList.sublist("Ifpack2 Settings").set("fact: ilut level-of-fill",2.0); + + builder.setParameterList(p); + + Teuchos::RCP > + lowsFactory = builder.createLinearSolveStrategy(""); + + model->set_W_factory(lowsFactory); + + // Create the initial guess + Teuchos::RCP > + initial_guess = model->getNominalValues().get_x()->clone_v(); + Thyra::V_S(initial_guess.ptr(),Teuchos::ScalarTraits::one()); + + const bool updatePreconditioner = false; // User controlled updating of preconditioner + Teuchos::RCP nox_group = + Teuchos::rcp(new NOX::Thyra::Group(*initial_guess, model, model->create_W_op(), lowsFactory, Teuchos::null, Teuchos::null, Teuchos::null, Teuchos::null, Teuchos::null, false, updatePreconditioner)); + + nox_group->computeF(); + + // Create the NOX status tests and the solver + // Create the convergence tests + Teuchos::RCP absresid = + Teuchos::rcp(new NOX::StatusTest::NormF(1.0e-8)); + Teuchos::RCP wrms = + Teuchos::rcp(new NOX::StatusTest::NormWRMS(1.0e-2, 1.0e-8)); + Teuchos::RCP converged = + Teuchos::rcp(new NOX::StatusTest::Combo(NOX::StatusTest::Combo::AND)); + converged->addStatusTest(absresid); + converged->addStatusTest(wrms); + Teuchos::RCP maxiters = + Teuchos::rcp(new NOX::StatusTest::MaxIters(20)); + Teuchos::RCP fv = + Teuchos::rcp(new NOX::StatusTest::FiniteValue); + Teuchos::RCP combo = + Teuchos::rcp(new NOX::StatusTest::Combo(NOX::StatusTest::Combo::OR)); + combo->addStatusTest(fv); + combo->addStatusTest(converged); + combo->addStatusTest(maxiters); + + // Create nox parameter list + Teuchos::RCP nl_params = Teuchos::parameterList(); + nl_params->set("Nonlinear Solver", "Line Search Based"); + nl_params->sublist("Direction").sublist("Newton").sublist("Linear Solver").set("Tolerance", 1.0e-4); + + // Add in Observsers. This test specifically adds in a + // preconditioner control observer + { + Teuchos::ParameterList precControlPL; + precControlPL.set("Update prec at start of nonlinear solve",true); + precControlPL.set("Update prec after this many nonlinear iterations",2); + precControlPL.set("Update prec after this many stalled linear solves",1); + precControlPL.set("Max linear iterations for stall",80); + auto pc_observer = NOX::createReusePreconditionerObserver(precControlPL); + auto print_observer = Teuchos::rcp(new NOX::ObserverPrint(utils)); + auto observers = Teuchos::rcp(new NOX::ObserverVector); + observers->pushBack(pc_observer); + observers->pushBack(print_observer); + nl_params->sublist("Solver Options").set>("Observer",observers); + } + + // Set output parameters + nl_params->sublist("Printing").sublist("Output Information").set("Debug",false); + nl_params->sublist("Printing").sublist("Output Information").set("Warning",false); + nl_params->sublist("Printing").sublist("Output Information").set("Error",false); + nl_params->sublist("Printing").sublist("Output Information").set("Test Details",false); + nl_params->sublist("Printing").sublist("Output Information").set("Details",false); + nl_params->sublist("Printing").sublist("Output Information").set("Parameters",false); + nl_params->sublist("Printing").sublist("Output Information").set("Linear Solver Details",false); + nl_params->sublist("Printing").sublist("Output Information").set("Inner Iteration",false); + nl_params->sublist("Printing").sublist("Output Information").set("Outer Iteration",false); + nl_params->sublist("Printing").sublist("Output Information").set("Outer Iteration StatusTest",false); + + // Create the solver + Teuchos::RCP solver = + NOX::Solver::buildSolver(nox_group, combo, nl_params); + NOX::StatusTest::StatusType solvStatus = solver->solve(); + + TEST_ASSERT(solvStatus == NOX::StatusTest::Converged); + + utils->out() << "\n"; + Teuchos::TimeMonitor::summarize(); +} From 5428049ad1f01a6016891e683e26b719fb967f14 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 23 Apr 2024 16:14:05 -0600 Subject: [PATCH 07/14] Panzer: add the nox preconditioner control to panzer example --- .../example/main_driver/CMakeLists.txt | 8 + .../main_driver/energy-ss-tp-delay-prec.xml | 315 ++++++++++++++++++ .../example/main_driver/main_driver.cpp | 8 +- .../user_app_NOXObserverFactory.hpp | 36 +- .../disc-fe/test/closure_model/CMakeLists.txt | 4 +- .../user_app_ClosureModel_Factory_impl.hpp | 16 +- .../closure_model/user_app_TSquaredModel.hpp | 71 ++++ .../user_app_TSquaredModel_impl.hpp | 86 +++++ 8 files changed, 532 insertions(+), 12 deletions(-) create mode 100644 packages/panzer/adapters-stk/example/main_driver/energy-ss-tp-delay-prec.xml create mode 100644 packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel.hpp create mode 100644 packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel_impl.hpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index cd943818310e..afecaa5c0ce3 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -20,6 +20,7 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files SOURCE_FILES energy-ss.xml energy-ss-tp.xml + energy-ss-tp-delay-prec.xml energy-ss-loca-eigenvalue.xml energy-ss-blocked.xml energy-transient-tempus-blocked.xml @@ -42,6 +43,13 @@ TRIBITS_ADD_ADVANCED_TEST( PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) +TRIBITS_ADD_ADVANCED_TEST( + main_driver_energy-ss-tp-delay-prec + TEST_0 EXEC main_driver + ARGS --i=energy-ss-tp-delay-prec.xml + PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." + ) + TRIBITS_ADD_ADVANCED_TEST( main_driver_energy-ss-blocked TEST_0 EXEC main_driver diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-ss-tp-delay-prec.xml b/packages/panzer/adapters-stk/example/main_driver/energy-ss-tp-delay-prec.xml new file mode 100644 index 000000000000..c2a0ac56daf6 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/energy-ss-tp-delay-prec.xml @@ -0,0 +1,315 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp index e6429b18c630..64961e2ab76c 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp @@ -112,6 +112,8 @@ int main(int argc, char *argv[]) int exodus_io_num_procs = 0; bool pauseToAttachOn = false; bool fluxCalculation = false; + bool printTimers = false; + bool printInputPL = false; { Teuchos::CommandLineProcessor clp; @@ -119,6 +121,8 @@ int main(int argc, char *argv[]) clp.setOption("exodus-io-num-procs", &exodus_io_num_procs, "Number of processes that can access the file system at the same time to read their portion of a sliced exodus file in parallel. Defaults to 0 - implies all processes for the run can access the file system at the same time."); clp.setOption("pause-to-attach","disable-pause-to-attach", &pauseToAttachOn, "Call pause to attach, default is off."); clp.setOption("flux-calc","disable-flux-calc", &fluxCalculation, "Enable the flux calculation."); + clp.setOption("time","no-time", &printTimers, "Print the timing information."); + clp.setOption("pl","no-pl", &printTimers, "Print the input ParameterList at the start of the run."); Teuchos::CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv,&std::cerr); @@ -345,7 +349,7 @@ int main(int argc, char *argv[]) TEUCHOS_ASSERT(response!=Teuchos::null); // should not be null! - *out << "Response Value \"" << responseIndexToName[i] << "\": " << *response << std::endl; + *out << "Response Value \"" << responseIndexToName[i] << "\": " << Thyra::get_ele(*response,0) << std::endl; } } @@ -383,7 +387,7 @@ int main(int argc, char *argv[]) } stackedTimer->stopBaseTimer(); - { + if (printTimers) { Teuchos::StackedTimer::OutputOptions options; options.output_fraction = true; options.output_minmax = true; diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_NOXObserverFactory.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_NOXObserverFactory.hpp index 159ff1d5fb7c..bdfabe2bfe27 100644 --- a/packages/panzer/adapters-stk/example/main_driver/user_app_NOXObserverFactory.hpp +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_NOXObserverFactory.hpp @@ -57,6 +57,8 @@ // Individual Observers #include "user_app_NOXObserver_WriteToExodus.hpp" #include "user_app_NOXObserver_NeumannBCAnalyticSystemTest.hpp" +#include "NOX_Observer_ReusePreconditionerFactory.hpp" +#include "NOX_Observer_Print.hpp" namespace user_app { @@ -87,13 +89,27 @@ namespace user_app { RCP observer = rcp(new NOX::PrePostOperatorVector); + // Delay the preconditioner update + if (this->getParameterList()->get("Delay Preconditioner Update")) { + Teuchos::ParameterList delay_list = this->getParameterList()->sublist("Delay Preconditioner Factory Sublist"); + auto ppo = NOX::createReusePreconditionerObserver(delay_list); + observer->pushBack(ppo); + } + + // New output format via observer + if (this->getParameterList()->get("New Output Format")) { + auto utils = Teuchos::rcp(new NOX::Utils(0,mesh->getComm()->getRank(),0,6)); + Teuchos::RCP po = + Teuchos::rcp(new NOX::ObserverPrint(utils)); + observer->pushBack(po); + } + // Exodus writer to output solution if (this->getParameterList()->get("Write Solution to Exodus File") == "ON") { Teuchos::RCP solution_writer = Teuchos::rcp(new user_app::NOXObserver_WriteToExodus(mesh,dof_manager,lof,stkIOResponseLibrary_)); observer->pushBack(solution_writer); } - // Neumann BC unit test if (this->getParameterList()->get("Neumann BC Analytic System Test") == "ON") { @@ -113,7 +129,7 @@ namespace user_app { using Teuchos::RCP; using Teuchos::rcp; - paramList->validateParametersAndSetDefaults(*(this->getValidParameters())); + paramList->validateParametersAndSetDefaults(*(this->getValidParameters()),0); setMyParamList(paramList); } @@ -123,14 +139,18 @@ namespace user_app { valid_params_ = Teuchos::rcp(new Teuchos::ParameterList); - valid_params_->set("Write Solution to Exodus File", "ON", - "Enables or disables writing of solution to Exodus file at end of NOX solve", - rcp(new Teuchos::StringValidator(Teuchos::tuple("ON", "OFF")))); + valid_params_->set("Write Solution to Exodus File", "ON", + "Enables or disables writing of solution to Exodus file at end of NOX solve", + rcp(new Teuchos::StringValidator(Teuchos::tuple("ON", "OFF")))); - valid_params_->set("Neumann BC Analytic System Test", "OFF", - "Checks solution values for Neumann BC Analytic System Test", - rcp(new Teuchos::StringValidator(Teuchos::tuple("ON", "OFF")))); + valid_params_->set("Neumann BC Analytic System Test", "OFF", + "Checks solution values for Neumann BC Analytic System Test", + rcp(new Teuchos::StringValidator(Teuchos::tuple("ON", "OFF")))); + + valid_params_->set("Delay Preconditioner Update",false,"Sets an observer that controls when to update the preconditioner."); + valid_params_->sublist("Delay Preconditioner Factory Sublist",false,"Sublist used to build the ReusePreconditioner observer."); + valid_params_->set("New Output Format",false,"Enables new output format from observer."); } return valid_params_; } diff --git a/packages/panzer/disc-fe/test/closure_model/CMakeLists.txt b/packages/panzer/disc-fe/test/closure_model/CMakeLists.txt index e3d53a065722..0c5f52d74f59 100644 --- a/packages/panzer/disc-fe/test/closure_model/CMakeLists.txt +++ b/packages/panzer/disc-fe/test/closure_model/CMakeLists.txt @@ -11,7 +11,9 @@ SET(SOURCES user_app_ClosureModel_Factory_TemplateBuilder.hpp user_app_ConstantModel.hpp user_app_ConstantModel_impl.hpp - ) + user_app_TSquaredModel.hpp + user_app_TSquaredModel_impl.hpp + ) TRIBITS_ADD_EXECUTABLE_AND_TEST( closure_model diff --git a/packages/panzer/disc-fe/test/closure_model/user_app_ClosureModel_Factory_impl.hpp b/packages/panzer/disc-fe/test/closure_model/user_app_ClosureModel_Factory_impl.hpp index 4feb592fcce3..8207dce02a64 100644 --- a/packages/panzer/disc-fe/test/closure_model/user_app_ClosureModel_Factory_impl.hpp +++ b/packages/panzer/disc-fe/test/closure_model/user_app_ClosureModel_Factory_impl.hpp @@ -57,6 +57,7 @@ // User application evaluators for this factory #include "user_app_ConstantModel.hpp" +#include "user_app_TSquaredModel.hpp" #include "Panzer_Parameter.hpp" #include "Panzer_GlobalStatistics.hpp" @@ -207,7 +208,20 @@ buildClosureModels(const std::string& model_id, found = true; } - + else if (plist.get("Type") == "T Squared Source") { + // Required parameters + plist.get("Multiplier"); + plist.get("Source Name"); + plist.get("Target Name"); + { + Teuchos::ParameterList input = plist; + input.set("Data Layout", ir->dl_scalar); + RCP< user_app::TSquaredModel > e = + rcp(new user_app::TSquaredModel(input)); + evaluators->push_back(e); + } + found = true; + } } else if (plist.isType("Value")) { { // at IP diff --git a/packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel.hpp b/packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel.hpp new file mode 100644 index 000000000000..1b62628dda28 --- /dev/null +++ b/packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel.hpp @@ -0,0 +1,71 @@ +// @HEADER +// *********************************************************************** +// +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// Copyright (2011) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and +// Eric C. Cyr (eccyr@sandia.gov) +// *********************************************************************** +// @HEADER + +#ifndef USER_APP_TSQUARED_MODEL_HPP +#define USER_APP_TSQUARED_MODEL_HPP + +#include "Phalanx_Evaluator_Macros.hpp" +#include "Phalanx_MDField.hpp" + +namespace user_app { + + /// Example evaluator that computes a source term k*T^2 at the integration points. + template + class TSquaredModel : public PHX::EvaluatorWithBaseImpl, + public PHX::EvaluatorDerived + { + public: + TSquaredModel(const Teuchos::ParameterList& p); + void evaluateFields(typename Traits::EvalData d); + + private: + using ScalarT = typename EvalT::ScalarT; + double k_; + PHX::MDField t_; + PHX::MDField k_t_squared_; + }; + +} // namespace user_app + +#include "user_app_TSquaredModel_impl.hpp" + +#endif diff --git a/packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel_impl.hpp b/packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel_impl.hpp new file mode 100644 index 000000000000..26710cb99f21 --- /dev/null +++ b/packages/panzer/disc-fe/test/closure_model/user_app_TSquaredModel_impl.hpp @@ -0,0 +1,86 @@ +// @HEADER +// *********************************************************************** +// +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// Copyright (2011) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Roger P. Pawlowski (rppawlo@sandia.gov) and +// Eric C. Cyr (eccyr@sandia.gov) +// *********************************************************************** +// @HEADER + +#ifndef USER_APP_T_SQUARED_MODEL_IMPL_HPP +#define USER_APP_T_SQUARED_MODEL_IMPL_HPP + +#include "Panzer_HierarchicParallelism.hpp" + +//********************************************************************** +template +user_app::TSquaredModel:: +TSquaredModel(const Teuchos::ParameterList& p) + : k_( p.get("Multiplier") ), + t_( p.get("Source Name"), + p.get< Teuchos::RCP >("Data Layout") ), + k_t_squared_( p.get("Target Name"), + p.get< Teuchos::RCP >("Data Layout") ) +{ + this->addDependentField(t_); + this->addEvaluatedField(k_t_squared_); + std::string n = "user_app::TSquared: " + k_t_squared_.fieldTag().name(); + this->setName(n); +} + +//********************************************************************** +template +void +user_app::TSquaredModel:: +evaluateFields(typename Traits::EvalData workset) +{ + const auto num_cells = workset.num_cells; + const int num_ip = static_cast(t_.extent(1)); + const double k = k_; + const auto t = t_; + const auto k_t_squared = k_t_squared_; + auto policy = panzer::HP::inst().teamPolicy(num_cells); + Kokkos::parallel_for("TSquared evaluator",policy,KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type& team){ + const int cell = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team,0,num_ip), [&] (const int pt) { + k_t_squared(cell,pt) = k * t(cell,pt)*t(cell,pt); + }); + }); +} + +//********************************************************************** + +#endif From 4e0325d46bce3ec167b689951bc468d5ef71140d Mon Sep 17 00:00:00 2001 From: iyamazaki Date: Tue, 23 Apr 2024 23:52:08 -0600 Subject: [PATCH 08/14] Amesos2:Pardiso_MKL : option to turn of matching --- .../amesos2/src/Amesos2_PardisoMKL_def.hpp | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp b/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp index 2e407d91052a..5b6010aa7003 100644 --- a/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp +++ b/packages/amesos2/src/Amesos2_PardisoMKL_def.hpp @@ -291,6 +291,7 @@ namespace Amesos2 { RCP valid_params = getValidParameters_impl(); + // Fill-in reordering: 0 = minimum degree, 2 = METIS 4.0.1 (default), 3 = METIS 5.1, 4 = AMD, if( parameterList->isParameter("IPARM(2)") ) { RCP fillin_validator = valid_params->getEntry("IPARM(2)").validator(); @@ -298,6 +299,7 @@ namespace Amesos2 { iparm_[1] = getIntegralValue(*parameterList, "IPARM(2)"); } + // Iterative-direct algorithm if( parameterList->isParameter("IPARM(4)") ) { RCP prec_validator = valid_params->getEntry("IPARM(4)").validator(); @@ -305,13 +307,15 @@ namespace Amesos2 { iparm_[3] = getIntegralValue(*parameterList, "IPARM(4)"); } + // Max numbers of iterative refinement steps if( parameterList->isParameter("IPARM(8)") ) { RCP refine_validator = valid_params->getEntry("IPARM(8)").validator(); parameterList->getEntry("IPARM(8)").setValidator(refine_validator); iparm_[7] = getIntegralValue(*parameterList, "IPARM(8)"); } - + + // Perturb the pivot elements if( parameterList->isParameter("IPARM(10)") ) { RCP pivot_perturb_validator = valid_params->getEntry("IPARM(10)").validator(); @@ -323,6 +327,7 @@ namespace Amesos2 { // Then solver specific options can override this. iparm_[11] = this->control_.useTranspose_ ? 2 : 0; + // Normal solve (0), or a transpose solve (1) if( parameterList->isParameter("IPARM(12)") ) { RCP trans_validator = valid_params->getEntry("IPARM(12)").validator(); @@ -330,6 +335,15 @@ namespace Amesos2 { iparm_[11] = getIntegralValue(*parameterList, "IPARM(12)"); } + // (Non-)symmetric matchings : detault 1 for nonsymmetric and 0 for symmetric matrix (default is nonsymmetric) + if( parameterList->isParameter("IPARM(13)") ) + { + RCP trans_validator = valid_params->getEntry("IPARM(13)").validator(); + parameterList->getEntry("IPARM(13)").setValidator(trans_validator); + iparm_[12] = getIntegralValue(*parameterList, "IPARM(13)"); + } + + // Output: Number of nonzeros in the factor LU if( parameterList->isParameter("IPARM(18)") ) { RCP report_validator = valid_params->getEntry("IPARM(18)").validator(); @@ -337,6 +351,7 @@ namespace Amesos2 { iparm_[17] = getIntegralValue(*parameterList, "IPARM(18)"); } + // Scheduling method for the parallel numerical factorization if( parameterList->isParameter("IPARM(24)") ) { RCP par_fact_validator = valid_params->getEntry("IPARM(24)").validator(); @@ -344,6 +359,7 @@ namespace Amesos2 { iparm_[23] = getIntegralValue(*parameterList, "IPARM(24)"); } + // Parallelization scheme for the forward and backward solv if( parameterList->isParameter("IPARM(25)") ) { RCP par_fbsolve_validator = valid_params->getEntry("IPARM(25)").validator(); @@ -351,6 +367,7 @@ namespace Amesos2 { iparm_[24] = getIntegralValue(*parameterList, "IPARM(25)"); } + // Graph compression scheme for METIS. if( parameterList->isParameter("IPARM(60)") ) { RCP ooc_validator = valid_params->getEntry("IPARM(60)").validator(); @@ -433,7 +450,14 @@ PardisoMKL::getValidParameters_impl() const "Transposed"), tuple(0, 1, 2), pl.getRawPtr()); - + + setStringToIntegralParameter("IPARM(13)", toString(iparm_temp[12]), + "Use weighted matching", + tuple("0", "1"), + tuple("No matching", "Use matching"), + tuple(0, 1), + pl.getRawPtr()); + setStringToIntegralParameter("IPARM(24)", toString(iparm_temp[23]), "Parallel factorization control", tuple("0", "1"), From 787ac44b1361ad95c3a32e423672c493f43fc4cc Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Wed, 24 Apr 2024 06:37:10 -0600 Subject: [PATCH 09/14] NOX: fix warning --- packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp b/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp index baff05432bb3..95e0caaf888f 100644 --- a/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp +++ b/packages/nox/src-thyra/NOX_Observer_ReusePreconditioner.cpp @@ -10,8 +10,8 @@ NOX::ObserverReusePreconditioner::ObserverReusePreconditioner() update_after_n_iterations_(false), num_iterations_for_update_(5), update_if_stalled_(false), - max_linear_iterations_for_stall_(-1), max_count_for_stall_(-1), + max_linear_iterations_for_stall_(-1), stall_count_(0), iterations_since_last_update_(0) {} From bee0495371b23a6f8bcbfcdb20dd71e33325326d Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Wed, 24 Apr 2024 06:37:30 -0600 Subject: [PATCH 10/14] Panzer: fix warning --- .../panzer/adapters-stk/example/main_driver/main_driver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp index 64961e2ab76c..c08845bfca73 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp @@ -142,7 +142,8 @@ int main(int argc, char *argv[]) Teuchos::RCP input_params = Teuchos::rcp(new Teuchos::ParameterList("User_App Parameters")); Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); - *out << *input_params << std::endl; + if (printInputPL) + *out << *input_params << std::endl; Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); input_params->remove("Solver Factories"); From 245a59875f03229838ef221304c4649593c424d6 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Wed, 24 Apr 2024 07:07:59 -0600 Subject: [PATCH 11/14] Phalanx: remove shadow warning --- .../test/ViewOfViews/tPhalanxViewOfViews.cpp | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp b/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp index e1326464ace5..2c5c20fec65c 100644 --- a/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp +++ b/packages/phalanx/test/ViewOfViews/tPhalanxViewOfViews.cpp @@ -684,16 +684,18 @@ TEUCHOS_UNIT_TEST(PhalanxViewOfViews,FadAndAssignment) { auto c = Mat_h(1,0); // Initialize a and b - Kokkos::MDRangePolicy> policy({0,0},{a.extent(0),a.extent(1)}); - Kokkos::parallel_for("FadAndAssignment init",policy,KOKKOS_LAMBDA(const int cell, const int pt) { - a(cell,pt).val() = double(cell) + double(pt); - a(cell,pt).fastAccessDx(0) = 0.0; - a(cell,pt).fastAccessDx(1) = 2.0 * double(cell) + double(pt); - b(cell,pt).val() = double(cell); - b(cell,pt).fastAccessDx(0) = 0.0; - b(cell,pt).fastAccessDx(1) = 0.0; - }); - PHX::Device::execution_space().fence(); + { + Kokkos::MDRangePolicy> policy({0,0},{a.extent(0),a.extent(1)}); + Kokkos::parallel_for("FadAndAssignment init",policy,KOKKOS_LAMBDA(const int cell, const int pt) { + a(cell,pt).val() = double(cell) + double(pt); + a(cell,pt).fastAccessDx(0) = 0.0; + a(cell,pt).fastAccessDx(1) = 2.0 * double(cell) + double(pt); + b(cell,pt).val() = double(cell); + b(cell,pt).fastAccessDx(0) = 0.0; + b(cell,pt).fastAccessDx(1) = 0.0; + }); + PHX::Device::execution_space().fence(); + } // Compute c using device vov auto Mat = vov.getViewDevice(); @@ -707,6 +709,7 @@ TEUCHOS_UNIT_TEST(PhalanxViewOfViews,FadAndAssignment) { }); }); } else { + Kokkos::MDRangePolicy> policy({0,0},{a.extent(0),a.extent(1)}); Kokkos::parallel_for("FadAndAssignement compute",policy,KOKKOS_LAMBDA(const int cell, const int pt) { Mat(1,0)(cell,pt) = Mat(0,0)(cell,pt) * Mat(0,1)(cell,pt); }); From 68680e78bdfd9b7b7d2c6d563a7573bb5f92554e Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Wed, 24 Apr 2024 07:09:34 -0600 Subject: [PATCH 12/14] Panzer: fix shadow warning --- .../test/fe_assembly/test_fe_projection.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/panzer/dof-mgr/test/fe_assembly/test_fe_projection.hpp b/packages/panzer/dof-mgr/test/fe_assembly/test_fe_projection.hpp index e1ba85267b7a..06cb52d37109 100644 --- a/packages/panzer/dof-mgr/test/fe_assembly/test_fe_projection.hpp +++ b/packages/panzer/dof-mgr/test/fe_assembly/test_fe_projection.hpp @@ -192,8 +192,8 @@ int feProjection(int argc, char *argv[]) { // parse command line arguments int degree = 2; local_ordinal_t nx = 2; - local_ordinal_t ny = nx; - local_ordinal_t nz = (dim == 3) ? nx :1; + local_ordinal_t ny = nx; + local_ordinal_t nz = (dim == 3) ? nx :1; int np = comm.getSize(); // number of processors int px = (dim == 2) ? std::sqrt(np) : std::cbrt(np); while(np%px!=0) --px; int py = (dim == 2) ? np/px : std::sqrt(np/px); while(np%py!=0) --py; @@ -881,7 +881,7 @@ int feProjection(int argc, char *argv[]) { } //evaluate the boundary terms in physical space - double nx(sideNormal[0]), ny(sideNormal[1]), nz(sideNormal[2]); + double n_x(sideNormal[0]), n_y(sideNormal[1]), n_z(sideNormal[2]); Kokkos::parallel_for(Kokkos::RangePolicy(0,numBoundarySides), KOKKOS_LAMBDA (const int &is) { Fun fun(functionSpace); @@ -898,7 +898,7 @@ int feProjection(int argc, char *argv[]) { case Intrepid2::FUNCTION_SPACE_HDIV: { //compute f \dot n double fx(fun(x,y,z,0)), fy(fun(x,y,z,1)), fz(fun(x,y,z,2)); - physTargetAtSideEvalPoints(is,pt) = (dim == 3) ? nx*fx+ny*fy+nz*fz : nx*fx+ny*fy; + physTargetAtSideEvalPoints(is,pt) = (dim == 3) ? n_x*fx+n_y*fy+n_z*fz : n_x*fx+n_y*fy; } break; case Intrepid2::FUNCTION_SPACE_HCURL: @@ -906,11 +906,11 @@ int feProjection(int argc, char *argv[]) { //compute f \times n double fx(fun(x,y,z,0)), fy(fun(x,y,z,1)), fz(fun(x,y,z,2)); if (dim == 3) { - physTargetAtSideEvalPoints(is,pt,0) = fy*nz-fz*ny; - physTargetAtSideEvalPoints(is,pt,1) = fz*nx-fx*nz; - physTargetAtSideEvalPoints(is,pt,2) = fx*ny-fy*nx; + physTargetAtSideEvalPoints(is,pt,0) = fy*n_z-fz*n_y; + physTargetAtSideEvalPoints(is,pt,1) = fz*n_x-fx*n_z; + physTargetAtSideEvalPoints(is,pt,2) = fx*n_y-fy*n_x; } else { - physTargetAtSideEvalPoints(is,pt) = fx*ny-fy*nx; + physTargetAtSideEvalPoints(is,pt) = fx*n_y-fy*n_x; } } break; From 1f31b52b946d9cf6bb281db46157f9a312124e68 Mon Sep 17 00:00:00 2001 From: Nate Roberts Date: Wed, 24 Apr 2024 13:12:38 -0600 Subject: [PATCH 13/14] Add intrepid2 sierra performance test (#12957) * First-pass effort at performance test targeting the Sierra gradient projection use case. Still need to make the timers imitate what's in packages/panzer/mini-em/example/BlockPrec/main.cpp more precisely. (What's here might work; I'm not sure.) * Some edits to SierraGradProjection test to make it look more like the Mini-EM performance test (stacked timer, etc.) Not sure what gets used by the testing infrastructure, though... * Fixed ComputeBasis perf test (build issues). * SierraGradProjection: added parallel_fors for things that set View values (to allow running on CUDA, etc.). Added CLI options, and made the stacked timer output to Watchr file. Got the individual timers to be picked up by the stacked timer. * Fixed some issues revealed by building with -DTrilinos_TEST_CATEGORIES=PERFORMANCE. * Tweaked SierraGradProjection to only use Teuchos::DefaultComm, not MpiComm. (Test failures apparently due to running in a build without MPI.) This test is only expected to be run on a single MPI rank, when MPI is available. * Fixing a CUDA build error. * Deleted some unnecessary "using namespace" declarations; one of them was leading to a compiler error on a test platform. --- packages/intrepid2/perf-test/CMakeLists.txt | 2 +- .../perf-test/ComputeBasis/test_hgrad.hpp | 6 +- .../intrepid2/perf-test/Sierra/CMakeLists.txt | 23 ++ .../perf-test/Sierra/SierraGradProjection.cpp | 206 ++++++++++++++++++ .../MonolithicExecutable/CMakeLists.txt | 32 ++- 5 files changed, 258 insertions(+), 11 deletions(-) create mode 100644 packages/intrepid2/perf-test/Sierra/CMakeLists.txt create mode 100644 packages/intrepid2/perf-test/Sierra/SierraGradProjection.cpp diff --git a/packages/intrepid2/perf-test/CMakeLists.txt b/packages/intrepid2/perf-test/CMakeLists.txt index 1fc39aecf0d9..a09e5c3af280 100644 --- a/packages/intrepid2/perf-test/CMakeLists.txt +++ b/packages/intrepid2/perf-test/CMakeLists.txt @@ -2,6 +2,6 @@ TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) ADD_SUBDIRECTORY(ComputeBasis) ADD_SUBDIRECTORY(DynRankView) - +ADD_SUBDIRECTORY(Sierra) diff --git a/packages/intrepid2/perf-test/ComputeBasis/test_hgrad.hpp b/packages/intrepid2/perf-test/ComputeBasis/test_hgrad.hpp index 13e38c3cc7a9..864f448c8e11 100644 --- a/packages/intrepid2/perf-test/ComputeBasis/test_hgrad.hpp +++ b/packages/intrepid2/perf-test/ComputeBasis/test_hgrad.hpp @@ -157,8 +157,10 @@ namespace Intrepid2 { refBasisValues("refBasisValues", numDofs, numPoints), refBasisGrads ("refBasisGrads", numDofs, numPoints, spaceDim); - ImplBasisType::getValues(refBasisValues, refPoints, OPERATOR_VALUE); - ImplBasisType::getValues(refBasisGrads, refPoints, OPERATOR_GRAD); + auto space = typename DeviceSpaceType::execution_space(); + + ImplBasisType::getValues(space, refBasisValues, refPoints, OPERATOR_VALUE); + ImplBasisType::getValues(space, refBasisGrads, refPoints, OPERATOR_GRAD); std::cout << " Ref completed\n"; diff --git a/packages/intrepid2/perf-test/Sierra/CMakeLists.txt b/packages/intrepid2/perf-test/Sierra/CMakeLists.txt new file mode 100644 index 000000000000..5071e95340e0 --- /dev/null +++ b/packages/intrepid2/perf-test/Sierra/CMakeLists.txt @@ -0,0 +1,23 @@ +TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +SET(SIERRA_GRAD_PROJECTION_SOURCES + SierraGradProjection.cpp + ) + +TRIBITS_ADD_EXECUTABLE( + SierraGradProjection + SOURCES ${SIERRA_GRAD_PROJECTION_SOURCES} + CATEGORIES BASIC PERFORMANCE + ) + +# Performance test representing gradient projection as used in Sierra +TRIBITS_ADD_TEST( + SierraGradProjection + NAME "SierraGradProjection" + ARGS "" + COMM mpi + NUM_MPI_PROCS 1 + CATEGORIES PERFORMANCE + RUN_SERIAL + ) + diff --git a/packages/intrepid2/perf-test/Sierra/SierraGradProjection.cpp b/packages/intrepid2/perf-test/Sierra/SierraGradProjection.cpp new file mode 100644 index 000000000000..b2c54231e487 --- /dev/null +++ b/packages/intrepid2/perf-test/Sierra/SierraGradProjection.cpp @@ -0,0 +1,206 @@ +#include "Teuchos_DefaultComm.hpp" +#include "Teuchos_GlobalMPISession.hpp" +#include "Teuchos_StackedTimer.hpp" +#include "Teuchos_TimeMonitor.hpp" + +#include "Intrepid2_ArrayTools.hpp" +#include "Intrepid2_CellGeometry.hpp" +#include "Intrepid2_CellTools.hpp" +#include "Intrepid2_FunctionSpaceTools.hpp" + +#include "Intrepid2_HGRAD_HEX_C1_FEM.hpp" + +#include "Teuchos_Array.hpp" +#include "Teuchos_UnitTestRepository.hpp" + +#include "Kokkos_Core.hpp" + +#include + +const int numFields = 8; +const int numNodes = 8; +const int numDims = 3; +const int numIntg = 8; + +using DeviceType = Kokkos::DefaultExecutionSpace::device_type; + +int worksetSize = 512; + +Kokkos::DynRankView points; // (P,D) or (C,P,D), allocated/initialized in main below +Kokkos::DynRankView gradient; // (C,F,P,D), allocated/initialized in main below + +Kokkos::DynRankView jacobian; +Kokkos::DynRankView jacobian_inv; +Kokkos::DynRankView tempGrad; + +shards::CellTopology cell_topology = shards::getCellTopologyData >(); + +using BasisFamily = Intrepid2::DerivedNodalBasisFamily; +BasisFamily::BasisPtr basis; + +/** \brief Create a uniform Cartesian mesh, with origin at 0, and domain extents and mesh widths that can be different in different coordinate dimensions. + \param [in] domainExtents - array specifying the extent of the domain in each coordinate dimension. + \param [in] gridCellCounts - array specifying the number of cells in each coordinate dimension. + \return a uniform Cartesion mesh, with origin at 0, with the specified domain extents and grid cell counts. +*/ +template +inline Intrepid2::CellGeometry uniformCartesianMesh(const Kokkos::Array &domainExtents, + const Kokkos::Array &gridCellCounts) +{ + Kokkos::Array origin; + for (int d=0; d; + const auto NO_SUBDIVISION = CellGeometry::NO_SUBDIVISION; + const auto HYPERCUBE_NODE_ORDER_CLASSIC_SHARDS = CellGeometry::HYPERCUBE_NODE_ORDER_CLASSIC_SHARDS; + + return CellGeometry(origin, domainExtents, gridCellCounts, NO_SUBDIVISION, HYPERCUBE_NODE_ORDER_CLASSIC_SHARDS); +} + +void intrepid2_gradient_operator(const int n, const double* coords, double* grad, double* det) +{ + using FunctionSpaceTools = Intrepid2::FunctionSpaceTools; + using CellToolsReal2 = Intrepid2::CellTools; + + using FieldContainer = Kokkos::DynRankView; + + const FieldContainer coordVec(const_cast(coords), n, numNodes, numDims); + + FieldContainer jacobian_det(det, n, numIntg); + + // compute the jacobian, its inverse, and its determinant + CellToolsReal2::setJacobian(jacobian, points, coordVec, basis); + CellToolsReal2::setJacobianInv(jacobian_inv, jacobian); + CellToolsReal2::setJacobianDet(jacobian_det, jacobian); + + // compute the Basis Function Gradients + FunctionSpaceTools::HGRADtransformGRAD(tempGrad, jacobian_inv, gradient); + + // + // Transform gradient operator from the intrepid order: ( Element, Node Number, Intg Point, Component) + // to index order consistent with the rest of Sierra: ( Element, Intg Point, Node Number, Component) + // + FieldContainer gradOpContainer(grad, n, numIntg, numNodes, numDims); + FieldContainer tempGradLocal = tempGrad; // local View to avoid CUDA complaints. + Kokkos::parallel_for("reorder gradients", n, + KOKKOS_LAMBDA(const int &ielem) + { + for (int ip = 0; ip < numIntg; ++ip) { + for (int inode = 0; inode < numNodes; ++inode) { + for (int d = 0; d stacked_timer; + + std::string test_name = "Intrepid2 Sierra Test"; + + { + // Note that the dtor for GlobalMPISession will call Kokkos::finalize_all() but ctor does not call Kokkos::initialize()... + Teuchos::GlobalMPISession mpiSession(&argc, &argv); + Kokkos::initialize(argc,argv); + Teuchos::UnitTestRepository::setGloballyReduceTestResult(true); + + Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcpFromRef(std::cout))); + auto comm = Teuchos::DefaultComm::getComm(); + + { + Teuchos::CommandLineProcessor cmdp(false,true); // false: don't throw exceptions; true: do return errors for unrecognized options + + cmdp.setOption("numInvocations", &numInvocations, "How many times to run the main loop"); + cmdp.setOption("worksetSize", &worksetSize, "The size of the workset - how many cells to process at a time"); + + if (cmdp.parse(argc,argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) + { + return -1; + } + } + + { + stacked_timer = rcp(new Teuchos::StackedTimer(test_name.c_str())); + Teuchos::RCP verbose_out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcpFromRef(std::cout))); + verbose_out->setShowProcRank(true); + stacked_timer->setVerboseOstream(verbose_out); + Teuchos::TimeMonitor::setStackedTimer(stacked_timer); + } + + Kokkos::DynRankView coords, det; + { + Teuchos::TimeMonitor setupTimer(*Teuchos::TimeMonitor::getNewTimer(std::string("allocations and other one-time setup"))); + + gradient = Kokkos::DynRankView("gradient", worksetSize, numFields, numIntg, numDims); + points = Kokkos::DynRankView("points", numIntg, numDims); // (P,D) + + jacobian = Kokkos::DynRankView("jacobian", worksetSize, numIntg, numDims, numDims); + jacobian_inv = Kokkos::DynRankView("jacobian_inv", worksetSize, numIntg, numDims, numDims); + tempGrad = Kokkos::DynRankView("tempGrad", worksetSize, numNodes, numIntg, numDims); + + basis = Teuchos::rcp( new Intrepid2::Basis_HGRAD_HEX_C1_FEM() ); + + const int cellCount = 25000; + + Kokkos::Array domainExtents{1.,1.,1.}; + Kokkos::Array gridCellCounts{50,50,10}; // 25000 total cells + + auto geometry = uniformCartesianMesh(domainExtents, gridCellCounts); + + det = Kokkos::DynRankView("det", worksetSize, numIntg); + coords = Kokkos::DynRankView("coords", cellCount, numNodes, numDims); + Kokkos::parallel_for("initialize coords", cellCount, + KOKKOS_LAMBDA(const int &cellOrdinal) + { + for (int nodeOrdinal=0; nodeOrdinal(); + points = Kokkos::DynRankView(); + basis = Teuchos::null; + jacobian = Kokkos::DynRankView(); + jacobian_inv = Kokkos::DynRankView(); + tempGrad = Kokkos::DynRankView(); + } + + stacked_timer->stop(test_name); + Teuchos::StackedTimer::OutputOptions options; + options.output_fraction = options.output_histogram = options.output_minmax = true; + stacked_timer->report(*out, comm, options); + + auto xmlOut = stacked_timer->reportWatchrXML(test_name + ' ' + std::to_string(comm->getSize()) + " ranks", comm); + if(xmlOut.length()) + std::cout << "\nAlso created Watchr performance report " << xmlOut << '\n'; + } + + return 0; +} + diff --git a/packages/intrepid2/unit-test/MonolithicExecutable/CMakeLists.txt b/packages/intrepid2/unit-test/MonolithicExecutable/CMakeLists.txt index 3b44c20a637c..4a6393836293 100644 --- a/packages/intrepid2/unit-test/MonolithicExecutable/CMakeLists.txt +++ b/packages/intrepid2/unit-test/MonolithicExecutable/CMakeLists.txt @@ -20,32 +20,48 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( # BasisEquivalenceTests TRIBITS_ADD_EXECUTABLE( BasisEquivalenceTests NOEXESUFFIX SOURCES BasisEquivalenceTests_Wedge.cpp BasisEquivalenceTests_Tri.cpp BasisEquivalenceTests_Tet.cpp BasisEquivalenceTests_Quad.cpp BasisEquivalenceTests_Line.cpp BasisEquivalenceTests_Hypercube.cpp BasisEquivalenceTests_Hex.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_BasisEquivalenceTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_BasisEquivalenceTests) + SET_TARGET_PROPERTIES( Intrepid2_BasisEquivalenceTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # BasisEquivalenceTests_Pyramid TRIBITS_ADD_EXECUTABLE( BasisEquivalenceTests_Pyramid NOEXESUFFIX SOURCES BasisEquivalenceTests_Pyramid.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_BasisEquivalenceTests_Pyramid PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_BasisEquivalenceTests_Pyramid) + SET_TARGET_PROPERTIES( Intrepid2_BasisEquivalenceTests_Pyramid PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # BasisValuesTests_Pyramid TRIBITS_ADD_EXECUTABLE( BasisValuesTests_Pyramid NOEXESUFFIX SOURCES BasisValuesTests_Pyramid.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_BasisValuesTests_Pyramid PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_BasisValuesTests_Pyramid) + SET_TARGET_PROPERTIES( Intrepid2_BasisValuesTests_Pyramid PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # BasisEquivalenceTests_Wedge TRIBITS_ADD_EXECUTABLE( BasisEquivalenceTests_Wedge NOEXESUFFIX SOURCES BasisEquivalenceTests_Wedge.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_BasisEquivalenceTests_Wedge PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_BasisEquivalenceTests_Wedge) + SET_TARGET_PROPERTIES( Intrepid2_BasisEquivalenceTests_Wedge PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # CellGeometry TRIBITS_ADD_EXECUTABLE( CellGeometryTests NOEXESUFFIX SOURCES CellGeometryTests.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_CellGeometryTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_CellGeometryTests) + SET_TARGET_PROPERTIES( Intrepid2_CellGeometryTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # DataTests TRIBITS_ADD_EXECUTABLE( DataTests NOEXESUFFIX SOURCES DataTests.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_DataTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_DataTests) + SET_TARGET_PROPERTIES( Intrepid2_DataTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # StructuredVersusStandard integration TRIBITS_ADD_EXECUTABLE( StructuredVersusStandard NOEXESUFFIX SOURCES StructuredIntegrationTests_StructuredVersusStandard.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_StructuredVersusStandard PROPERTIES EXCLUDE_FROM_ALL TRUE) +IF (TARGET Intrepid2_StructuredVersusStandard) + SET_TARGET_PROPERTIES( Intrepid2_StructuredVersusStandard PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() # SubBasisInclusionTests TRIBITS_ADD_EXECUTABLE( SubBasisInclusionTests NOEXESUFFIX SOURCES SubBasisInclusionTests.cpp UnitTestMain.cpp ) -SET_TARGET_PROPERTIES( Intrepid2_SubBasisInclusionTests PROPERTIES EXCLUDE_FROM_ALL TRUE) \ No newline at end of file +IF (TARGET Intrepid2_SubBasisInclusionTests) + SET_TARGET_PROPERTIES( Intrepid2_SubBasisInclusionTests PROPERTIES EXCLUDE_FROM_ALL TRUE) +ENDIF() \ No newline at end of file From ccc05fef7978b133c0700aa1612fc2b178538d56 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 24 Apr 2024 15:51:49 -0600 Subject: [PATCH 14/14] Create SECURITY.md (#12897) Co-authored-by: trilinos-autotester --- SECURITY.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000000..3cca180ccf1c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security Policy + +## Supported Versions + +The latest released version of Trilinos is the only version supported with security updates. You can find that at: https://github.com/trilinos/Trilinos/releases. + +## Reporting a Vulnerability + +For run-of-the mill memory errors, etc., please file a Github issue. For specific, sensitive security issues, please click on "Report a vulnerability" at https://github.com/trilinos/Trilinos/security.