From 4cd148191e82ef849e33009b59cdb1ac5803f819 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Tue, 6 Aug 2024 19:59:42 -0600 Subject: [PATCH 01/16] add protego submodule --- .gitmodules | 3 +++ protego-mech | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 protego-mech diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..22f340bc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "protego-mech"] + path = protego-mech + url = git@github.com:aperijake/protego-mech.git diff --git a/protego-mech b/protego-mech new file mode 160000 index 00000000..43593162 --- /dev/null +++ b/protego-mech @@ -0,0 +1 @@ +Subproject commit 43593162153f2aa4564cf5fb4b8df9f1834aecf1 From d872dd4dfa73b8d978de00b8db0eb5acd28955ff Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Tue, 6 Aug 2024 20:03:30 -0600 Subject: [PATCH 02/16] update protego with basic first file --- protego-mech | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protego-mech b/protego-mech index 43593162..db094b6d 160000 --- a/protego-mech +++ b/protego-mech @@ -1 +1 @@ -Subproject commit 43593162153f2aa4564cf5fb4b8df9f1834aecf1 +Subproject commit db094b6d32529f1ef40787bf01180057a62af68e From e071e9871f6680b09aec630feb10773df878bff9 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 11:16:05 -0600 Subject: [PATCH 03/16] fixed hanging issue in regression tests --- test/utils/regression_test/regression_test.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/utils/regression_test/regression_test.py b/test/utils/regression_test/regression_test.py index 02545cc7..8e4815ae 100644 --- a/test/utils/regression_test/regression_test.py +++ b/test/utils/regression_test/regression_test.py @@ -7,6 +7,7 @@ # trunk-ignore(bandit/B404) import subprocess import sys +import threading import time import psutil @@ -17,6 +18,12 @@ def _log_output(log_file, message): f.write(message) +def _read_output(pipe, log_file): + with pipe: + for line in iter(pipe.readline, b""): + _log_output(log_file, line.decode("utf-8")) + + def _run_executable( command_pre, executable_path, command_args, log_file, check_memory=False ): @@ -40,6 +47,15 @@ def _run_executable( ps_process = psutil.Process(process.pid) peak_memory = 0 + stdout_thread = threading.Thread( + target=_read_output, args=(process.stdout, log_file) + ) + stderr_thread = threading.Thread( + target=_read_output, args=(process.stderr, log_file) + ) + stdout_thread.start() + stderr_thread.start() + while process.poll() is None: try: total_memory = ps_process.memory_info().rss @@ -50,7 +66,8 @@ def _run_executable( pass time.sleep(0.01) - stdout, stderr = process.communicate() + stdout_thread.join() + stderr_thread.join() return_code = process.wait() peak_memory_mb = peak_memory / (1024 * 1024) From 4bc124b6ec2904a63d690438e2e9f25197638bbe Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 11:39:10 -0600 Subject: [PATCH 04/16] hook in protego-mech --- .vscode/settings.json | 1 + CMakeLists.txt | 5 ++++ cmake/build_settings.cmake | 2 +- cmake/code_coverage.cmake | 2 +- cmake/targets.cmake | 7 +++++ cmake/version_info.cmake | 2 +- do_configure | 61 ++++++++++++++++++++------------------ include/MathUtils.h | 9 ++++++ protego-mech | 2 +- 9 files changed, 58 insertions(+), 33 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8c2375ef..8f6fb536 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -108,6 +108,7 @@ "mpirun", "pamgen", "pnetcdf", + "protego", "SIDESET", "spack", "spyhis", diff --git a/CMakeLists.txt b/CMakeLists.txt index 0717231e..f84bad81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,11 @@ cmake_minimum_required(VERSION 3.20) project(aperi-mech) +# Add options +option(CHECK_CODE_COVERAGE "Enable code coverage" OFF) +option(USE_GPU "Enable GPU support" OFF) +option(USE_PROTEGO_MECH "Include protego-mech library" OFF) + # Set the flags for the RelWithDebInfo configuration set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -pg") diff --git a/cmake/build_settings.cmake b/cmake/build_settings.cmake index 7f8dc27e..563ed1bf 100644 --- a/cmake/build_settings.cmake +++ b/cmake/build_settings.cmake @@ -5,7 +5,7 @@ set(CMAKE_CUDA_STANDARD 14) # Set the project languages enable_language(CXX) -IF(GPU) +IF(USE_GPU) enable_language(CUDA) set(CUDA_SEPARABLE_COMPILATION ON) ENDIF() diff --git a/cmake/code_coverage.cmake b/cmake/code_coverage.cmake index d96463b4..b2163c14 100644 --- a/cmake/code_coverage.cmake +++ b/cmake/code_coverage.cmake @@ -1,5 +1,5 @@ ############# CODE COVERAGE ############# -if (CODE_COVERAGE) +if (CHECK_CODE_COVERAGE) list(APPEND CMAKE_PREFIX_PATH "${LCOV_BIN_DIR}") include(CodeCoverage) append_coverage_compiler_flags() diff --git a/cmake/targets.cmake b/cmake/targets.cmake index fef591d0..6f79990c 100644 --- a/cmake/targets.cmake +++ b/cmake/targets.cmake @@ -24,6 +24,13 @@ target_include_directories(aperimech PRIVATE ${MPI_INCLUDE_PATH} ) +# Conditionally include protego-mech +if(USE_PROTEGO_MECH) + add_subdirectory(protego-mech) + target_link_libraries(aperimech protegomech) + target_compile_definitions(aperimech PRIVATE USE_PROTEGO_MECH) +endif() + # Add the executable set(MAIN_SOURCES src/main.cpp; diff --git a/cmake/version_info.cmake b/cmake/version_info.cmake index 3aaba399..37f82629 100644 --- a/cmake/version_info.cmake +++ b/cmake/version_info.cmake @@ -42,7 +42,7 @@ set(BUILD_TYPE ${CMAKE_BUILD_TYPE}) string(TIMESTAMP BUILD_DATE "%Y-%m-%d") string(TIMESTAMP BUILD_TIME "%H:%M:%S") -IF(GPU) +IF(USE_GPU) set(GPU_OR_CPU "GPU") else() set(GPU_OR_CPU "CPU") diff --git a/do_configure b/do_configure index c57e1762..13d44966 100755 --- a/do_configure +++ b/do_configure @@ -3,8 +3,9 @@ set -e # Initialize variables with default values BUILD_TYPE="Release" -CODE_COVERAGE=false -GPU=false +CHECK_CODE_COVERAGE=OFF +USE_GPU=OFF +USE_PROTEGO_MECH=OFF USE_LOCAL_COMPADRE=false # Get the current directory @@ -20,16 +21,21 @@ while [[ $# -gt 0 ]]; do shift ;; -c | --code-coverage) - CODE_COVERAGE=true + CHECK_CODE_COVERAGE=ON BUILD_TYPE="Debug" echo "Requested code coverage. Build type set to Debug" shift ;; -g | --gpu) - GPU=true + USE_GPU=ON echo "Requested GPU support." shift ;; + -p | --protego-mech) + USE_PROTEGO_MECH=ON + echo "Including protego-mech." + shift + ;; -l | --local-compadre) USE_LOCAL_COMPADRE=true echo "Using local compadre." @@ -42,22 +48,9 @@ while [[ $# -gt 0 ]]; do esac done -# Build directory base -BUILD_DIR="build/" - -# Append to build directory based on build type -BUILD_DIR+="${BUILD_TYPE}" - # If some of the spack commmands below fail, make sure the correct environment is activated. e.g.: # spack env activate aperi-mech -CUDA_PATH="" -# Set build directory and GPU option -if ${GPU}; then - BUILD_DIR+="_gpu" - CUDA_PATH=$(spack location -i cuda) -fi - # Check for cmake in the spack environment, if not found, use the system cmake if spack find -p cmake; then cmake=$(spack location -i --first cmake)/bin/cmake @@ -68,7 +61,8 @@ fi # Configure CMake with specified build type and other options cmake_command="${cmake}" -cmake_command+=" -D GPU:BOOL=${GPU}" +cmake_command+=" -D USE_GPU:BOOL=${USE_GPU}" +cmake_command+=" -D USE_PROTEGO_MECH:BOOL=${USE_PROTEGO_MECH}" cmake_command+=" -D CMAKE_BUILD_TYPE:STRING=\"${BUILD_TYPE}\"" cmake_command+=" -D TRILINOS_PATH:FILEPATH=$(spack location -i trilinos)" cmake_command+=" -D Kokkos_ROOT:FILEPATH=$(spack location -i kokkos)" @@ -76,16 +70,10 @@ cmake_command+=" -D EIGEN_PATH:FILEPATH=$(spack location -i eigen)" cmake_command+=" -D GTEST_PATH:FILEPATH=$(spack location -i googletest)" cmake_command+=" -D YAML-CPP_PATH:FILEPATH=$(spack location -i yaml-cpp)" cmake_command+=" -D OPENMPI_PATH:FILEPATH=$(spack location -i openmpi)" - -if ${USE_LOCAL_COMPADRE}; then - COMPADRE_PATH="/home/azureuser/projects/compadre/${BUILD_DIR}/install" -else - COMPADRE_PATH=$(spack location -i compadre) -fi -cmake_command+=" -D COMPADRE_PATH:FILEPATH=${COMPADRE_PATH}" +cmake_command+=" -D COMPADRE_PATH:FILEPATH=$(spack location -i compadre)" # Add the lcov path to the cmake command -if ${CODE_COVERAGE}; then +if [[ ${CHECK_CODE_COVERAGE} == "ON" ]]; then # Verify that lcov is in /usr/bin if [[ -f /usr/bin/lcov ]]; then cmake_command+=" -D LCOV_BIN_DIR:FILEPATH=/usr/bin" @@ -101,9 +89,11 @@ if ${CODE_COVERAGE}; then fi fi fi -cmake_command+=" -D CODE_COVERAGE:BOOL=${CODE_COVERAGE}" +cmake_command+=" -D CHECK_CODE_COVERAGE:BOOL=${CHECK_CODE_COVERAGE}" -if ${GPU}; then +if [[ ${USE_GPU} == "ON" ]]; then + # Add the cuda path to the cmake command + CUDA_PATH=$(spack location -i cuda) # Add the cuda compiler to the cmake command cmake_command+=" -D CMAKE_CUDA_COMPILER:FILEPATH=${CUDA_PATH}/bin/nvcc" # Add the cuda flags to the cmake command. There has got to be a better way to do this. Needed? @@ -134,7 +124,20 @@ cmake_command+=" -D CMAKE_CXX_EXTENSIONS=Off" cmake_command+=" ${SCRIPT_DIR}" # Create build directory -[[ ${CODE_COVERAGE} == true ]] && BUILD_DIR+="_cov" +# Build directory base +BUILD_DIR="build/${BUILD_TYPE}" + +# Append to build directory based on GPU usage +if [[ ${USE_GPU} == "ON" ]]; then + BUILD_DIR+="_gpu" +fi + +# Append to build directory based on code coverage +if [[ ${CHECK_CODE_COVERAGE} == "ON" ]]; then + BUILD_DIR+="_cov" +fi + +# Create the build directory and change to it mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" || exit diff --git a/include/MathUtils.h b/include/MathUtils.h index ebff6278..37c7d8b1 100644 --- a/include/MathUtils.h +++ b/include/MathUtils.h @@ -8,6 +8,10 @@ #include #include +#ifdef USE_PROTEGO_MECH +#include "ProtegoMathUtils.h" +#endif + namespace aperi { // Compute the cross product of two vectors @@ -239,6 +243,7 @@ KOKKOS_FORCEINLINE_FUNCTION Eigen::Matrix InvertMatrix(const #endif } +#ifndef USE_PROTEGO_MECH KOKKOS_INLINE_FUNCTION double ComputeKernel(const Eigen::Vector &vector_neighbor_to_point, double R) { const double normalized_radius = vector_neighbor_to_point.norm() / (R); // Calculate the kernel value using a cubic b-spline kernel @@ -250,6 +255,10 @@ KOKKOS_INLINE_FUNCTION double ComputeKernel(const Eigen::Vector &vect return 0.0; } +#else // USE_PROTEGO_MECH +using protego::ComputeKernel; +#endif + template KOKKOS_FUNCTION constexpr auto DetApIm1(const Eigen::Matrix &A) { // From the Cayley-Hamilton theorem, we get that for any N by N matrix A, diff --git a/protego-mech b/protego-mech index db094b6d..eb932551 160000 --- a/protego-mech +++ b/protego-mech @@ -1 +1 @@ -Subproject commit db094b6d32529f1ef40787bf01180057a62af68e +Subproject commit eb9325514aa7026d78c4764816e95c1b38e44d50 From 9cb9e5166a5e5e280204dc589a6e1b101e005b6f Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 11:50:51 -0600 Subject: [PATCH 05/16] put protego-mech message in version --- cmake/version_info.cmake | 6 ++++++ include/git_commit.h.in | 3 ++- src/main.cpp | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmake/version_info.cmake b/cmake/version_info.cmake index 37f82629..7b67f3fb 100644 --- a/cmake/version_info.cmake +++ b/cmake/version_info.cmake @@ -48,6 +48,12 @@ else() set(GPU_OR_CPU "CPU") endif() +IF(USE_PROTEGO_MECH) + set(PROTEGO_MECH "ON") +else() + set(PROTEGO_MECH "OFF") +endif() + # Generate git_commit.h with the current hash configure_file( ${CMAKE_SOURCE_DIR}/include/git_commit.h.in diff --git a/include/git_commit.h.in b/include/git_commit.h.in index f8ea0ad6..498c1a1f 100644 --- a/include/git_commit.h.in +++ b/include/git_commit.h.in @@ -5,4 +5,5 @@ #define BUILD_DATE "@BUILD_DATE@" #define BUILD_TIME "@BUILD_TIME@" #define GIT_DIRTY "@GIT_DIRTY@" -#define GPU_OR_CPU "@GPU_OR_CPU@" \ No newline at end of file +#define GPU_OR_CPU "@GPU_OR_CPU@" +#define PROTEGO_MECH "@PROTEGO_MECH@" \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index d247c154..deb5f6e9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -19,6 +19,9 @@ void RunApplication(const std::string& input_filename, MPI_Comm comm) { void PrintVersion() { std::string version; version += std::string(GIT_COMMIT_HASH) + "-" + std::string(BUILD_TYPE) + "-" + std::string(GPU_OR_CPU); + if (std::string(PROTEGO_MECH) == "ON") { + version += "-protego-mech"; + } if (std::string(GIT_DIRTY) == "dirty") { version += "-uncommitted_changes"; } @@ -35,6 +38,9 @@ void PrintHeader() { aperi::CoutP0() << " Git Tag: " << GIT_TAG << std::endl; aperi::CoutP0() << " Build Date: " << BUILD_DATE << std::endl; aperi::CoutP0() << " Build Time: " << BUILD_TIME << std::endl; + if (std::string(PROTEGO_MECH) == "ON") { + aperi::CoutP0() << " With protego-mech" << std::endl; + } aperi::CoutP0() << "############################################\n" << std::endl; } From cff43c1e11f9a5dc3b5666a8c7c5ccbf50b3c631 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 12:05:48 -0600 Subject: [PATCH 06/16] split some processors into different files --- include/ElementReproducingKernel.h | 1 + include/FunctionValueStorageProcessor.h | 156 ++++++++ include/NeighborSearchProcessor.h | 358 ------------------ include/ValueFromGeneralizedFieldProcessor.h | 262 +++++++++++++ src/MassUtils.cpp | 2 +- src/Solver.cpp | 2 +- ...FunctionValueStorageProcessorTestFixture.h | 1 + ...ValueFromGeneralizedFieldProcessorTest.cpp | 1 + 8 files changed, 423 insertions(+), 360 deletions(-) create mode 100644 include/FunctionValueStorageProcessor.h create mode 100644 include/ValueFromGeneralizedFieldProcessor.h diff --git a/include/ElementReproducingKernel.h b/include/ElementReproducingKernel.h index c24faf98..13550898 100644 --- a/include/ElementReproducingKernel.h +++ b/include/ElementReproducingKernel.h @@ -10,6 +10,7 @@ #include "ElementProcessor.h" #include "ElementUtils.h" #include "FieldData.h" +#include "FunctionValueStorageProcessor.h" #include "Kokkos_Core.hpp" #include "Material.h" #include "MeshData.h" diff --git a/include/FunctionValueStorageProcessor.h b/include/FunctionValueStorageProcessor.h new file mode 100644 index 00000000..9a47733f --- /dev/null +++ b/include/FunctionValueStorageProcessor.h @@ -0,0 +1,156 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AperiStkUtils.h" +#include "FieldData.h" +#include "LogUtils.h" +#include "MathUtils.h" +#include "MeshData.h" + +namespace aperi { + +class FunctionValueStorageProcessor { + typedef stk::mesh::Field DoubleField; + typedef stk::mesh::NgpField NgpDoubleField; + typedef stk::mesh::Field UnsignedField; + typedef stk::mesh::NgpField NgpUnsignedField; + + public: + FunctionValueStorageProcessor(std::shared_ptr mesh_data, const std::vector &sets = {}) : m_mesh_data(mesh_data), m_sets(sets) { + // Throw an exception if the mesh data is null. + if (mesh_data == nullptr) { + throw std::runtime_error("Mesh data is null."); + } + m_bulk_data = mesh_data->GetBulkData(); + m_ngp_mesh = stk::mesh::get_updated_ngp_mesh(*m_bulk_data); + stk::mesh::MetaData *meta_data = &m_bulk_data->mesh_meta_data(); + m_selector = StkGetSelector(sets, meta_data); + // Warn if the selector is empty. + if (m_selector.is_empty(stk::topology::ELEMENT_RANK)) { + aperi::CoutP0() << "Warning: NeighborSearchProcessor selector is empty." << std::endl; + } + + stk::mesh::Selector full_owned_selector = m_bulk_data->mesh_meta_data().locally_owned_part(); + m_owned_selector = m_selector & full_owned_selector; + + // Get the number of neighbors field + m_num_neighbors_field = StkGetField(FieldQueryData{"num_neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_num_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_num_neighbors_field); + + // Get the neighbors field + m_neighbors_field = StkGetField(FieldQueryData{"neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_neighbors_field); + + // Get the coordinates field + m_coordinates_field = StkGetField(FieldQueryData{mesh_data->GetCoordinatesFieldName(), FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_coordinates_field = &stk::mesh::get_updated_ngp_field(*m_coordinates_field); + + // Get the function values field + m_function_values_field = StkGetField(FieldQueryData{"function_values", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_function_values_field = &stk::mesh::get_updated_ngp_field(*m_function_values_field); + + // Get the kernel radius field + m_kernel_radius_field = StkGetField(FieldQueryData{"kernel_radius", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_kernel_radius_field = &stk::mesh::get_updated_ngp_field(*m_kernel_radius_field); + } + + // use_evaluation_point_kernels is a flag to center the kernel at the evaluation point instead of the neighbor node. This is to match Compadre. + template + void compute_and_store_function_values(FunctionFunctor &function_functor, const bool use_evaluation_point_kernels = false) { + auto ngp_mesh = m_ngp_mesh; + // Get the ngp fields + auto ngp_num_neighbors_field = *m_ngp_num_neighbors_field; + auto ngp_neighbors_field = *m_ngp_neighbors_field; + auto ngp_coordinates_field = *m_ngp_coordinates_field; + auto ngp_function_values_field = *m_ngp_function_values_field; + auto npg_kernel_radius_field = *m_ngp_kernel_radius_field; + + stk::mesh::for_each_entity_run( + ngp_mesh, stk::topology::NODE_RANK, m_selector, + KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex &node_index) { + // Get the number of neighbors + size_t num_neighbors = ngp_num_neighbors_field(node_index, 0); + + Eigen::Matrix coordinates; + for (size_t j = 0; j < 3; ++j) { + coordinates(0, j) = ngp_coordinates_field(node_index, j); + } + + Eigen::Matrix shifted_neighbor_coordinates; + Eigen::Matrix kernel_values; + + double kernel_radius = npg_kernel_radius_field(node_index, 0); // for use_evaluation_point_kernels = true, to match Compadre + + for (size_t i = 0; i < num_neighbors; ++i) { + // Create the entity + stk::mesh::Entity entity(ngp_neighbors_field(node_index, i)); + stk::mesh::FastMeshIndex neighbor_index = ngp_mesh.fast_mesh_index(entity); + // Get the neighbor's coordinates + for (size_t j = 0; j < 3; ++j) { + shifted_neighbor_coordinates(i, j) = coordinates(0, j) - ngp_coordinates_field(neighbor_index, j); + } + // Get the neighbor's kernel radius + if (!use_evaluation_point_kernels) { + kernel_radius = npg_kernel_radius_field(neighbor_index, 0); + } + // Compute the kernel value + kernel_values(i, 0) = aperi::ComputeKernel(shifted_neighbor_coordinates.row(i), kernel_radius); + } + + // Compute the function values + Eigen::Matrix function_values = function_functor.values(kernel_values, shifted_neighbor_coordinates, num_neighbors); + + for (size_t i = 0; i < num_neighbors; ++i) { + ngp_function_values_field(node_index, i) = function_values(i, 0); + } + }); + m_ngp_function_values_field->clear_sync_state(); + m_ngp_function_values_field->modify_on_device(); + } + + void SyncFieldsToHost() { + m_ngp_function_values_field->sync_to_host(); + } + + void CommunicateAllFieldData() const { + stk::mesh::communicate_field_data(*m_bulk_data, {m_function_values_field}); + } + + private: + std::shared_ptr m_mesh_data; // The mesh data object. + std::vector m_sets; // The sets to process. + stk::mesh::BulkData *m_bulk_data; // The bulk data object. + stk::mesh::Selector m_selector; // The selector + stk::mesh::Selector m_owned_selector; // The local selector + stk::mesh::NgpMesh m_ngp_mesh; // The ngp mesh object. + UnsignedField *m_num_neighbors_field; // The number of neighbors field + UnsignedField *m_neighbors_field; // The neighbors field + DoubleField *m_coordinates_field; // The coordinates field + DoubleField *m_function_values_field; // The function values field + DoubleField *m_kernel_radius_field; // The kernel radius field + NgpUnsignedField *m_ngp_num_neighbors_field; // The ngp number of neighbors field + NgpUnsignedField *m_ngp_neighbors_field; // The ngp neighbors field + NgpDoubleField *m_ngp_coordinates_field; // The ngp coordinates field + NgpDoubleField *m_ngp_function_values_field; // The ngp function values field + NgpDoubleField *m_ngp_kernel_radius_field; // The ngp kernel radius field +}; + +} // namespace aperi \ No newline at end of file diff --git a/include/NeighborSearchProcessor.h b/include/NeighborSearchProcessor.h index 0ab71ecc..92a02d71 100644 --- a/include/NeighborSearchProcessor.h +++ b/include/NeighborSearchProcessor.h @@ -612,362 +612,4 @@ class NeighborSearchProcessor { NgpDoubleField *m_ngp_kernel_radius_field; // The ngp kernel radius field }; -class FunctionValueStorageProcessor { - typedef stk::mesh::Field DoubleField; - typedef stk::mesh::NgpField NgpDoubleField; - typedef stk::mesh::Field UnsignedField; - typedef stk::mesh::NgpField NgpUnsignedField; - - public: - FunctionValueStorageProcessor(std::shared_ptr mesh_data, const std::vector &sets = {}) : m_mesh_data(mesh_data), m_sets(sets) { - // Throw an exception if the mesh data is null. - if (mesh_data == nullptr) { - throw std::runtime_error("Mesh data is null."); - } - m_bulk_data = mesh_data->GetBulkData(); - m_ngp_mesh = stk::mesh::get_updated_ngp_mesh(*m_bulk_data); - stk::mesh::MetaData *meta_data = &m_bulk_data->mesh_meta_data(); - m_selector = StkGetSelector(sets, meta_data); - // Warn if the selector is empty. - if (m_selector.is_empty(stk::topology::ELEMENT_RANK)) { - aperi::CoutP0() << "Warning: NeighborSearchProcessor selector is empty." << std::endl; - } - - stk::mesh::Selector full_owned_selector = m_bulk_data->mesh_meta_data().locally_owned_part(); - m_owned_selector = m_selector & full_owned_selector; - - // Get the number of neighbors field - m_num_neighbors_field = StkGetField(FieldQueryData{"num_neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_num_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_num_neighbors_field); - - // Get the neighbors field - m_neighbors_field = StkGetField(FieldQueryData{"neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_neighbors_field); - - // Get the coordinates field - m_coordinates_field = StkGetField(FieldQueryData{mesh_data->GetCoordinatesFieldName(), FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_coordinates_field = &stk::mesh::get_updated_ngp_field(*m_coordinates_field); - - // Get the function values field - m_function_values_field = StkGetField(FieldQueryData{"function_values", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_function_values_field = &stk::mesh::get_updated_ngp_field(*m_function_values_field); - - // Get the kernel radius field - m_kernel_radius_field = StkGetField(FieldQueryData{"kernel_radius", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_kernel_radius_field = &stk::mesh::get_updated_ngp_field(*m_kernel_radius_field); - } - - // use_evaluation_point_kernels is a flag to center the kernel at the evaluation point instead of the neighbor node. This is to match Compadre. - template - void compute_and_store_function_values(FunctionFunctor &function_functor, const bool use_evaluation_point_kernels = false) { - auto ngp_mesh = m_ngp_mesh; - // Get the ngp fields - auto ngp_num_neighbors_field = *m_ngp_num_neighbors_field; - auto ngp_neighbors_field = *m_ngp_neighbors_field; - auto ngp_coordinates_field = *m_ngp_coordinates_field; - auto ngp_function_values_field = *m_ngp_function_values_field; - auto npg_kernel_radius_field = *m_ngp_kernel_radius_field; - - stk::mesh::for_each_entity_run( - ngp_mesh, stk::topology::NODE_RANK, m_selector, - KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex &node_index) { - // Get the number of neighbors - size_t num_neighbors = ngp_num_neighbors_field(node_index, 0); - - Eigen::Matrix coordinates; - for (size_t j = 0; j < 3; ++j) { - coordinates(0, j) = ngp_coordinates_field(node_index, j); - } - - Eigen::Matrix shifted_neighbor_coordinates; - Eigen::Matrix kernel_values; - - double kernel_radius = npg_kernel_radius_field(node_index, 0); // for use_evaluation_point_kernels = true, to match Compadre - - for (size_t i = 0; i < num_neighbors; ++i) { - // Create the entity - stk::mesh::Entity entity(ngp_neighbors_field(node_index, i)); - stk::mesh::FastMeshIndex neighbor_index = ngp_mesh.fast_mesh_index(entity); - // Get the neighbor's coordinates - for (size_t j = 0; j < 3; ++j) { - shifted_neighbor_coordinates(i, j) = coordinates(0, j) - ngp_coordinates_field(neighbor_index, j); - } - // Get the neighbor's kernel radius - if (!use_evaluation_point_kernels) { - kernel_radius = npg_kernel_radius_field(neighbor_index, 0); - } - // Compute the kernel value - kernel_values(i, 0) = aperi::ComputeKernel(shifted_neighbor_coordinates.row(i), kernel_radius); - } - - // Compute the function values - Eigen::Matrix function_values = function_functor.values(kernel_values, shifted_neighbor_coordinates, num_neighbors); - - for (size_t i = 0; i < num_neighbors; ++i) { - ngp_function_values_field(node_index, i) = function_values(i, 0); - } - }); - m_ngp_function_values_field->clear_sync_state(); - m_ngp_function_values_field->modify_on_device(); - } - - void SyncFieldsToHost() { - m_ngp_function_values_field->sync_to_host(); - } - - void CommunicateAllFieldData() const { - stk::mesh::communicate_field_data(*m_bulk_data, {m_function_values_field}); - } - - private: - std::shared_ptr m_mesh_data; // The mesh data object. - std::vector m_sets; // The sets to process. - stk::mesh::BulkData *m_bulk_data; // The bulk data object. - stk::mesh::Selector m_selector; // The selector - stk::mesh::Selector m_owned_selector; // The local selector - stk::mesh::NgpMesh m_ngp_mesh; // The ngp mesh object. - UnsignedField *m_num_neighbors_field; // The number of neighbors field - UnsignedField *m_neighbors_field; // The neighbors field - DoubleField *m_coordinates_field; // The coordinates field - DoubleField *m_function_values_field; // The function values field - DoubleField *m_kernel_radius_field; // The kernel radius field - NgpUnsignedField *m_ngp_num_neighbors_field; // The ngp number of neighbors field - NgpUnsignedField *m_ngp_neighbors_field; // The ngp neighbors field - NgpDoubleField *m_ngp_coordinates_field; // The ngp coordinates field - NgpDoubleField *m_ngp_function_values_field; // The ngp function values field - NgpDoubleField *m_ngp_kernel_radius_field; // The ngp kernel radius field -}; - -template -class ValueFromGeneralizedFieldProcessor { - typedef stk::mesh::Field DoubleField; - typedef stk::mesh::NgpField NgpDoubleField; - typedef stk::mesh::Field UnsignedField; - typedef stk::mesh::NgpField NgpUnsignedField; - - public: - ValueFromGeneralizedFieldProcessor(const std::array, NumFields> source_field_query_data, const std::array, NumFields> destination_field_query_data, std::shared_ptr mesh_data, const std::vector &sets = {}) { - // Throw an exception if the mesh data is null. - if (mesh_data == nullptr) { - throw std::runtime_error("Mesh data is null."); - } - m_bulk_data = mesh_data->GetBulkData(); - m_ngp_mesh = stk::mesh::get_updated_ngp_mesh(*m_bulk_data); - stk::mesh::MetaData *meta_data = &m_bulk_data->mesh_meta_data(); - m_selector = StkGetSelector(sets, meta_data); - // Warn if the selector is empty. - if (m_selector.is_empty(stk::topology::ELEMENT_RANK)) { - aperi::CoutP0() << "Warning: NeighborSearchProcessor selector is empty." << std::endl; - } - - stk::mesh::Selector full_owned_selector = m_bulk_data->mesh_meta_data().locally_owned_part(); - m_owned_selector = m_selector & full_owned_selector; - - // Get the number of neighbors field - m_num_neighbors_field = StkGetField(FieldQueryData{"num_neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_num_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_num_neighbors_field); - - // Get the neighbors field - m_neighbors_field = StkGetField(FieldQueryData{"neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_neighbors_field); - - // Get the function values field - m_function_values_field = StkGetField(FieldQueryData{"function_values", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); - m_ngp_function_values_field = &stk::mesh::get_updated_ngp_field(*m_function_values_field); - - // Get the source (generalized) and destination fields - for (size_t i = 0; i < NumFields; ++i) { - m_source_fields.push_back(StkGetField(source_field_query_data[i], meta_data)); - m_ngp_source_fields[i] = &stk::mesh::get_updated_ngp_field(*m_source_fields.back()); - m_destination_fields.push_back(StkGetField(destination_field_query_data[i], meta_data)); - m_ngp_destination_fields[i] = &stk::mesh::get_updated_ngp_field(*m_destination_fields.back()); - } - } - - void compute_value_from_generalized_field() { - // destination_fields(i) = /sum_{j=0}^{num_neighbors} source_fields(neighbors(i, j)) * function_values(i, j) - - auto ngp_mesh = m_ngp_mesh; - // Get the ngp fields - auto ngp_num_neighbors_field = *m_ngp_num_neighbors_field; - auto ngp_neighbors_field = *m_ngp_neighbors_field; - auto ngp_function_values_field = *m_ngp_function_values_field; - Kokkos::Array ngp_source_fields; - Kokkos::Array ngp_destination_fields; - for (size_t i = 0; i < NumFields; i++) { - ngp_source_fields[i] = *m_ngp_source_fields[i]; - ngp_destination_fields[i] = *m_ngp_destination_fields[i]; - } - - stk::mesh::for_each_entity_run( - ngp_mesh, stk::topology::NODE_RANK, m_selector, - KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex &node_index) { - // Get the number of neighbors - size_t num_neighbors = ngp_num_neighbors_field(node_index, 0); - - const int num_components = 3; // Hardcoded 3 (vector field) for now. TODO(jake): Make this more general - - // If there are no neighbors, set the destination field to the source field - if (num_neighbors == 0) { - for (size_t i = 0; i < NumFields; ++i) { - for (size_t j = 0; j < num_components; ++j) { - ngp_destination_fields[i](node_index, j) = ngp_source_fields[i](node_index, j); - } - } - return; - } else { // Zero out the destination field and prepare for the sum in the next loop - for (size_t i = 0; i < NumFields; ++i) { - for (size_t j = 0; j < num_components; ++j) { - ngp_destination_fields[i](node_index, j) = 0.0; - } - } - } - - // If there are neighbors, compute the destination field from the function values and source fields. - // Do in reverse order. Adding smaller function_value terms first to help with parallel consistency - for (size_t k = num_neighbors; k-- > 0;) { - // Create the entity - stk::mesh::Entity entity(ngp_neighbors_field(node_index, k)); - stk::mesh::FastMeshIndex neighbor_index = ngp_mesh.fast_mesh_index(entity); - - // Get the function value - double function_value = ngp_function_values_field(node_index, k); - - // Get the source field values - for (size_t i = 0; i < NumFields; ++i) { - for (size_t j = 0; j < num_components; ++j) { - double source_value = ngp_source_fields[i](neighbor_index, j); - ngp_destination_fields[i](node_index, j) += source_value * function_value; - } - } - } - }); - } - - // Marking modified on device - void MarkDestinationFieldModifiedOnDevice(size_t field_index) { - m_ngp_destination_fields[field_index]->clear_sync_state(); - m_ngp_destination_fields[field_index]->modify_on_device(); - } - - void MarkSourceFieldModifiedOnDevice(size_t field_index) { - m_ngp_source_fields[field_index]->clear_sync_state(); - m_ngp_source_fields[field_index]->modify_on_device(); - } - - void MarkAllDestinationFieldsModifiedOnDevice() { - for (size_t i = 0; i < NumFields; i++) { - MarkDestinationFieldModifiedOnDevice(i); - } - } - - void MarkAllSourceFieldsModifiedOnDevice() { - for (size_t i = 0; i < NumFields; i++) { - MarkSourceFieldModifiedOnDevice(i); - } - } - - // Mark modified on host - void MarkDestinationFieldModifiedOnHost(size_t field_index) { - m_ngp_destination_fields[field_index]->modify_on_host(); - } - - void MarkSourceFieldModifiedOnHost(size_t field_index) { - m_ngp_source_fields[field_index]->modify_on_host(); - } - - void MarkAllDestinationFieldsModifiedOnHost() { - for (size_t i = 0; i < NumFields; i++) { - MarkDestinationFieldModifiedOnHost(i); - } - } - - void MarkAllSourceFieldsModifiedOnHost() { - for (size_t i = 0; i < NumFields; i++) { - MarkSourceFieldModifiedOnHost(i); - } - } - - // Syncing, device to host - void SyncDestinationFieldDeviceToHost(size_t field_index) { - m_ngp_destination_fields[field_index]->sync_to_host(); - } - - void SyncSourceFieldDeviceToHost(size_t field_index) { - m_ngp_source_fields[field_index]->sync_to_host(); - } - - void SyncAllDestinationFieldsDeviceToHost() { - for (size_t i = 0; i < NumFields; i++) { - SyncDestinationFieldDeviceToHost(i); - } - } - - void SyncAllSourceFieldsDeviceToHost() { - for (size_t i = 0; i < NumFields; i++) { - SyncSourceFieldDeviceToHost(i); - } - } - - // Syncing, host to device - void SyncDestinationFieldHostToDevice(size_t field_index) { - m_ngp_destination_fields[field_index]->sync_to_device(); - } - - void SyncSourceFieldHostToDevice(size_t field_index) { - m_ngp_source_fields[field_index]->sync_to_device(); - } - - void SyncAllDestinationFieldsHostToDevice() { - for (size_t i = 0; i < NumFields; i++) { - SyncDestinationFieldHostToDevice(i); - } - } - - void SyncAllSourceFieldsHostToDevice() { - for (size_t i = 0; i < NumFields; i++) { - SyncSourceFieldHostToDevice(i); - } - } - - // Parallel communication - void CommunicateDestinationFieldData(int field_index) const { - stk::mesh::communicate_field_data(*m_bulk_data, {m_destination_fields[field_index]}); - } - - void CommunicateSourceFieldData(int field_index) const { - stk::mesh::communicate_field_data(*m_bulk_data, {m_source_fields[field_index]}); - } - - void CommunicateAllDestinationFieldData() const { - for (size_t i = 0; i < NumFields; i++) { - CommunicateDestinationFieldData(i); - } - } - - void CommunicateAllSourceFieldData() const { - for (size_t i = 0; i < NumFields; i++) { - CommunicateSourceFieldData(i); - } - } - - private: - std::shared_ptr m_mesh_data; // The mesh data object. - std::vector m_sets; // The sets to process. - stk::mesh::BulkData *m_bulk_data; // The bulk data object. - stk::mesh::Selector m_selector; // The selector - stk::mesh::Selector m_owned_selector; // The local selector - stk::mesh::NgpMesh m_ngp_mesh; // The ngp mesh object. - UnsignedField *m_num_neighbors_field; // The number of neighbors field - UnsignedField *m_neighbors_field; // The neighbors field - DoubleField *m_function_values_field; // The function values field - NgpUnsignedField *m_ngp_num_neighbors_field; // The ngp number of neighbors field - NgpUnsignedField *m_ngp_neighbors_field; // The ngp neighbors field - NgpDoubleField *m_ngp_function_values_field; // The ngp function values field - std::vector m_source_fields; // The fields to process - Kokkos::Array m_ngp_source_fields; // The ngp fields to process - std::vector m_destination_fields; // The fields to process - Kokkos::Array m_ngp_destination_fields; // The ngp fields to process -}; - } // namespace aperi \ No newline at end of file diff --git a/include/ValueFromGeneralizedFieldProcessor.h b/include/ValueFromGeneralizedFieldProcessor.h new file mode 100644 index 00000000..13f4148c --- /dev/null +++ b/include/ValueFromGeneralizedFieldProcessor.h @@ -0,0 +1,262 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AperiStkUtils.h" +#include "FieldData.h" +#include "LogUtils.h" +#include "MathUtils.h" +#include "MeshData.h" + +namespace aperi { + +template +class ValueFromGeneralizedFieldProcessor { + typedef stk::mesh::Field DoubleField; + typedef stk::mesh::NgpField NgpDoubleField; + typedef stk::mesh::Field UnsignedField; + typedef stk::mesh::NgpField NgpUnsignedField; + + public: + ValueFromGeneralizedFieldProcessor(const std::array, NumFields> source_field_query_data, const std::array, NumFields> destination_field_query_data, std::shared_ptr mesh_data, const std::vector &sets = {}) { + // Throw an exception if the mesh data is null. + if (mesh_data == nullptr) { + throw std::runtime_error("Mesh data is null."); + } + m_bulk_data = mesh_data->GetBulkData(); + m_ngp_mesh = stk::mesh::get_updated_ngp_mesh(*m_bulk_data); + stk::mesh::MetaData *meta_data = &m_bulk_data->mesh_meta_data(); + m_selector = StkGetSelector(sets, meta_data); + // Warn if the selector is empty. + if (m_selector.is_empty(stk::topology::ELEMENT_RANK)) { + aperi::CoutP0() << "Warning: NeighborSearchProcessor selector is empty." << std::endl; + } + + stk::mesh::Selector full_owned_selector = m_bulk_data->mesh_meta_data().locally_owned_part(); + m_owned_selector = m_selector & full_owned_selector; + + // Get the number of neighbors field + m_num_neighbors_field = StkGetField(FieldQueryData{"num_neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_num_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_num_neighbors_field); + + // Get the neighbors field + m_neighbors_field = StkGetField(FieldQueryData{"neighbors", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_neighbors_field = &stk::mesh::get_updated_ngp_field(*m_neighbors_field); + + // Get the function values field + m_function_values_field = StkGetField(FieldQueryData{"function_values", FieldQueryState::None, FieldDataTopologyRank::NODE}, meta_data); + m_ngp_function_values_field = &stk::mesh::get_updated_ngp_field(*m_function_values_field); + + // Get the source (generalized) and destination fields + for (size_t i = 0; i < NumFields; ++i) { + m_source_fields.push_back(StkGetField(source_field_query_data[i], meta_data)); + m_ngp_source_fields[i] = &stk::mesh::get_updated_ngp_field(*m_source_fields.back()); + m_destination_fields.push_back(StkGetField(destination_field_query_data[i], meta_data)); + m_ngp_destination_fields[i] = &stk::mesh::get_updated_ngp_field(*m_destination_fields.back()); + } + } + + void compute_value_from_generalized_field() { + // destination_fields(i) = /sum_{j=0}^{num_neighbors} source_fields(neighbors(i, j)) * function_values(i, j) + + auto ngp_mesh = m_ngp_mesh; + // Get the ngp fields + auto ngp_num_neighbors_field = *m_ngp_num_neighbors_field; + auto ngp_neighbors_field = *m_ngp_neighbors_field; + auto ngp_function_values_field = *m_ngp_function_values_field; + Kokkos::Array ngp_source_fields; + Kokkos::Array ngp_destination_fields; + for (size_t i = 0; i < NumFields; i++) { + ngp_source_fields[i] = *m_ngp_source_fields[i]; + ngp_destination_fields[i] = *m_ngp_destination_fields[i]; + } + + stk::mesh::for_each_entity_run( + ngp_mesh, stk::topology::NODE_RANK, m_selector, + KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex &node_index) { + // Get the number of neighbors + size_t num_neighbors = ngp_num_neighbors_field(node_index, 0); + + const int num_components = 3; // Hardcoded 3 (vector field) for now. TODO(jake): Make this more general + + // If there are no neighbors, set the destination field to the source field + if (num_neighbors == 0) { + for (size_t i = 0; i < NumFields; ++i) { + for (size_t j = 0; j < num_components; ++j) { + ngp_destination_fields[i](node_index, j) = ngp_source_fields[i](node_index, j); + } + } + return; + } else { // Zero out the destination field and prepare for the sum in the next loop + for (size_t i = 0; i < NumFields; ++i) { + for (size_t j = 0; j < num_components; ++j) { + ngp_destination_fields[i](node_index, j) = 0.0; + } + } + } + + // If there are neighbors, compute the destination field from the function values and source fields. + // Do in reverse order. Adding smaller function_value terms first to help with parallel consistency + for (size_t k = num_neighbors; k-- > 0;) { + // Create the entity + stk::mesh::Entity entity(ngp_neighbors_field(node_index, k)); + stk::mesh::FastMeshIndex neighbor_index = ngp_mesh.fast_mesh_index(entity); + + // Get the function value + double function_value = ngp_function_values_field(node_index, k); + + // Get the source field values + for (size_t i = 0; i < NumFields; ++i) { + for (size_t j = 0; j < num_components; ++j) { + double source_value = ngp_source_fields[i](neighbor_index, j); + ngp_destination_fields[i](node_index, j) += source_value * function_value; + } + } + } + }); + } + + // Marking modified on device + void MarkDestinationFieldModifiedOnDevice(size_t field_index) { + m_ngp_destination_fields[field_index]->clear_sync_state(); + m_ngp_destination_fields[field_index]->modify_on_device(); + } + + void MarkSourceFieldModifiedOnDevice(size_t field_index) { + m_ngp_source_fields[field_index]->clear_sync_state(); + m_ngp_source_fields[field_index]->modify_on_device(); + } + + void MarkAllDestinationFieldsModifiedOnDevice() { + for (size_t i = 0; i < NumFields; i++) { + MarkDestinationFieldModifiedOnDevice(i); + } + } + + void MarkAllSourceFieldsModifiedOnDevice() { + for (size_t i = 0; i < NumFields; i++) { + MarkSourceFieldModifiedOnDevice(i); + } + } + + // Mark modified on host + void MarkDestinationFieldModifiedOnHost(size_t field_index) { + m_ngp_destination_fields[field_index]->modify_on_host(); + } + + void MarkSourceFieldModifiedOnHost(size_t field_index) { + m_ngp_source_fields[field_index]->modify_on_host(); + } + + void MarkAllDestinationFieldsModifiedOnHost() { + for (size_t i = 0; i < NumFields; i++) { + MarkDestinationFieldModifiedOnHost(i); + } + } + + void MarkAllSourceFieldsModifiedOnHost() { + for (size_t i = 0; i < NumFields; i++) { + MarkSourceFieldModifiedOnHost(i); + } + } + + // Syncing, device to host + void SyncDestinationFieldDeviceToHost(size_t field_index) { + m_ngp_destination_fields[field_index]->sync_to_host(); + } + + void SyncSourceFieldDeviceToHost(size_t field_index) { + m_ngp_source_fields[field_index]->sync_to_host(); + } + + void SyncAllDestinationFieldsDeviceToHost() { + for (size_t i = 0; i < NumFields; i++) { + SyncDestinationFieldDeviceToHost(i); + } + } + + void SyncAllSourceFieldsDeviceToHost() { + for (size_t i = 0; i < NumFields; i++) { + SyncSourceFieldDeviceToHost(i); + } + } + + // Syncing, host to device + void SyncDestinationFieldHostToDevice(size_t field_index) { + m_ngp_destination_fields[field_index]->sync_to_device(); + } + + void SyncSourceFieldHostToDevice(size_t field_index) { + m_ngp_source_fields[field_index]->sync_to_device(); + } + + void SyncAllDestinationFieldsHostToDevice() { + for (size_t i = 0; i < NumFields; i++) { + SyncDestinationFieldHostToDevice(i); + } + } + + void SyncAllSourceFieldsHostToDevice() { + for (size_t i = 0; i < NumFields; i++) { + SyncSourceFieldHostToDevice(i); + } + } + + // Parallel communication + void CommunicateDestinationFieldData(int field_index) const { + stk::mesh::communicate_field_data(*m_bulk_data, {m_destination_fields[field_index]}); + } + + void CommunicateSourceFieldData(int field_index) const { + stk::mesh::communicate_field_data(*m_bulk_data, {m_source_fields[field_index]}); + } + + void CommunicateAllDestinationFieldData() const { + for (size_t i = 0; i < NumFields; i++) { + CommunicateDestinationFieldData(i); + } + } + + void CommunicateAllSourceFieldData() const { + for (size_t i = 0; i < NumFields; i++) { + CommunicateSourceFieldData(i); + } + } + + private: + std::shared_ptr m_mesh_data; // The mesh data object. + std::vector m_sets; // The sets to process. + stk::mesh::BulkData *m_bulk_data; // The bulk data object. + stk::mesh::Selector m_selector; // The selector + stk::mesh::Selector m_owned_selector; // The local selector + stk::mesh::NgpMesh m_ngp_mesh; // The ngp mesh object. + UnsignedField *m_num_neighbors_field; // The number of neighbors field + UnsignedField *m_neighbors_field; // The neighbors field + DoubleField *m_function_values_field; // The function values field + NgpUnsignedField *m_ngp_num_neighbors_field; // The ngp number of neighbors field + NgpUnsignedField *m_ngp_neighbors_field; // The ngp neighbors field + NgpDoubleField *m_ngp_function_values_field; // The ngp function values field + std::vector m_source_fields; // The fields to process + Kokkos::Array m_ngp_source_fields; // The ngp fields to process + std::vector m_destination_fields; // The fields to process + Kokkos::Array m_ngp_destination_fields; // The ngp fields to process +}; + +} // namespace aperi \ No newline at end of file diff --git a/src/MassUtils.cpp b/src/MassUtils.cpp index 355c20cc..f9ccfcd5 100644 --- a/src/MassUtils.cpp +++ b/src/MassUtils.cpp @@ -8,7 +8,7 @@ #include "LogUtils.h" #include "MathUtils.h" #include "MeshData.h" -#include "NeighborSearchProcessor.h" +#include "ValueFromGeneralizedFieldProcessor.h" namespace aperi { diff --git a/src/Solver.cpp b/src/Solver.cpp index 19334105..b5423b20 100644 --- a/src/Solver.cpp +++ b/src/Solver.cpp @@ -14,9 +14,9 @@ #include "Material.h" #include "MathUtils.h" #include "MeshData.h" -#include "NeighborSearchProcessor.h" #include "Scheduler.h" #include "TimeStepper.h" +#include "ValueFromGeneralizedFieldProcessor.h" namespace aperi { diff --git a/test/unit_tests/FunctionValueStorageProcessorTestFixture.h b/test/unit_tests/FunctionValueStorageProcessorTestFixture.h index 7b881f7f..efd2bd1a 100644 --- a/test/unit_tests/FunctionValueStorageProcessorTestFixture.h +++ b/test/unit_tests/FunctionValueStorageProcessorTestFixture.h @@ -14,6 +14,7 @@ #include #include "Constants.h" +#include "FunctionValueStorageProcessor.h" #include "NeighborSearchProcessorTestFixture.h" #include "ShapeFunctionsFunctorReproducingKernel.h" diff --git a/test/unit_tests/ValueFromGeneralizedFieldProcessorTest.cpp b/test/unit_tests/ValueFromGeneralizedFieldProcessorTest.cpp index c5cdaa0a..f93e223f 100644 --- a/test/unit_tests/ValueFromGeneralizedFieldProcessorTest.cpp +++ b/test/unit_tests/ValueFromGeneralizedFieldProcessorTest.cpp @@ -10,6 +10,7 @@ #include "MeshData.h" #include "NeighborSearchProcessorTestFixture.h" #include "UnitTestUtils.h" +#include "ValueFromGeneralizedFieldProcessor.h" struct FillLinearFieldFunctor { explicit FillLinearFieldFunctor(double slope) : m_slope(slope) {} From 6553fdda3a424116ad956b245a84335f92b2c022 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 12:28:06 -0600 Subject: [PATCH 07/16] change scope of protego-mech include --- include/FunctionValueStorageProcessor.h | 10 ++++++++++ include/MathUtils.h | 9 --------- protego-mech | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/FunctionValueStorageProcessor.h b/include/FunctionValueStorageProcessor.h index 9a47733f..ef0350ee 100644 --- a/include/FunctionValueStorageProcessor.h +++ b/include/FunctionValueStorageProcessor.h @@ -25,8 +25,14 @@ #include "MathUtils.h" #include "MeshData.h" +#ifdef USE_PROTEGO_MECH +#include "ProtegoFunctionValueStorageProcessor.h" +#endif + namespace aperi { +#ifndef USE_PROTEGO_MECH + class FunctionValueStorageProcessor { typedef stk::mesh::Field DoubleField; typedef stk::mesh::NgpField NgpDoubleField; @@ -153,4 +159,8 @@ class FunctionValueStorageProcessor { NgpDoubleField *m_ngp_kernel_radius_field; // The ngp kernel radius field }; +#else // USE_PROTEGO_MECH +using protego::FunctionValueStorageProcessor; +#endif + } // namespace aperi \ No newline at end of file diff --git a/include/MathUtils.h b/include/MathUtils.h index 37c7d8b1..ebff6278 100644 --- a/include/MathUtils.h +++ b/include/MathUtils.h @@ -8,10 +8,6 @@ #include #include -#ifdef USE_PROTEGO_MECH -#include "ProtegoMathUtils.h" -#endif - namespace aperi { // Compute the cross product of two vectors @@ -243,7 +239,6 @@ KOKKOS_FORCEINLINE_FUNCTION Eigen::Matrix InvertMatrix(const #endif } -#ifndef USE_PROTEGO_MECH KOKKOS_INLINE_FUNCTION double ComputeKernel(const Eigen::Vector &vector_neighbor_to_point, double R) { const double normalized_radius = vector_neighbor_to_point.norm() / (R); // Calculate the kernel value using a cubic b-spline kernel @@ -255,10 +250,6 @@ KOKKOS_INLINE_FUNCTION double ComputeKernel(const Eigen::Vector &vect return 0.0; } -#else // USE_PROTEGO_MECH -using protego::ComputeKernel; -#endif - template KOKKOS_FUNCTION constexpr auto DetApIm1(const Eigen::Matrix &A) { // From the Cayley-Hamilton theorem, we get that for any N by N matrix A, diff --git a/protego-mech b/protego-mech index eb932551..12e26035 160000 --- a/protego-mech +++ b/protego-mech @@ -1 +1 @@ -Subproject commit eb9325514aa7026d78c4764816e95c1b38e44d50 +Subproject commit 12e26035f394e82eae830dd778ed4271fc1c24cf From 0584139803e6406b7b12401f4bfc007786da16ad Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 14:24:50 -0600 Subject: [PATCH 08/16] move some of FieldData into a src file --- include/FieldData.h | 34 ++------------------------- protego-mech | 2 +- src/FieldData.cpp | 57 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 src/FieldData.cpp diff --git a/include/FieldData.h b/include/FieldData.h index ea2e4953..abbfe73e 100644 --- a/include/FieldData.h +++ b/include/FieldData.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -99,37 +100,6 @@ struct FieldQueryData { * @brief Function to get default field data. * @return A vector of default FieldData. */ -inline std::vector GetFieldData(bool use_strain_smoothing = true) { - std::vector field_data; - - // Node data - field_data.push_back(FieldData("velocity", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The velocity field, generalized - field_data.push_back(FieldData("displacement", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The displacement field, generalized - field_data.push_back(FieldData("acceleration", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The acceleration field, generalized - field_data.push_back(FieldData("force", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The force field, generalized - field_data.push_back(FieldData("mass_from_elements", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The mass as determined from the attached elements - field_data.push_back(FieldData("mass", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The mass field (mass_from_elements as coefficients on the approximation functions) - - // Element data - field_data.push_back(FieldData("volume", FieldDataRank::SCALAR, FieldDataTopologyRank::ELEMENT, 1, std::vector{})); - - // TODO(jake): Add ability to turn this on / off per part - if (use_strain_smoothing) { - // TODO(jake): Some of these fields are only really needed for RK, but NeighborSearchProcessor needs to be refactored to allow for this - // Node neighbor data. - field_data.push_back(FieldData("num_neighbors", FieldDataRank::SCALAR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The number of neighbors for the node - field_data.push_back(FieldData("neighbors", FieldDataRank::CUSTOM, FieldDataTopologyRank::NODE, 1, MAX_NODE_NUM_NEIGHBORS, std::vector{})); // The neighbors of the node - field_data.push_back(FieldData("function_values", FieldDataRank::CUSTOM, FieldDataTopologyRank::NODE, 1, MAX_NODE_NUM_NEIGHBORS, std::vector{})); // The function values of neighbors at the node - field_data.push_back(FieldData("kernel_radius", FieldDataRank::SCALAR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The kernel radius for the node - - // Cell neighbor data. - field_data.push_back(FieldData("num_neighbors", FieldDataRank::SCALAR, FieldDataTopologyRank::ELEMENT, 1, std::vector{})); // The number of neighbors for the cell - field_data.push_back(FieldData("neighbors", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The neighbors of the cell - field_data.push_back(FieldData("function_derivatives_x", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The function derivatives in x of neighbors at the cell - field_data.push_back(FieldData("function_derivatives_y", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The function derivatives in y of neighbors at the cell - field_data.push_back(FieldData("function_derivatives_z", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The function derivatives in z of neighbors at the cell - } - return field_data; -} +std::vector GetFieldData(bool use_strain_smoothing = true); } // namespace aperi diff --git a/protego-mech b/protego-mech index 12e26035..dd67f361 160000 --- a/protego-mech +++ b/protego-mech @@ -1 +1 @@ -Subproject commit 12e26035f394e82eae830dd778ed4271fc1c24cf +Subproject commit dd67f36102a5c668e28ba739e759118043a92335 diff --git a/src/FieldData.cpp b/src/FieldData.cpp new file mode 100644 index 00000000..3763e308 --- /dev/null +++ b/src/FieldData.cpp @@ -0,0 +1,57 @@ +#include "FieldData.h" + +#include +#include +#include + +#include "Constants.h" + +#ifdef USE_PROTEGO_MECH +#include "ProtegoFieldData.h" +#endif + +namespace aperi { + +/** + * @brief Function to get default field data. + * @return A vector of default FieldData. + */ +std::vector GetFieldData(bool use_strain_smoothing) { + std::vector field_data; + + // Node data + field_data.push_back(FieldData("velocity", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The velocity field, generalized + field_data.push_back(FieldData("displacement", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The displacement field, generalized + field_data.push_back(FieldData("acceleration", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The acceleration field, generalized + field_data.push_back(FieldData("force", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 2, std::vector{})); // The force field, generalized + field_data.push_back(FieldData("mass_from_elements", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The mass as determined from the attached elements + field_data.push_back(FieldData("mass", FieldDataRank::VECTOR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The mass field (mass_from_elements as coefficients on the approximation functions) + + // Element data + field_data.push_back(FieldData("volume", FieldDataRank::SCALAR, FieldDataTopologyRank::ELEMENT, 1, std::vector{})); + + // TODO(jake): Add ability to turn this on / off per part + if (use_strain_smoothing) { + // TODO(jake): Some of these fields are only really needed for RK, but NeighborSearchProcessor needs to be refactored to allow for this + // Node neighbor data. + field_data.push_back(FieldData("num_neighbors", FieldDataRank::SCALAR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The number of neighbors for the node + field_data.push_back(FieldData("neighbors", FieldDataRank::CUSTOM, FieldDataTopologyRank::NODE, 1, MAX_NODE_NUM_NEIGHBORS, std::vector{})); // The neighbors of the node + field_data.push_back(FieldData("function_values", FieldDataRank::CUSTOM, FieldDataTopologyRank::NODE, 1, MAX_NODE_NUM_NEIGHBORS, std::vector{})); // The function values of neighbors at the node + field_data.push_back(FieldData("kernel_radius", FieldDataRank::SCALAR, FieldDataTopologyRank::NODE, 1, std::vector{})); // The kernel radius for the node + + // Cell neighbor data. + field_data.push_back(FieldData("num_neighbors", FieldDataRank::SCALAR, FieldDataTopologyRank::ELEMENT, 1, std::vector{})); // The number of neighbors for the cell + field_data.push_back(FieldData("neighbors", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The neighbors of the cell + field_data.push_back(FieldData("function_derivatives_x", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The function derivatives in x of neighbors at the cell + field_data.push_back(FieldData("function_derivatives_y", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The function derivatives in y of neighbors at the cell + field_data.push_back(FieldData("function_derivatives_z", FieldDataRank::CUSTOM, FieldDataTopologyRank::ELEMENT, 1, MAX_CELL_NUM_NEIGHBORS, std::vector{})); // The function derivatives in z of neighbors at the cell + +#ifdef USE_PROTEGO_MECH + std::vector protego_field_data = protego::GetFieldData(); + field_data.insert(field_data.end(), protego_field_data.begin(), protego_field_data.end()); +#endif + } + return field_data; +} + +} // namespace aperi From 36fb19c8ab424dd52e97056c61800739a65abde5 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 15:17:49 -0600 Subject: [PATCH 09/16] start a preprocessor --- include/BoundaryCondition.h | 6 +++++- include/Preprocessor.h | 33 +++++++++++++++++++++++++++++++++ protego-mech | 2 +- src/Application.cpp | 4 ++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 include/Preprocessor.h diff --git a/include/BoundaryCondition.h b/include/BoundaryCondition.h index f7236267..468e41cf 100644 --- a/include/BoundaryCondition.h +++ b/include/BoundaryCondition.h @@ -15,7 +15,7 @@ namespace aperi { class BoundaryCondition { public: - BoundaryCondition(std::vector> components_and_values, std::pair, std::function> time_functions, std::vector sets, std::shared_ptr mesh_data) : m_components_and_values(components_and_values), m_velocity_time_function(time_functions.first), m_acceleration_time_function(time_functions.second) { + BoundaryCondition(std::vector> components_and_values, std::pair, std::function> time_functions, std::vector sets, std::shared_ptr mesh_data) : m_components_and_values(components_and_values), m_velocity_time_function(time_functions.first), m_acceleration_time_function(time_functions.second), m_sets(sets) { const std::array, 1> velocity_field_query_data_vec = {FieldQueryData{"velocity", FieldQueryState::NP1}}; const std::array, 1> acceleration_field_query_data_vec = {FieldQueryData{"acceleration", FieldQueryState::NP1}}; m_node_processor_velocity = std::make_shared>(velocity_field_query_data_vec, mesh_data, sets); @@ -30,10 +30,14 @@ class BoundaryCondition { // Apply the acceleration boundary condition to the field void ApplyAcceleration(double time); + // Get the set names + std::vector GetSetNames() const { return m_sets; } + private: std::vector> m_components_and_values; std::function m_velocity_time_function; std::function m_acceleration_time_function; + std::vector m_sets; std::shared_ptr> m_node_processor_velocity; std::shared_ptr> m_node_processor_acceleration; }; diff --git a/include/Preprocessor.h b/include/Preprocessor.h new file mode 100644 index 00000000..11981199 --- /dev/null +++ b/include/Preprocessor.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#ifdef USE_PROTEGO_MECH +#include "ProtegoPreprocessor.h" +#endif + +namespace aperi { + +class BoundaryCondition; +class IoMesh; +class InternalForceContribution; +class ExternalForceContribution; + +/** + * @brief Creates a solver object. + * + * This function creates a solver object with the given parameters. + * + * @param io_mesh The input/output mesh object. + * @param force_contributions The vector of internal force contributions. + * @param external_force_contributions The vector of external force contributions. + * @return A unique pointer to the created solver object. + */ +void DoPreprocessing(std::shared_ptr io_mesh, std::vector> force_contributions, std::vector> external_force_contributions, std::vector> boundary_conditions) { +#ifdef USE_PROTEGO_MECH + protego::DoPreprocessing(io_mesh, force_contributions, external_force_contributions, boundary_conditions); +#endif +} + +} // namespace aperi diff --git a/protego-mech b/protego-mech index dd67f361..22184b22 160000 --- a/protego-mech +++ b/protego-mech @@ -1 +1 @@ -Subproject commit dd67f36102a5c668e28ba739e759118043a92335 +Subproject commit 22184b2281ac883437f8fb8d072134eff0359af7 diff --git a/src/Application.cpp b/src/Application.cpp index bc2ce91e..7f3d77df 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -10,6 +10,7 @@ #include "IoInputFile.h" #include "IoMesh.h" #include "LogUtils.h" +#include "Preprocessor.h" #include "Scheduler.h" #include "Solver.h" #include "TimeStepper.h" @@ -90,6 +91,9 @@ void Application::Run(const std::string& input_filename) { // Get the output scheduler std::shared_ptr output_scheduler = CreateScheduler(m_io_input_file->GetOutputScheduler(procedure_id)); + // Run pre-processing + aperi::DoPreprocessing(m_io_mesh, m_internal_force_contributions, m_external_force_contributions, m_boundary_conditions); + // Create solver m_solver = aperi::CreateSolver(m_io_mesh, m_internal_force_contributions, m_external_force_contributions, m_boundary_conditions, time_stepper, output_scheduler); From fea2193f0cfd7dfc5b6fb95af138ccd005125c8b Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Wed, 7 Aug 2024 15:23:51 -0600 Subject: [PATCH 10/16] finish remove local compadre flag --- do_configure | 6 ------ 1 file changed, 6 deletions(-) diff --git a/do_configure b/do_configure index 13d44966..86973988 100755 --- a/do_configure +++ b/do_configure @@ -6,7 +6,6 @@ BUILD_TYPE="Release" CHECK_CODE_COVERAGE=OFF USE_GPU=OFF USE_PROTEGO_MECH=OFF -USE_LOCAL_COMPADRE=false # Get the current directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" @@ -36,11 +35,6 @@ while [[ $# -gt 0 ]]; do echo "Including protego-mech." shift ;; - -l | --local-compadre) - USE_LOCAL_COMPADRE=true - echo "Using local compadre." - shift - ;; *) echo "Unknown option: $1" exit 1 From 562b412bffb95b23b26d8aa0ee3b2a5e5a5a021b Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Thu, 8 Aug 2024 11:06:57 -0600 Subject: [PATCH 11/16] split preprocessing for force contribs out and put in preprocessor --- include/ForceContribution.h | 1 + include/InternalForceContribution.h | 6 +++--- include/Preprocessor.h | 8 +++++++- protego-mech | 2 +- src/InternalForceContribution.cpp | 7 +------ test/unit_tests/SolverTestFixture.h | 4 ++++ 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/include/ForceContribution.h b/include/ForceContribution.h index 3d50b426..7cded651 100644 --- a/include/ForceContribution.h +++ b/include/ForceContribution.h @@ -11,6 +11,7 @@ class ForceContribution { public: ForceContribution() = default; ~ForceContribution() = default; + virtual void Preprocess(){}; virtual void ComputeForce() = 0; }; diff --git a/include/InternalForceContribution.h b/include/InternalForceContribution.h index 8c801ab9..db33be3e 100644 --- a/include/InternalForceContribution.h +++ b/include/InternalForceContribution.h @@ -27,7 +27,7 @@ class InternalForceContribution : public ForceContribution { * * @param parameters The parameters associated with the force contribution. */ - InternalForceContribution(InternalForceContributionParameters parameters); + InternalForceContribution(InternalForceContributionParameters parameters) : m_internal_force_contribution_parameters(std::move(parameters)) {} /** * @brief Computes the internal forces. @@ -68,9 +68,9 @@ class InternalForceContribution : public ForceContribution { return m_element->UsesGeneralizedFields(); } - protected: - void SetupInternalForceContribution(); + void Preprocess() override; + protected: InternalForceContributionParameters m_internal_force_contribution_parameters; ///< The parameters associated with the force contribution. size_t m_num_nodes_per_element; ///< The number of nodes per element. std::shared_ptr m_element; ///< The element associated with the force contribution. diff --git a/include/Preprocessor.h b/include/Preprocessor.h index 11981199..23d87e94 100644 --- a/include/Preprocessor.h +++ b/include/Preprocessor.h @@ -24,10 +24,16 @@ class ExternalForceContribution; * @param external_force_contributions The vector of external force contributions. * @return A unique pointer to the created solver object. */ -void DoPreprocessing(std::shared_ptr io_mesh, std::vector> force_contributions, std::vector> external_force_contributions, std::vector> boundary_conditions) { +inline void DoPreprocessing(std::shared_ptr io_mesh, std::vector> force_contributions, std::vector> external_force_contributions, std::vector> boundary_conditions) { #ifdef USE_PROTEGO_MECH protego::DoPreprocessing(io_mesh, force_contributions, external_force_contributions, boundary_conditions); #endif + for (const auto& force_contribution : force_contributions) { + force_contribution->Preprocess(); + } + for (const auto& external_force_contribution : external_force_contributions) { + external_force_contribution->Preprocess(); + } } } // namespace aperi diff --git a/protego-mech b/protego-mech index 22184b22..880c5842 160000 --- a/protego-mech +++ b/protego-mech @@ -1 +1 @@ -Subproject commit 22184b2281ac883437f8fb8d072134eff0359af7 +Subproject commit 880c58427ec730a41bed4a33da8764b90a8a0201 diff --git a/src/InternalForceContribution.cpp b/src/InternalForceContribution.cpp index fbdf5a18..405688f3 100644 --- a/src/InternalForceContribution.cpp +++ b/src/InternalForceContribution.cpp @@ -11,12 +11,7 @@ namespace aperi { -InternalForceContribution::InternalForceContribution(InternalForceContributionParameters parameters) : m_internal_force_contribution_parameters(std::move(parameters)) { - // Setup the internal force contribution - SetupInternalForceContribution(); -} - -void InternalForceContribution::SetupInternalForceContribution() { +void InternalForceContribution::Preprocess() { // Get the number of nodes per element m_num_nodes_per_element = m_internal_force_contribution_parameters.mesh_data->GetNumNodesPerElement(m_internal_force_contribution_parameters.part_name); if (m_num_nodes_per_element != 4) { diff --git a/test/unit_tests/SolverTestFixture.h b/test/unit_tests/SolverTestFixture.h index 84ae40fd..01dfc711 100644 --- a/test/unit_tests/SolverTestFixture.h +++ b/test/unit_tests/SolverTestFixture.h @@ -14,6 +14,7 @@ #include "IoInputFile.h" #include "IoMesh.h" #include "Material.h" +#include "Preprocessor.h" #include "Scheduler.h" #include "Solver.h" #include "TimeStepper.h" @@ -95,6 +96,9 @@ class SolverTest : public ApplicationTest { // Get the output scheduler std::shared_ptr output_scheduler = aperi::CreateScheduler(m_io_input_file->GetOutputScheduler(0)); + // Do preprocessing + aperi::DoPreprocessing(m_io_mesh, m_internal_force_contributions, m_external_force_contributions, m_boundary_conditions); + // Create solver m_solver = aperi::CreateSolver(m_io_mesh, m_internal_force_contributions, m_external_force_contributions, m_boundary_conditions, time_stepper, output_scheduler); From b937e32056af0ae8bc3ffa939e138e944ca08b5e Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Thu, 8 Aug 2024 11:33:05 -0600 Subject: [PATCH 12/16] add submmodule config to ci/cd --- .github/workflows/ci-cd-pipeline.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-cd-pipeline.yaml b/.github/workflows/ci-cd-pipeline.yaml index 0219afca..c65de2df 100644 --- a/.github/workflows/ci-cd-pipeline.yaml +++ b/.github/workflows/ci-cd-pipeline.yaml @@ -85,6 +85,9 @@ jobs: git checkout main git pull origin main --rebase + git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + git submodule update --init --recursive + echo "Checking out appropriate branch..." if [ "${{ github.event_name }}" = "pull_request" ]; then git checkout ${{ github.sha }} From 1a0d8c872ba8ddcc3740301e0108f4792716cc03 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Thu, 8 Aug 2024 11:53:10 -0600 Subject: [PATCH 13/16] attempt to fix ci/cd secret issue --- .github/workflows/ci-cd-pipeline.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd-pipeline.yaml b/.github/workflows/ci-cd-pipeline.yaml index c65de2df..96211ed5 100644 --- a/.github/workflows/ci-cd-pipeline.yaml +++ b/.github/workflows/ci-cd-pipeline.yaml @@ -85,7 +85,10 @@ jobs: git checkout main git pull origin main --rebase - git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + git config --global url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf "https://github.com/" + + echo "Initializing and updating submodules..." + git submodule sync git submodule update --init --recursive echo "Checking out appropriate branch..." From 3a7c6b39cf1068c570e890fbafb0bd17843ef7a1 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Thu, 8 Aug 2024 12:11:28 -0600 Subject: [PATCH 14/16] second attempt to fix ci/cd secret issue --- .github/workflows/ci-cd-pipeline.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd-pipeline.yaml b/.github/workflows/ci-cd-pipeline.yaml index 96211ed5..eafc0d4d 100644 --- a/.github/workflows/ci-cd-pipeline.yaml +++ b/.github/workflows/ci-cd-pipeline.yaml @@ -85,7 +85,10 @@ jobs: git checkout main git pull origin main --rebase - git config --global url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf "https://github.com/" + # Ensure the git config is only set once + if ! git config --global --get url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf; then + git config --global url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf "https://github.com/" + fi echo "Initializing and updating submodules..." git submodule sync From f8e2e8d7d19155a610826bdd74b5a60528039b36 Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Thu, 8 Aug 2024 12:27:36 -0600 Subject: [PATCH 15/16] third attempt to fix ci/cd secret issue --- .github/workflows/ci-cd-pipeline.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-cd-pipeline.yaml b/.github/workflows/ci-cd-pipeline.yaml index eafc0d4d..cb8b9ac8 100644 --- a/.github/workflows/ci-cd-pipeline.yaml +++ b/.github/workflows/ci-cd-pipeline.yaml @@ -85,25 +85,31 @@ jobs: git checkout main git pull origin main --rebase - # Ensure the git config is only set once + # Configure git to use the CICD_REPO_SECRET for fetching submodules if ! git config --global --get url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf; then git config --global url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf "https://github.com/" fi + # For authenticated access to the repository the secret will be changed periodically and the git config will have to be updated + # On the VM, do: + # git config --global --get-regexp url.*.insteadOf + # to see the current git config. And then remove the config with: + # git config --global --unset-all url.https://OLD_SECRET@github.com/.insteadof + # Where OLD_SECRET is the old secret from the first command. Then add the new secret with: + # git config --global url."https://NEW_SECRET@github.com/".insteadOf " + # Where the NEW_SECRET is genertated in developer settings on GitHub and added to the repository secrets. + # or, it will be automatically updated when this action runs again. echo "Initializing and updating submodules..." - git submodule sync - git submodule update --init --recursive + git submodule update --init --recursive --remote echo "Checking out appropriate branch..." if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch origin ${{ github.sha }} git checkout ${{ github.sha }} - git pull origin ${{ github.sha }} --rebase elif [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/main" ]; then git checkout main - git pull origin main --rebase else git checkout ${{ github.ref }} - git pull origin ${{ github.ref }} --rebase fi git lfs pull # Pull LFS files rm -rf build # Remove build directory, this prevents some false positives in tests From 1734c9f730e862c2f03484337eb704354e1214de Mon Sep 17 00:00:00 2001 From: Jake Koester Date: Thu, 8 Aug 2024 15:34:39 -0600 Subject: [PATCH 16/16] fix performance pipeline too --- .github/workflows/performance-pipeline.yaml | 61 ++++++++++++++------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/.github/workflows/performance-pipeline.yaml b/.github/workflows/performance-pipeline.yaml index ba74f1d0..962627b0 100644 --- a/.github/workflows/performance-pipeline.yaml +++ b/.github/workflows/performance-pipeline.yaml @@ -74,28 +74,47 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} run: | ssh -T -o ConnectTimeout=10 ${{ secrets.VM_USERNAME }}@${{ secrets.VM_IP }} << 'EOF' - set -e # Exit on error - cd ~/aperi-mech - echo "Fetching git branches..." - git fetch --all - - echo "Checking out main branch and pulling latest changes..." - git checkout main - git pull origin main - - echo "Checking out appropriate branch..." - if [ "${{ github.event_name }}" = "pull_request" ]; then - git checkout ${{ github.sha }} - git pull origin ${{ github.sha }} - elif [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/main" ]; then + set -e # Exit on error + + cd ~/aperi-mech + echo "Fetching git branches..." + git fetch --all + + echo "Stashing any unstaged changes..." + git stash --include-untracked + + echo "Checking out main branch and pulling latest changes..." git checkout main - git pull origin main - else - git checkout ${{ github.ref }} - git pull origin ${{ github.ref }} - fi - git lfs pull # Pull LFS files - rm -rf build # Remove build directory, this prevents some false positives in tests + git pull origin main --rebase + + # Configure git to use the CICD_REPO_SECRET for fetching submodules + if ! git config --global --get url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf; then + git config --global url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf "https://github.com/" + fi + # For authenticated access to the repository the secret will be changed periodically and the git config will have to be updated + # On the VM, do: + # git config --global --get-regexp url.*.insteadOf + # to see the current git config. And then remove the config with: + # git config --global --unset-all url.https://OLD_SECRET@github.com/.insteadof + # Where OLD_SECRET is the old secret from the first command. Then add the new secret with: + # git config --global url."https://NEW_SECRET@github.com/".insteadOf " + # Where the NEW_SECRET is genertated in developer settings on GitHub and added to the repository secrets. + # or, it will be automatically updated when this action runs again. + + echo "Initializing and updating submodules..." + git submodule update --init --recursive --remote + + echo "Checking out appropriate branch..." + if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch origin ${{ github.sha }} + git checkout ${{ github.sha }} + elif [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/main" ]; then + git checkout main + else + git checkout ${{ github.ref }} + fi + git lfs pull # Pull LFS files + rm -rf build # Remove build directory, this prevents some false positives in tests EOF - name: Skipping Checkout Code on VM