diff --git a/src/serac/infrastructure/CMakeLists.txt b/src/serac/infrastructure/CMakeLists.txt index 934ec14d4..e82587911 100644 --- a/src/serac/infrastructure/CMakeLists.txt +++ b/src/serac/infrastructure/CMakeLists.txt @@ -41,9 +41,10 @@ set(infrastructure_sources terminator.cpp ) -set(infrastructure_depends axom::inlet axom::fmt axom::cli11 mfem ${serac_device_depends}) +set(infrastructure_depends axom::inlet axom::fmt axom::cli11 camp mfem ${serac_device_depends}) blt_list_append(TO infrastructure_depends ELEMENTS tribol IF TRIBOL_FOUND) blt_list_append(TO infrastructure_depends ELEMENTS caliper adiak::adiak IF SERAC_ENABLE_PROFILING) +blt_list_append(TO infrastructure_depends ELEMENTS blt::openmp IF SERAC_ENABLE_OPENMP) list(APPEND infrastructure_depends blt::mpi) blt_add_library( diff --git a/src/serac/infrastructure/accelerator.hpp b/src/serac/infrastructure/accelerator.hpp index 6da2af89d..0169a10e4 100644 --- a/src/serac/infrastructure/accelerator.hpp +++ b/src/serac/infrastructure/accelerator.hpp @@ -55,6 +55,7 @@ #include "axom/core.hpp" #include "serac/infrastructure/logger.hpp" +#include "serac/infrastructure/memory.hpp" #include "serac/infrastructure/profiling.hpp" /** @@ -121,7 +122,7 @@ void zero_out(axom::Array& arr) /// @brief set the contents of an array to zero, byte-wise template -void zero_out(axom::ArrayView& arr) +void zero_out(axom::ArrayView& arr) { std::memset(arr.data(), 0, static_cast(arr.size()) * sizeof(T)); } diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 7bfc1f983..3382e6138 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -13,6 +13,9 @@ #include #include +#include "serac/infrastructure/memory.hpp" +#include "serac/numerics/functional/element_restriction.hpp" + /** * @brief write an array of values out to file, in a space-separated format * @tparam T the type of each value in the array @@ -68,7 +71,7 @@ std::ostream& operator<<(std::ostream& out, DoF dof) * @param filename the name of the output file */ template -void write_to_file(axom::Array arr, std::string filename) +void write_to_file(axom::Array arr, std::string filename) { std::ofstream outfile(filename); @@ -91,7 +94,7 @@ void write_to_file(axom::Array arr, std::string f * @param filename the name of the output file */ template -void write_to_file(axom::Array arr, std::string filename) +void write_to_file(axom::Array arr, std::string filename) { std::ofstream outfile(filename); diff --git a/src/serac/infrastructure/memory.hpp b/src/serac/infrastructure/memory.hpp new file mode 100644 index 000000000..4970b67db --- /dev/null +++ b/src/serac/infrastructure/memory.hpp @@ -0,0 +1,31 @@ +// Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and +// other Serac Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/** + * @file memory.hpp + * + * @brief This file defines the host memory space + */ + +#pragma once + +#include "axom/core.hpp" + +#include "serac/serac_config.hpp" + +namespace serac { + +namespace detail { + +#ifdef SERAC_USE_UMPIRE +constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Host; +#else +constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Dynamic; +#endif + +} // namespace detail + +} // namespace serac diff --git a/src/serac/numerics/functional/domain_integral_kernels.hpp b/src/serac/numerics/functional/domain_integral_kernels.hpp index 74a749c1d..ffcd50fd7 100644 --- a/src/serac/numerics/functional/domain_integral_kernels.hpp +++ b/src/serac/numerics/functional/domain_integral_kernels.hpp @@ -10,7 +10,9 @@ #include "serac/numerics/functional/quadrature_data.hpp" #include "serac/numerics/functional/function_signature.hpp" #include "serac/numerics/functional/differentiate_wrt.hpp" +#ifdef SERAC_USE_RAJA #include "RAJA/RAJA.hpp" +#endif #include #include diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index d5e721312..e5c613364 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -216,7 +216,8 @@ std::vector > geom_local_face_dofs(int p) return output; } -axom::Array GetElementRestriction(const serac::fes_t* fes, mfem::Geometry::Type geom) +axom::Array GetElementRestriction(const serac::fes_t* fes, + mfem::Geometry::Type geom) { std::vector elem_dofs{}; mfem::Mesh* mesh = fes->GetMesh(); @@ -269,17 +270,17 @@ axom::Array GetElementRestriction(const serac:: } if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { - uint64_t dofs_per_elem = elem_dofs.size() / n; - axom::Array output(n, dofs_per_elem); + uint64_t dofs_per_elem = elem_dofs.size() / n; + axom::Array output(n, dofs_per_elem); std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -axom::Array GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom, - const std::vector& mfem_elem_ids) +axom::Array GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom, + const std::vector& mfem_elem_ids) { std::vector elem_dofs{}; @@ -335,17 +336,17 @@ axom::Array GetElementDofs(const serac::fes_t* } if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { - uint64_t dofs_per_elem = elem_dofs.size() / n; - axom::Array output(n, dofs_per_elem); + uint64_t dofs_per_elem = elem_dofs.size() / n; + axom::Array output(n, dofs_per_elem); std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - FaceType type) +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, FaceType type) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -450,17 +451,18 @@ axom::Array GetFaceDofs(const serac::fes_t* fes delete face_to_elem; if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { - uint64_t dofs_per_face = face_dofs.size() / n; - axom::Array output(n, dofs_per_face); + uint64_t dofs_per_face = face_dofs.size() / n; + axom::Array output(n, dofs_per_face); std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } } -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - const std::vector& mfem_face_ids) +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, + const std::vector& mfem_face_ids) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -617,10 +619,10 @@ axom::Array GetFaceDofs(const serac::fes_t* fes delete face_to_elem; if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { - uint64_t dofs_per_face = face_dofs.size() / n; - axom::Array output(n, dofs_per_face); + uint64_t dofs_per_face = face_dofs.size() / n; + axom::Array output(n, dofs_per_face); std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index dc2e1c336..9444c08a9 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -7,6 +7,7 @@ #include "geometry.hpp" #include "domain.hpp" +#include "serac/infrastructure/memory.hpp" #include "serac/numerics/functional/typedefs.hpp" inline bool isH1(const mfem::FiniteElementSpace& fes) @@ -197,7 +198,7 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space - axom::Array dof_info; + axom::Array dof_info; /// whether the underlying dofs are arranged "byNodes" or "byVDim" mfem::Ordering::Type ordering; @@ -242,7 +243,8 @@ struct BlockElementRestriction { * @param fes the finite element space containing the dof information * @param geom the kind of element geometry */ -axom::Array GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom); +axom::Array GetElementDofs(const serac::fes_t* fes, + mfem::Geometry::Type geom); /** * @brief Get the list of dofs for each face element (of the specified geometry) from the fes_t @@ -251,9 +253,10 @@ axom::Array GetElementDofs(const serac::fes_t* * @param geom the kind of element geometry * @param type whether the face is of interior or boundary type */ -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - FaceType type); +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, FaceType type); /// @overload -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - const std::vector& mfem_face_ids); +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, + const std::vector& mfem_face_ids); diff --git a/src/serac/numerics/functional/tests/dg_restriction_operators.cpp b/src/serac/numerics/functional/tests/dg_restriction_operators.cpp index 74cadd1a1..4edcee9ee 100644 --- a/src/serac/numerics/functional/tests/dg_restriction_operators.cpp +++ b/src/serac/numerics/functional/tests/dg_restriction_operators.cpp @@ -229,7 +229,7 @@ mfem::Mesh generate_permuted_mesh(mfem::Geometry::Type geom, int i) return {}; } -std::ostream& operator<<(std::ostream& out, axom::Array arr) +std::ostream& operator<<(std::ostream& out, axom::Array arr) { for (int i = 0; i < arr.shape()[0]; i++) { for (int j = 0; j < arr.shape()[1]; j++) { diff --git a/src/serac/numerics/functional/tests/element_restriction_tests.cpp b/src/serac/numerics/functional/tests/element_restriction_tests.cpp index ec543cdd2..8d858a4ad 100644 --- a/src/serac/numerics/functional/tests/element_restriction_tests.cpp +++ b/src/serac/numerics/functional/tests/element_restriction_tests.cpp @@ -5,7 +5,7 @@ using namespace serac; -std::ostream& operator<<(std::ostream& out, axom::Array arr) +std::ostream& operator<<(std::ostream& out, axom::Array arr) { for (int i = 0; i < arr.shape()[0]; i++) { for (int j = 0; j < arr.shape()[1]; j++) {