Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes for builds with no raja or umpire #1307

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/serac/infrastructure/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ set(infrastructure_sources
terminator.cpp
)

set(infrastructure_depends axom::inlet axom::fmt axom::cli11 mfem ${serac_device_depends})
set(infrastructure_depends axom::inlet axom::fmt axom::cli11 camp mfem ${serac_device_depends})
blt_list_append(TO infrastructure_depends ELEMENTS tribol IF TRIBOL_FOUND)
blt_list_append(TO infrastructure_depends ELEMENTS caliper adiak::adiak IF SERAC_ENABLE_PROFILING)
blt_list_append(TO infrastructure_depends ELEMENTS blt::openmp IF SERAC_ENABLE_OPENMP)
Copy link
Member Author

@ebchin ebchin Jan 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this is needed, but the openmp dependency for mfem wasn't picked up through FindMFEM.cmake. Any ideas on how to fix this @white238 @chapman39 ? This is with clang 16.0.6 on Ubuntu 24.04

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens when you call blt_print_target_properties(TARGET mfem)? also can you check to see if mfem was built with +openmp in the concretizer tree?

list(APPEND infrastructure_depends blt::mpi)

blt_add_library(
Expand Down
3 changes: 2 additions & 1 deletion src/serac/infrastructure/accelerator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
#include "axom/core.hpp"

#include "serac/infrastructure/logger.hpp"
#include "serac/infrastructure/memory.hpp"
#include "serac/infrastructure/profiling.hpp"

/**
Expand Down Expand Up @@ -121,7 +122,7 @@ void zero_out(axom::Array<T, dim, space>& arr)

/// @brief set the contents of an array to zero, byte-wise
template <typename T, int dim>
void zero_out(axom::ArrayView<T, dim, axom::MemorySpace::Host>& arr)
void zero_out(axom::ArrayView<T, dim, detail::host_memory_space>& arr)
{
std::memset(arr.data(), 0, static_cast<std::size_t>(arr.size()) * sizeof(T));
}
Expand Down
7 changes: 5 additions & 2 deletions src/serac/infrastructure/debug_print.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
#include <iomanip>
#include <vector>

#include "serac/infrastructure/memory.hpp"
#include "serac/numerics/functional/element_restriction.hpp"

/**
* @brief write an array of values out to file, in a space-separated format
* @tparam T the type of each value in the array
Expand Down Expand Up @@ -68,7 +71,7 @@ std::ostream& operator<<(std::ostream& out, DoF dof)
* @param filename the name of the output file
*/
template <typename T>
void write_to_file(axom::Array<T, 2, axom::MemorySpace::Host> arr, std::string filename)
void write_to_file(axom::Array<T, 2, serac::detail::host_memory_space> arr, std::string filename)
{
std::ofstream outfile(filename);

Expand All @@ -91,7 +94,7 @@ void write_to_file(axom::Array<T, 2, axom::MemorySpace::Host> arr, std::string f
* @param filename the name of the output file
*/
template <typename T>
void write_to_file(axom::Array<T, 3, axom::MemorySpace::Host> arr, std::string filename)
void write_to_file(axom::Array<T, 3, serac::detail::host_memory_space> arr, std::string filename)
{
std::ofstream outfile(filename);

Expand Down
31 changes: 31 additions & 0 deletions src/serac/infrastructure/memory.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and
// other Serac Project Developers. See the top-level LICENSE file for
// details.
//
// SPDX-License-Identifier: (BSD-3-Clause)

/**
* @file memory.hpp
*
* @brief This file defines the host memory space
*/

#pragma once

#include "axom/core.hpp"

#include "serac/serac_config.hpp"

namespace serac {

namespace detail {

#ifdef SERAC_USE_UMPIRE
constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Host;
#else
constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Dynamic;
#endif

} // namespace detail

} // namespace serac
2 changes: 2 additions & 0 deletions src/serac/numerics/functional/domain_integral_kernels.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
#include "serac/numerics/functional/quadrature_data.hpp"
#include "serac/numerics/functional/function_signature.hpp"
#include "serac/numerics/functional/differentiate_wrt.hpp"
#ifdef SERAC_USE_RAJA
#include "RAJA/RAJA.hpp"
#endif

#include <array>
#include <cstdint>
Expand Down
40 changes: 21 additions & 19 deletions src/serac/numerics/functional/element_restriction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,8 @@ std::vector<Array2D<int> > geom_local_face_dofs(int p)
return output;
}

axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementRestriction(const serac::fes_t* fes, mfem::Geometry::Type geom)
axom::Array<DoF, 2, serac::detail::host_memory_space> GetElementRestriction(const serac::fes_t* fes,
mfem::Geometry::Type geom)
{
std::vector<DoF> elem_dofs{};
mfem::Mesh* mesh = fes->GetMesh();
Expand Down Expand Up @@ -269,17 +270,17 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementRestriction(const serac::
}

if (n == 0) {
return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
} else {
uint64_t dofs_per_elem = elem_dofs.size() / n;
axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_elem);
uint64_t dofs_per_elem = elem_dofs.size() / n;
axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_elem);
std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem);
return output;
}
}

axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom,
const std::vector<int>& mfem_elem_ids)
axom::Array<DoF, 2, serac::detail::host_memory_space> GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom,
const std::vector<int>& mfem_elem_ids)

{
std::vector<DoF> elem_dofs{};
Expand Down Expand Up @@ -335,17 +336,17 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t*
}

if (n == 0) {
return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
} else {
uint64_t dofs_per_elem = elem_dofs.size() / n;
axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_elem);
uint64_t dofs_per_elem = elem_dofs.size() / n;
axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_elem);
std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem);
return output;
}
}

axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
FaceType type)
axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
mfem::Geometry::Type face_geom, FaceType type)
{
std::vector<DoF> face_dofs;
mfem::Mesh* mesh = fes->GetMesh();
Expand Down Expand Up @@ -450,17 +451,18 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes
delete face_to_elem;

if (n == 0) {
return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
} else {
uint64_t dofs_per_face = face_dofs.size() / n;
axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_face);
uint64_t dofs_per_face = face_dofs.size() / n;
axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_face);
std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face);
return output;
}
}

axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
const std::vector<int>& mfem_face_ids)
axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
mfem::Geometry::Type face_geom,
const std::vector<int>& mfem_face_ids)
{
std::vector<DoF> face_dofs;
mfem::Mesh* mesh = fes->GetMesh();
Expand Down Expand Up @@ -617,10 +619,10 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes
delete face_to_elem;

if (n == 0) {
return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
} else {
uint64_t dofs_per_face = face_dofs.size() / n;
axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_face);
uint64_t dofs_per_face = face_dofs.size() / n;
axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_face);
std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face);
return output;
}
Expand Down
15 changes: 9 additions & 6 deletions src/serac/numerics/functional/element_restriction.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "geometry.hpp"
#include "domain.hpp"

#include "serac/infrastructure/memory.hpp"
#include "serac/numerics/functional/typedefs.hpp"

inline bool isH1(const mfem::FiniteElementSpace& fes)
Expand Down Expand Up @@ -197,7 +198,7 @@ struct ElementRestriction {
uint64_t nodes_per_elem;

/// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space
axom::Array<DoF, 2, axom::MemorySpace::Host> dof_info;
axom::Array<DoF, 2, serac::detail::host_memory_space> dof_info;

/// whether the underlying dofs are arranged "byNodes" or "byVDim"
mfem::Ordering::Type ordering;
Expand Down Expand Up @@ -242,7 +243,8 @@ struct BlockElementRestriction {
* @param fes the finite element space containing the dof information
* @param geom the kind of element geometry
*/
axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom);
axom::Array<DoF, 2, serac::detail::host_memory_space> GetElementDofs(const serac::fes_t* fes,
mfem::Geometry::Type geom);

/**
* @brief Get the list of dofs for each face element (of the specified geometry) from the fes_t
Expand All @@ -251,9 +253,10 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t*
* @param geom the kind of element geometry
* @param type whether the face is of interior or boundary type
*/
axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
FaceType type);
axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
mfem::Geometry::Type face_geom, FaceType type);

/// @overload
axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
const std::vector<int>& mfem_face_ids);
axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
mfem::Geometry::Type face_geom,
const std::vector<int>& mfem_face_ids);
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ mfem::Mesh generate_permuted_mesh(mfem::Geometry::Type geom, int i)
return {};
}

std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, axom::MemorySpace::Host> arr)
std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, serac::detail::host_memory_space> arr)
{
for (int i = 0; i < arr.shape()[0]; i++) {
for (int j = 0; j < arr.shape()[1]; j++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

using namespace serac;

std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, axom::MemorySpace::Host> arr)
std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, serac::detail::host_memory_space> arr)
{
for (int i = 0; i < arr.shape()[0]; i++) {
for (int j = 0; j < arr.shape()[1]; j++) {
Expand Down
Loading