From b48471b2524d32f7d17c9dc81c5de74e7b85db51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20H=C3=BCbner?= Date: Tue, 19 Mar 2024 14:03:21 +0100 Subject: [PATCH 1/5] Rework the allgatherv example The allgatherv example is designed as the first example the user should read. It does not assume any prior knowledge of terminology. It should highlight the main features of the KaMPIng API. --- examples/usage/allgatherv_example.cpp | 45 ++++++++++++++++++--------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/examples/usage/allgatherv_example.cpp b/examples/usage/allgatherv_example.cpp index 795a5770e..ec48a73af 100644 --- a/examples/usage/allgatherv_example.cpp +++ b/examples/usage/allgatherv_example.cpp @@ -27,44 +27,59 @@ int main() { using namespace kamping; - kamping::Environment e; + + // The Environment class is a RAII wrapper around MPI_Init and MPI_Finalize. + kamping::Environment e; + + // A kamping::Communicator abstracts away an MPI_Comm; here MPI_COMM_WORLD. kamping::Communicator comm; - std::vector input(comm.rank(), comm.rank_signed()); - { - // simply return the recv buffer + // Note, that the size of the input vector is different for each rank. + std::vector input(comm.rank(), comm.rank_signed()); + + { // Gather the input from all ranks to the root rank. auto recv_buffer = comm.allgatherv(send_buf(input)); - print_result(recv_buffer, comm); } - { - // return recv buffer and recv_counts + { // We can also request the number of elements received from each rank. The recv_buf will always be the first out + // parameter. After that, the output parameters are ordered as they are in the function call. auto [recv_buffer, recv_counts] = comm.allgatherv(send_buf(input), recv_counts_out()); - print_result(recv_buffer, comm); - print_result(recv_counts, comm); } - { - // write result to an exisiting container + { // To re-use memory, we can provide an already allocated container to the MPI call. std::vector recv_buffer; + // Let KaMPIng resize the recv_buffer to the correct size. Other possibilities are no_resize and grow_only. comm.allgatherv(send_buf(input), recv_buf(recv_buffer)); - print_result(recv_buffer, comm); - // additionally, receive counts and/or receive displacements can be provided + // We can also re-use already allocated containers for the other output parameters, e.g. recv_counts. std::vector recv_counts(comm.size()); std::iota(recv_counts.begin(), recv_counts.end(), 0); + comm.allgatherv(send_buf(input), recv_buf(recv_buffer), kamping::recv_counts(recv_counts)); + std::vector recv_displs(comm.size()); std::exclusive_scan(recv_counts.begin(), recv_counts.end(), recv_displs.begin(), 0); recv_buffer.clear(); + // In this example, we combine all of the concepts mentioned above: + // - Se input as the send buffer + // - Receive all elements into recv_buffer, resizing it to fit exactly the number of elements received. + // - Output the number of elements received from each rank into recv_counts. + // - Output the displacement of the first element received from each rank into recv_displs. comm.allgatherv( send_buf(input), recv_buf(recv_buffer), kamping::recv_counts(recv_counts), kamping::recv_displs(recv_displs) ); - print_result(recv_buffer, comm); + } - return 0; + { // If we have many out parameters, we can replace the structured bindings with extract_*() calls in order to + // increase readability. + auto result = comm.allgatherv(send_buf(input), recv_counts_out(), recv_displs_out()); + auto const recv_buffer = result.extract_recv_buffer(); + auto const recv_counts = result.extract_recv_counts(); + auto const recv_displs = result.extract_recv_displs(); } + + return 0; } From 9dffd62e524e6b9c5be97c6e7d00bd58192acefe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20H=C3=BCbner?= Date: Tue, 19 Mar 2024 15:33:03 +0100 Subject: [PATCH 2/5] Add section describing how to send a span of the send_buf --- examples/usage/allgatherv_example.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/examples/usage/allgatherv_example.cpp b/examples/usage/allgatherv_example.cpp index ec48a73af..aaf16c340 100644 --- a/examples/usage/allgatherv_example.cpp +++ b/examples/usage/allgatherv_example.cpp @@ -81,5 +81,15 @@ int main() { auto const recv_displs = result.extract_recv_displs(); } + { // You can also use views to send parts of the data. + input.resize(comm.rank() + 1, comm.rank_signed()); + + // Note, if you're on C++ >= 20 you can use std::span instead. + comm.allgatherv(send_buf(kamping::Span(input).subspan(0, comm.rank()))); + + // Alternatively + comm.allgatherv(send_buf(input), send_count(comm.rank_signed())); + } + return 0; } From 9253bdd540d6dfbe664f2e15ff4b2e12dca2a755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20H=C3=BCbner?= Date: Wed, 20 Mar 2024 10:36:23 +0100 Subject: [PATCH 3/5] Add Matthias' feedback to allgatherv example --- examples/usage/allgatherv_example.cpp | 60 +++++++++++++++++++-------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/examples/usage/allgatherv_example.cpp b/examples/usage/allgatherv_example.cpp index aaf16c340..a4793be14 100644 --- a/examples/usage/allgatherv_example.cpp +++ b/examples/usage/allgatherv_example.cpp @@ -15,8 +15,11 @@ #include #include +#include #include +#include "cereal/types/string.hpp" +#include "cereal/types/unordered_map.hpp" #include "helpers_for_examples.hpp" #include "kamping/checking_casts.hpp" #include "kamping/collectives/allgather.hpp" @@ -37,51 +40,55 @@ int main() { // Note, that the size of the input vector is different for each rank. std::vector input(comm.rank(), comm.rank_signed()); - { // Gather the input from all ranks to the root rank. - auto recv_buffer = comm.allgatherv(send_buf(input)); + { // Basic use case; gather the inputs across all ranks to all ranks. + auto const output = comm.allgatherv(send_buf(input)); + print_result_on_root(output, comm); + print_on_root("-----", comm); } { // We can also request the number of elements received from each rank. The recv_buf will always be the first out - // parameter. After that, the output parameters are ordered as they are in the function call. + // parameter. After that, the output parameters are ordered as they appear in the function call. + // Communicating KaMPIng calls like allgatherv return a result object which can be decomposed using structured + // bindings (here) or explicit extract_*() calls (see below). auto [recv_buffer, recv_counts] = comm.allgatherv(send_buf(input), recv_counts_out()); } { // To re-use memory, we can provide an already allocated container to the MPI call. - std::vector recv_buffer; + std::vector output; // Let KaMPIng resize the recv_buffer to the correct size. Other possibilities are no_resize and grow_only. - comm.allgatherv(send_buf(input), recv_buf(recv_buffer)); + comm.allgatherv(send_buf(input), recv_buf(output)); // We can also re-use already allocated containers for the other output parameters, e.g. recv_counts. - std::vector recv_counts(comm.size()); - std::iota(recv_counts.begin(), recv_counts.end(), 0); - comm.allgatherv(send_buf(input), recv_buf(recv_buffer), kamping::recv_counts(recv_counts)); + std::vector output_counts(comm.size()); + std::iota(output_counts.begin(), output_counts.end(), 0); + comm.allgatherv(send_buf(input), recv_buf(output), output_counts(output_counts)); - std::vector recv_displs(comm.size()); - std::exclusive_scan(recv_counts.begin(), recv_counts.end(), recv_displs.begin(), 0); - recv_buffer.clear(); + std::vector displacements(comm.size()); + std::exclusive_scan(output_counts.begin(), output_counts.end(), displacements.begin(), 0); + output.clear(); // In this example, we combine all of the concepts mentioned above: - // - Se input as the send buffer + // - Use input as the send buffer // - Receive all elements into recv_buffer, resizing it to fit exactly the number of elements received. // - Output the number of elements received from each rank into recv_counts. // - Output the displacement of the first element received from each rank into recv_displs. comm.allgatherv( send_buf(input), - recv_buf(recv_buffer), - kamping::recv_counts(recv_counts), - kamping::recv_displs(recv_displs) + recv_buf(output), + recv_counts(output_counts), + recv_displs(displacements) ); } - { // If we have many out parameters, we can replace the structured bindings with extract_*() calls in order to - // increase readability. + { // It is also possible to use result.extract_*() calls instead of decomposing the result object using structured + // bindings in order to increase readability. auto result = comm.allgatherv(send_buf(input), recv_counts_out(), recv_displs_out()); auto const recv_buffer = result.extract_recv_buffer(); auto const recv_counts = result.extract_recv_counts(); auto const recv_displs = result.extract_recv_displs(); } - { // You can also use views to send parts of the data. + { // C++ views can be used to send parts of the data. input.resize(comm.rank() + 1, comm.rank_signed()); // Note, if you're on C++ >= 20 you can use std::span instead. @@ -91,5 +98,22 @@ int main() { comm.allgatherv(send_buf(input), send_count(comm.rank_signed())); } + { // KaMPIng also provides serialization/deserialization using Cereal + using dict_type = std::unordered_map; + dict_type data = {{"rank", comm.rank()}}; + if (comm.root()) { + data.insert({"size", comm.size()}); + data.insert({"root", comm.root()}); + } + + auto const output = comm.allgatherv(send_buf(as_serialized(data)), recv_buf(as_deserializable())); + + if (comm.root()) { + for (auto const& [key, value]: output) { + std::cout << key << " -> " << value << std::endl; + } + } + } + return 0; } From d50ca662e5193661d3eeba40e0477b0211476515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20H=C3=BCbner?= Date: Wed, 20 Mar 2024 10:40:14 +0100 Subject: [PATCH 4/5] Remove serialization example from allgatherv example --- examples/usage/allgatherv_example.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/examples/usage/allgatherv_example.cpp b/examples/usage/allgatherv_example.cpp index a4793be14..8a2e74739 100644 --- a/examples/usage/allgatherv_example.cpp +++ b/examples/usage/allgatherv_example.cpp @@ -98,22 +98,5 @@ int main() { comm.allgatherv(send_buf(input), send_count(comm.rank_signed())); } - { // KaMPIng also provides serialization/deserialization using Cereal - using dict_type = std::unordered_map; - dict_type data = {{"rank", comm.rank()}}; - if (comm.root()) { - data.insert({"size", comm.size()}); - data.insert({"root", comm.root()}); - } - - auto const output = comm.allgatherv(send_buf(as_serialized(data)), recv_buf(as_deserializable())); - - if (comm.root()) { - for (auto const& [key, value]: output) { - std::cout << key << " -> " << value << std::endl; - } - } - } - return 0; } From 4b227d95dc86db21337b39306bb6f2a3a9a66e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20H=C3=BCbner?= Date: Wed, 20 Mar 2024 10:41:19 +0100 Subject: [PATCH 5/5] Add example of serialization to allgatherv example --- examples/usage/allgatherv_example.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/usage/allgatherv_example.cpp b/examples/usage/allgatherv_example.cpp index 8a2e74739..a4793be14 100644 --- a/examples/usage/allgatherv_example.cpp +++ b/examples/usage/allgatherv_example.cpp @@ -98,5 +98,22 @@ int main() { comm.allgatherv(send_buf(input), send_count(comm.rank_signed())); } + { // KaMPIng also provides serialization/deserialization using Cereal + using dict_type = std::unordered_map; + dict_type data = {{"rank", comm.rank()}}; + if (comm.root()) { + data.insert({"size", comm.size()}); + data.insert({"root", comm.root()}); + } + + auto const output = comm.allgatherv(send_buf(as_serialized(data)), recv_buf(as_deserializable())); + + if (comm.root()) { + for (auto const& [key, value]: output) { + std::cout << key << " -> " << value << std::endl; + } + } + } + return 0; }