14 #include <dolfinx/graph/AdjacencyList.h>
18 #include <type_traits>
22 #define MPICH_IGNORE_CXX_SEEK 1
39 explicit Comm(MPI_Comm
comm,
bool duplicate =
true);
57 MPI_Comm
comm()
const;
65 static int rank(MPI_Comm comm);
69 static int size(MPI_Comm comm);
90 const std::set<int>& edges);
98 const std::vector<int>& send_offsets,
99 const std::vector<T>& send_data);
106 static std::tuple<std::vector<int>, std::vector<int>>
111 static std::size_t
global_offset(MPI_Comm comm, std::size_t range,
116 static std::array<std::int64_t, 2>
local_range(
int process, std::int64_t N,
126 template <
typename T>
132 template <
typename T>
136 throw std::runtime_error(
"MPI data type unknown");
145 inline MPI_Datatype MPI::mpi_type<float>()
150 inline MPI_Datatype MPI::mpi_type<double>()
155 inline MPI_Datatype MPI::mpi_type<std::complex<double>>()
157 return MPI_DOUBLE_COMPLEX;
160 inline MPI_Datatype MPI::mpi_type<short int>()
165 inline MPI_Datatype MPI::mpi_type<int>()
170 inline MPI_Datatype MPI::mpi_type<unsigned int>()
175 inline MPI_Datatype MPI::mpi_type<long int>()
180 inline MPI_Datatype MPI::mpi_type<unsigned long>()
182 return MPI_UNSIGNED_LONG;
185 inline MPI_Datatype MPI::mpi_type<long long>()
187 return MPI_LONG_LONG;
190 inline MPI_Datatype MPI::mpi_type<unsigned long long>()
192 return MPI_UNSIGNED_LONG_LONG;
195 inline MPI_Datatype MPI::mpi_type<bool>()
201 template <
typename T>
202 graph::AdjacencyList<T>
206 const std::vector<std::int32_t>& send_offsets = send_data.
offsets();
207 const std::vector<T>& values_in = send_data.
array();
210 assert(send_data.
num_nodes() == comm_size);
213 std::vector<int> send_size(comm_size);
214 std::adjacent_difference(std::next(send_offsets.begin(), +1),
215 send_offsets.end(), send_size.begin());
218 std::vector<int> recv_size(comm_size);
219 MPI_Alltoall(send_size.data(), 1, mpi_type<int>(), recv_size.data(), 1,
220 mpi_type<int>(), comm);
223 std::vector<std::int32_t> recv_offset(comm_size + 1);
225 std::partial_sum(recv_size.begin(), recv_size.end(),
226 std::next(recv_offset.begin()));
229 std::vector<T> recv_values(recv_offset[comm_size]);
230 MPI_Alltoallv(values_in.data(), send_size.data(), send_offsets.data(),
231 mpi_type<T>(), recv_values.data(), recv_size.data(),
232 recv_offset.data(), mpi_type<T>(), comm);
235 std::move(recv_offset));
238 template <
typename T>
241 const std::vector<int>& send_offsets,
242 const std::vector<T>& send_data)
245 int indegree(-1), outdegree(-2), weighted(-1);
246 MPI_Dist_graph_neighbors_count(neighbor_comm, &indegree, &outdegree,
249 assert((
int)send_data.size() == send_offsets.back());
250 assert(send_offsets[0] == 0);
253 std::vector<int> send_sizes(outdegree, 0);
254 std::vector<int> recv_sizes(indegree);
255 std::adjacent_difference(std::next(send_offsets.begin()), send_offsets.end(),
257 MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI::mpi_type<int>(),
258 recv_sizes.data(), 1, MPI::mpi_type<int>(),
262 std::vector<int> recv_offsets(recv_sizes.size() + 1);
264 std::partial_sum(recv_sizes.begin(), recv_sizes.end(),
265 std::next(recv_offsets.begin(), 1));
267 std::vector<T> recv_data(recv_offsets[recv_offsets.size() - 1]);
268 MPI_Neighbor_alltoallv(
269 send_data.data(), send_sizes.data(), send_offsets.data(),
270 MPI::mpi_type<T>(), recv_data.data(), recv_sizes.data(),
271 recv_offsets.data(), MPI::mpi_type<T>(), neighbor_comm);
A duplicate MPI communicator and manage lifetime of the communicator.
Definition: MPI.h:36
~Comm()
Destructor (frees wrapped communicator)
Definition: MPI.cpp:43
Comm(MPI_Comm comm, bool duplicate=true)
Duplicate communicator and wrap duplicate.
Definition: MPI.cpp:13
MPI_Comm comm() const
Return the underlying MPI_Comm object.
Definition: MPI.cpp:77
This class provides utility functions for easy communication with MPI and handles cases when DOLFINX ...
Definition: MPI.h:31
static std::vector< int > compute_graph_edges(MPI_Comm comm, const std::set< int > &edges)
Definition: MPI.cpp:139
static MPI_Datatype mpi_type()
MPI Type.
Definition: MPI.h:133
static std::array< std::int64_t, 2 > local_range(int process, std::int64_t N, int size)
Return local range for given process, splitting [0, N - 1] into size() portions of almost equal size.
Definition: MPI.cpp:105
static int rank(MPI_Comm comm)
Return process rank for the communicator.
Definition: MPI.cpp:79
static int size(MPI_Comm comm)
Return size of the group (number of processes) associated with the communicator.
Definition: MPI.cpp:87
static int index_owner(int size, std::size_t index, std::size_t N)
Return which process owns index (inverse of local_range)
Definition: MPI.cpp:123
static std::size_t global_offset(MPI_Comm comm, std::size_t range, bool exclusive)
Find global offset (index) (wrapper for MPI_(Ex)Scan with MPI_SUM as reduction op)
Definition: MPI.cpp:94
static graph::AdjacencyList< T > neighbor_all_to_all(MPI_Comm neighbor_comm, const std::vector< int > &send_offsets, const std::vector< T > &send_data)
neighborhood all-to-all. Send data to neighbors using offsets into contiguous data array....
Definition: MPI.h:240
static std::tuple< std::vector< int >, std::vector< int > > neighbors(MPI_Comm neighbor_comm)
Definition: MPI.cpp:161
static graph::AdjacencyList< T > all_to_all(MPI_Comm comm, const graph::AdjacencyList< T > &send_data)
Send in_values[p0] to process p0 and receive values from process p1 in out_values[p1].
Definition: MPI.h:203
This class provides a static adjacency list data structure. It is commonly used to store directed gra...
Definition: AdjacencyList.h:46
const std::vector< std::int32_t > & offsets() const
Offset for each node in array() (const version)
Definition: AdjacencyList.h:151
std::int32_t num_nodes() const
Get the number of nodes.
Definition: AdjacencyList.h:113
const std::vector< T > & array() const
Return contiguous array of links for all nodes (const version)
Definition: AdjacencyList.h:145