18 #include <deal.II/base/partitioner.templates.h>
20 #include <boost/serialization/utility.hpp>
34 , n_ghost_indices_data(0)
35 , n_import_indices_data(0)
36 , n_ghost_indices_in_larger_set(0)
39 , communicator(MPI_COMM_SELF)
40 , have_ghost_indices(false)
47 , locally_owned_range_data(size)
50 , n_ghost_indices_data(0)
51 , n_import_indices_data(0)
52 , n_ghost_indices_in_larger_set(0)
55 , communicator(MPI_COMM_SELF)
56 , have_ghost_indices(false)
58 locally_owned_range_data.add_range(0, size);
59 locally_owned_range_data.compress();
60 ghost_indices_data.set_size(size);
67 const MPI_Comm & communicator)
70 , locally_owned_range_data(global_size)
71 , local_range_data{0, local_size}
72 , n_ghost_indices_data(ghost_size)
73 , n_import_indices_data(0)
74 , n_ghost_indices_in_larger_set(0)
77 , communicator(communicator)
78 , have_ghost_indices(true)
82 #ifdef DEAL_II_WITH_MPI
84 MPI_Exscan(&local_size,
93 local_range_data = {prefix_sum, prefix_sum + local_size};
95 locally_owned_range_data.add_range(prefix_sum, prefix_sum + local_size);
96 locally_owned_range_data.compress();
103 const MPI_Comm &communicator_in)
106 , n_ghost_indices_data(0)
107 , n_import_indices_data(0)
108 , n_ghost_indices_in_larger_set(0)
111 , communicator(communicator_in)
112 , have_ghost_indices(false)
114 set_owned_indices(locally_owned_indices);
115 set_ghost_indices(ghost_indices_in);
121 const MPI_Comm &communicator_in)
124 , n_ghost_indices_data(0)
125 , n_import_indices_data(0)
126 , n_ghost_indices_in_larger_set(0)
129 , communicator(communicator_in)
130 , have_ghost_indices(false)
132 set_owned_indices(locally_owned_indices);
139 const IndexSet &read_write_vector_index_set,
140 const MPI_Comm &communicator_in)
142 have_ghost_indices =
false;
143 communicator = communicator_in;
144 set_owned_indices(vector_space_vector_index_set);
145 set_ghost_indices(read_write_vector_index_set);
151 Partitioner::set_owned_indices(
const IndexSet &locally_owned_indices)
166 ExcMessage(
"The index set specified in locally_owned_indices "
167 "is not contiguous."));
171 std::pair<types::global_dof_index, types::global_dof_index>(
176 local_range_data.second - local_range_data.first <
180 "Index overflow: This class supports at most 2^32-1 locally owned vector entries"));
181 locally_owned_range_data.set_size(locally_owned_indices.
size());
182 locally_owned_range_data.add_range(local_range_data.first,
183 local_range_data.second);
184 locally_owned_range_data.compress();
186 ghost_indices_data.set_size(locally_owned_indices.
size());
192 Partitioner::set_ghost_indices(
const IndexSet &ghost_indices_in,
193 const IndexSet &larger_ghost_index_set)
199 ghost_indices_in.
size() == locally_owned_range_data.size(),
201 locally_owned_range_data.size()));
203 ghost_indices_data = ghost_indices_in;
204 if (ghost_indices_data.size() != locally_owned_range_data.size())
205 ghost_indices_data.
set_size(locally_owned_range_data.size());
206 ghost_indices_data.subtract_set(locally_owned_range_data);
207 ghost_indices_data.compress();
209 ghost_indices_data.n_elements() <
213 "Index overflow: This class supports at most 2^32-1 ghost elements"));
214 n_ghost_indices_data = ghost_indices_data.n_elements();
227 #ifdef DEAL_II_WITH_MPI
242 my_size += local_range_data.first;
246 const int ierr = MPI_Exscan(&my_size,
261 my_shift = local_range_data.first;
265 if (local_range_data.first == 0 && my_shift != 0)
268 locally_owned_size();
269 local_range_data.first = my_shift;
270 local_range_data.second = my_shift + old_locally_owned_size;
273 std::vector<unsigned int> owning_ranks_of_ghosts(
274 ghost_indices_data.n_elements());
277 internal::ComputeIndexOwner::ConsensusAlgorithmsPayload process(
278 locally_owned_range_data,
281 owning_ranks_of_ghosts,
287 ConsensusAlgorithms::Selector<
289 std::pair<types::global_dof_index, types::global_dof_index>>,
290 std::vector<unsigned int>>
292 consensus_algorithm.run(process, communicator);
295 ghost_targets_data = {};
297 if (owning_ranks_of_ghosts.size() > 0)
299 ghost_targets_data.emplace_back(owning_ranks_of_ghosts[0], 0);
300 for (
auto i : owning_ranks_of_ghosts)
302 Assert(i >= ghost_targets_data.back().first,
304 "Expect result of ConsensusAlgorithms::Process to be "
306 if (i == ghost_targets_data.back().first)
307 ghost_targets_data.back().second++;
309 ghost_targets_data.emplace_back(i, 1);
315 std::map<unsigned int, IndexSet> import_data = process.get_requesters();
318 n_import_indices_data = 0;
319 import_targets_data = {};
320 import_targets_data.reserve(import_data.size());
321 import_indices_chunks_by_rank_data = {};
322 import_indices_chunks_by_rank_data.reserve(import_data.size());
323 import_indices_chunks_by_rank_data.resize(1);
324 for (
const auto &i : import_data)
325 if (i.second.n_elements() > 0)
327 import_targets_data.emplace_back(i.first, i.second.n_elements());
328 n_import_indices_data += i.second.n_elements();
329 import_indices_chunks_by_rank_data.push_back(
330 import_indices_chunks_by_rank_data.back() +
331 i.second.n_intervals());
335 import_indices_data = {};
336 import_indices_data.reserve(import_indices_chunks_by_rank_data.back());
337 for (
const auto &i : import_data)
339 Assert((i.second & locally_owned_range_data) == i.second,
341 for (
auto interval = i.second.begin_intervals();
342 interval != i.second.end_intervals();
344 import_indices_data.emplace_back(*interval->begin() -
345 local_range_data.first,
346 interval->last() + 1 -
347 local_range_data.first);
368 const std::vector<types::global_dof_index> ghost_indices_ref =
369 ghost_indices_data.get_index_vector();
371 std::vector<types::global_dof_index> indices_to_send(n_import_indices());
372 std::vector<types::global_dof_index> ghost_indices(n_ghost_indices());
374 const std::vector<types::global_dof_index> my_indices =
375 locally_owned_range_data.get_index_vector();
376 std::vector<MPI_Request> requests;
377 n_ghost_indices_in_larger_set = n_ghost_indices_data;
378 export_to_ghosted_array_start(127,
380 my_indices.data(), my_indices.size()),
384 export_to_ghosted_array_finish(
make_array_view(ghost_indices), requests);
386 const int ierr = MPI_Testall(requests.size(),
389 MPI_STATUSES_IGNORE);
393 "MPI found unfinished requests. Check communication setup"));
395 for (
unsigned int i = 0; i < ghost_indices.size(); ++i)
402 if (larger_ghost_index_set.
size() == 0)
404 ghost_indices_subset_chunks_by_rank_data.clear();
405 ghost_indices_subset_data.emplace_back(0, n_ghost_indices());
406 n_ghost_indices_in_larger_set = n_ghost_indices_data;
411 ghost_indices_data.size());
413 (larger_ghost_index_set & locally_owned_range_data).n_elements() ==
415 ExcMessage(
"Ghost index set should not overlap with owned set."));
416 Assert((larger_ghost_index_set & ghost_indices_data) ==
418 ExcMessage(
"Larger ghost index set must contain the tight "
419 "ghost index set."));
421 n_ghost_indices_in_larger_set = larger_ghost_index_set.
n_elements();
425 std::vector<unsigned int> expanded_numbering;
429 ExcMessage(
"The given larger ghost index set must contain "
430 "all indices in the actual index set."));
436 "Index overflow: This class supports at most 2^32-1 ghost elements"));
437 expanded_numbering.push_back(
442 std::vector<std::pair<unsigned int, unsigned int>>
443 ghost_indices_subset;
444 ghost_indices_subset_chunks_by_rank_data.resize(
445 ghost_targets_data.size() + 1);
447 ghost_indices_subset_chunks_by_rank_data[0] = 0;
448 unsigned int shift = 0;
449 for (
unsigned int p = 0; p < ghost_targets_data.size(); ++p)
452 for (
unsigned int ii = 0; ii < ghost_targets_data[p].second; ++ii)
454 const unsigned int i =
shift + ii;
455 if (expanded_numbering[i] == last_index + 1)
457 ghost_indices_subset.back().second++;
460 ghost_indices_subset.emplace_back(expanded_numbering[i],
461 expanded_numbering[i] +
463 last_index = expanded_numbering[i];
465 shift += ghost_targets_data[p].second;
466 ghost_indices_subset_chunks_by_rank_data[p + 1] =
467 ghost_indices_subset.size();
469 ghost_indices_subset_data = ghost_indices_subset;
476 Partitioner::is_compatible(
const Partitioner &part)
const
482 #ifdef DEAL_II_WITH_MPI
485 int communicators_same = 0;
486 const int ierr = MPI_Comm_compare(part.communicator,
488 &communicators_same);
490 if (!(communicators_same == MPI_IDENT ||
491 communicators_same == MPI_CONGRUENT))
495 return (global_size == part.global_size &&
496 local_range_data == part.local_range_data &&
497 ghost_indices_data == part.ghost_indices_data);
503 Partitioner::is_globally_compatible(
const Partitioner &part)
const
515 memory += locally_owned_range_data.memory_consumption();
517 memory += ghost_indices_data.memory_consumption();
518 memory +=
sizeof(n_ghost_indices_data);
521 memory +=
sizeof(import_indices_plain_dev) +
522 sizeof(*import_indices_plain_dev.begin()) *
523 import_indices_plain_dev.capacity();
527 import_indices_chunks_by_rank_data);
531 ghost_indices_subset_chunks_by_rank_data);
544 Partitioner::initialize_import_indices_plain_dev()
const
546 const unsigned int n_import_targets = import_targets_data.size();
547 import_indices_plain_dev.reserve(n_import_targets);
548 for (
unsigned int i = 0; i < n_import_targets; ++i)
551 std::vector<std::pair<unsigned int, unsigned int>>::const_iterator
552 my_imports = import_indices_data.begin() +
553 import_indices_chunks_by_rank_data[i],
554 end_my_imports = import_indices_data.begin() +
555 import_indices_chunks_by_rank_data[i + 1];
556 std::vector<unsigned int> import_indices_plain_host;
557 for (; my_imports != end_my_imports; ++my_imports)
560 my_imports->second - my_imports->first;
562 import_indices_plain_host.push_back(my_imports->first + j);
566 const auto chunk_size = import_indices_plain_host.size();
567 import_indices_plain_dev.emplace_back(
"import_indices_plain_dev" +
570 Kokkos::deep_copy(import_indices_plain_dev.back(),
571 Kokkos::View<unsigned int *, Kokkos::HostSpace>(
572 import_indices_plain_host.data(),
chunk_size));
583 #include "partitioner.inst"
ArrayView< typename std::remove_reference< typename std::iterator_traits< Iterator >::reference >::type, MemorySpaceType > make_array_view(const Iterator begin, const Iterator end)
bool is_contiguous() const
size_type index_within_set(const size_type global_index) const
size_type n_elements() const
bool is_element(const size_type index) const
void set_size(const size_type size)
size_type nth_index_in_set(const size_type local_index) const
types::global_dof_index size_type
SymmetricTensor< rank, dim, Number > sum(const SymmetricTensor< rank, dim, Number > &local, const MPI_Comm &mpi_communicator)
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcDimensionMismatch(std::size_t arg1, std::size_t arg2)
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
std::enable_if_t< std::is_fundamental< T >::value, std::size_t > memory_consumption(const T &t)
unsigned int this_mpi_process(const MPI_Comm &mpi_communicator)
T min(const T &t, const MPI_Comm &mpi_communicator)
T sum(const T &t, const MPI_Comm &mpi_communicator)
const MPI_Datatype mpi_type_id_for_type
unsigned int n_mpi_processes(const MPI_Comm &mpi_communicator)
T max(const T &t, const MPI_Comm &mpi_communicator)
void reinit(MatrixBlock< MatrixType > &v, const BlockSparsityPattern &p)
static const unsigned int invalid_unsigned_int
unsigned int global_dof_index
#define DEAL_II_DOF_INDEX_MPI_TYPE