18#include <deal.II/base/partitioner.templates.h>
20#include <boost/serialization/utility.hpp>
22#include <Kokkos_Core.hpp>
37 , n_ghost_indices_data(0)
38 , n_import_indices_data(0)
39 , n_ghost_indices_in_larger_set(0)
42 , communicator(MPI_COMM_SELF)
43 , have_ghost_indices(false)
48 Partitioner::Partitioner(
const unsigned int size)
50 , locally_owned_range_data(
size)
53 , n_ghost_indices_data(0)
54 , n_import_indices_data(0)
55 , n_ghost_indices_in_larger_set(0)
58 , communicator(MPI_COMM_SELF)
59 , have_ghost_indices(false)
61 locally_owned_range_data.add_range(0,
size);
62 locally_owned_range_data.compress();
63 ghost_indices_data.set_size(
size);
73 , locally_owned_range_data(global_size)
74 , local_range_data{0, local_size}
75 , n_ghost_indices_data(ghost_size)
76 , n_import_indices_data(0)
77 , n_ghost_indices_in_larger_set(0)
80 , communicator(communicator)
81 , have_ghost_indices(true)
85# ifdef DEAL_II_WITH_MPI
87 MPI_Exscan(&local_size,
96 local_range_data = {prefix_sum, prefix_sum + local_size};
98 locally_owned_range_data.add_range(prefix_sum, prefix_sum + local_size);
99 locally_owned_range_data.compress();
104 Partitioner::Partitioner(
const IndexSet &locally_owned_indices,
109 , n_ghost_indices_data(0)
110 , n_import_indices_data(0)
111 , n_ghost_indices_in_larger_set(0)
114 , communicator(communicator_in)
115 , have_ghost_indices(false)
117 set_owned_indices(locally_owned_indices);
118 set_ghost_indices(ghost_indices_in);
123 Partitioner::Partitioner(
const IndexSet &locally_owned_indices,
127 , n_ghost_indices_data(0)
128 , n_import_indices_data(0)
129 , n_ghost_indices_in_larger_set(0)
132 , communicator(communicator_in)
133 , have_ghost_indices(false)
135 set_owned_indices(locally_owned_indices);
141 Partitioner::reinit(
const IndexSet &locally_owned_indices,
145 have_ghost_indices =
false;
146 communicator = communicator_in;
147 set_owned_indices(locally_owned_indices);
148 set_ghost_indices(ghost_indices);
154 Partitioner::set_owned_indices(
const IndexSet &locally_owned_indices)
161 ExcMessage(
"The index set specified in locally_owned_indices "
162 "is not contiguous."));
166 std::pair<types::global_dof_index, types::global_dof_index>(
171 local_range_data.second - local_range_data.first <
173 std::numeric_limits<unsigned int>::max()),
175 "Index overflow: This class supports at most 2^32-1 locally owned vector entries"));
176 locally_owned_range_data.set_size(locally_owned_indices.
size());
177 locally_owned_range_data.add_range(local_range_data.first,
178 local_range_data.second);
179 locally_owned_range_data.compress();
181 ghost_indices_data.set_size(locally_owned_indices.
size());
187 Partitioner::set_ghost_indices(
const IndexSet &ghost_indices_in,
188 const IndexSet &larger_ghost_index_set)
194 ghost_indices_in.
size() == locally_owned_range_data.size(),
196 locally_owned_range_data.size()));
198 ghost_indices_data = ghost_indices_in;
199 if (ghost_indices_data.size() != locally_owned_range_data.size())
200 ghost_indices_data.
set_size(locally_owned_range_data.size());
201 ghost_indices_data.subtract_set(locally_owned_range_data);
202 ghost_indices_data.compress();
204 ghost_indices_data.n_elements() <
206 std::numeric_limits<unsigned int>::max()),
208 "Index overflow: This class supports at most 2^32-1 ghost elements"));
209 n_ghost_indices_data = ghost_indices_data.n_elements();
222# ifdef DEAL_II_WITH_MPI
237 my_size += local_range_data.first;
241 const int ierr = MPI_Exscan(
245 Utilities::MPI::mpi_type_id_for_type<types::global_dof_index>,
257 my_shift = local_range_data.first;
261 if (local_range_data.first == 0 && my_shift != 0)
265 local_range_data.first = my_shift;
266 local_range_data.second = my_shift + old_locally_owned_size;
269 const auto [owning_ranks_of_ghosts, import_data] =
271 locally_owned_range_data, ghost_indices_data, communicator);
275 ghost_targets_data = {};
277 if (owning_ranks_of_ghosts.size() > 0)
279 ghost_targets_data.emplace_back(owning_ranks_of_ghosts[0], 0);
280 for (
auto i : owning_ranks_of_ghosts)
282 Assert(i >= ghost_targets_data.back().first,
284 "Expect result of ConsensusAlgorithms::Process to be "
286 if (i == ghost_targets_data.back().first)
287 ghost_targets_data.back().second++;
289 ghost_targets_data.emplace_back(i, 1);
295 n_import_indices_data = 0;
296 import_targets_data = {};
297 import_targets_data.reserve(import_data.size());
298 import_indices_chunks_by_rank_data = {};
299 import_indices_chunks_by_rank_data.reserve(import_data.size());
300 import_indices_chunks_by_rank_data.resize(1);
301 for (
const auto &i : import_data)
302 if (i.
second.n_elements() > 0)
304 import_targets_data.emplace_back(i.first, i.second.n_elements());
305 n_import_indices_data += i.second.n_elements();
306 import_indices_chunks_by_rank_data.push_back(
307 import_indices_chunks_by_rank_data.back() +
308 i.second.n_intervals());
312 import_indices_data = {};
313 import_indices_data.reserve(import_indices_chunks_by_rank_data.back());
314 for (
const auto &i : import_data)
316 Assert((i.second & locally_owned_range_data) == i.second,
318 for (
auto interval = i.second.begin_intervals();
319 interval != i.second.end_intervals();
321 import_indices_data.emplace_back(*interval->begin() -
322 local_range_data.first,
323 interval->last() + 1 -
324 local_range_data.first);
346 const std::vector<types::global_dof_index> ghost_indices_ref =
347 ghost_indices_data.get_index_vector();
349 std::vector<types::global_dof_index> indices_to_send(
351 std::vector<types::global_dof_index> ghost_indices(n_ghost_indices());
353 const std::vector<types::global_dof_index> my_indices =
354 locally_owned_range_data.get_index_vector();
355 std::vector<MPI_Request> requests;
356 n_ghost_indices_in_larger_set = n_ghost_indices_data;
357 export_to_ghosted_array_start(
367 const int ierr = MPI_Testall(requests.size(),
370 MPI_STATUSES_IGNORE);
374 "MPI found unfinished requests. Check communication setup"));
376 for (
unsigned int i = 0; i < ghost_indices.size(); ++i)
382 if (larger_ghost_index_set.
size() == 0)
384 ghost_indices_subset_chunks_by_rank_data.clear();
385 ghost_indices_subset_data.emplace_back(0, n_ghost_indices());
386 n_ghost_indices_in_larger_set = n_ghost_indices_data;
391 ghost_indices_data.size());
393 (larger_ghost_index_set & locally_owned_range_data).n_elements() ==
395 ExcMessage(
"Ghost index set should not overlap with owned set."));
396 Assert((larger_ghost_index_set & ghost_indices_data) ==
398 ExcMessage(
"Larger ghost index set must contain the tight "
399 "ghost index set."));
401 n_ghost_indices_in_larger_set = larger_ghost_index_set.
n_elements();
405 std::vector<unsigned int> expanded_numbering;
406 for (const ::IndexSet::size_type index : ghost_indices_data)
409 ExcMessage(
"The given larger ghost index set must contain "
410 "all indices in the actual index set."));
414 std::numeric_limits<unsigned int>::max()),
416 "Index overflow: This class supports at most 2^32-1 ghost elements"));
417 expanded_numbering.push_back(
422 std::vector<std::pair<unsigned int, unsigned int>>
423 ghost_indices_subset;
424 ghost_indices_subset_chunks_by_rank_data.resize(
425 ghost_targets_data.size() + 1);
427 ghost_indices_subset_chunks_by_rank_data[0] = 0;
428 unsigned int shift = 0;
429 for (
unsigned int p = 0; p < ghost_targets_data.size(); ++p)
432 for (
unsigned int ii = 0; ii < ghost_targets_data[p].second; ++ii)
434 const unsigned int i =
shift + ii;
435 if (expanded_numbering[i] == last_index + 1)
437 ghost_indices_subset.back().second++;
440 ghost_indices_subset.emplace_back(expanded_numbering[i],
441 expanded_numbering[i] +
443 last_index = expanded_numbering[i];
445 shift += ghost_targets_data[p].second;
446 ghost_indices_subset_chunks_by_rank_data[p + 1] =
447 ghost_indices_subset.size();
449 ghost_indices_subset_data = ghost_indices_subset;
456 Partitioner::is_compatible(
const Partitioner &part)
const
462# ifdef DEAL_II_WITH_MPI
465 int communicators_same = 0;
466 const int ierr = MPI_Comm_compare(part.communicator,
468 &communicators_same);
470 if (!(communicators_same == MPI_IDENT ||
471 communicators_same == MPI_CONGRUENT))
475 return (global_size == part.global_size &&
476 local_range_data == part.local_range_data &&
477 ghost_indices_data == part.ghost_indices_data);
483 Partitioner::is_globally_compatible(
const Partitioner &part)
const
492 Partitioner::memory_consumption()
const
495 memory += locally_owned_range_data.memory_consumption();
497 memory += ghost_indices_data.memory_consumption();
498 memory +=
sizeof(n_ghost_indices_data);
501 memory +=
sizeof(import_indices_plain_dev) +
502 sizeof(*import_indices_plain_dev.begin()) *
503 import_indices_plain_dev.capacity();
507 import_indices_chunks_by_rank_data);
511 ghost_indices_subset_chunks_by_rank_data);
524 Partitioner::initialize_import_indices_plain_dev()
const
526 const unsigned int n_import_targets = import_targets_data.size();
527 import_indices_plain_dev.reserve(n_import_targets);
528 for (
unsigned int i = 0; i < n_import_targets; ++i)
531 std::vector<std::pair<unsigned int, unsigned int>>::const_iterator
532 my_imports = import_indices_data.begin() +
533 import_indices_chunks_by_rank_data[i],
534 end_my_imports = import_indices_data.begin() +
535 import_indices_chunks_by_rank_data[i + 1];
536 std::vector<unsigned int> import_indices_plain_host;
537 for (; my_imports != end_my_imports; ++my_imports)
539 const unsigned int chunk_size =
540 my_imports->second - my_imports->first;
541 for (
unsigned int j = 0; j < chunk_size; ++j)
542 import_indices_plain_host.push_back(my_imports->first + j);
546 const auto chunk_size = import_indices_plain_host.size();
547 import_indices_plain_dev.emplace_back(
"import_indices_plain_dev" +
550 Kokkos::deep_copy(import_indices_plain_dev.back(),
551 Kokkos::View<unsigned int *, Kokkos::HostSpace>(
552 import_indices_plain_host.data(), chunk_size));
563#include "base/partitioner.inst"
ArrayView< std::remove_reference_t< typename std::iterator_traits< Iterator >::reference >, MemorySpaceType > make_array_view(const Iterator begin, const Iterator end)
bool is_contiguous() const
size_type index_within_set(const size_type global_index) const
size_type n_elements() const
bool is_element(const size_type index) const
void set_size(const size_type size)
size_type nth_index_in_set(const size_type local_index) const
#define DEAL_II_NAMESPACE_OPEN
constexpr bool running_in_debug_mode()
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcDimensionMismatch(std::size_t arg1, std::size_t arg2)
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
types::global_dof_index locally_owned_size
const unsigned int n_procs
std::enable_if_t< std::is_fundamental_v< T >, std::size_t > memory_consumption(const T &t)
std::pair< std::vector< unsigned int >, std::map< unsigned int, IndexSet > > compute_index_owner_and_requesters(const IndexSet &owned_indices, const IndexSet &indices_to_look_up, const MPI_Comm &comm)
T sum(const T &t, const MPI_Comm mpi_communicator)
unsigned int n_mpi_processes(const MPI_Comm mpi_communicator)
T max(const T &t, const MPI_Comm mpi_communicator)
T min(const T &t, const MPI_Comm mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
const MPI_Datatype mpi_type_id_for_type
constexpr unsigned int invalid_unsigned_int
unsigned int global_dof_index