717 namespace ComputeIndexOwner
719 class FlexibleIndexStorage
722 using index_type =
unsigned int;
726 FlexibleIndexStorage(
const bool use_vector =
true);
729 reinit(
const bool use_vector,
730 const bool index_range_contiguous,
731 const std::size_t size);
734 fill(
const std::size_t start,
735 const std::size_t end,
736 const index_type &value);
739 operator[](
const std::size_t index);
742 operator[](
const std::size_t index)
const;
745 entry_has_been_set(
const std::size_t index)
const;
829 std::pair<types::global_dof_index, types::global_dof_index>
870 get_index_offset(
const unsigned int rank);
878 get_owning_rank_index(
const unsigned int rank_in_owned_indices,
879 const unsigned int guess = 0);
898 class ConsensusAlgorithmsPayload
901 using RequestType = std::vector<
902 std::pair<types::global_dof_index, types::global_dof_index>>;
903 using AnswerType = std::vector<unsigned int>;
908 ConsensusAlgorithmsPayload(
const IndexSet &owned_indices,
911 std::vector<unsigned int> &owning_ranks,
912 const bool track_index_requesters =
false);
970 std::vector<std::vector<
971 std::pair<
unsigned int,
981 std::map<
unsigned int,
982 std::pair<std::vector<types::global_dof_index>,
983 std::vector<unsigned int>>>
989 std::vector<unsigned int>
996 std::pair<types::global_dof_index, types::global_dof_index>>
997 create_request(
const unsigned int other_rank);
1002 std::vector<unsigned int>
1004 const unsigned int other_rank,
1013 process_answer(
const unsigned int other_rank,
1014 const std::vector<unsigned int> &recv_buffer);
1031 std::map<unsigned int, IndexSet>
1049 append_index_origin(
1051 const unsigned int rank_of_request,
1052 const unsigned int rank_of_owner,
1053 unsigned int &owner_index_guess);
1068 Dictionary::get_index_offset(
const unsigned int rank)
1080 Dictionary::get_owning_rank_index(
1081 const unsigned int rank_in_owned_indices,
1082 const unsigned int guess)
1091 rank_in_owned_indices);
1099 const FlexibleIndexStorage::index_type
1100 FlexibleIndexStorage::invalid_index_value;
1104 FlexibleIndexStorage::FlexibleIndexStorage(
const bool use_vector)
1112 FlexibleIndexStorage::reinit(
const bool use_vector,
1113 const bool index_range_contiguous,
1114 const std::size_t
size)
1131 FlexibleIndexStorage::fill(
1132 const std::size_t start,
1133 const std::size_t end,
1134 const FlexibleIndexStorage::index_type &value)
1141 if (
data.empty() && end > start)
1153 std::fill(
data.begin() + start,
1165 std::fill(
data.begin() + start,
data.begin() + end, value);
1170 for (
auto i = start; i <
end; ++i)
1177 FlexibleIndexStorage::index_type &
1178 FlexibleIndexStorage::operator[](
const std::size_t index)
1197 FlexibleIndexStorage::entry_has_been_set(
const std::size_t index)
const
1224 std::map<
unsigned int,
1229 const auto owned_indices_size_actual =
1234 owned_indices_size_actual ==
1247 std::pair<types::global_dof_index, types::global_dof_index>
1248 index_range(*interval->begin(), interval->last() + 1);
1252 while (index_range.first != index_range.second)
1254 Assert(index_range.first < index_range.second,
1257 const unsigned int owner =
1258 dof_to_dict_rank(index_range.first);
1263 std::min(get_index_offset(owner + 1), index_range.second);
1285 dic_local_received += next_index - index_range.first;
1290 buffers[owner].emplace_back(index_range.first, next_index);
1292 index_range.first = next_index;
1296#ifdef DEAL_II_WITH_MPI
1298 std::vector<MPI_Request> request;
1312 static CollectiveMutex mutex;
1313 CollectiveMutex::ScopedLock lock(mutex,
comm);
1320 for (
const auto &rank_pair : buffers)
1322 request.push_back(MPI_Request());
1324 MPI_Isend(rank_pair.second.data(),
1325 rank_pair.second.size() * 2,
1336 while (this->locally_owned_size != dic_local_received)
1340 int ierr = MPI_Probe(MPI_ANY_SOURCE, mpi_tag,
comm, &status);
1345 ierr = MPI_Get_count(&status,
1351 const auto other_rank = status.MPI_SOURCE;
1357 std::pair<types::global_dof_index, types::global_dof_index>>
1358 buffer(number_amount / 2);
1359 ierr = MPI_Recv(buffer.data(),
1369 for (
auto interval : buffer)
1375 i < interval.second;
1393 dic_local_received += interval.second - interval.first;
1405 using RequestType = std::vector<
1406 std::pair<types::global_dof_index, types::global_dof_index>>;
1408 ConsensusAlgorithms::selector<RequestType>(
1411 std::vector<unsigned int> targets;
1412 targets.reserve(buffers.size());
1413 for (
const auto &rank_pair : buffers)
1414 targets.emplace_back(rank_pair.
first);
1420 [&buffers](
const unsigned int target_rank) -> RequestType {
1421 return buffers.at(target_rank);
1425 [&](
const unsigned int source_rank,
1426 const RequestType &request) ->
void {
1428 for (
auto interval : request)
1434 i < interval.second;
1440 "Multiple processes seem to own the same global index. "
1441 "A possible reason is that the sets of locally owned "
1442 "indices are not distinct."));
1443 Assert(interval.first < interval.second,
1449 "The specified interval is not handled by the current process."));
1472 if (request.size() > 0)
1474 const int ierr = MPI_Waitall(request.size(),
1476 MPI_STATUSES_IGNORE);
1483 (void)dic_local_received;
1514 ConsensusAlgorithmsPayload::ConsensusAlgorithmsPayload(
1533 std::vector<unsigned int>
1534 ConsensusAlgorithmsPayload::compute_targets()
1536 std::vector<unsigned int> targets;
1539 unsigned int index = 0;
1540 unsigned int owner_index_guess = 0;
1543 unsigned int other_rank =
dict.dof_to_dict_rank(i);
1547 dict.actually_owning_ranks[i -
dict.local_range.first];
1549 append_index_origin(i -
dict.local_range.first,
1556 if (targets.empty() || targets.back() != other_rank)
1557 targets.push_back(other_rank);
1559 indices.first.push_back(i);
1560 indices.second.push_back(index);
1573 std::vector<std::pair<types::global_dof_index, types::global_dof_index>>
1574 ConsensusAlgorithmsPayload::create_request(
1575 const unsigned int other_rank)
1578 std::pair<types::global_dof_index, types::global_dof_index>>
1584 is.add_indices(indices_i.begin(), indices_i.end());
1587 for (
auto interval = is.begin_intervals();
1588 interval != is.end_intervals();
1590 send_buffer.emplace_back(*interval->begin(), interval->last() + 1);
1597 std::vector<unsigned int>
1598 ConsensusAlgorithmsPayload::answer_request(
1599 const unsigned int other_rank,
1603 std::vector<unsigned int> request_buffer;
1605 unsigned int owner_index_guess = 0;
1606 for (
const auto &interval : buffer_recv)
1607 for (auto i = interval.
first; i < interval.second; ++i)
1609 const unsigned int actual_owner =
1610 dict.actually_owning_ranks[i -
dict.local_range.first];
1611 request_buffer.push_back(actual_owner);
1614 append_index_origin(i -
dict.local_range.first,
1620 return request_buffer;
1626 ConsensusAlgorithmsPayload::process_answer(
1627 const unsigned int other_rank,
1628 const std::vector<unsigned int> &recv_buffer)
1630 const auto &recv_indices =
1633 for (
unsigned int j = 0; j < recv_indices.size(); ++j)
1639 std::map<unsigned int, IndexSet>
1640 ConsensusAlgorithmsPayload::get_requesters()
1643 ExcMessage(
"Must enable index range tracking in "
1644 "constructor of ConsensusAlgorithmProcess"));
1646 std::map<unsigned int, ::IndexSet> requested_indices;
1648#ifdef DEAL_II_WITH_MPI
1650 static CollectiveMutex mutex;
1651 CollectiveMutex::ScopedLock lock(mutex,
comm);
1660 std::vector<MPI_Request> send_requests;
1671 std::vector<std::vector<types::global_dof_index>> send_data(
1673 for (
unsigned int i = 0; i <
requesters.size(); ++i)
1682 IndexSet &my_index_set = requested_indices[j.first];
1684 for (
const auto &interval : j.
second)
1685 my_index_set.add_range(index_offset + interval.
first,
1686 index_offset + interval.
second);
1693 send_data[i].push_back(j.first);
1694 send_data[i].push_back(j.second.size());
1695 for (
const auto &interval : j.
second)
1697 send_data[i].push_back(interval.first);
1698 send_data[i].push_back(interval.second);
1701 send_requests.push_back(MPI_Request());
1703 MPI_Isend(send_data[i].
data(),
1704 send_data[i].
size(),
1707 dict.actually_owning_rank_list[i],
1710 &send_requests.back());
1716 for (
unsigned int c = 0; c <
dict.n_dict_procs_in_owned_indices; ++c)
1720 int ierr = MPI_Probe(MPI_ANY_SOURCE, mpi_tag,
comm, &status);
1725 ierr = MPI_Get_count(
1727 Utilities::MPI::mpi_type_id_for_type<types::global_dof_index>,
1734 std::pair<types::global_dof_index, types::global_dof_index>>
1735 buffer(number_amount / 2);
1739 Utilities::MPI::mpi_type_id_for_type<types::global_dof_index>,
1749 dict.get_index_offset(status.MPI_SOURCE);
1750 unsigned int offset = 0;
1751 while (offset < buffer.size())
1757 for (
unsigned int i = offset + 1;
1758 i < offset + buffer[offset].second + 1;
1760 my_index_set.
add_range(index_offset + buffer[i].first,
1761 index_offset + buffer[i].second);
1766 IndexSet &index_set = requested_indices[buffer[offset].first];
1767 if (index_set.
size() == 0)
1771 offset += buffer[offset].second + 1;
1776 if (send_requests.size() > 0)
1778 const auto ierr = MPI_Waitall(send_requests.size(),
1779 send_requests.data(),
1780 MPI_STATUSES_IGNORE);
1787 for (
const auto &it : requested_indices)
1793 "The indices requested from the current "
1794 "MPI rank should be locally owned here!"));
1800 return requested_indices;
1806 ConsensusAlgorithmsPayload::append_index_origin(
1808 const unsigned int rank_of_request,
1809 const unsigned int rank_of_owner,
1810 unsigned int &owner_index_guess)
1817 dict.get_owning_rank_index(rank_of_owner, owner_index_guess);
1819 auto &request =
requesters[owner_index_guess];
1820 if (request.empty() || request.back().first != rank_of_request)
1821 request.emplace_back(
1824 std::pair<types::global_dof_index, types::global_dof_index>>());
1826 auto &intervals = request.back().second;
1827 if (intervals.empty() || intervals.back().second != index_within_dict)
1828 intervals.emplace_back(index_within_dict, index_within_dict + 1);
1830 ++intervals.back().second;