mpi.cc File Reference

#include <deal.II/base/exceptions.h>
#include <deal.II/base/index_set.h>
#include <deal.II/base/mpi.h>
#include <deal.II/base/mpi.templates.h>
#include <deal.II/base/mpi_consensus_algorithms.h>
#include <deal.II/base/mpi_large_count.h>
#include <deal.II/base/mpi_tags.h>
#include <deal.II/base/utilities.h>
#include <deal.II/lac/full_matrix.h>
#include <deal.II/lac/lapack_full_matrix.h>
#include <boost/serialization/utility.hpp>
#include <iostream>
#include <limits>
#include <numeric>
#include <set>
#include <vector>
#include "mpi.inst"

Go to the source code of this file.

Namespaces
namespace	Utilities
namespace	Utilities::MPI
namespace	Utilities::MPI::ComputeIndexOwner
namespace	Utilities::MPI::internal
namespace	Utilities::MPI::internal::CollectiveMutexImplementation

Functions
IndexSet	Utilities::create_evenly_distributed_partitioning (const unsigned int my_partition_id, const unsigned int n_partitions, const types::global_dof_index total_size)
MinMaxAvg	Utilities::MPI::min_max_avg (const double my_value, const MPI_Comm mpi_communicator)
std::vector< MinMaxAvg >	Utilities::MPI::min_max_avg (const std::vector< double > &my_value, const MPI_Comm mpi_communicator)
unsigned int	Utilities::MPI::n_mpi_processes (const MPI_Comm mpi_communicator)
unsigned int	Utilities::MPI::this_mpi_process (const MPI_Comm mpi_communicator)
std::vector< unsigned int >	Utilities::MPI::mpi_processes_within_communicator (const MPI_Comm comm_large, const MPI_Comm comm_small)
MPI_Comm	Utilities::MPI::duplicate_communicator (const MPI_Comm mpi_communicator)
void	Utilities::MPI::free_communicator (MPI_Comm mpi_communicator)
std::vector< IndexSet >	Utilities::MPI::create_ascending_partitioning (const MPI_Comm comm, const types::global_dof_index locally_owned_size)
IndexSet	Utilities::MPI::create_evenly_distributed_partitioning (const MPI_Comm comm, const types::global_dof_index total_size)
std::unique_ptr< MPI_Datatype, void(*)(MPI_Datatype *)>	Utilities::MPI::create_mpi_data_type_n_bytes (const std::size_t n_bytes)
std::vector< unsigned int >	Utilities::MPI::compute_point_to_point_communication_pattern (const MPI_Comm mpi_comm, const std::vector< unsigned int > &destinations)
unsigned int	Utilities::MPI::compute_n_point_to_point_communications (const MPI_Comm mpi_comm, const std::vector< unsigned int > &destinations)
void	Utilities::MPI::min_max_avg (const ArrayView< const double > &my_values, const ArrayView< MinMaxAvg > &result, const MPI_Comm mpi_communicator)
bool	Utilities::MPI::job_supports_mpi ()
std::vector< unsigned int >	Utilities::MPI::compute_index_owner (const IndexSet &owned_indices, const IndexSet &indices_to_look_up, const MPI_Comm comm)
std::pair< std::vector< unsigned int >, std::map< unsigned int, IndexSet > >	Utilities::MPI::compute_index_owner_and_requesters (const IndexSet &owned_indices, const IndexSet &indices_to_look_up, const MPI_Comm &comm)
void	Utilities::MPI::internal::CollectiveMutexImplementation::check_exception ()

Variable Documentation

invalid_index_value

const index_type invalid_index_value

static

Initial value:

=
            numbers::invalid_unsigned_int

Definition at line 708 of file mpi.cc.

use_vector

bool use_vector

private

Definition at line 733 of file mpi.cc.

size

types::global_dof_index size

private

The global size of the index space.

Definition at line 734 of file mpi.cc.

data

std::vector<index_type> data

private

Definition at line 735 of file mpi.cc.

data_map

std::map<std::size_t, index_type> data_map

private

Definition at line 736 of file mpi.cc.

range_minimum_grain_size

constexpr unsigned int range_minimum_grain_size = 64

staticconstexpr

The minimum grain size for the intervals.

We choose to limit the smallest size an interval for the two-stage lookup can have with the following two conflicting goals in mind: On the one hand, we do not want intervals in the dictionary to become too short. For uneven distributions of unknowns (some ranks with several thousands of unknowns, others with none), the lookup DoFs -> dictionary then involves sending from one MPI rank to many other MPI ranks holding dictionary intervals, leading to an exceedingly high number of messages some ranks have to send. Also, fewer longer intervals are generally more efficient to look up. On the other hand, a range size too large leads to opposite effect of many messages that come into a particular dictionary owner in the lookup DoFs -> dictionary. With the current setting, we get at most 64 messages coming to a single MPI rank in case there is 1 dof per MPI rank, which is reasonably low. At the same time, uneven distributions up to factors of 4096 can be handled with at most 64 messages as well.

Definition at line 769 of file mpi.cc.

sparsity_factor

constexpr unsigned int sparsity_factor = 4

staticconstexpr

Factor that determines if an index set is sparse or not. An index set if sparse if less than 25% of the indices are owned by any process. If the index set is sparse, we switch the internal storage from a fast storage (vector) to a memory-efficient storage (map).

Definition at line 777 of file mpi.cc.

actually_owning_ranks

FlexibleIndexStorage actually_owning_ranks

A vector with as many entries as there are dofs in the dictionary of the current process, and each entry containing the rank of the owner of that dof in the IndexSet owned_indices. This is queried in the index lookup, so we keep an expanded list.

Definition at line 793 of file mpi.cc.

actually_owning_rank_list

std::vector<unsigned int> actually_owning_rank_list

A sorted vector containing the MPI ranks appearing in actually_owning_ranks.

Definition at line 799 of file mpi.cc.

dofs_per_process

types::global_dof_index dofs_per_process

The number of unknowns in the dictionary for on each MPI rank used for the index space splitting. For simplicity of index lookup without additional communication, this number is the same on all MPI ranks.

Definition at line 807 of file mpi.cc.

local_range

std::pair<types::global_dof_index, types::global_dof_index> local_range

The local range of the global index space that is represented in the dictionary, computed from dofs_per_process, the current MPI rank, and range_minimum_grain_size.

Definition at line 815 of file mpi.cc.

locally_owned_size

types::global_dof_index locally_owned_size

The actual size, computed as the minimum of dofs_per_process and the possible end of the index space. Equivalent to local_range.second - local_range.first.

Definition at line 822 of file mpi.cc.

n_dict_procs_in_owned_indices

unsigned int n_dict_procs_in_owned_indices

The number of ranks the owned_indices IndexSet is distributed among.

Definition at line 833 of file mpi.cc.

stride_small_size

unsigned int stride_small_size

A stride to distribute the work more evenly over MPI ranks in case the grain size forces us to have fewer ranges than we have processes.

Definition at line 840 of file mpi.cc.

owned_indices

const IndexSet& owned_indices

The index space which describes the locally owned space.

Definition at line 902 of file mpi.cc.

indices_to_look_up

const IndexSet& indices_to_look_up

The indices which are "ghosts" on a given rank and should be looked up in terms of their owner rank from owned_indices.

Definition at line 908 of file mpi.cc.

comm

const MPI_Comm comm

The underlying MPI communicator.

Definition at line 913 of file mpi.cc.

my_rank

const unsigned int my_rank

The present MPI rank.

Definition at line 918 of file mpi.cc.

n_procs

const unsigned int n_procs

The total number of ranks participating in the MPI communicator comm.

Definition at line 924 of file mpi.cc.

track_index_requesters

const bool track_index_requesters

Controls whether we should record a list of ranks who sent requests to the present MPI process when looking up their remote indices, and what those indices were. If true, it will be added into requesters and can be queried by get_requesters().

Definition at line 932 of file mpi.cc.

owning_ranks

std::vector<unsigned int>& owning_ranks

The result of the index owner computation: To each index contained in indices_to_look_up, this vector contains the MPI rank of the owner in owned_indices.

Definition at line 939 of file mpi.cc.

dict

Dictionary dict

The dictionary handling the requests.

Definition at line 944 of file mpi.cc.

requesters

std::vector<std::vector< std::pair<unsigned int, std::vector<std::pair<types::global_dof_index, types::global_dof_index> > > > > requesters

Keeps track of the origin of the requests. The layout of the data structure is as follows: The outermost vector has as many entries as Dictionary::actually_owning_rank_list and represents the information we should send back to the owners from the present dictionary entry. The second vector then collects a list of MPI ranks that have requested data, using the rank in the first pair entry and a list of index ranges as the second entry.

Definition at line 959 of file mpi.cc.

indices_to_look_up_by_dict_rank

std::map<unsigned int, std::pair<std::vector<types::global_dof_index>, std::vector<unsigned int> > > indices_to_look_up_by_dict_rank

Array to collect the indices to look up (first vector) and their local index among indices (second vector), sorted by the rank in the dictionary.

Definition at line 969 of file mpi.cc.