#include <deal.II/lac/la_parallel_vector.h>

Inheritance diagram for LinearAlgebra::distributed::Vector< Number, MemorySpace >:

Public Types
using	memory_space = MemorySpace

using	value_type = Number

using	pointer = value_type *

using	const_pointer = const value_type *

using	iterator = value_type *

using	const_iterator = const value_type *

using	reference = value_type &

using	const_reference = const value_type &

using	size_type = types::global_dof_index

using	real_type = typename numbers::NumberTraits< Number >::real_type

Public Member Functions
template<typename number >
Vector &	operator= (const ::Vector< number > &v)

virtual void	compress (VectorOperation::values)

1: Basic Object-handling
	Vector ()

	Vector (const Vector< Number, MemorySpace > &in_vector)

	Vector (const size_type size)

	Vector (const IndexSet &local_range, const IndexSet &ghost_indices, const MPI_Comm &communicator)

	Vector (const IndexSet &local_range, const MPI_Comm &communicator)

	Vector (const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner)

virtual	~Vector () override

void	reinit (const size_type size, const bool omit_zeroing_entries=false)

template<typename Number2 >
void	reinit (const Vector< Number2, MemorySpace > &in_vector, const bool omit_zeroing_entries=false)

void	reinit (const IndexSet &local_range, const IndexSet &ghost_indices, const MPI_Comm &communicator)

void	reinit (const IndexSet &local_range, const MPI_Comm &communicator)

void	reinit (const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner, const MPI_Comm &comm_sm=MPI_COMM_SELF)

void	reinit (const types::global_dof_index local_size, const types::global_dof_index ghost_size, const MPI_Comm &comm, const MPI_Comm &comm_sm=MPI_COMM_SELF)

void	swap (Vector< Number, MemorySpace > &v)

Vector< Number, MemorySpace > &	operator= (const Vector< Number, MemorySpace > &in_vector)

template<typename Number2 >
Vector< Number, MemorySpace > &	operator= (const Vector< Number2, MemorySpace > &in_vector)

2: Parallel data exchange
virtual void	compress (::VectorOperation::values operation) override

void	update_ghost_values () const

void	compress_start (const unsigned int communication_channel=0, ::VectorOperation::values operation=VectorOperation::add)

void	compress_finish (::VectorOperation::values operation)

void	update_ghost_values_start (const unsigned int communication_channel=0) const

void	update_ghost_values_finish () const

void	zero_out_ghosts () const

void	zero_out_ghost_values () const

bool	has_ghost_elements () const

template<typename Number2 >
void	copy_locally_owned_data_from (const Vector< Number2, MemorySpace > &src)

template<typename MemorySpace2 >
void	import (const Vector< Number, MemorySpace2 > &src, VectorOperation::values operation)

3: Implementation of VectorSpaceVector
virtual void	reinit (const VectorSpaceVector< Number > &V, const bool omit_zeroing_entries=false) override

virtual Vector< Number, MemorySpace > &	operator*= (const Number factor) override

virtual Vector< Number, MemorySpace > &	operator/= (const Number factor) override

virtual Vector< Number, MemorySpace > &	operator+= (const VectorSpaceVector< Number > &V) override

virtual Vector< Number, MemorySpace > &	operator-= (const VectorSpaceVector< Number > &V) override

virtual void	import (const LinearAlgebra::ReadWriteVector< Number > &V, VectorOperation::values operation, std::shared_ptr< const Utilities::MPI::CommunicationPatternBase > communication_pattern={}) override

virtual Number	operator* (const VectorSpaceVector< Number > &V) const override

virtual void	add (const Number a) override

virtual void	add (const Number a, const VectorSpaceVector< Number > &V) override

virtual void	add (const Number a, const VectorSpaceVector< Number > &V, const Number b, const VectorSpaceVector< Number > &W) override

virtual void	add (const std::vector< size_type > &indices, const std::vector< Number > &values)

virtual void	sadd (const Number s, const Number a, const VectorSpaceVector< Number > &V) override

virtual void	scale (const VectorSpaceVector< Number > &scaling_factors) override

virtual void	equ (const Number a, const VectorSpaceVector< Number > &V) override

virtual real_type	l1_norm () const override

virtual real_type	l2_norm () const override

real_type	norm_sqr () const

virtual real_type	linfty_norm () const override

virtual Number	add_and_dot (const Number a, const VectorSpaceVector< Number > &V, const VectorSpaceVector< Number > &W) override

virtual size_type	size () const override

virtual ::IndexSet	locally_owned_elements () const override

virtual void	print (std::ostream &out, const unsigned int precision=3, const bool scientific=true, const bool across=true) const override

virtual std::size_t	memory_consumption () const override

4: Other vector operations not included in VectorSpaceVector
virtual Vector< Number, MemorySpace > &	operator= (const Number s) override

template<typename OtherNumber >
void	add (const std::vector< size_type > &indices, const ::Vector< OtherNumber > &values)

template<typename OtherNumber >
void	add (const size_type n_elements, const size_type indices, const OtherNumber values)

void	sadd (const Number s, const Vector< Number, MemorySpace > &V)

5: Entry access and local data representation
size_type	local_size () const

size_type	locally_owned_size () const

bool	in_local_range (const size_type global_index) const

iterator	begin ()

const_iterator	begin () const

iterator	end ()

const_iterator	end () const

Number	operator() (const size_type global_index) const

Number &	operator() (const size_type global_index)

Number	operator[] (const size_type global_index) const

Number &	operator[] (const size_type global_index)

Number	local_element (const size_type local_index) const

Number &	local_element (const size_type local_index)

Number *	get_values () const

template<typename OtherNumber >
void	extract_subvector_to (const std::vector< size_type > &indices, std::vector< OtherNumber > &values) const

template<typename ForwardIterator , typename OutputIterator >
void	extract_subvector_to (ForwardIterator indices_begin, const ForwardIterator indices_end, OutputIterator values_begin) const

virtual bool	all_zero () const override

virtual Number	mean_value () const override

real_type	lp_norm (const real_type p) const

6: Mixed stuff
std::shared_ptr< const Utilities::MPI::Partitioner >	partitioner

size_type	allocated_size

mutable ::MemorySpace::MemorySpaceData< Number, MemorySpace >	data

std::shared_ptr<::parallel::internal::TBBPartitioner >	thread_loop_partitioner

mutable ::MemorySpace::MemorySpaceData< Number, MemorySpace >	import_data

bool	vector_is_ghosted

std::vector< MPI_Request >	compress_requests

std::vector< MPI_Request >	update_ghost_values_requests

std::mutex	mutex

MPI_Comm	comm_sm

template<typename Number2 , typename MemorySpace2 >
class	Vector

template<typename Number2 >
class	BlockVector

const MPI_Comm &	get_mpi_communicator () const

const std::shared_ptr< const Utilities::MPI::Partitioner > &	get_partitioner () const

bool	partitioners_are_compatible (const Utilities::MPI::Partitioner &part) const

bool	partitioners_are_globally_compatible (const Utilities::MPI::Partitioner &part) const

void	set_ghost_state (const bool ghosted) const

const std::vector< ArrayView< const Number > > &	shared_vector_data () const

static ::ExceptionBase &	ExcVectorTypeNotCompatible ()

static ::ExceptionBase &	ExcNotAllowedForCuda ()

static ::ExceptionBase &	ExcNonMatchingElements (Number arg1, Number arg2, unsigned int arg3)

static ::ExceptionBase &	ExcAccessToNonLocalElement (size_type arg1, size_type arg2, size_type arg3, size_type arg4)

void	add_local (const Number a, const VectorSpaceVector< Number > &V)

void	sadd_local (const Number s, const Number a, const VectorSpaceVector< Number > &V)

template<typename Number2 >
Number	inner_product_local (const Vector< Number2, MemorySpace > &V) const

real_type	norm_sqr_local () const

Number	mean_value_local () const

real_type	l1_norm_local () const

real_type	lp_norm_local (const real_type p) const

real_type	linfty_norm_local () const

Number	add_and_dot_local (const Number a, const Vector< Number, MemorySpace > &V, const Vector< Number, MemorySpace > &W)

void	clear_mpi_requests ()

void	resize_val (const size_type new_allocated_size, const MPI_Comm &comm_sm=MPI_COMM_SELF)

Subscriptor functionality
Classes derived from Subscriptor provide a facility to subscribe to this object. This is mostly used by the SmartPointer class.
std::atomic< unsigned int >	counter

std::map< std::string, unsigned int >	counter_map

std::vector< std::atomic< bool > * >	validity_pointers

const std::type_info *	object_info

void	subscribe (std::atomic< bool > *const validity, const std::string &identifier="") const

void	unsubscribe (std::atomic< bool > *const validity, const std::string &identifier="") const

unsigned int	n_subscriptions () const

template<typename StreamType >
void	list_subscribers (StreamType &stream) const

void	list_subscribers () const

template<class Archive >
void	serialize (Archive &ar, const unsigned int version)

static ::ExceptionBase &	ExcInUse (int arg1, std::string arg2, std::string arg3)

static ::ExceptionBase &	ExcNoSubscriber (std::string arg1, std::string arg2)

void	check_no_subscribers () const noexcept

using	map_value_type = decltype(counter_map)::value_type

using	map_iterator = decltype(counter_map)::iterator

Detailed Description

template<typename Number, typename MemorySpace = MemorySpace::Host>
class LinearAlgebra::distributed::Vector< Number, MemorySpace >

Implementation of a parallel vector class. The design of this class is similar to the standard Vector class in deal.II, with the exception that storage is distributed with MPI.

The vector is designed for the following scheme of parallel partitioning:

The indices held by individual processes (locally owned part) in the MPI parallelization form a contiguous range [my_first_index,my_last_index).
Ghost indices residing on arbitrary positions of other processors are allowed. It is in general more efficient if ghost indices are clustered, since they are stored as a set of intervals. The communication pattern of the ghost indices is determined when calling the function reinit (locally_owned, ghost_indices, communicator), and retained until the partitioning is changed. This allows for efficient parallel communication of indices. In particular, it stores the communication pattern, rather than having to compute it again for every communication. For more information on ghost vectors, see also the glossary entry on vectors with ghost elements.
Besides the usual global access operator() it is also possible to access vector entries in the local index space with the function local_element(). Locally owned indices are placed first, [0, locally_owned_size()), and then all ghost indices follow after them contiguously, [locally_owned_size(), locally_owned_size()+n_ghost_entries()).

Functions related to parallel functionality:

The function compress() goes through the data associated with ghost indices and communicates it to the owner process, which can then add it to the correct position. This can be used e.g. after having run an assembly routine involving ghosts that fill this vector. Note that the insert mode of compress() does not set the elements included in ghost entries but simply discards them, assuming that the owning processor has set them to the desired value already (See also the glossary entry on compress).
The update_ghost_values() function imports the data from the owning processor to the ghost indices in order to provide read access to the data associated with ghosts.
It is possible to split the above functions into two phases, where the first initiates the communication and the second one finishes it. These functions can be used to overlap communication with computations in other parts of the code.
Of course, reduction operations (like norms) make use of collective all-to-all MPI communications.

This vector can take two different states with respect to ghost elements:

After creation and whenever zero_out_ghost_values() is called (or operator= (0.)), the vector does only allow writing into ghost elements but not reading from ghost elements.
After a call to update_ghost_values(), the vector does not allow writing into ghost elements but only reading from them. This is to avoid undesired ghost data artifacts when calling compress() after modifying some vector entries. The current status of the ghost entries (read mode or write mode) can be queried by the method has_ghost_elements(), which returns true exactly when ghost elements have been updated and false otherwise, irrespective of the actual number of ghost entries in the vector layout (for that information, use n_ghost_entries() instead).

This vector uses the facilities of the class ::Vector<Number> for implementing the operations on the local range of the vector. In particular, it also inherits thread parallelism that splits most vector-vector operations into smaller chunks if the program uses multiple threads. This may or may not be desired when working also with MPI.

Limitations regarding the vector size

This vector class is based on two different number types for indexing. The so-called global index type encodes the overall size of the vector. Its type is types::global_dof_index. The largest possible value is 2^32-1 or approximately 4 billion in case 64 bit integers are disabled at configuration of deal.II (default case) or 2^64-1 or approximately 10^19 if 64 bit integers are enabled (see the glossary entry on When to use types::global_dof_index instead of unsigned int for further information).

The second relevant index type is the local index used within one MPI rank. As opposed to the global index, the implementation assumes 32-bit unsigned integers unconditionally. In other words, to actually use a vector with more than four billion entries, you need to use MPI with more than one rank (which in general is a safe assumption since four billion entries consume at least 16 GB of memory for floats or 32 GB of memory for doubles) and enable 64-bit indices. If more than 4 billion local elements are present, the implementation tries to detect that, which triggers an exception and aborts the code. Note, however, that the detection of overflow is tricky and the detection mechanism might fail in some circumstances. Therefore, it is strongly recommended to not rely on this class to automatically detect the unsupported case.

CUDA support

This vector class supports two different memory spaces: Host and CUDA. By default, the memory space is Host and all the data are allocated on the CPU. When the memory space is CUDA, all the data is allocated on the GPU. The operations on the vector are performed on the chosen memory space. * From the host, there are two methods to access the elements of the Vector when using the CUDA memory space:

use get_values():
Vector<double, MemorySpace::CUDA> vector(local_range, comm);

double* vector_dev = vector.get_values();

std::vector<double> vector_host(local_range.n_elements(), 1.);

Utilities::CUDA::copy_to_dev(vector_host, vector_dev);

LinearAlgebra::distributed::Vector
Definition: la_parallel_vector.h:250

Utilities::CUDA::copy_to_dev
void copy_to_dev(const ArrayView< const T, MemorySpace::Host > &in, ArrayView< T, MemorySpace::CUDA > &out)
Definition: cuda.h:148

comm
*braid_SplitCommworld & comm
Definition: parallel_in_time.h:1946
use import():
Vector<double, MemorySpace::CUDA> vector(local_range, comm);

ReadWriteVector<double> rw_vector(local_range);

for (auto & val : rw_vector)

val = 1.;

vector.import(rw_vector, VectorOperations::insert);

LinearAlgebra::ReadWriteVector
Definition: read_write_vector.h:134

VectorTools::EvaluationFlags::insert
@ insert
Definition: vector_tools_evaluate.h:63

The import method is a lot safer and will perform an MPI communication if necessary. Since an MPI communication may be performed, import needs to be called on all the processors.

Note: By default, all the ranks will try to access the device 0. This is fine is if you have one rank per node and one gpu per node. If you have multiple GPUs on one node, we need each process to access a different GPU. If each node has the same number of GPUs, this can be done as follows: int n_devices = 0; cudaGetDeviceCount(&n_devices); int device_id = my_rank % n_devices; cudaSetDevice(device_id);

MPI-3 shared-memory support

In Host mode, this class allows to use MPI-3 shared-memory features by providing a separate MPI communicator that consists of processes on the same shared-memory domain. By calling vector.shared_vector_data();, users have read-only access to both locally-owned and ghost values of processes combined in the shared-memory communicator (comm_sm in reinit()).

For this to work, you have to call the constructor or one of the reinit() functions of this class with a non-default value for the comm_sm argument, where the argument corresponds to a communicator consisting of all processes on the same shared-memory domain. This kind of communicator can be created using the following code snippet:

MPI_Comm comm_sm;
MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL,
                    &comm_sm);

See also: CUDAWrappers

Definition at line 248 of file la_parallel_vector.h.

Member Function Documentation

◆ operator=()

template<typename Number , typename MemorySpace = MemorySpace::Host>

template<typename number >

Vector & LinearAlgebra::distributed::Vector< Number, MemorySpace >::operator= ( const ::Vector< number > & v )

Definition at line 508 of file trilinos_vector.cc.

The documentation for this class was generated from the following files:

include/deal.II/lac/la_parallel_vector.h
source/lac/trilinos_vector.cc

Public Types

Public Member Functions