16#ifndef dealii_aligned_vector_h
17#define dealii_aligned_vector_h
29#include <boost/version.hpp>
30#if BOOST_VERSION >= 106400
31# include <boost/serialization/array_wrapper.hpp>
33# include <boost/serialization/array.hpp>
35#include <boost/serialization/split_member.hpp>
90 typename RandomAccessIterator,
91 typename = std::enable_if_t<std::is_convertible_v<
92 typename std::iterator_traits<RandomAccessIterator>::iterator_category,
93 std::random_access_iterator_tag>>>
251 template <
typename ForwardIterator>
264 typename RandomAccessIterator,
265 typename = std::enable_if_t<std::is_convertible_v<
266 typename std::iterator_traits<RandomAccessIterator>::iterator_category,
267 std::random_access_iterator_tag>>>
270 RandomAccessIterator
begin,
271 RandomAccessIterator
end);
385 const unsigned int root_process);
473 template <
class Archive>
475 save(Archive &ar,
const unsigned int version)
const;
482 template <
class Archive>
484 load(Archive &ar,
const unsigned int version);
492 template <
class Archive>
494 serialize(Archive &archive,
const unsigned int version);
498 BOOST_SERIALIZATION_SPLIT_MEMBER()
508 "Changing the vector after a call to "
509 "replicate_across_communicator() is not allowed.");
518 const std::size_t new_size,
519 const std::size_t new_allocated_size);
622#ifdef DEAL_II_WITH_MPI
631 const bool is_shmem_root,
632 T *aligned_shmem_pointer,
634 MPI_Win shmem_window);
676#ifdef DEAL_II_WITH_MPI
776 template <
typename RandomAccessIterator,
typename T>
781 160000 /
sizeof(T) + 1;
794 RandomAccessIterator source_end,
795 T *
const destination)
800 Assert(source_end == source_begin || destination !=
nullptr,
802 const std::size_t
size = source_end - source_begin;
815 const std::size_t end)
const override
821 if constexpr (std::is_trivially_copyable_v<T> ==
true &&
822 (std::is_same_v<T *, RandomAccessIterator> ||
823 std::is_same_v<const T *, RandomAccessIterator>) ==
true)
826 (end - begin) *
sizeof(T));
828 for (std::size_t i = begin; i < end; ++i)
844 template <
typename RandomAccessIterator,
typename T>
849 160000 /
sizeof(T) + 1;
862 RandomAccessIterator source_end,
863 T *
const destination)
868 Assert(source_end == source_begin || destination !=
nullptr,
870 const std::size_t
size = source_end - source_begin;
883 const std::size_t end)
const override
889 if constexpr (std::is_trivially_copyable_v<T> ==
true &&
890 (std::is_same_v<T *, RandomAccessIterator> ||
891 std::is_same_v<const T *, RandomAccessIterator>) ==
true)
894 (end - begin) *
sizeof(T));
898 for (std::size_t i = begin; i < end; ++i)
925 template <
typename T,
bool initialize_memory>
929 160000 /
sizeof(T) + 1;
938 T *
const destination)
950 if constexpr (std::is_trivially_default_constructible_v<T> ==
true &&
951 std::is_same_v<T, long double> ==
false)
953 const unsigned char zero[
sizeof(T)] = {};
954 if (std::memcmp(zero, &element,
sizeof(T)) == 0)
968 const std::size_t end)
const override
972 if constexpr (std::is_trivially_default_constructible_v<T> ==
true)
975 std::memset(
destination_ + begin, 0, (end - begin) *
sizeof(T));
981 std::bool_constant<initialize_memory>());
992 const std::size_t end,
993 std::bool_constant<false>)
const
995 for (std::size_t i = begin; i < end; ++i)
1002 const std::size_t end,
1003 std::bool_constant<true>)
const
1005 for (std::size_t i = begin; i < end; ++i)
1024 template <
typename T,
bool initialize_memory>
1029 160000 /
sizeof(T) + 1;
1054 const std::size_t end)
const override
1058 if constexpr (std::is_trivially_default_constructible_v<T> ==
true)
1059 std::memset(
destination_ + begin, 0, (end - begin) *
sizeof(T));
1063 std::bool_constant<initialize_memory>());
1072 const std::size_t end,
1073 std::bool_constant<false>)
const
1075 for (std::size_t i = begin; i < end; ++i)
1082 const std::size_t end,
1083 std::bool_constant<true>)
const
1085 for (std::size_t i = begin; i < end; ++i)
1097template <
typename T>
1099 : deleter_action_object(nullptr)
1100 , owning_aligned_vector(owning_object)
1104# ifdef DEAL_II_WITH_MPI
1106template <
typename T>
1108 const bool is_shmem_root,
1109 T *aligned_shmem_pointer,
1111 MPI_Win shmem_window)
1112 : deleter_action_object(
1113 std::make_unique<MPISharedMemDeleterAction>(is_shmem_root,
1114 aligned_shmem_pointer,
1115 shmem_group_communicator,
1117 , owning_aligned_vector(owning_object)
1122template <
typename T>
1128 if (deleter_action_object ==
nullptr)
1132 Assert(owning_aligned_vector->used_elements_end !=
nullptr,
1135 if (std::is_trivially_destructible_v<T> ==
false)
1136 for (T *p = owning_aligned_vector->used_elements_end - 1; p >= ptr;
1145 deleter_action_object->delete_array(owning_aligned_vector, ptr);
1150template <
typename T>
1155 owning_aligned_vector = new_aligned_vector_ptr;
1159# ifdef DEAL_II_WITH_MPI
1161template <
typename T>
1164 T *aligned_shmem_pointer,
1166 MPI_Win shmem_window)
1167 : is_shmem_root(is_shmem_root)
1168 , aligned_shmem_pointer(aligned_shmem_pointer)
1169 , shmem_group_communicator(shmem_group_communicator)
1170 , shmem_window(shmem_window)
1175template <
typename T>
1192 if (std::is_trivially_destructible_v<T> ==
false)
1197 ierr = MPI_Win_free(&shmem_window);
1208 : elements(nullptr, Deleter(this))
1209 , used_elements_end(nullptr)
1210 , allocated_elements_end(nullptr)
1211 , replicated_across_communicator(false)
1217template <
typename RandomAccessIterator,
typename>
1219 RandomAccessIterator end)
1220 : elements(nullptr, Deleter(this))
1221 , used_elements_end(nullptr)
1222 , allocated_elements_end(nullptr)
1223 , replicated_across_communicator(false)
1225 allocate_and_move(0u, end - begin, end - begin);
1226 used_elements_end = allocated_elements_end;
1235 : elements(nullptr, Deleter(this))
1236 , used_elements_end(nullptr)
1237 , allocated_elements_end(nullptr)
1238 , replicated_across_communicator(false)
1248 : elements(nullptr, Deleter(this))
1249 , used_elements_end(nullptr)
1250 , allocated_elements_end(nullptr)
1251 , replicated_across_communicator(false)
1254 reserve(vec.
size());
1255 used_elements_end = allocated_elements_end;
1268 *
this = std::move(vec);
1291 used_elements_end = elements.get() + new_size;
1310 elements = std::move(vec.
elements);
1311 elements.get_deleter().reset_owning_object(
this);
1333 else if (new_size == old_size)
1336 else if (new_size < old_size)
1341 if (std::is_trivially_destructible_v<T> ==
false)
1342 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1344 used_elements_end = elements.get() + new_size;
1350 used_elements_end = elements.get() + new_size;
1354 if (std::is_trivially_default_constructible_v<T> ==
false)
1356 new_size - old_size, elements.get() + old_size);
1370 else if (new_size == old_size)
1373 else if (new_size < old_size)
1378 if (std::is_trivially_destructible_v<T> ==
false)
1379 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1381 used_elements_end = elements.get() + new_size;
1387 used_elements_end = elements.get() + new_size;
1391 new_size - old_size, elements.get() + old_size);
1405 else if (new_size == old_size)
1408 else if (new_size < old_size)
1413 if (std::is_trivially_destructible_v<T> ==
false)
1414 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1416 used_elements_end = elements.get() + new_size;
1422 used_elements_end = elements.get() + new_size;
1426 new_size - old_size, init, elements.get() + old_size);
1435 const std::size_t new_size,
1436 const std::size_t new_allocated_size)
1443 new_size *
sizeof(T));
1450 Deleter deleter(
this);
1453 if (new_allocated_size > 0)
1455 elements.get(), elements.get() + old_size, new_data_ptr);
1466 elements =
decltype(elements)(new_data_ptr, std::move(deleter));
1467 used_elements_end = elements.get() + old_size;
1468 allocated_elements_end = elements.get() + new_size;
1477 const size_type old_size = used_elements_end - elements.get();
1478 const size_type old_allocated_size = allocated_elements_end - elements.get();
1479 if (new_allocated_size > old_allocated_size)
1485 std::max(new_allocated_size, 2 * old_allocated_size);
1487 allocate_and_move(old_size, new_size, new_allocated_size);
1489 else if (new_allocated_size == 0)
1504 Assert(replicated_across_communicator ==
false,
1505 ExcAlignedVectorChangeAfterReplication());
1507 const size_type used_size = used_elements_end - elements.get();
1508 const size_type allocated_size = allocated_elements_end - elements.get();
1509 if (allocated_size > used_size)
1510 allocate_and_move(used_size, used_size, used_size);
1527 used_elements_end =
nullptr;
1528 allocated_elements_end =
nullptr;
1538 if (used_elements_end == allocated_elements_end)
1540 new (used_elements_end++)
T(in_data);
1550 T *field = used_elements_end - 1;
1561 const T *field = used_elements_end - 1;
1568template <
typename ForwardIterator>
1573 reserve(old_size + (end - begin));
1575 new (used_elements_end)
T(*begin);
1581template <
typename RandomAccessIterator,
typename>
1584 RandomAccessIterator begin,
1585 RandomAccessIterator end)
1587 Assert(replicated_across_communicator ==
false,
1588 ExcAlignedVectorChangeAfterReplication());
1589 Assert(this->begin() <= position && position <= this->
end(),
1590 ExcMessage(
"The position iterator is not valid."));
1591 const auto offset = position - this->begin();
1595 const size_type new_size = old_size + range_size;
1596 if (range_size != 0)
1601 T *new_data_ptr =
nullptr;
1603 reinterpret_cast<void **
>(&new_data_ptr), 64, new_size *
sizeof(T));
1609 elements.get(), elements.get() + offset, new_data_ptr);
1611 temporary.begin(), temporary.end(), new_data_ptr + offset);
1613 elements.get() + offset,
1614 elements.get() + old_size,
1615 new_data_ptr + offset + range_size);
1617 Deleter deleter(
this);
1618 elements =
decltype(elements)(new_data_ptr, std::move(deleter));
1619 used_elements_end = elements.get() + new_size;
1620 allocated_elements_end = elements.get() + new_size;
1622 return this->begin() + offset;
1651 const unsigned int root_process)
1653# ifdef DEAL_II_WITH_MPI
1674 used_elements_end =
nullptr;
1675 allocated_elements_end =
nullptr;
1699 MPI_Comm shmem_group_communicator_temp;
1700 int ierr = MPI_Comm_split_type(communicator,
1701 MPI_COMM_TYPE_SHARED,
1704 &shmem_group_communicator_temp);
1709 ierr = MPI_Comm_split(shmem_group_communicator_temp,
1712 &shmem_group_communicator);
1723 const bool is_shmem_root =
1757 const int ierr = MPI_Comm_split(communicator,
1759 (is_shmem_root ? 0 : 1),
1761 &shmem_roots_communicator);
1770 const unsigned int shmem_roots_root_rank = 0;
1771 const bool is_shmem_roots_root =
1773 shmem_roots_communicator) == shmem_roots_root_rank));
1780 if (std::is_trivially_copyable_v<T> ==
true)
1792 shmem_roots_root_rank);
1793 if (is_shmem_roots_root ==
false)
1797 int ierr = MPI_Bcast(elements.get(),
1798 sizeof(T) * new_size,
1800 shmem_roots_root_rank,
1801 shmem_roots_communicator);
1818 shmem_roots_root_rank);
1822 shmem_roots_root_rank);
1858 MPI_Win shmem_window;
1860 const MPI_Aint align_by = 64;
1861 const MPI_Aint alloc_size =
1863 (
size() *
sizeof(T) + (align_by - 1)),
1870 ierr = MPI_Info_create(&mpi_info);
1872 ierr = MPI_Info_set(mpi_info,
1873 "mpi_minimum_memory_alignment",
1874 std::to_string(align_by).c_str());
1876 ierr = MPI_Win_allocate_shared((is_shmem_root ? alloc_size : 0),
1879 shmem_group_communicator,
1884 ierr = MPI_Info_free(&mpi_info);
1907 if (is_shmem_root ==
false)
1910 MPI_Aint alloc_size;
1911 const int ierr = MPI_Win_shared_query(
1912 shmem_window, MPI_PROC_NULL, &alloc_size, &disp_unit, &base_ptr);
1932 std::size_t available_space = alloc_size;
1933 void *base_ptr_backup = base_ptr;
1934 T *aligned_shmem_pointer =
static_cast<T *
>(
1935 std::align(align_by, new_size *
sizeof(T), base_ptr, available_space));
1965 aligned_shmem_pointer =
static_cast<T *
>(base_ptr_backup);
1973 if (std::is_trivially_copyable_v<T> ==
true)
1974 std::memcpy(aligned_shmem_pointer, elements.get(),
sizeof(T) *
size());
1976 for (std::size_t i = 0; i <
size(); ++i)
1977 new (&aligned_shmem_pointer[i])
T(std::move(elements[i]));
1982 const int ierr = MPI_Barrier(shmem_group_communicator);
2002 elements =
decltype(elements)(aligned_shmem_pointer,
2005 aligned_shmem_pointer,
2006 shmem_group_communicator,
2013 used_elements_end = elements.get() + new_size;
2014 allocated_elements_end = used_elements_end;
2021 replicated_across_communicator =
true;
2024 std::accumulate(packed_data.begin(), packed_data.end(),
int(0));
2045 elements.get_deleter().reset_owning_object(
this);
2046 vec.
elements.get_deleter().reset_owning_object(&vec);
2059 return used_elements_end == elements.get();
2068 return used_elements_end - elements.get();
2077 return allocated_elements_end - elements.get();
2087 return elements[
index];
2097 return elements[
index];
2102template <
typename T>
2106 return elements.get();
2111template <
typename T>
2115 return elements.get();
2124 return elements.get();
2133 return used_elements_end;
2142 return elements.get();
2151 return used_elements_end;
2157template <
class Archive>
2164 ar &boost::serialization::make_array(elements.get(), vec_size);
2170template <
class Archive>
2180 ar &boost::serialization::make_array(elements.get(), vec_size);
2181 used_elements_end = elements.get() + vec_size;
2192 for (
const T *t = elements.get(); t != used_elements_end; ++t)
2194 memory +=
sizeof(
T) * (allocated_elements_end - used_elements_end);
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
virtual ~DeleterActionBase()=default
virtual void delete_array(const AlignedVector< T > *owning_aligned_vector, T *ptr)=0
MPI_Comm shmem_group_communicator
virtual void delete_array(const AlignedVector< T > *aligned_vector, T *ptr)
MPISharedMemDeleterAction(const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
T * aligned_shmem_pointer
Deleter(AlignedVector< T > *owning_object, const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
Deleter(AlignedVector< T > *owning_object)
std::unique_ptr< DeleterActionBase > deleter_action_object
void reset_owning_object(const AlignedVector< T > *new_aligned_vector_ptr)
const AlignedVector< T > * owning_aligned_vector
void replicate_across_communicator(const MPI_Comm communicator, const unsigned int root_process)
size_type memory_consumption() const
void resize_fast(const size_type new_size)
std::unique_ptr< T[], Deleter > elements
reference operator[](const size_type index)
void fill(const T &element)
const_iterator end() const
void allocate_and_move(const std::size_t old_size, const std::size_t new_size, const std::size_t new_allocated_size)
AlignedVector(AlignedVector< T > &&vec) noexcept
void reserve(const size_type new_allocated_size)
void serialize(Archive &archive, const unsigned int version)
bool operator!=(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const_reference operator[](const size_type index) const
void resize(const size_type new_size, const T &init)
AlignedVector & operator=(AlignedVector< T > &&vec) noexcept
AlignedVector(RandomAccessIterator begin, RandomAccessIterator end)
size_type capacity() const
AlignedVector & operator=(const AlignedVector< T > &vec)
void swap(AlignedVector< T > &vec) noexcept
const value_type * const_pointer
bool replicated_across_communicator
void push_back(const T in_data)
iterator insert(const_iterator position, RandomAccessIterator begin, RandomAccessIterator end)
const_iterator begin() const
AlignedVector(const size_type size, const T &init=T())
T * allocated_elements_end
AlignedVector(const AlignedVector< T > &vec)
const_reference back() const
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const value_type * const_iterator
void resize(const size_type new_size)
void load(Archive &ar, const unsigned int version)
void save(Archive &ar, const unsigned int version) const
void insert_back(ForwardIterator begin, ForwardIterator end)
const value_type & const_reference
const_pointer data() const
AlignedVectorCopyConstruct(RandomAccessIterator source_begin, RandomAccessIterator source_end, T *const destination)
RandomAccessIterator source_
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
AlignedVectorDefaultInitialize(const std::size_t size, T *const destination)
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< false >) const
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< true >) const
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< false >) const
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< true >) const
AlignedVectorInitialize(const std::size_t size, const T &element, T *const destination)
RandomAccessIterator source_
AlignedVectorMoveConstruct(RandomAccessIterator source_begin, RandomAccessIterator source_end, T *const destination)
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
static const std::size_t minimum_parallel_grain_size
#define DEAL_II_NAMESPACE_OPEN
constexpr bool running_in_debug_mode()
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
#define DeclExceptionMsg(Exception, defaulttext)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
static ::ExceptionBase & ExcAlignedVectorChangeAfterReplication()
std::vector< index_type > data
types::global_dof_index size_type
std::enable_if_t< std::is_fundamental_v< T >, std::size_t > memory_consumption(const T &t)
VectorType::value_type * end(VectorType &V)
VectorType::value_type * begin(VectorType &V)
T max(const T &t, const MPI_Comm mpi_communicator)
T min(const T &t, const MPI_Comm mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
void free_communicator(MPI_Comm mpi_communicator)
T broadcast(const MPI_Comm comm, const T &object_to_send, const unsigned int root_process=0)
void posix_memalign(void **memptr, std::size_t alignment, std::size_t size)
std::size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
::VectorizedArray< Number, width > max(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
void apply_parallel(const std::size_t begin, const std::size_t end, const std::size_t minimum_parallel_grain_size) const