16#ifndef dealii_aligned_vector_h
17#define dealii_aligned_vector_h
29#include <boost/version.hpp>
30#if BOOST_VERSION >= 106400
31# include <boost/serialization/array_wrapper.hpp>
33# include <boost/serialization/array.hpp>
35#include <boost/serialization/split_member.hpp>
90 typename RandomAccessIterator,
91 typename = std::enable_if_t<std::is_convertible_v<
92 typename std::iterator_traits<RandomAccessIterator>::iterator_category,
93 std::random_access_iterator_tag>>>
251 template <
typename ForwardIterator>
264 typename RandomAccessIterator,
265 typename = std::enable_if_t<std::is_convertible_v<
266 typename std::iterator_traits<RandomAccessIterator>::iterator_category,
267 std::random_access_iterator_tag>>>
270 RandomAccessIterator
begin,
271 RandomAccessIterator
end);
385 const unsigned int root_process);
473 template <
class Archive>
475 save(Archive &ar,
const unsigned int version)
const;
482 template <
class Archive>
484 load(Archive &ar,
const unsigned int version);
492 template <
class Archive>
494 serialize(Archive &archive,
const unsigned int version);
498 BOOST_SERIALIZATION_SPLIT_MEMBER()
508 "Changing the vector after a call to "
509 "replicate_across_communicator() is not allowed.");
518 const size_t new_size,
519 const size_t new_allocated_size);
622#ifdef DEAL_II_WITH_MPI
631 const bool is_shmem_root,
632 T *aligned_shmem_pointer,
634 MPI_Win shmem_window);
676#ifdef DEAL_II_WITH_MPI
776 template <
typename RandomAccessIterator,
typename T>
781 160000 /
sizeof(T) + 1;
794 RandomAccessIterator source_end,
795 T *
const destination)
800 Assert(source_end == source_begin || destination !=
nullptr,
802 const std::size_t
size = source_end - source_begin;
815 const std::size_t end)
const override
821 if constexpr (std::is_trivially_copyable_v<T> ==
true &&
822 (std::is_same_v<T *, RandomAccessIterator> ||
823 std::is_same_v<const T *, RandomAccessIterator>) ==
true)
826 (end - begin) *
sizeof(T));
828 for (std::size_t i = begin; i < end; ++i)
844 template <
typename RandomAccessIterator,
typename T>
849 160000 /
sizeof(T) + 1;
862 RandomAccessIterator source_end,
863 T *
const destination)
868 Assert(source_end == source_begin || destination !=
nullptr,
870 const std::size_t
size = source_end - source_begin;
883 const std::size_t end)
const override
889 if constexpr (std::is_trivially_copyable_v<T> ==
true &&
890 (std::is_same_v<T *, RandomAccessIterator> ||
891 std::is_same_v<const T *, RandomAccessIterator>) ==
true)
894 (end - begin) *
sizeof(T));
898 for (std::size_t i = begin; i < end; ++i)
925 template <
typename T,
bool initialize_memory>
929 160000 /
sizeof(T) + 1;
938 T *
const destination)
950 if constexpr (std::is_trivially_default_constructible_v<T> ==
true &&
951 std::is_same_v<T, long double> ==
false)
953 const unsigned char zero[
sizeof(T)] = {};
954 if (std::memcmp(zero, &element,
sizeof(T)) == 0)
968 const std::size_t end)
const override
972 if constexpr (std::is_trivially_default_constructible_v<T> ==
true)
975 std::memset(
destination_ + begin, 0, (end - begin) *
sizeof(T));
981 std::bool_constant<initialize_memory>());
992 const std::size_t end,
993 std::bool_constant<false>)
const
995 for (std::size_t i = begin; i < end; ++i)
1002 const std::size_t end,
1003 std::bool_constant<true>)
const
1005 for (std::size_t i = begin; i < end; ++i)
1024 template <
typename T,
bool initialize_memory>
1029 160000 /
sizeof(T) + 1;
1054 const std::size_t end)
const override
1058 if constexpr (std::is_trivially_default_constructible_v<T> ==
true)
1059 std::memset(
destination_ + begin, 0, (end - begin) *
sizeof(T));
1063 std::bool_constant<initialize_memory>());
1072 const std::size_t end,
1073 std::bool_constant<false>)
const
1075 for (std::size_t i = begin; i < end; ++i)
1082 const std::size_t end,
1083 std::bool_constant<true>)
const
1085 for (std::size_t i = begin; i < end; ++i)
1097template <
typename T>
1099 : deleter_action_object(nullptr)
1100 , owning_aligned_vector(owning_object)
1104# ifdef DEAL_II_WITH_MPI
1106template <
typename T>
1108 const bool is_shmem_root,
1109 T *aligned_shmem_pointer,
1111 MPI_Win shmem_window)
1112 : deleter_action_object(
1113 std::make_unique<MPISharedMemDeleterAction>(is_shmem_root,
1114 aligned_shmem_pointer,
1115 shmem_group_communicator,
1117 , owning_aligned_vector(owning_object)
1122template <
typename T>
1128 if (deleter_action_object ==
nullptr)
1132 Assert(owning_aligned_vector->used_elements_end !=
nullptr,
1135 if (std::is_trivially_destructible_v<T> ==
false)
1136 for (T *p = owning_aligned_vector->used_elements_end - 1; p >= ptr;
1145 deleter_action_object->delete_array(owning_aligned_vector, ptr);
1150template <
typename T>
1155 owning_aligned_vector = new_aligned_vector_ptr;
1159# ifdef DEAL_II_WITH_MPI
1161template <
typename T>
1164 T *aligned_shmem_pointer,
1166 MPI_Win shmem_window)
1167 : is_shmem_root(is_shmem_root)
1168 , aligned_shmem_pointer(aligned_shmem_pointer)
1169 , shmem_group_communicator(shmem_group_communicator)
1170 , shmem_window(shmem_window)
1175template <
typename T>
1192 if (std::is_trivially_destructible_v<T> ==
false)
1197 ierr = MPI_Win_free(&shmem_window);
1208 : elements(nullptr, Deleter(this))
1209 , used_elements_end(nullptr)
1210 , allocated_elements_end(nullptr)
1211 , replicated_across_communicator(false)
1217template <
typename RandomAccessIterator,
typename>
1219 RandomAccessIterator end)
1220 : elements(nullptr, Deleter(this))
1221 , used_elements_end(nullptr)
1222 , allocated_elements_end(nullptr)
1223 , replicated_across_communicator(false)
1225 allocate_and_move(0u, end - begin, end - begin);
1226 used_elements_end = allocated_elements_end;
1235 : elements(nullptr, Deleter(this))
1236 , used_elements_end(nullptr)
1237 , allocated_elements_end(nullptr)
1238 , replicated_across_communicator(false)
1248 : elements(nullptr, Deleter(this))
1249 , used_elements_end(nullptr)
1250 , allocated_elements_end(nullptr)
1251 , replicated_across_communicator(false)
1254 reserve(vec.
size());
1255 used_elements_end = allocated_elements_end;
1268 *
this = std::move(vec);
1291 used_elements_end = elements.get() + new_size;
1310 elements = std::move(vec.
elements);
1311 elements.get_deleter().reset_owning_object(
this);
1333 else if (new_size == old_size)
1336 else if (new_size < old_size)
1341 if (std::is_trivially_destructible_v<T> ==
false)
1342 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1344 used_elements_end = elements.get() + new_size;
1350 used_elements_end = elements.get() + new_size;
1354 if (std::is_trivially_default_constructible_v<T> ==
false)
1356 new_size - old_size, elements.get() + old_size);
1370 else if (new_size == old_size)
1373 else if (new_size < old_size)
1378 if (std::is_trivially_destructible_v<T> ==
false)
1379 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1381 used_elements_end = elements.get() + new_size;
1387 used_elements_end = elements.get() + new_size;
1391 new_size - old_size, elements.get() + old_size);
1405 else if (new_size == old_size)
1408 else if (new_size < old_size)
1413 if (std::is_trivially_destructible_v<T> ==
false)
1414 for (T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1416 used_elements_end = elements.get() + new_size;
1422 used_elements_end = elements.get() + new_size;
1426 new_size - old_size, init, elements.get() + old_size);
1435 const size_t new_size,
1436 const size_t new_allocated_size)
1443 new_size *
sizeof(T));
1450 Deleter deleter(
this);
1453 if (new_allocated_size > 0)
1455 elements.get(), elements.get() + old_size, new_data_ptr);
1466 elements =
decltype(elements)(new_data_ptr, std::move(deleter));
1467 used_elements_end = elements.get() + old_size;
1468 allocated_elements_end = elements.get() + new_size;
1477 const size_type old_size = used_elements_end - elements.get();
1478 const size_type old_allocated_size = allocated_elements_end - elements.get();
1479 if (new_allocated_size > old_allocated_size)
1485 std::max(new_allocated_size, 2 * old_allocated_size);
1487 allocate_and_move(old_size, new_size, new_allocated_size);
1489 else if (new_allocated_size == 0)
1503 Assert(replicated_across_communicator ==
false,
1504 ExcAlignedVectorChangeAfterReplication());
1506 const size_type used_size = used_elements_end - elements.get();
1507 const size_type allocated_size = allocated_elements_end - elements.get();
1508 if (allocated_size > used_size)
1509 allocate_and_move(used_size, used_size, used_size);
1526 used_elements_end =
nullptr;
1527 allocated_elements_end =
nullptr;
1537 if (used_elements_end == allocated_elements_end)
1539 new (used_elements_end++) T(in_data);
1549 T *field = used_elements_end - 1;
1560 const T *field = used_elements_end - 1;
1567template <
typename ForwardIterator>
1572 reserve(old_size + (end - begin));
1574 new (used_elements_end) T(*begin);
1580template <
typename RandomAccessIterator,
typename>
1583 RandomAccessIterator begin,
1584 RandomAccessIterator end)
1586 Assert(replicated_across_communicator ==
false,
1587 ExcAlignedVectorChangeAfterReplication());
1588 Assert(this->begin() <= position && position <= this->
end(),
1589 ExcMessage(
"The position iterator is not valid."));
1590 const auto offset = position - this->begin();
1594 const size_type new_size = old_size + range_size;
1595 if (range_size != 0)
1600 T *new_data_ptr =
nullptr;
1602 reinterpret_cast<void **
>(&new_data_ptr), 64, new_size *
sizeof(T));
1608 elements.get(), elements.get() + offset, new_data_ptr);
1610 temporary.begin(), temporary.end(), new_data_ptr + offset);
1612 elements.get() + offset,
1613 elements.get() + old_size,
1614 new_data_ptr + offset + range_size);
1616 Deleter deleter(
this);
1617 elements =
decltype(elements)(new_data_ptr, std::move(deleter));
1618 used_elements_end = elements.get() + new_size;
1619 allocated_elements_end = elements.get() + new_size;
1621 return this->begin() + offset;
1650 const unsigned int root_process)
1652# ifdef DEAL_II_WITH_MPI
1673 used_elements_end =
nullptr;
1674 allocated_elements_end =
nullptr;
1698 MPI_Comm shmem_group_communicator_temp;
1699 int ierr = MPI_Comm_split_type(communicator,
1700 MPI_COMM_TYPE_SHARED,
1703 &shmem_group_communicator_temp);
1708 ierr = MPI_Comm_split(shmem_group_communicator_temp,
1711 &shmem_group_communicator);
1722 const bool is_shmem_root =
1756 const int ierr = MPI_Comm_split(communicator,
1758 (is_shmem_root ? 0 : 1),
1760 &shmem_roots_communicator);
1769 const unsigned int shmem_roots_root_rank = 0;
1770 const bool is_shmem_roots_root =
1772 shmem_roots_communicator) == shmem_roots_root_rank));
1779 if (std::is_trivially_copyable_v<T> ==
true)
1791 shmem_roots_root_rank);
1792 if (is_shmem_roots_root ==
false)
1796 int ierr = MPI_Bcast(elements.get(),
1797 sizeof(T) * new_size,
1799 shmem_roots_root_rank,
1800 shmem_roots_communicator);
1817 shmem_roots_root_rank);
1821 shmem_roots_root_rank);
1857 MPI_Win shmem_window;
1859 const MPI_Aint align_by = 64;
1860 const MPI_Aint alloc_size =
1862 (
size() *
sizeof(T) + (align_by - 1)),
1869 ierr = MPI_Info_create(&mpi_info);
1871 ierr = MPI_Info_set(mpi_info,
1872 "mpi_minimum_memory_alignment",
1873 std::to_string(align_by).c_str());
1875 ierr = MPI_Win_allocate_shared((is_shmem_root ? alloc_size : 0),
1878 shmem_group_communicator,
1883 ierr = MPI_Info_free(&mpi_info);
1906 if (is_shmem_root ==
false)
1909 MPI_Aint alloc_size;
1910 const int ierr = MPI_Win_shared_query(
1911 shmem_window, MPI_PROC_NULL, &alloc_size, &disp_unit, &base_ptr);
1931 std::size_t available_space = alloc_size;
1932 void *base_ptr_backup = base_ptr;
1933 T *aligned_shmem_pointer =
static_cast<T *
>(
1934 std::align(align_by, new_size *
sizeof(T), base_ptr, available_space));
1964 aligned_shmem_pointer =
static_cast<T *
>(base_ptr_backup);
1972 if (std::is_trivially_copyable_v<T> ==
true)
1973 std::memcpy(aligned_shmem_pointer, elements.get(),
sizeof(T) *
size());
1975 for (std::size_t i = 0; i <
size(); ++i)
1976 new (&aligned_shmem_pointer[i]) T(std::move(elements[i]));
1981 const int ierr = MPI_Barrier(shmem_group_communicator);
2001 elements =
decltype(elements)(aligned_shmem_pointer,
2004 aligned_shmem_pointer,
2005 shmem_group_communicator,
2012 used_elements_end = elements.get() + new_size;
2013 allocated_elements_end = used_elements_end;
2019 replicated_across_communicator =
true;
2022 std::accumulate(packed_data.begin(), packed_data.end(),
int(0));
2042 elements.get_deleter().reset_owning_object(
this);
2043 vec.
elements.get_deleter().reset_owning_object(&vec);
2056 return used_elements_end == elements.get();
2065 return used_elements_end - elements.get();
2074 return allocated_elements_end - elements.get();
2084 return elements[
index];
2094 return elements[
index];
2099template <
typename T>
2103 return elements.get();
2108template <
typename T>
2112 return elements.get();
2121 return elements.get();
2130 return used_elements_end;
2139 return elements.get();
2148 return used_elements_end;
2154template <
class Archive>
2161 ar &boost::serialization::make_array(elements.get(), vec_size);
2167template <
class Archive>
2177 ar &boost::serialization::make_array(elements.get(), vec_size);
2178 used_elements_end = elements.get() + vec_size;
2189 for (
const T *t = elements.get(); t != used_elements_end; ++t)
2191 memory +=
sizeof(T) * (allocated_elements_end - used_elements_end);
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
virtual ~DeleterActionBase()=default
virtual void delete_array(const AlignedVector< T > *owning_aligned_vector, T *ptr)=0
MPI_Comm shmem_group_communicator
virtual void delete_array(const AlignedVector< T > *aligned_vector, T *ptr)
MPISharedMemDeleterAction(const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
T * aligned_shmem_pointer
Deleter(AlignedVector< T > *owning_object, const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
Deleter(AlignedVector< T > *owning_object)
std::unique_ptr< DeleterActionBase > deleter_action_object
void reset_owning_object(const AlignedVector< T > *new_aligned_vector_ptr)
const AlignedVector< T > * owning_aligned_vector
void replicate_across_communicator(const MPI_Comm communicator, const unsigned int root_process)
size_type memory_consumption() const
void resize_fast(const size_type new_size)
std::unique_ptr< T[], Deleter > elements
reference operator[](const size_type index)
void fill(const T &element)
const_iterator end() const
AlignedVector(AlignedVector< T > &&vec) noexcept
void reserve(const size_type new_allocated_size)
void serialize(Archive &archive, const unsigned int version)
bool operator!=(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const_reference operator[](const size_type index) const
void resize(const size_type new_size, const T &init)
AlignedVector & operator=(AlignedVector< T > &&vec) noexcept
AlignedVector(RandomAccessIterator begin, RandomAccessIterator end)
size_type capacity() const
AlignedVector & operator=(const AlignedVector< T > &vec)
void swap(AlignedVector< T > &vec) noexcept
const value_type * const_pointer
bool replicated_across_communicator
void push_back(const T in_data)
iterator insert(const_iterator position, RandomAccessIterator begin, RandomAccessIterator end)
const_iterator begin() const
AlignedVector(const size_type size, const T &init=T())
T * allocated_elements_end
AlignedVector(const AlignedVector< T > &vec)
const_reference back() const
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const value_type * const_iterator
void resize(const size_type new_size)
void load(Archive &ar, const unsigned int version)
void allocate_and_move(const size_t old_size, const size_t new_size, const size_t new_allocated_size)
void save(Archive &ar, const unsigned int version) const
void insert_back(ForwardIterator begin, ForwardIterator end)
const value_type & const_reference
const_pointer data() const
AlignedVectorCopyConstruct(RandomAccessIterator source_begin, RandomAccessIterator source_end, T *const destination)
RandomAccessIterator source_
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
AlignedVectorDefaultInitialize(const std::size_t size, T *const destination)
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< false >) const
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< true >) const
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< false >) const
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::bool_constant< true >) const
AlignedVectorInitialize(const std::size_t size, const T &element, T *const destination)
RandomAccessIterator source_
AlignedVectorMoveConstruct(RandomAccessIterator source_begin, RandomAccessIterator source_end, T *const destination)
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
static const std::size_t minimum_parallel_grain_size
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
#define DeclExceptionMsg(Exception, defaulttext)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
static ::ExceptionBase & ExcAlignedVectorChangeAfterReplication()
std::vector< index_type > data
types::global_dof_index size_type
std::enable_if_t< std::is_fundamental_v< T >, std::size_t > memory_consumption(const T &t)
VectorType::value_type * end(VectorType &V)
VectorType::value_type * begin(VectorType &V)
T max(const T &t, const MPI_Comm mpi_communicator)
T min(const T &t, const MPI_Comm mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
void free_communicator(MPI_Comm mpi_communicator)
T broadcast(const MPI_Comm comm, const T &object_to_send, const unsigned int root_process=0)
void posix_memalign(void **memptr, std::size_t alignment, std::size_t size)
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
::VectorizedArray< Number, width > max(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
void apply_parallel(const std::size_t begin, const std::size_t end, const std::size_t minimum_parallel_grain_size) const