17 #ifndef dealii_aligned_vector_h
18 #define dealii_aligned_vector_h
30 #include <boost/version.hpp>
31 #if BOOST_VERSION >= 106400
32 # include <boost/serialization/array_wrapper.hpp>
34 # include <boost/serialization/array.hpp>
36 #include <boost/serialization/split_member.hpp>
40 #include <type_traits>
240 template <
typename ForwardIterator>
356 const unsigned int root_process);
444 template <
class Archive>
446 save(Archive &ar,
const unsigned int version)
const;
453 template <
class Archive>
455 load(Archive &ar,
const unsigned int version);
463 template <
class Archive>
465 serialize(Archive &archive,
const unsigned int version);
469 BOOST_SERIALIZATION_SPLIT_MEMBER()
479 "Changing the vector after a call to "
480 "replicate_across_communicator() is not allowed.");
489 const size_t new_size,
490 const size_t new_allocated_size);
593 #ifdef DEAL_II_WITH_MPI
602 const bool is_shmem_root,
603 T *aligned_shmem_pointer,
604 MPI_Comm shmem_group_communicator,
605 MPI_Win shmem_window);
647 #ifdef DEAL_II_WITH_MPI
747 template <
typename T>
752 160000 /
sizeof(
T) + 1;
765 const T *
const source_end,
766 T *
const destination)
771 Assert(source_end == source_begin || destination !=
nullptr,
773 const std::size_t size = source_end - source_begin;
786 const std::size_t
end)
const override
795 if (std::is_trivial_v<T> ==
true)
800 for (std::size_t i =
begin; i <
end; ++i)
816 template <
typename T>
821 160000 /
sizeof(
T) + 1;
835 T *
const destination)
840 Assert(source_end == source_begin || destination !=
nullptr,
842 const std::size_t size = source_end - source_begin;
855 const std::size_t
end)
const override
863 if (std::is_trivial_v<T> ==
true)
870 for (std::size_t i =
begin; i <
end; ++i)
897 template <
typename T,
bool initialize_memory>
901 160000 /
sizeof(
T) + 1;
910 T *
const destination)
922 if (std::is_trivial_v<T> ==
true &&
923 std::is_same_v<T, long double> ==
false)
925 const unsigned char zero[
sizeof(
T)] = {};
929 if (std::memcmp(
zero,
930 static_cast<const void *
>(&element),
945 const std::size_t
end)
const override
957 begin,
end, std::integral_constant<bool, initialize_memory>());
968 const std::size_t
end,
969 std::integral_constant<bool, false>)
const
971 for (std::size_t i =
begin; i <
end; ++i)
978 const std::size_t
end,
979 std::integral_constant<bool, true>)
const
981 for (std::size_t i =
begin; i <
end; ++i)
1000 template <
typename T,
bool initialize_memory>
1005 160000 /
sizeof(
T) + 1;
1030 const std::size_t
end)
const override
1036 if (std::is_trivial_v<T> ==
true)
1042 begin,
end, std::integral_constant<bool, initialize_memory>());
1051 const std::size_t
end,
1052 std::integral_constant<bool, false>)
const
1054 for (std::size_t i =
begin; i <
end; ++i)
1061 const std::size_t
end,
1062 std::integral_constant<bool, true>)
const
1064 for (std::size_t i =
begin; i <
end; ++i)
1076 template <
typename T>
1078 : deleter_action_object(nullptr)
1079 , owning_aligned_vector(owning_object)
1083 # ifdef DEAL_II_WITH_MPI
1085 template <
typename T>
1087 const bool is_shmem_root,
1088 T *aligned_shmem_pointer,
1089 MPI_Comm shmem_group_communicator,
1090 MPI_Win shmem_window)
1091 : deleter_action_object(
1092 std::make_unique<MPISharedMemDeleterAction>(is_shmem_root,
1093 aligned_shmem_pointer,
1094 shmem_group_communicator,
1096 , owning_aligned_vector(owning_object)
1101 template <
typename T>
1107 if (deleter_action_object ==
nullptr)
1111 Assert(owning_aligned_vector->used_elements_end !=
nullptr,
1114 if (std::is_trivial_v<T> ==
false)
1115 for (
T *p = owning_aligned_vector->used_elements_end - 1; p >= ptr;
1124 deleter_action_object->delete_array(owning_aligned_vector, ptr);
1129 template <
typename T>
1134 owning_aligned_vector = new_aligned_vector_ptr;
1138 # ifdef DEAL_II_WITH_MPI
1140 template <
typename T>
1143 T *aligned_shmem_pointer,
1144 MPI_Comm shmem_group_communicator,
1145 MPI_Win shmem_window)
1146 : is_shmem_root(is_shmem_root)
1147 , aligned_shmem_pointer(aligned_shmem_pointer)
1148 , shmem_group_communicator(shmem_group_communicator)
1149 , shmem_window(shmem_window)
1154 template <
typename T>
1171 if (std::is_trivial_v<T> ==
false)
1176 ierr = MPI_Win_free(&shmem_window);
1187 : elements(nullptr, Deleter(this))
1188 , used_elements_end(nullptr)
1189 , allocated_elements_end(nullptr)
1191 , replicated_across_communicator(false)
1199 : elements(nullptr, Deleter(this))
1200 , used_elements_end(nullptr)
1201 , allocated_elements_end(nullptr)
1203 , replicated_across_communicator(false)
1214 : elements(nullptr, Deleter(this))
1215 , used_elements_end(nullptr)
1216 , allocated_elements_end(nullptr)
1218 , replicated_across_communicator(false)
1222 reserve(vec.
size());
1223 used_elements_end = allocated_elements_end;
1236 *
this = std::move(vec);
1259 used_elements_end = elements.get() + new_size;
1278 elements = std::move(vec.
elements);
1279 elements.get_deleter().reset_owning_object(
this);
1301 else if (new_size == old_size)
1304 else if (new_size < old_size)
1309 if (std::is_trivial_v<T> ==
false)
1310 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1312 used_elements_end = elements.get() + new_size;
1318 used_elements_end = elements.get() + new_size;
1322 if (std::is_trivial_v<T> ==
false)
1324 new_size - old_size, elements.get() + old_size);
1338 else if (new_size == old_size)
1341 else if (new_size < old_size)
1346 if (std::is_trivial_v<T> ==
false)
1347 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1349 used_elements_end = elements.get() + new_size;
1355 used_elements_end = elements.get() + new_size;
1359 new_size - old_size, elements.get() + old_size);
1373 else if (new_size == old_size)
1376 else if (new_size < old_size)
1381 if (std::is_trivial_v<T> ==
false)
1382 for (
T *p = used_elements_end - 1; p >= elements.get() + new_size; --p)
1384 used_elements_end = elements.get() + new_size;
1390 used_elements_end = elements.get() + new_size;
1394 new_size - old_size, init, elements.get() + old_size);
1403 const size_t new_size,
1404 const size_t new_allocated_size)
1411 new_size *
sizeof(
T));
1418 Deleter deleter(
this);
1421 if (new_allocated_size > 0)
1423 elements.get() + old_size,
1435 elements = decltype(elements)(new_data_ptr, std::move(deleter));
1436 used_elements_end = elements.get() + old_size;
1437 allocated_elements_end = elements.get() + new_size;
1446 const size_type old_size = used_elements_end - elements.get();
1447 const size_type old_allocated_size = allocated_elements_end - elements.get();
1448 if (new_allocated_size > old_allocated_size)
1454 std::max(new_allocated_size, 2 * old_allocated_size);
1456 allocate_and_move(old_size, new_size, new_allocated_size);
1458 else if (new_allocated_size == 0)
1472 Assert(replicated_across_communicator ==
false,
1473 ExcAlignedVectorChangeAfterReplication());
1475 const size_type used_size = used_elements_end - elements.get();
1476 const size_type allocated_size = allocated_elements_end - elements.get();
1477 if (allocated_size > used_size)
1478 allocate_and_move(used_size, used_size, used_size);
1495 used_elements_end =
nullptr;
1496 allocated_elements_end =
nullptr;
1506 if (used_elements_end == allocated_elements_end)
1508 if (std::is_trivial_v<T> ==
false)
1509 new (used_elements_end++)
T(in_data);
1511 *used_elements_end++ = in_data;
1521 T *field = used_elements_end - 1;
1532 const T *field = used_elements_end - 1;
1539 template <
typename ForwardIterator>
1547 if (std::is_trivial_v<T> ==
false)
1548 new (used_elements_end)
T;
1549 *used_elements_end = *
begin;
1579 const unsigned int root_process)
1581 # ifdef DEAL_II_WITH_MPI
1602 used_elements_end =
nullptr;
1603 allocated_elements_end =
nullptr;
1625 MPI_Comm shmem_group_communicator;
1627 MPI_Comm shmem_group_communicator_temp;
1628 int ierr = MPI_Comm_split_type(communicator,
1629 MPI_COMM_TYPE_SHARED,
1632 &shmem_group_communicator_temp);
1637 ierr = MPI_Comm_split(shmem_group_communicator_temp,
1640 &shmem_group_communicator);
1651 const bool is_shmem_root =
1680 MPI_Comm shmem_roots_communicator;
1685 const int ierr = MPI_Comm_split(communicator,
1687 (is_shmem_root ? 0 : 1),
1689 &shmem_roots_communicator);
1698 const unsigned int shmem_roots_root_rank = 0;
1699 const bool is_shmem_roots_root =
1701 shmem_roots_communicator) == shmem_roots_root_rank));
1708 if (std::is_trivial_v<T>)
1720 shmem_roots_root_rank);
1721 if (is_shmem_roots_root ==
false)
1725 int ierr = MPI_Bcast(elements.get(),
1726 sizeof(
T) * new_size,
1728 shmem_roots_root_rank,
1729 shmem_roots_communicator);
1746 shmem_roots_root_rank);
1750 shmem_roots_root_rank);
1786 MPI_Win shmem_window;
1788 const MPI_Aint align_by = 64;
1789 const MPI_Aint alloc_size =
1791 (size() *
sizeof(
T) + (align_by - 1)),
1798 ierr = MPI_Info_create(&mpi_info);
1800 ierr = MPI_Info_set(mpi_info,
1801 "mpi_minimum_memory_alignment",
1804 ierr = MPI_Win_allocate_shared((is_shmem_root ? alloc_size : 0),
1807 shmem_group_communicator,
1812 ierr = MPI_Info_free(&mpi_info);
1835 if (is_shmem_root ==
false)
1838 MPI_Aint alloc_size;
1839 const int ierr = MPI_Win_shared_query(
1840 shmem_window, MPI_PROC_NULL, &alloc_size, &disp_unit, &base_ptr);
1860 std::size_t available_space = alloc_size;
1861 void *base_ptr_backup = base_ptr;
1862 T *aligned_shmem_pointer =
static_cast<T *
>(
1863 std::align(align_by, new_size *
sizeof(
T), base_ptr, available_space));
1893 aligned_shmem_pointer =
static_cast<T *
>(base_ptr_backup);
1901 if (std::is_trivial_v<T> ==
true)
1902 std::memcpy(aligned_shmem_pointer, elements.get(),
sizeof(
T) * size());
1904 for (std::size_t i = 0; i < size(); ++i)
1905 new (&aligned_shmem_pointer[i])
T(std::move(elements[i]));
1910 const int ierr = MPI_Barrier(shmem_group_communicator);
1930 elements = decltype(elements)(aligned_shmem_pointer,
1933 aligned_shmem_pointer,
1934 shmem_group_communicator,
1941 used_elements_end = elements.get() + new_size;
1942 allocated_elements_end = used_elements_end;
1948 replicated_across_communicator =
true;
1951 std::accumulate(packed_data.begin(), packed_data.end(),
int(0));
1971 elements.get_deleter().reset_owning_object(
this);
1972 vec.
elements.get_deleter().reset_owning_object(&vec);
1985 return used_elements_end == elements.get();
1994 return used_elements_end - elements.get();
2003 return allocated_elements_end - elements.get();
2013 return elements[
index];
2023 return elements[
index];
2028 template <
typename T>
2032 return elements.get();
2037 template <
typename T>
2041 return elements.get();
2050 return elements.get();
2059 return used_elements_end;
2068 return elements.get();
2077 return used_elements_end;
2083 template <
class Archive>
2090 ar &boost::serialization::make_array(elements.get(), vec_size);
2096 template <
class Archive>
2106 ar &boost::serialization::make_array(elements.get(), vec_size);
2107 used_elements_end = elements.get() + vec_size;
2118 for (
const T *t = elements.get(); t != used_elements_end; ++t)
2120 memory +=
sizeof(
T) * (allocated_elements_end - used_elements_end);
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
virtual ~DeleterActionBase()=default
virtual void delete_array(const AlignedVector< T > *owning_aligned_vector, T *ptr)=0
MPI_Comm shmem_group_communicator
virtual void delete_array(const AlignedVector< T > *aligned_vector, T *ptr)
MPISharedMemDeleterAction(const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
T * aligned_shmem_pointer
Deleter(AlignedVector< T > *owning_object, const bool is_shmem_root, T *aligned_shmem_pointer, MPI_Comm shmem_group_communicator, MPI_Win shmem_window)
Deleter(AlignedVector< T > *owning_object)
std::unique_ptr< DeleterActionBase > deleter_action_object
void reset_owning_object(const AlignedVector< T > *new_aligned_vector_ptr)
const AlignedVector< T > * owning_aligned_vector
void replicate_across_communicator(const MPI_Comm communicator, const unsigned int root_process)
size_type memory_consumption() const
void resize_fast(const size_type new_size)
std::unique_ptr< T[], Deleter > elements
reference operator[](const size_type index)
void fill(const T &element)
AlignedVector & operator=(AlignedVector< T > &&vec) noexcept
const_iterator end() const
AlignedVector(AlignedVector< T > &&vec) noexcept
void reserve(const size_type new_allocated_size)
void serialize(Archive &archive, const unsigned int version)
bool operator!=(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const_reference operator[](const size_type index) const
void swap(AlignedVector< T > &vec)
void resize(const size_type new_size, const T &init)
size_type capacity() const
const value_type * const_pointer
bool replicated_across_communicator
void push_back(const T in_data)
AlignedVector & operator=(const AlignedVector< T > &vec)
const_iterator begin() const
AlignedVector(const size_type size, const T &init=T())
T * allocated_elements_end
AlignedVector(const AlignedVector< T > &vec)
const_reference back() const
bool operator==(const AlignedVector< T > &lhs, const AlignedVector< T > &rhs)
const value_type * const_iterator
void resize(const size_type new_size)
void load(Archive &ar, const unsigned int version)
void allocate_and_move(const size_t old_size, const size_t new_size, const size_t new_allocated_size)
void save(Archive &ar, const unsigned int version) const
void insert_back(ForwardIterator begin, ForwardIterator end)
const value_type & const_reference
const_pointer data() const
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
AlignedVectorCopyConstruct(const T *const source_begin, const T *const source_end, T *const destination)
static const std::size_t minimum_parallel_grain_size
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, true >) const
void default_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, false >) const
AlignedVectorDefaultInitialize(const std::size_t size, T *const destination)
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, true >) const
static const std::size_t minimum_parallel_grain_size
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
void copy_construct_or_assign(const std::size_t begin, const std::size_t end, std::integral_constant< bool, false >) const
AlignedVectorInitialize(const std::size_t size, const T &element, T *const destination)
static const std::size_t minimum_parallel_grain_size
AlignedVectorMoveConstruct(T *const source_begin, T *const source_end, T *const destination)
virtual void apply_to_subrange(const std::size_t begin, const std::size_t end) const override
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcAlignedVectorChangeAfterReplication()
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
#define DeclExceptionMsg(Exception, defaulttext)
static const types::blas_int zero
types::global_dof_index size_type
std::enable_if_t< std::is_fundamental_v< T >, std::size_t > memory_consumption(const T &t)
void swap(MemorySpaceData< T, MemorySpace > &u, MemorySpaceData< T, MemorySpace > &v)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::enable_if_t< is_mpi_type< T >==false, T > broadcast(const MPI_Comm comm, const T &object_to_send, const unsigned int root_process=0)
T max(const T &t, const MPI_Comm mpi_communicator)
T min(const T &t, const MPI_Comm mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
void free_communicator(MPI_Comm mpi_communicator)
void posix_memalign(void **memptr, std::size_t alignment, std::size_t size)
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
void apply_parallel(const std::size_t begin, const std::size_t end, const std::size_t minimum_parallel_grain_size) const