17#ifndef dealii_matrix_free_vector_access_internal_h
18#define dealii_matrix_free_vector_access_internal_h
29# include <boost/algorithm/string/join.hpp>
42 template <
typename VectorType,
43 typename std::enable_if<!has_local_element<VectorType>,
44 VectorType>::type * =
nullptr>
45 inline typename VectorType::value_type
56 template <
typename VectorType,
57 typename std::enable_if<!has_local_element<VectorType>,
58 VectorType>::type * =
nullptr>
59 inline typename VectorType::value_type &
70 template <
typename VectorType,
71 typename std::enable_if<has_local_element<VectorType>,
72 VectorType>::type * =
nullptr>
73 inline typename VectorType::value_type &
76 return vec.local_element(entry);
82 template <
typename VectorType,
83 typename std::enable_if<has_local_element<VectorType>,
84 VectorType>::type * =
nullptr>
85 inline typename VectorType::value_type
86 vector_access(
const VectorType &vec,
const unsigned int entry)
88 return vec.local_element(entry);
93 template <
typename VectorType,
94 typename std::enable_if<has_add_local_element<VectorType>,
95 VectorType>::type * =
nullptr>
98 const unsigned int entry,
99 const typename VectorType::value_type &val)
101 vec.add_local_element(entry, val);
106 template <
typename VectorType,
107 typename std::enable_if<!has_add_local_element<VectorType>,
108 VectorType>::type * =
nullptr>
111 const unsigned int entry,
112 const typename VectorType::value_type &val)
119 template <
typename VectorType,
120 typename std::enable_if<has_add_local_element<VectorType>,
121 VectorType>::type * =
nullptr>
125 const typename VectorType::value_type &val)
132 template <
typename VectorType,
133 typename std::enable_if<!has_add_local_element<VectorType>,
134 VectorType>::type * =
nullptr>
138 const typename VectorType::value_type &val)
145 template <
typename VectorType,
146 typename std::enable_if<has_set_local_element<VectorType>,
147 VectorType>::type * =
nullptr>
150 const unsigned int entry,
151 const typename VectorType::value_type &val)
153 vec.set_local_element(entry, val);
158 template <
typename VectorType,
159 typename std::enable_if<!has_set_local_element<VectorType>,
160 VectorType>::type * =
nullptr>
163 const unsigned int entry,
164 const typename VectorType::value_type &val)
178 typename VectorizedArrayType,
180 typename std::enable_if<!has_partitioners_are_compatible<VectorType>,
181 VectorType>::type * =
nullptr>
184 const VectorType & vec,
200 typename VectorizedArrayType,
202 typename std::enable_if<has_partitioners_are_compatible<VectorType>,
203 VectorType>::type * =
nullptr>
206 const VectorType & vec,
219 for (
unsigned int i = 0; i < matrix_free.
n_components(); ++i)
228 std::vector<std::string> dof_indices_with_compatible_partitioners;
230 for (
unsigned int i = 0; i < matrix_free.
n_components(); ++i)
231 if (vec.partitioners_are_compatible(
233 dof_indices_with_compatible_partitioners.push_back(
236 if (dof_indices_with_compatible_partitioners.empty())
239 ExcMessage(
"The parallel layout of the given vector is "
240 "compatible neither with the Partitioner of the "
241 "current FEEvaluation with dof_handler_index=" +
242 std::to_string(dof_index) +
243 " nor with any Partitioner in MatrixFree. A "
244 "potential reason is that you did not use "
245 "MatrixFree::initialize_dof_vector() to get a "
246 "compatible vector."));
253 "The parallel layout of the given vector is "
254 "not compatible with the Partitioner of the "
255 "current FEEvaluation with dof_handler_index=" +
256 std::to_string(dof_index) +
257 ". However, the underlying "
258 "MatrixFree contains Partitioner objects that are compatible. "
259 "They have the following dof_handler_index values: " +
260 boost::algorithm::join(dof_indices_with_compatible_partitioners,
262 ". Did you want to pass any of these values to the "
263 "constructor of the current FEEvaluation object or "
264 "did you not use MatrixFree::initialize_dof_vector() "
265 "with dof_handler_index=" +
266 std::to_string(dof_index) +
268 "compatible vector?"));
282 template <
typename Number,
typename VectorizedArrayType>
285 template <
typename VectorType>
288 const VectorType & vec,
296 template <
typename VectorNumberType>
305 template <
typename VectorType>
308 const unsigned int dof_index,
310 VectorizedArrayType *dof_values,
311 std::integral_constant<bool, true>)
const
320 std::integral_constant<bool, false>());
322 const Number *vec_ptr = vec.begin() + dof_index;
323 for (
unsigned int i = 0; i < dofs_per_cell;
324 ++i, vec_ptr += VectorizedArrayType::size())
325 dof_values[i].load(vec_ptr);
331 template <
typename VectorType>
334 const unsigned int dof_index,
335 const VectorType & vec,
336 VectorizedArrayType *dof_values,
337 std::integral_constant<bool, false>)
const
339 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
340 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
342 vector_access(vec, dof_index + v + i * VectorizedArrayType::size());
347 template <
typename VectorType>
350 const unsigned int * dof_indices,
352 VectorizedArrayType *dof_values,
353 std::integral_constant<bool, true>)
const
363 template <
typename VectorType>
366 const unsigned int * dof_indices,
367 const VectorType & vec,
368 VectorizedArrayType *dof_values,
369 std::integral_constant<bool, false>)
const
371 for (
unsigned int d = 0; d < dofs_per_cell; ++d)
372 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
378 template <
typename Number2>
381 const unsigned int dofs_per_cell,
382 const std::array<Number2 *, VectorizedArrayType::size()> &global_ptr,
383 VectorizedArrayType * dof_values,
384 std::integral_constant<bool, true>)
const
393 template <
typename Number2>
397 const std::array<Number2 *, VectorizedArrayType::size()> &,
398 VectorizedArrayType *,
399 std::integral_constant<bool, false>)
const
408 template <
typename VectorType>
412 const unsigned int constant_offset,
413 VectorizedArrayType &res,
414 std::integral_constant<bool, true>)
const
423 std::integral_constant<bool, false>());
425 res.gather(vec.begin() + constant_offset, indices);
433 template <
typename VectorType>
436 const VectorType & vec,
437 const unsigned int constant_offset,
438 VectorizedArrayType &res,
439 std::integral_constant<bool, false>)
const
441 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
447 template <
typename VectorType>
450 const VectorType & vec,
466 template <
typename VectorType>
470 const VectorType & vec,
489 res = VectorizedArrayType();
497 template <
typename Number,
typename VectorizedArrayType>
500 template <
typename VectorType>
508 template <
typename VectorNumberType>
517 template <
typename VectorType>
520 const unsigned int dof_index,
522 VectorizedArrayType *dof_values,
523 std::integral_constant<bool, true>)
const
525 Number *vec_ptr = vec.begin() + dof_index;
526 for (
unsigned int i = 0; i < dofs_per_cell;
527 ++i, vec_ptr += VectorizedArrayType::size())
529 VectorizedArrayType tmp;
531 tmp += dof_values[i];
538 template <
typename VectorType>
541 const unsigned int dof_index,
543 VectorizedArrayType *dof_values,
544 std::integral_constant<bool, false>)
const
546 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
547 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
549 dof_index + v + i * VectorizedArrayType::size(),
555 template <
typename VectorType>
558 const unsigned int * dof_indices,
560 VectorizedArrayType *dof_values,
561 std::integral_constant<bool, true>)
const
564 true, dofs_per_cell, dof_values, dof_indices, vec.begin());
569 template <
typename VectorType>
572 const unsigned int * dof_indices,
574 VectorizedArrayType *dof_values,
575 std::integral_constant<bool, false>)
const
577 for (
unsigned int d = 0; d < dofs_per_cell; ++d)
578 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
584 template <
typename Number2>
587 const unsigned int dofs_per_cell,
588 std::array<Number2 *, VectorizedArrayType::size()> &global_ptr,
589 VectorizedArrayType * dof_values,
590 std::integral_constant<bool, true>)
const
600 template <
typename Number2>
604 std::array<Number2 *, VectorizedArrayType::size()> &,
605 VectorizedArrayType *,
606 std::integral_constant<bool, false>)
const
615 template <
typename VectorType>
619 const unsigned int constant_offset,
620 VectorizedArrayType &res,
621 std::integral_constant<bool, true>)
const
623#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS < 512
624 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
628 VectorizedArrayType tmp;
629 tmp.gather(vec.begin() + constant_offset, indices);
631 tmp.scatter(indices, vec.begin() + constant_offset);
639 template <
typename VectorType>
643 const unsigned int constant_offset,
644 VectorizedArrayType &res,
645 std::integral_constant<bool, false>)
const
647 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
653 template <
typename VectorType>
672 template <
typename VectorType>
698 template <
typename Number,
typename VectorizedArrayType>
701 template <
typename VectorType>
710 template <
typename VectorNumberType>
719 template <
typename VectorType>
722 const unsigned int dof_index,
724 VectorizedArrayType *dof_values,
725 std::integral_constant<bool, true>)
const
727 Number *vec_ptr = vec.begin() + dof_index;
728 for (
unsigned int i = 0; i < dofs_per_cell;
729 ++i, vec_ptr += VectorizedArrayType::size())
730 dof_values[i].store(vec_ptr);
735 template <
typename VectorType>
738 const unsigned int dof_index,
740 VectorizedArrayType *dof_values,
741 std::integral_constant<bool, false>)
const
743 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
744 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
745 vector_access(vec, dof_index + v + i * VectorizedArrayType::size()) =
751 template <
typename VectorType>
754 const unsigned int * dof_indices,
756 VectorizedArrayType *dof_values,
757 std::integral_constant<bool, true>)
const
760 false, dofs_per_cell, dof_values, dof_indices, vec.begin());
765 template <
typename VectorType,
bool booltype>
768 const unsigned int * dof_indices,
770 VectorizedArrayType *dof_values,
771 std::integral_constant<bool, false>)
const
773 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
774 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
780 template <
typename Number2>
783 const unsigned int dofs_per_cell,
784 std::array<Number2 *, VectorizedArrayType::size()> &global_ptr,
785 VectorizedArrayType * dof_values,
786 std::integral_constant<bool, true>)
const
796 template <
typename Number2>
800 std::array<Number2 *, VectorizedArrayType::size()> &,
801 VectorizedArrayType *,
802 std::integral_constant<bool, false>)
const
809 template <
typename VectorType>
813 const unsigned int constant_offset,
814 VectorizedArrayType &res,
815 std::integral_constant<bool, true>)
const
817 res.scatter(indices, vec.begin() + constant_offset);
822 template <
typename VectorType>
826 const unsigned int constant_offset,
827 VectorizedArrayType &res,
828 std::integral_constant<bool, false>)
const
830 for (
unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
836 template <
typename VectorType>
853 template <
typename VectorType>
const internal::MatrixFreeFunctions::DoFInfo & get_dof_info(const unsigned int dof_handler_index_component=0) const
unsigned int n_components() const
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcNotImplemented()
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
void vector_access_add(VectorType &vec, const unsigned int entry, const typename VectorType::value_type &val)
void check_vector_compatibility(const VectorType &vec, const MatrixFree< dim, Number, VectorizedArrayType > &matrix_free, const internal::MatrixFreeFunctions::DoFInfo &dof_info)
void vector_access_add_global(VectorType &vec, const types::global_dof_index entry, const typename VectorType::value_type &val)
VectorType::value_type vector_access(const VectorType &vec, const unsigned int entry)
void vector_access_set(VectorType &vec, const unsigned int entry, const typename VectorType::value_type &val)
static const unsigned int invalid_unsigned_int
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType &res, std::integral_constant< bool, true >) const
void pre_constraints(const Number &input, Number &res) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_dofs_vectorized_transpose(const unsigned int, std::array< Number2 *, VectorizedArrayType::size()> &, VectorizedArrayType *, std::integral_constant< bool, false >) const
void process_empty(VectorizedArrayType &) const
void process_dof(const unsigned int index, VectorType &vec, Number &res) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void post_constraints(const Number &, Number &) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, std::array< Number2 *, VectorizedArrayType::size()> &global_ptr, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType &res, std::integral_constant< bool, false >) const
void process_dof(VectorNumberType &global, Number &local) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, false >) const
void process_constraint(const unsigned int index, const Number weight, VectorType &vec, Number &res) const
void process_dof_global(const types::global_dof_index index, VectorType &vec, Number &res) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType &res, std::integral_constant< bool, true >) const
void process_dof_gather(const unsigned int *indices, const VectorType &vec, const unsigned int constant_offset, VectorizedArrayType &res, std::integral_constant< bool, false >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void pre_constraints(const Number &, Number &res) const
void post_constraints(const Number &sum, Number &write_pos) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, const VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, false >) const
void process_dof_global(const types::global_dof_index index, const VectorType &vec, Number &res) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, const VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, false >) const
void process_constraint(const unsigned int index, const Number weight, const VectorType &vec, Number &res) const
void process_dof(const VectorNumberType &global, Number &local) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const std::array< Number2 *, VectorizedArrayType::size()> &global_ptr, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_empty(VectorizedArrayType &res) const
void process_dofs_vectorized_transpose(const unsigned int, const std::array< Number2 *, VectorizedArrayType::size()> &, VectorizedArrayType *, std::integral_constant< bool, false >) const
void process_dof(const unsigned int index, const VectorType &vec, Number &res) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_empty(VectorizedArrayType &) const
void process_dofs_vectorized_transpose(const unsigned int, std::array< Number2 *, VectorizedArrayType::size()> &, VectorizedArrayType *, std::integral_constant< bool, false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, false >) const
void process_dof_global(const types::global_dof_index index, VectorType &vec, Number &res) const
void pre_constraints(const Number &, Number &) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, std::array< Number2 *, VectorizedArrayType::size()> &global_ptr, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_dofs_vectorized(const unsigned int dofs_per_cell, const unsigned int dof_index, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, false >) const
void process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, const unsigned int *dof_indices, VectorType &vec, VectorizedArrayType *dof_values, std::integral_constant< bool, true >) const
void process_constraint(const unsigned int, const Number, VectorType &, Number &) const
void process_dof(const unsigned int index, VectorType &vec, Number &res) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType &res, std::integral_constant< bool, true >) const
void process_dof_gather(const unsigned int *indices, VectorType &vec, const unsigned int constant_offset, VectorizedArrayType &res, std::integral_constant< bool, false >) const
void post_constraints(const Number &, Number &) const
void process_dof(VectorNumberType &global, Number &local) const
void vectorized_load_and_transpose(const unsigned int n_entries, const Number *in, const unsigned int *offsets, VectorizedArray< Number, width > *out)
void vectorized_transpose_and_store(const bool add_into, const unsigned int n_entries, const VectorizedArray< Number, width > *in, const unsigned int *offsets, Number *out)