17 #ifndef dealii_matrix_free_h
18 #define dealii_matrix_free_h
116 std::is_same<Number, typename VectorizedArrayType::value_type>::value,
117 "Type of Number and of VectorizedArrayType do not match.");
580 template <
typename QuadratureType,
typename number2,
typename MappingType>
585 const QuadratureType & quad,
609 template <
typename QuadratureType,
typename number2,
typename MappingType>
614 const std::vector<QuadratureType> & quad,
624 template <
typename QuadratureType,
typename number2,
typename MappingType>
629 const QuadratureType & quad,
786 template <
typename OutVector,
typename InVector>
792 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
794 const InVector & src,
795 const bool zero_dst_vector =
false)
const;
843 template <
typename CLASS,
typename OutVector,
typename InVector>
849 const std::pair<unsigned int, unsigned int> &)
const,
850 const CLASS * owning_class,
853 const bool zero_dst_vector =
false)
const;
858 template <
typename CLASS,
typename OutVector,
typename InVector>
864 const std::pair<unsigned int, unsigned int> &),
865 CLASS * owning_class,
868 const bool zero_dst_vector =
false)
const;
954 template <
typename CLASS,
typename OutVector,
typename InVector>
960 const std::pair<unsigned int, unsigned int> &)
const,
961 const CLASS * owning_class,
964 const std::function<
void(
const unsigned int,
const unsigned int)>
965 &operation_before_loop,
966 const std::function<
void(
const unsigned int,
const unsigned int)>
967 & operation_after_loop,
968 const unsigned int dof_handler_index_pre_post = 0)
const;
973 template <
typename CLASS,
typename OutVector,
typename InVector>
979 const std::pair<unsigned int, unsigned int> &),
980 CLASS * owning_class,
983 const std::function<
void(
const unsigned int,
const unsigned int)>
984 &operation_before_loop,
985 const std::function<
void(
const unsigned int,
const unsigned int)>
986 & operation_after_loop,
987 const unsigned int dof_handler_index_pre_post = 0)
const;
993 template <
typename OutVector,
typename InVector>
999 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1001 const InVector & src,
1002 const std::function<
void(
const unsigned int,
const unsigned int)>
1003 &operation_before_loop,
1004 const std::function<
void(
const unsigned int,
const unsigned int)>
1005 & operation_after_loop,
1006 const unsigned int dof_handler_index_pre_post = 0)
const;
1083 template <
typename OutVector,
typename InVector>
1089 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1090 const std::function<
1094 const std::pair<unsigned int, unsigned int> &)> &face_operation,
1095 const std::function<
void(
1099 const std::pair<unsigned int, unsigned int> &)> &boundary_operation,
1101 const InVector & src,
1102 const bool zero_dst_vector =
false,
1193 template <
typename CLASS,
typename OutVector,
typename InVector>
1196 void (CLASS::*cell_operation)(
const MatrixFree &,
1199 const std::pair<unsigned int, unsigned int> &)
1201 void (CLASS::*face_operation)(
const MatrixFree &,
1204 const std::pair<unsigned int, unsigned int> &)
1206 void (CLASS::*boundary_operation)(
1210 const std::pair<unsigned int, unsigned int> &)
const,
1211 const CLASS * owning_class,
1213 const InVector & src,
1214 const bool zero_dst_vector =
false,
1223 template <
typename CLASS,
typename OutVector,
typename InVector>
1225 loop(
void (CLASS::*cell_operation)(
1229 const std::pair<unsigned int, unsigned int> &),
1230 void (CLASS::*face_operation)(
1234 const std::pair<unsigned int, unsigned int> &),
1235 void (CLASS::*boundary_operation)(
1239 const std::pair<unsigned int, unsigned int> &),
1240 CLASS * owning_class,
1242 const InVector & src,
1243 const bool zero_dst_vector =
false,
1313 template <
typename CLASS,
typename OutVector,
typename InVector>
1319 const std::pair<unsigned int, unsigned int> &)
const,
1320 const CLASS * owning_class,
1322 const InVector & src,
1323 const bool zero_dst_vector =
false,
1330 template <
typename CLASS,
typename OutVector,
typename InVector>
1336 const std::pair<unsigned int, unsigned int> &),
1337 CLASS * owning_class,
1339 const InVector & src,
1340 const bool zero_dst_vector =
false,
1347 template <
typename OutVector,
typename InVector>
1353 const std::pair<unsigned int, unsigned int> &)>
1356 const InVector & src,
1357 const bool zero_dst_vector =
false,
1368 std::pair<unsigned int, unsigned int>
1370 const unsigned int fe_degree,
1371 const unsigned int dof_handler_index = 0)
const;
1379 std::pair<unsigned int, unsigned int>
1381 const std::pair<unsigned int, unsigned int> &range,
1382 const unsigned int fe_index,
1383 const unsigned int dof_handler_index = 0)
const;
1396 const std::pair<unsigned int, unsigned int> range)
const;
1403 const bool is_interior_face =
true)
const;
1416 template <
typename T>
1425 template <
typename T>
1442 template <
typename VectorType>
1445 const unsigned int dof_handler_index = 0)
const;
1464 template <
typename Number2>
1467 const unsigned int dof_handler_index = 0)
const;
1479 const std::shared_ptr<const Utilities::MPI::Partitioner> &
1503 const std::vector<unsigned int> &
1518 const unsigned int dof_handler_index = 0);
1529 template <
int spacedim>
1624 const unsigned int face_number)
const;
1647 const unsigned int lane_index,
1648 const unsigned int dof_handler_index = 0)
const;
1657 const unsigned int lane_index)
const;
1671 std::pair<typename DoFHandler<dim>::cell_iterator,
unsigned int>
1673 const unsigned int lane_index,
1674 const bool interior =
true,
1675 const unsigned int fe_component = 0)
const;
1721 const unsigned int hp_active_fe_index = 0)
const;
1728 const unsigned int hp_active_fe_index = 0)
const;
1736 const unsigned int hp_active_fe_index = 0)
const;
1744 const unsigned int hp_active_fe_index = 0)
const;
1751 const unsigned int hp_active_fe_index = 0)
const;
1758 const unsigned int hp_active_fe_index = 0)
const;
1774 const std::pair<unsigned int, unsigned int> cell_batch_range)
const;
1780 std::pair<unsigned int, unsigned int>
1782 const std::pair<unsigned int, unsigned int> face_batch_range)
const;
1802 std::pair<unsigned int, unsigned int>
1836 template <
typename StreamType>
1864 const internal::MatrixFreeFunctions::
1865 MappingInfo<dim, Number, VectorizedArrayType> &
1900 const unsigned int quad_index = 0,
1901 const unsigned int fe_base_element = 0,
1902 const unsigned int hp_active_fe_index = 0,
1903 const unsigned int hp_active_quad_index = 0)
const;
1909 VectorizedArrayType::size()> &
1969 template <
typename number2,
int q_dim>
1975 const std::vector<IndexSet> & locally_owned_set,
1985 template <
typename number2>
1989 const std::vector<IndexSet> & locally_owned_set,
2009 std::vector<internal::MatrixFreeFunctions::DoFInfo>
dof_info;
2087 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>>
2094 mutable std::list<std::pair<bool, AlignedVector<Number>>>
2111 template <
int dim,
typename Number,
typename VectorizedArrayType>
2112 template <
typename T>
2122 template <
int dim,
typename Number,
typename VectorizedArrayType>
2123 template <
typename T>
2134 template <
int dim,
typename Number,
typename VectorizedArrayType>
2135 template <
typename VectorType>
2139 const unsigned int comp)
const
2142 "This function is not supported for block vectors.");
2145 ExcMessage(
"This function can only be used in serial."));
2148 vec.reinit(
dof_info[comp].vector_partitioner->size());
2153 template <
int dim,
typename Number,
typename VectorizedArrayType>
2154 template <
typename Number2>
2158 const unsigned int comp)
const
2166 template <
int dim,
typename Number,
typename VectorizedArrayType>
2167 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
2169 const unsigned int comp)
const
2172 return dof_info[comp].vector_partitioner;
2177 template <
int dim,
typename Number,
typename VectorizedArrayType>
2178 inline const std::vector<unsigned int> &
2180 const unsigned int comp)
const
2183 return dof_info[comp].constrained_dofs;
2188 template <
int dim,
typename Number,
typename VectorizedArrayType>
2198 template <
int dim,
typename Number,
typename VectorizedArrayType>
2201 const unsigned int dof_no)
const
2205 return dof_handlers[dof_no]->get_fe().n_base_elements();
2210 template <
int dim,
typename Number,
typename VectorizedArrayType>
2219 template <
int dim,
typename Number,
typename VectorizedArrayType>
2228 template <
int dim,
typename Number,
typename VectorizedArrayType>
2237 template <
int dim,
typename Number,
typename VectorizedArrayType>
2247 template <
int dim,
typename Number,
typename VectorizedArrayType>
2258 template <
int dim,
typename Number,
typename VectorizedArrayType>
2270 template <
int dim,
typename Number,
typename VectorizedArrayType>
2281 template <
int dim,
typename Number,
typename VectorizedArrayType>
2284 const unsigned int face_batch_index)
const
2296 template <
int dim,
typename Number,
typename VectorizedArrayType>
2299 const unsigned int cell_batch_index,
2300 const unsigned int face_number)
const
2308 for (
unsigned int v = 0;
2318 template <
int dim,
typename Number,
typename VectorizedArrayType>
2319 inline const internal::MatrixFreeFunctions::
2320 MappingInfo<dim, Number, VectorizedArrayType> &
2328 template <
int dim,
typename Number,
typename VectorizedArrayType>
2331 const unsigned int dof_index)
const
2339 template <
int dim,
typename Number,
typename VectorizedArrayType>
2348 template <
int dim,
typename Number,
typename VectorizedArrayType>
2349 inline const Number *
2351 const unsigned int row)
const
2361 template <
int dim,
typename Number,
typename VectorizedArrayType>
2362 inline const Number *
2364 const unsigned int row)
const
2374 template <
int dim,
typename Number,
typename VectorizedArrayType>
2375 inline std::pair<unsigned int, unsigned int>
2377 const std::pair<unsigned int, unsigned int> &range,
2378 const unsigned int degree,
2379 const unsigned int dof_handler_component)
const
2381 if (
dof_info[dof_handler_component].cell_active_fe_index.empty())
2384 dof_info[dof_handler_component].fe_index_conversion.size(), 1);
2386 dof_info[dof_handler_component].fe_index_conversion[0].size(), 1);
2387 if (
dof_info[dof_handler_component].fe_index_conversion[0][0] == degree)
2390 return {range.second, range.second};
2393 const unsigned int fe_index =
2394 dof_info[dof_handler_component].fe_index_from_degree(0, degree);
2395 if (fe_index >=
dof_info[dof_handler_component].max_fe_index)
2396 return {range.second, range.second};
2400 dof_handler_component);
2405 template <
int dim,
typename Number,
typename VectorizedArrayType>
2408 const unsigned int cell_batch_index)
const
2411 return VectorizedArrayType::size() > 1 &&
2414 VectorizedArrayType::size() -
2420 template <
int dim,
typename Number,
typename VectorizedArrayType>
2428 template <
int dim,
typename Number,
typename VectorizedArrayType>
2431 const std::pair<unsigned int, unsigned int> range)
const
2433 const auto &fe_indices =
dof_info[0].cell_active_fe_index;
2435 if (fe_indices.empty() ==
true)
2438 const auto index = fe_indices[range.first];
2440 for (
unsigned int i = range.first; i < range.second; ++i)
2448 template <
int dim,
typename Number,
typename VectorizedArrayType>
2451 const std::pair<unsigned int, unsigned int> range,
2452 const bool is_interior_face)
const
2454 const auto &fe_indices =
dof_info[0].cell_active_fe_index;
2456 if (fe_indices.empty() ==
true)
2459 if (is_interior_face)
2461 const unsigned int index =
2463 VectorizedArrayType::size()];
2465 for (
unsigned int i = range.first; i < range.second; ++i)
2468 VectorizedArrayType::size()]);
2474 const unsigned int index =
2476 VectorizedArrayType::size()];
2478 for (
unsigned int i = range.first; i < range.second; ++i)
2481 VectorizedArrayType::size()]);
2489 template <
int dim,
typename Number,
typename VectorizedArrayType>
2492 const unsigned int cell_batch_index)
const
2496 const std::vector<unsigned char> &n_lanes_filled =
2497 dof_info[0].n_vectorization_lanes_filled
2501 return n_lanes_filled[cell_batch_index];
2506 template <
int dim,
typename Number,
typename VectorizedArrayType>
2509 const unsigned int face_batch_index)
const
2513 const std::vector<unsigned char> &n_lanes_filled =
2514 dof_info[0].n_vectorization_lanes_filled
2517 return n_lanes_filled[face_batch_index];
2522 template <
int dim,
typename Number,
typename VectorizedArrayType>
2525 const unsigned int dof_handler_index,
2526 const unsigned int active_fe_index)
const
2528 return dof_info[dof_handler_index].dofs_per_cell[active_fe_index];
2533 template <
int dim,
typename Number,
typename VectorizedArrayType>
2536 const unsigned int quad_index,
2537 const unsigned int active_fe_index)
const
2541 .descriptor[active_fe_index]
2547 template <
int dim,
typename Number,
typename VectorizedArrayType>
2550 const unsigned int dof_handler_index,
2551 const unsigned int active_fe_index)
const
2553 return dof_info[dof_handler_index].dofs_per_face[active_fe_index];
2558 template <
int dim,
typename Number,
typename VectorizedArrayType>
2561 const unsigned int quad_index,
2562 const unsigned int active_fe_index)
const
2566 .descriptor[active_fe_index]
2572 template <
int dim,
typename Number,
typename VectorizedArrayType>
2575 const unsigned int dof_handler_index)
const
2577 return dof_info[dof_handler_index].vector_partitioner->locally_owned_range();
2582 template <
int dim,
typename Number,
typename VectorizedArrayType>
2585 const unsigned int dof_handler_index)
const
2587 return dof_info[dof_handler_index].vector_partitioner->ghost_indices();
2592 template <
int dim,
typename Number,
typename VectorizedArrayType>
2595 const unsigned int dof_handler_index,
2596 const unsigned int index_quad,
2597 const unsigned int index_fe,
2598 const unsigned int active_fe_index,
2599 const unsigned int active_quad_index)
const
2602 const unsigned int ind =
2603 dof_info[dof_handler_index].global_base_element_offset + index_fe;
2608 return shape_info(ind, index_quad, active_fe_index, active_quad_index);
2613 template <
int dim,
typename Number,
typename VectorizedArrayType>
2615 VectorizedArrayType::size()> &
2617 const unsigned int face_batch_index)
const
2625 template <
int dim,
typename Number,
typename VectorizedArrayType>
2635 template <
int dim,
typename Number,
typename VectorizedArrayType>
2638 const unsigned int quad_index,
2639 const unsigned int active_fe_index)
const
2643 .descriptor[active_fe_index]
2649 template <
int dim,
typename Number,
typename VectorizedArrayType>
2652 const unsigned int quad_index,
2653 const unsigned int active_fe_index)
const
2657 .descriptor[active_fe_index]
2663 template <
int dim,
typename Number,
typename VectorizedArrayType>
2666 const std::pair<unsigned int, unsigned int> range)
const
2670 for (
unsigned int i = range.first; i < range.second; ++i)
2678 template <
int dim,
typename Number,
typename VectorizedArrayType>
2679 inline std::pair<unsigned int, unsigned int>
2681 const std::pair<unsigned int, unsigned int> range)
const
2685 for (
unsigned int i = range.first; i < range.second; ++i)
2696 template <
int dim,
typename Number,
typename VectorizedArrayType>
2699 const unsigned int cell_batch_index)
const
2703 if (
dof_info[0].cell_active_fe_index.empty())
2706 return dof_info[0].cell_active_fe_index[cell_batch_index];
2711 template <
int dim,
typename Number,
typename VectorizedArrayType>
2712 inline std::pair<unsigned int, unsigned int>
2714 const unsigned int face_batch_index)
const
2717 if (
dof_info[0].cell_active_fe_index.empty())
2718 return std::make_pair(0
U, 0
U);
2720 std::pair<unsigned int, unsigned int> result = std::make_pair(0
U, 0
U);
2721 for (
unsigned int v = 0;
2722 v < VectorizedArrayType::size() &&
2729 .cells_interior[v] /
2730 VectorizedArrayType::size()]);
2733 for (
unsigned int v = 0;
2734 v < VectorizedArrayType::size() &&
2741 .cells_exterior[v] /
2742 VectorizedArrayType::size()]);
2750 template <
int dim,
typename Number,
typename VectorizedArrayType>
2759 template <
int dim,
typename Number,
typename VectorizedArrayType>
2767 template <
int dim,
typename Number,
typename VectorizedArrayType>
2776 template <
int dim,
typename Number,
typename VectorizedArrayType>
2781 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>;
2783 for (
typename list_type::iterator it = data.begin(); it != data.end(); ++it)
2784 if (it->first ==
false)
2790 return &data.front().second;
2795 template <
int dim,
typename Number,
typename VectorizedArrayType>
2801 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>;
2803 for (
typename list_type::iterator it = data.begin(); it != data.end(); ++it)
2804 if (&it->second == scratch)
2815 template <
int dim,
typename Number,
typename VectorizedArrayType>
2824 if (it->first ==
false)
2836 template <
int dim,
typename Number,
typename VectorizedArrayType>
2846 if (&it->second == scratch)
2861 namespace MatrixFreeImplementation
2863 template <
int dim,
int spacedim>
2864 inline std::vector<IndexSet>
2865 extract_locally_owned_index_sets(
2866 const std::vector<const ::DoFHandler<dim, spacedim> *> &dofh,
2867 const unsigned int level)
2869 std::vector<IndexSet> locally_owned_set;
2870 locally_owned_set.reserve(dofh.size());
2871 for (
unsigned int j = 0; j < dofh.size(); ++j)
2873 locally_owned_set.push_back(dofh[j]->locally_owned_dofs());
2875 locally_owned_set.push_back(dofh[j]->locally_owned_mg_dofs(
level));
2876 return locally_owned_set;
2883 template <
int dim,
typename Number,
typename VectorizedArrayType>
2884 template <
typename QuadratureType,
typename number2,
typename MappingType>
2887 const MappingType & mapping,
2890 const QuadratureType & quad,
2894 std::vector<const DoFHandler<dim, dim> *>
dof_handlers;
2895 std::vector<const AffineConstraints<number2> *> constraints;
2898 constraints.push_back(&constraints_in);
2900 std::vector<IndexSet> locally_owned_sets =
2901 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
2904 std::vector<hp::QCollection<dim>> quad_hp;
2905 quad_hp.emplace_back(quad);
2917 template <
int dim,
typename Number,
typename VectorizedArrayType>
2918 template <
typename QuadratureType,
typename number2,
typename MappingType>
2921 const MappingType & mapping,
2924 const QuadratureType & quad,
2928 std::vector<IndexSet> locally_owned_set =
2929 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
2930 dof_handler, additional_data.
mg_level);
2931 std::vector<hp::QCollection<dim>> quad_hp;
2932 quad_hp.emplace_back(quad);
2944 template <
int dim,
typename Number,
typename VectorizedArrayType>
2945 template <
typename QuadratureType,
typename number2,
typename MappingType>
2948 const MappingType & mapping,
2951 const std::vector<QuadratureType> & quad,
2955 std::vector<IndexSet> locally_owned_set =
2956 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
2957 dof_handler, additional_data.
mg_level);
2958 std::vector<hp::QCollection<dim>> quad_hp;
2959 for (
unsigned int q = 0; q < quad.size(); ++q)
2960 quad_hp.emplace_back(quad[q]);
2987 template <
int dim,
typename Number,
typename VectorizedArrayType>
2988 struct VectorDataExchange
2995 static constexpr
unsigned int channel_shift = 20;
3004 const ::MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
3005 const typename ::MatrixFree<dim, Number, VectorizedArrayType>::
3006 DataAccessOnFaces vector_face_access,
3007 const unsigned int n_components)
3008 : matrix_free(matrix_free)
3009 , vector_face_access(
3010 matrix_free.get_task_info().face_partition_data.empty() ?
3011 ::
MatrixFree<dim, Number, VectorizedArrayType>::
3012 DataAccessOnFaces::unspecified :
3014 , ghosts_were_set(false)
3015 # ifdef DEAL_II_WITH_MPI
3016 , tmp_data(n_components)
3017 , requests(n_components)
3021 if (this->vector_face_access !=
3023 DataAccessOnFaces::unspecified)
3024 for (
unsigned int c = 0; c < matrix_free.n_components(); ++c)
3026 matrix_free.get_dof_info(c).vector_exchanger_face_variants.size(),
3035 ~VectorDataExchange()
3037 # ifdef DEAL_II_WITH_MPI
3038 for (
unsigned int i = 0; i < tmp_data.size(); ++i)
3039 if (tmp_data[i] !=
nullptr)
3040 matrix_free.release_scratch_data_non_threadsafe(tmp_data[i]);
3050 template <
typename VectorType>
3052 find_vector_in_mf(
const VectorType &vec,
3053 const bool check_global_compatibility =
true)
const
3056 for (
unsigned int c = 0; c < matrix_free.n_components(); ++c)
3057 if (vec.get_partitioner().get() ==
3058 matrix_free.get_dof_info(c).vector_partitioner.get())
3062 for (
unsigned int c = 0; c < matrix_free.n_components(); ++c)
3063 if (check_global_compatibility ?
3064 vec.get_partitioner()->is_globally_compatible(
3065 *matrix_free.get_dof_info(c).vector_partitioner) :
3066 vec.get_partitioner()->is_compatible(
3067 *matrix_free.get_dof_info(c).vector_partitioner))
3083 get_partitioner(
const unsigned int mf_component)
const
3086 .vector_exchanger_face_variants.size(),
3088 if (vector_face_access ==
3091 return *matrix_free.get_dof_info(mf_component)
3092 .vector_exchanger_face_variants[0];
3093 else if (vector_face_access ==
3096 return *matrix_free.get_dof_info(mf_component)
3097 .vector_exchanger_face_variants[1];
3098 else if (vector_face_access ==
3101 return *matrix_free.get_dof_info(mf_component)
3102 .vector_exchanger_face_variants[2];
3103 else if (vector_face_access ==
3105 DataAccessOnFaces::values_all_faces)
3106 return *matrix_free.get_dof_info(mf_component)
3107 .vector_exchanger_face_variants[3];
3108 else if (vector_face_access ==
3110 DataAccessOnFaces::gradients_all_faces)
3111 return *matrix_free.get_dof_info(mf_component)
3112 .vector_exchanger_face_variants[4];
3114 return *matrix_free.get_dof_info(mf_component).vector_exchanger.get();
3122 template <
typename VectorType,
3123 typename std::enable_if<is_not_parallel_vector<VectorType>,
3124 VectorType>::type * =
nullptr>
3126 update_ghost_values_start(
const unsigned int ,
3127 const VectorType & )
3136 typename VectorType,
3137 typename std::enable_if<!has_update_ghost_values_start<VectorType> &&
3138 !is_not_parallel_vector<VectorType>,
3139 VectorType>::type * =
nullptr>
3141 update_ghost_values_start(
const unsigned int component_in_block_vector,
3142 const VectorType & vec)
3144 (void)component_in_block_vector;
3145 bool ghosts_set = vec.has_ghost_elements();
3147 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3148 ghosts_set ==
false,
3152 ghosts_were_set =
true;
3154 vec.update_ghost_values();
3165 typename VectorType,
3166 typename std::enable_if<has_update_ghost_values_start<VectorType> &&
3167 !has_exchange_on_subset<VectorType>,
3168 VectorType>::type * =
nullptr>
3170 update_ghost_values_start(
const unsigned int component_in_block_vector,
3171 const VectorType & vec)
3173 (void)component_in_block_vector;
3174 bool ghosts_set = vec.has_ghost_elements();
3176 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3177 ghosts_set ==
false,
3181 ghosts_were_set =
true;
3183 vec.update_ghost_values_start(component_in_block_vector + channel_shift);
3195 typename VectorType,
3196 typename std::enable_if<has_update_ghost_values_start<VectorType> &&
3197 has_exchange_on_subset<VectorType>,
3198 VectorType>::type * =
nullptr>
3200 update_ghost_values_start(
const unsigned int component_in_block_vector,
3201 const VectorType & vec)
3204 std::is_same<Number, typename VectorType::value_type>::value,
3205 "Type mismatch between VectorType and VectorDataExchange");
3206 (void)component_in_block_vector;
3207 bool ghosts_set = vec.has_ghost_elements();
3209 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3210 ghosts_set ==
false,
3214 ghosts_were_set =
true;
3216 if (vec.size() != 0)
3218 # ifdef DEAL_II_WITH_MPI
3219 const unsigned int mf_component = find_vector_in_mf(vec);
3221 const auto &part = get_partitioner(mf_component);
3223 if (part.n_ghost_indices() == 0 && part.n_import_indices() == 0 &&
3224 part.n_import_sm_procs() == 0)
3227 tmp_data[component_in_block_vector] =
3228 matrix_free.acquire_scratch_data_non_threadsafe();
3229 tmp_data[component_in_block_vector]->resize_fast(
3230 part.n_import_indices());
3233 part.export_to_ghosted_array_start(
3234 component_in_block_vector * 2 + channel_shift,
3236 vec.shared_vector_data(),
3238 part.locally_owned_size(),
3239 matrix_free.get_dof_info(mf_component)
3240 .vector_partitioner->n_ghost_indices()),
3242 part.n_import_indices()),
3243 this->requests[component_in_block_vector]);
3255 typename VectorType,
3256 typename std::enable_if<!has_update_ghost_values_start<VectorType>,
3257 VectorType>::type * =
nullptr>
3259 update_ghost_values_finish(
const unsigned int ,
3260 const VectorType & )
3271 typename VectorType,
3272 typename std::enable_if<has_update_ghost_values_start<VectorType> &&
3273 !has_exchange_on_subset<VectorType>,
3274 VectorType>::type * =
nullptr>
3276 update_ghost_values_finish(
const unsigned int component_in_block_vector,
3277 const VectorType & vec)
3279 (void)component_in_block_vector;
3280 vec.update_ghost_values_finish();
3292 typename VectorType,
3293 typename std::enable_if<has_update_ghost_values_start<VectorType> &&
3294 has_exchange_on_subset<VectorType>,
3295 VectorType>::type * =
nullptr>
3297 update_ghost_values_finish(
const unsigned int component_in_block_vector,
3298 const VectorType & vec)
3301 std::is_same<Number, typename VectorType::value_type>::value,
3302 "Type mismatch between VectorType and VectorDataExchange");
3303 (void)component_in_block_vector;
3305 if (vec.size() != 0)
3307 # ifdef DEAL_II_WITH_MPI
3311 const unsigned int mf_component = find_vector_in_mf(vec);
3313 const auto &part = get_partitioner(mf_component);
3315 if (part.n_ghost_indices() != 0 || part.n_import_indices() != 0 ||
3316 part.n_import_sm_procs() != 0)
3318 part.export_to_ghosted_array_finish(
3320 vec.shared_vector_data(),
3322 part.locally_owned_size(),
3323 matrix_free.get_dof_info(mf_component)
3324 .vector_partitioner->n_ghost_indices()),
3325 this->requests[component_in_block_vector]);
3327 matrix_free.release_scratch_data_non_threadsafe(
3328 tmp_data[component_in_block_vector]);
3329 tmp_data[component_in_block_vector] =
nullptr;
3335 vec.set_ghost_state(
true);
3343 template <
typename VectorType,
3344 typename std::enable_if<is_not_parallel_vector<VectorType>,
3345 VectorType>::type * =
nullptr>
3347 compress_start(
const unsigned int ,
3357 template <
typename VectorType,
3358 typename std::enable_if<!has_compress_start<VectorType> &&
3359 !is_not_parallel_vector<VectorType>,
3360 VectorType>::type * =
nullptr>
3362 compress_start(
const unsigned int component_in_block_vector,
3365 (void)component_in_block_vector;
3377 template <
typename VectorType,
3378 typename std::enable_if<has_compress_start<VectorType> &&
3379 !has_exchange_on_subset<VectorType>,
3380 VectorType>::type * =
nullptr>
3382 compress_start(
const unsigned int component_in_block_vector,
3385 (void)component_in_block_vector;
3387 vec.compress_start(component_in_block_vector + channel_shift);
3398 template <
typename VectorType,
3399 typename std::enable_if<has_compress_start<VectorType> &&
3400 has_exchange_on_subset<VectorType>,
3401 VectorType>::type * =
nullptr>
3403 compress_start(
const unsigned int component_in_block_vector,
3407 std::is_same<Number, typename VectorType::value_type>::value,
3408 "Type mismatch between VectorType and VectorDataExchange");
3409 (void)component_in_block_vector;
3412 if (vec.size() != 0)
3414 # ifdef DEAL_II_WITH_MPI
3415 const unsigned int mf_component = find_vector_in_mf(vec);
3417 const auto &part = get_partitioner(mf_component);
3419 if (part.n_ghost_indices() == 0 && part.n_import_indices() == 0 &&
3420 part.n_import_sm_procs() == 0)
3423 tmp_data[component_in_block_vector] =
3424 matrix_free.acquire_scratch_data_non_threadsafe();
3425 tmp_data[component_in_block_vector]->resize_fast(
3426 part.n_import_indices());
3429 part.import_from_ghosted_array_start(
3431 component_in_block_vector * 2 + channel_shift,
3433 vec.shared_vector_data(),
3435 matrix_free.get_dof_info(mf_component)
3436 .vector_partitioner->n_ghost_indices()),
3438 part.n_import_indices()),
3439 this->requests[component_in_block_vector]);
3450 template <
typename VectorType,
3451 typename std::enable_if<!has_compress_start<VectorType>,
3452 VectorType>::type * =
nullptr>
3454 compress_finish(
const unsigned int ,
3465 template <
typename VectorType,
3466 typename std::enable_if<has_compress_start<VectorType> &&
3467 !has_exchange_on_subset<VectorType>,
3468 VectorType>::type * =
nullptr>
3470 compress_finish(
const unsigned int component_in_block_vector,
3473 (void)component_in_block_vector;
3485 template <
typename VectorType,
3486 typename std::enable_if<has_compress_start<VectorType> &&
3487 has_exchange_on_subset<VectorType>,
3488 VectorType>::type * =
nullptr>
3490 compress_finish(
const unsigned int component_in_block_vector,
3494 std::is_same<Number, typename VectorType::value_type>::value,
3495 "Type mismatch between VectorType and VectorDataExchange");
3496 (void)component_in_block_vector;
3497 if (vec.size() != 0)
3499 # ifdef DEAL_II_WITH_MPI
3503 const unsigned int mf_component = find_vector_in_mf(vec);
3505 const auto &part = get_partitioner(mf_component);
3507 if (part.n_ghost_indices() != 0 || part.n_import_indices() != 0 ||
3508 part.n_import_sm_procs() != 0)
3510 part.import_from_ghosted_array_finish(
3513 vec.shared_vector_data(),
3515 matrix_free.get_dof_info(mf_component)
3516 .vector_partitioner->n_ghost_indices()),
3518 tmp_data[component_in_block_vector]->begin(),
3519 part.n_import_indices()),
3520 this->requests[component_in_block_vector]);
3522 matrix_free.release_scratch_data_non_threadsafe(
3523 tmp_data[component_in_block_vector]);
3524 tmp_data[component_in_block_vector] =
nullptr;
3530 MPI_Barrier(matrix_free.get_task_info().communicator_sm);
3542 template <
typename VectorType,
3543 typename std::enable_if<is_not_parallel_vector<VectorType>,
3544 VectorType>::type * =
nullptr>
3546 reset_ghost_values(
const VectorType & )
const
3555 template <
typename VectorType,
3556 typename std::enable_if<!has_exchange_on_subset<VectorType> &&
3557 !is_not_parallel_vector<VectorType>,
3558 VectorType>::type * =
nullptr>
3560 reset_ghost_values(
const VectorType &vec)
const
3562 if (ghosts_were_set ==
true)
3565 vec.zero_out_ghost_values();
3575 template <
typename VectorType,
3576 typename std::enable_if<has_exchange_on_subset<VectorType>,
3577 VectorType>::type * =
nullptr>
3579 reset_ghost_values(
const VectorType &vec)
const
3582 std::is_same<Number, typename VectorType::value_type>::value,
3583 "Type mismatch between VectorType and VectorDataExchange");
3584 if (ghosts_were_set ==
true)
3587 if (vec.size() != 0)
3589 # ifdef DEAL_II_WITH_MPI
3592 const unsigned int mf_component = find_vector_in_mf(vec);
3594 const auto &part = get_partitioner(mf_component);
3596 if (part.n_ghost_indices() > 0)
3601 part.locally_owned_size(),
3602 matrix_free.get_dof_info(mf_component)
3603 .vector_partitioner->n_ghost_indices()));
3609 vec.set_ghost_state(
false);
3619 template <
typename VectorType,
3620 typename std::enable_if<has_exchange_on_subset<VectorType>,
3621 VectorType>::type * =
nullptr>
3623 zero_vector_region(
const unsigned int range_index, VectorType &vec)
const
3626 std::is_same<Number, typename VectorType::value_type>::value,
3627 "Type mismatch between VectorType and VectorDataExchange");
3632 const unsigned int mf_component = find_vector_in_mf(vec,
false);
3634 matrix_free.get_dof_info(mf_component);
3642 for (
unsigned int id =
3661 template <
typename VectorType,
3662 typename std::enable_if<!has_exchange_on_subset<VectorType>,
3663 VectorType>::type * =
nullptr,
3664 typename VectorType::value_type * =
nullptr>
3666 zero_vector_region(
const unsigned int range_index, VectorType &vec)
const
3669 vec =
typename VectorType::value_type();
3679 zero_vector_region(
const unsigned int, ...)
const
3683 "which provide VectorType::value_type"));
3688 const ::MatrixFree<dim, Number, VectorizedArrayType> &matrix_free;
3689 const typename ::MatrixFree<dim, Number, VectorizedArrayType>::
3690 DataAccessOnFaces vector_face_access;
3691 bool ghosts_were_set;
3692 # ifdef DEAL_II_WITH_MPI
3693 std::vector<AlignedVector<Number> *> tmp_data;
3694 std::vector<std::vector<MPI_Request>> requests;
3698 template <
typename VectorStruct>
3700 n_components(
const VectorStruct &vec);
3702 template <
typename VectorStruct>
3704 n_components_block(
const VectorStruct &vec,
3705 std::integral_constant<bool, true>)
3707 unsigned int components = 0;
3708 for (
unsigned int bl = 0; bl < vec.n_blocks(); ++bl)
3709 components += n_components(vec.block(bl));
3713 template <
typename VectorStruct>
3715 n_components_block(
const VectorStruct &, std::integral_constant<bool, false>)
3720 template <
typename VectorStruct>
3722 n_components(
const VectorStruct &vec)
3724 return n_components_block(
3728 template <
typename VectorStruct>
3730 n_components(
const std::vector<VectorStruct> &vec)
3732 unsigned int components = 0;
3733 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3734 components += n_components_block(
3740 template <
typename VectorStruct>
3742 n_components(
const std::vector<VectorStruct *> &vec)
3744 unsigned int components = 0;
3745 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3746 components += n_components_block(
3759 template <
typename VectorStruct,
3760 typename std::enable_if<!has_communication_block_size<VectorStruct>,
3761 VectorStruct>::type * =
nullptr>
3762 constexpr
unsigned int
3763 get_communication_block_size(
const VectorStruct &)
3770 template <
typename VectorStruct,
3771 typename std::enable_if<has_communication_block_size<VectorStruct>,
3772 VectorStruct>::type * =
nullptr>
3773 constexpr
unsigned int
3774 get_communication_block_size(
const VectorStruct &)
3776 return VectorStruct::communication_block_size;
3791 typename VectorStruct,
3793 typename VectorizedArrayType,
3794 typename std::enable_if<IsBlockVector<VectorStruct>::value,
3795 VectorStruct>::type * =
nullptr>
3797 update_ghost_values_start(
3798 const VectorStruct & vec,
3799 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
3800 const unsigned int channel = 0)
3802 if (get_communication_block_size(vec) < vec.n_blocks())
3806 exchanger.ghosts_were_set = vec.has_ghost_elements();
3807 vec.update_ghost_values();
3811 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
3812 update_ghost_values_start(vec.block(i), exchanger, channel + i);
3820 typename VectorStruct,
3822 typename VectorizedArrayType,
3823 typename std::enable_if<!IsBlockVector<VectorStruct>::value,
3824 VectorStruct>::type * =
nullptr>
3826 update_ghost_values_start(
3827 const VectorStruct & vec,
3828 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
3829 const unsigned int channel = 0)
3831 exchanger.update_ghost_values_start(channel, vec);
3838 typename VectorStruct,
3840 typename VectorizedArrayType>
3842 update_ghost_values_start(
3843 const std::vector<VectorStruct> & vec,
3844 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
3846 unsigned int component_index = 0;
3847 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3849 update_ghost_values_start(vec[comp], exchanger, component_index);
3850 component_index += n_components(vec[comp]);
3858 typename VectorStruct,
3860 typename VectorizedArrayType>
3862 update_ghost_values_start(
3863 const std::vector<VectorStruct *> & vec,
3864 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
3866 unsigned int component_index = 0;
3867 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3869 update_ghost_values_start(*vec[comp], exchanger, component_index);
3870 component_index += n_components(*vec[comp]);
3882 typename VectorStruct,
3884 typename VectorizedArrayType,
3885 typename std::enable_if<IsBlockVector<VectorStruct>::value,
3886 VectorStruct>::type * =
nullptr>
3888 update_ghost_values_finish(
3889 const VectorStruct & vec,
3890 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
3891 const unsigned int channel = 0)
3893 if (get_communication_block_size(vec) < vec.n_blocks())
3899 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
3900 update_ghost_values_finish(vec.block(i), exchanger, channel + i);
3907 typename VectorStruct,
3909 typename VectorizedArrayType,
3910 typename std::enable_if<!IsBlockVector<VectorStruct>::value,
3911 VectorStruct>::type * =
nullptr>
3913 update_ghost_values_finish(
3914 const VectorStruct & vec,
3915 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
3916 const unsigned int channel = 0)
3918 exchanger.update_ghost_values_finish(channel, vec);
3925 typename VectorStruct,
3927 typename VectorizedArrayType>
3929 update_ghost_values_finish(
3930 const std::vector<VectorStruct> & vec,
3931 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
3933 unsigned int component_index = 0;
3934 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3936 update_ghost_values_finish(vec[comp], exchanger, component_index);
3937 component_index += n_components(vec[comp]);
3945 typename VectorStruct,
3947 typename VectorizedArrayType>
3949 update_ghost_values_finish(
3950 const std::vector<VectorStruct *> & vec,
3951 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
3953 unsigned int component_index = 0;
3954 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
3956 update_ghost_values_finish(*vec[comp], exchanger, component_index);
3957 component_index += n_components(*vec[comp]);
3969 typename VectorStruct,
3971 typename VectorizedArrayType,
3972 typename std::enable_if<IsBlockVector<VectorStruct>::value,
3973 VectorStruct>::type * =
nullptr>
3977 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
3978 const unsigned int channel = 0)
3980 if (get_communication_block_size(vec) < vec.n_blocks())
3983 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
3984 compress_start(vec.block(i), exchanger, channel + i);
3991 typename VectorStruct,
3993 typename VectorizedArrayType,
3994 typename std::enable_if<!IsBlockVector<VectorStruct>::value,
3995 VectorStruct>::type * =
nullptr>
3999 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4000 const unsigned int channel = 0)
4002 exchanger.compress_start(channel, vec);
4009 typename VectorStruct,
4011 typename VectorizedArrayType>
4014 std::vector<VectorStruct> & vec,
4015 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4017 unsigned int component_index = 0;
4018 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4020 compress_start(vec[comp], exchanger, component_index);
4021 component_index += n_components(vec[comp]);
4029 typename VectorStruct,
4031 typename VectorizedArrayType>
4034 std::vector<VectorStruct *> & vec,
4035 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4037 unsigned int component_index = 0;
4038 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4040 compress_start(*vec[comp], exchanger, component_index);
4041 component_index += n_components(*vec[comp]);
4053 typename VectorStruct,
4055 typename VectorizedArrayType,
4056 typename std::enable_if<IsBlockVector<VectorStruct>::value,
4057 VectorStruct>::type * =
nullptr>
4061 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4062 const unsigned int channel = 0)
4064 if (get_communication_block_size(vec) < vec.n_blocks())
4070 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4071 compress_finish(vec.block(i), exchanger, channel + i);
4078 typename VectorStruct,
4080 typename VectorizedArrayType,
4081 typename std::enable_if<!IsBlockVector<VectorStruct>::value,
4082 VectorStruct>::type * =
nullptr>
4086 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4087 const unsigned int channel = 0)
4089 exchanger.compress_finish(channel, vec);
4096 typename VectorStruct,
4098 typename VectorizedArrayType>
4101 std::vector<VectorStruct> & vec,
4102 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4104 unsigned int component_index = 0;
4105 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4107 compress_finish(vec[comp], exchanger, component_index);
4108 component_index += n_components(vec[comp]);
4116 typename VectorStruct,
4118 typename VectorizedArrayType>
4121 std::vector<VectorStruct *> & vec,
4122 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4124 unsigned int component_index = 0;
4125 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4127 compress_finish(*vec[comp], exchanger, component_index);
4128 component_index += n_components(*vec[comp]);
4144 typename VectorStruct,
4146 typename VectorizedArrayType,
4147 typename std::enable_if<IsBlockVector<VectorStruct>::value,
4148 VectorStruct>::type * =
nullptr>
4151 const VectorStruct & vec,
4152 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4155 if (exchanger.ghosts_were_set ==
true)
4158 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4159 reset_ghost_values(vec.block(i), exchanger);
4166 typename VectorStruct,
4168 typename VectorizedArrayType,
4169 typename std::enable_if<!IsBlockVector<VectorStruct>::value,
4170 VectorStruct>::type * =
nullptr>
4173 const VectorStruct & vec,
4174 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4176 exchanger.reset_ghost_values(vec);
4183 typename VectorStruct,
4185 typename VectorizedArrayType>
4188 const std::vector<VectorStruct> & vec,
4189 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4192 if (exchanger.ghosts_were_set ==
true)
4195 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4196 reset_ghost_values(vec[comp], exchanger);
4203 typename VectorStruct,
4205 typename VectorizedArrayType>
4208 const std::vector<VectorStruct *> & vec,
4209 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4212 if (exchanger.ghosts_were_set ==
true)
4215 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4216 reset_ghost_values(*vec[comp], exchanger);
4227 typename VectorStruct,
4229 typename VectorizedArrayType,
4230 typename std::enable_if<IsBlockVector<VectorStruct>::value,
4231 VectorStruct>::type * =
nullptr>
4234 const unsigned int range_index,
4236 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4238 for (
unsigned int i = 0; i < vec.n_blocks(); ++i)
4239 exchanger.zero_vector_region(range_index, vec.block(i));
4246 typename VectorStruct,
4248 typename VectorizedArrayType,
4249 typename std::enable_if<!IsBlockVector<VectorStruct>::value,
4250 VectorStruct>::type * =
nullptr>
4253 const unsigned int range_index,
4255 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4257 exchanger.zero_vector_region(range_index, vec);
4264 typename VectorStruct,
4266 typename VectorizedArrayType>
4269 const unsigned int range_index,
4270 std::vector<VectorStruct> & vec,
4271 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4273 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4274 zero_vector_region(range_index, vec[comp], exchanger);
4281 typename VectorStruct,
4283 typename VectorizedArrayType>
4286 const unsigned int range_index,
4287 std::vector<VectorStruct *> & vec,
4288 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4290 for (
unsigned int comp = 0; comp < vec.size(); ++comp)
4291 zero_vector_region(range_index, *vec[comp], exchanger);
4299 template <
typename VectorStruct1,
typename VectorStruct2>
4301 apply_operation_to_constrained_dofs(
const std::vector<unsigned int> &,
4302 const VectorStruct1 &,
4306 template <
typename Number>
4308 apply_operation_to_constrained_dofs(
4309 const std::vector<unsigned int> & constrained_dofs,
4313 for (
const unsigned int i : constrained_dofs)
4318 namespace MatrixFreeFunctions
4322 template <
typename,
typename,
typename,
typename,
bool>
4323 struct InterfaceSelector
4327 template <
typename MF,
4331 struct InterfaceSelector<MF, InVector, OutVector, Container, true>
4333 using function_type = void (Container::*)(
4337 const std::pair<unsigned int, unsigned int> &)
const;
4341 template <
typename MF,
4345 struct InterfaceSelector<MF, InVector, OutVector, Container, false>
4347 using function_type =
4348 void (Container::*)(
const MF &,
4351 const std::pair<unsigned int, unsigned int> &);
4359 template <
typename MF,
4364 class MFWorker :
public MFWorkerInterface
4368 using function_type =
typename MatrixFreeFunctions::
4369 InterfaceSelector<MF, InVector, OutVector, Container, is_constant>::
4373 MFWorker(
const MF & matrix_free,
4374 const InVector & src,
4376 const bool zero_dst_vector_setting,
4377 const Container & container,
4378 function_type cell_function,
4379 function_type face_function,
4380 function_type boundary_function,
4381 const typename MF::DataAccessOnFaces src_vector_face_access =
4383 const typename MF::DataAccessOnFaces dst_vector_face_access =
4385 const std::function<
void(
const unsigned int,
const unsigned int)>
4386 &operation_before_loop = {},
4387 const std::function<void(
const unsigned int,
const unsigned int)>
4388 & operation_after_loop = {},
4389 const unsigned int dof_handler_index_pre_post = 0)
4390 : matrix_free(matrix_free)
4391 , container(
const_cast<Container &
>(container))
4392 , cell_function(cell_function)
4393 , face_function(face_function)
4394 , boundary_function(boundary_function)
4397 , src_data_exchanger(matrix_free,
4398 src_vector_face_access,
4400 , dst_data_exchanger(matrix_free,
4401 dst_vector_face_access,
4404 , zero_dst_vector_setting(zero_dst_vector_setting &&
4405 !src_and_dst_are_same)
4406 , operation_before_loop(operation_before_loop)
4407 , operation_after_loop(operation_after_loop)
4408 , dof_handler_index_pre_post(dof_handler_index_pre_post)
4413 cell(
const std::pair<unsigned int, unsigned int> &cell_range)
override
4415 if (cell_function !=
nullptr && cell_range.second > cell_range.first)
4416 for (
unsigned int i = 0; i < matrix_free.n_active_fe_indices(); ++i)
4418 const auto cell_subrange =
4419 matrix_free.create_cell_subrange_hp_by_index(cell_range, i);
4421 if (cell_subrange.second <= cell_subrange.first)
4425 cell_function)(matrix_free, this->dst, this->src, cell_subrange);
4430 cell(
const unsigned int range_index)
override
4432 process_range(cell_function,
4433 matrix_free.get_task_info().cell_partition_data_hp_ptr,
4434 matrix_free.get_task_info().cell_partition_data_hp,
4439 face(
const unsigned int range_index)
override
4441 process_range(face_function,
4442 matrix_free.get_task_info().face_partition_data_hp_ptr,
4443 matrix_free.get_task_info().face_partition_data_hp,
4448 boundary(
const unsigned int range_index)
override
4450 process_range(boundary_function,
4451 matrix_free.get_task_info().boundary_partition_data_hp_ptr,
4452 matrix_free.get_task_info().boundary_partition_data_hp,
4458 process_range(
const function_type & fu,
4459 const std::vector<unsigned int> &ptr,
4460 const std::vector<unsigned int> &data,
4461 const unsigned int range_index)
4467 for (
unsigned int i = ptr[range_index]; i < ptr[range_index + 1]; ++i)
4470 (container.*fu)(matrix_free,
4473 std::make_pair(data[2 * i], data[2 * i + 1]));
4485 vector_update_ghosts_start()
override
4487 if (!src_and_dst_are_same)
4488 internal::update_ghost_values_start(src, src_data_exchanger);
4493 vector_update_ghosts_finish()
override
4495 if (!src_and_dst_are_same)
4496 internal::update_ghost_values_finish(src, src_data_exchanger);
4501 vector_compress_start()
override
4503 internal::compress_start(dst, dst_data_exchanger);
4508 vector_compress_finish()
override
4510 internal::compress_finish(dst, dst_data_exchanger);
4511 if (!src_and_dst_are_same)
4512 internal::reset_ghost_values(src, src_data_exchanger);
4517 zero_dst_vector_range(
const unsigned int range_index)
override
4519 if (zero_dst_vector_setting)
4520 internal::zero_vector_region(range_index, dst, dst_data_exchanger);
4524 cell_loop_pre_range(
const unsigned int range_index)
override
4526 if (operation_before_loop)
4529 matrix_free.get_dof_info(dof_handler_index_pre_post);
4536 operation_before_loop,
4543 for (
unsigned int id =
4554 cell_loop_post_range(
const unsigned int range_index)
override
4556 if (operation_after_loop)
4560 const std::vector<unsigned int> &partition_row_index =
4561 matrix_free.get_task_info().partition_row_index;
4563 partition_row_index[partition_row_index.size() - 2] - 1)
4564 apply_operation_to_constrained_dofs(
4565 matrix_free.get_constrained_dofs(dof_handler_index_pre_post),
4570 matrix_free.get_dof_info(dof_handler_index_pre_post);
4577 operation_after_loop,
4584 for (
unsigned int id =
4595 const MF & matrix_free;
4596 Container & container;
4597 function_type cell_function;
4598 function_type face_function;
4599 function_type boundary_function;
4601 const InVector &src;
4603 VectorDataExchange<MF::dimension,
4604 typename MF::value_type,
4605 typename MF::vectorized_value_type>
4607 VectorDataExchange<MF::dimension,
4608 typename MF::value_type,
4609 typename MF::vectorized_value_type>
4611 const bool src_and_dst_are_same;
4612 const bool zero_dst_vector_setting;
4613 const std::function<void(
const unsigned int,
const unsigned int)>
4614 operation_before_loop;
4615 const std::function<void(
const unsigned int,
const unsigned int)>
4616 operation_after_loop;
4617 const unsigned int dof_handler_index_pre_post;
4626 template <
class MF,
typename InVector,
typename OutVector>
4627 struct MFClassWrapper
4629 using function_type =
4630 std::function<void(
const MF &,
4633 const std::pair<unsigned int, unsigned int> &)>;
4635 MFClassWrapper(
const function_type cell,
4636 const function_type face,
4637 const function_type boundary)
4640 , boundary(boundary)
4644 cell_integrator(
const MF & mf,
4646 const InVector & src,
4647 const std::pair<unsigned int, unsigned int> &range)
const
4650 cell(mf, dst, src, range);
4654 face_integrator(
const MF & mf,
4656 const InVector & src,
4657 const std::pair<unsigned int, unsigned int> &range)
const
4660 face(mf, dst, src, range);
4664 boundary_integrator(
4667 const InVector & src,
4668 const std::pair<unsigned int, unsigned int> &range)
const
4671 boundary(mf, dst, src, range);
4674 const function_type cell;
4675 const function_type face;
4676 const function_type boundary;
4683 template <
int dim,
typename Number,
typename VectorizedArrayType>
4684 template <
typename OutVector,
typename InVector>
4690 const std::pair<unsigned int, unsigned int> &)>
4693 const InVector &src,
4694 const bool zero_dst_vector)
const
4697 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
4700 Wrapper wrap(cell_operation,
nullptr,
nullptr);
4701 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4711 &Wrapper::cell_integrator,
4712 &Wrapper::face_integrator,
4713 &Wrapper::boundary_integrator);
4720 template <
int dim,
typename Number,
typename VectorizedArrayType>
4721 template <
typename OutVector,
typename InVector>
4727 const std::pair<unsigned int, unsigned int> &)>
4730 const InVector &src,
4731 const std::function<
void(
const unsigned int,
const unsigned int)>
4732 &operation_before_loop,
4733 const std::function<
void(
const unsigned int,
const unsigned int)>
4734 & operation_after_loop,
4735 const unsigned int dof_handler_index_pre_post)
const
4738 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
4741 Wrapper wrap(cell_operation,
nullptr,
nullptr);
4742 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4752 &Wrapper::cell_integrator,
4753 &Wrapper::face_integrator,
4754 &Wrapper::boundary_integrator,
4757 operation_before_loop,
4758 operation_after_loop,
4759 dof_handler_index_pre_post);
4766 template <
int dim,
typename Number,
typename VectorizedArrayType>
4767 template <
typename OutVector,
typename InVector>
4773 const std::pair<unsigned int, unsigned int> &)>
4778 const std::pair<unsigned int, unsigned int> &)>
4783 const std::pair<unsigned int, unsigned int> &)>
4784 & boundary_operation,
4786 const InVector & src,
4787 const bool zero_dst_vector,
4788 const DataAccessOnFaces dst_vector_face_access,
4789 const DataAccessOnFaces src_vector_face_access)
const
4792 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
4795 Wrapper wrap(cell_operation, face_operation, boundary_operation);
4796 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4806 &Wrapper::cell_integrator,
4807 &Wrapper::face_integrator,
4808 &Wrapper::boundary_integrator,
4809 src_vector_face_access,
4810 dst_vector_face_access);
4817 template <
int dim,
typename Number,
typename VectorizedArrayType>
4818 template <
typename CLASS,
typename OutVector,
typename InVector>
4821 void (CLASS::*function_pointer)(
4825 const std::pair<unsigned int, unsigned int> &)
const,
4826 const CLASS * owning_class,
4828 const InVector &src,
4829 const bool zero_dst_vector)
const
4831 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4849 template <
int dim,
typename Number,
typename VectorizedArrayType>
4850 template <
typename CLASS,
typename OutVector,
typename InVector>
4853 void (CLASS::*function_pointer)(
4857 const std::pair<unsigned int, unsigned int> &)
const,
4858 const CLASS * owning_class,
4860 const InVector &src,
4861 const std::function<
void(
const unsigned int,
const unsigned int)>
4862 &operation_before_loop,
4863 const std::function<
void(
const unsigned int,
const unsigned int)>
4864 & operation_after_loop,
4865 const unsigned int dof_handler_index_pre_post)
const
4867 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4882 operation_before_loop,
4883 operation_after_loop,
4884 dof_handler_index_pre_post);
4890 template <
int dim,
typename Number,
typename VectorizedArrayType>
4891 template <
typename CLASS,
typename OutVector,
typename InVector>
4894 void (CLASS::*cell_operation)(
4898 const std::pair<unsigned int, unsigned int> &)
const,
4899 void (CLASS::*face_operation)(
4903 const std::pair<unsigned int, unsigned int> &)
const,
4904 void (CLASS::*boundary_operation)(
4908 const std::pair<unsigned int, unsigned int> &)
const,
4909 const CLASS * owning_class,
4911 const InVector & src,
4912 const bool zero_dst_vector,
4913 const DataAccessOnFaces dst_vector_face_access,
4914 const DataAccessOnFaces src_vector_face_access)
const
4916 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4929 src_vector_face_access,
4930 dst_vector_face_access);
4936 template <
int dim,
typename Number,
typename VectorizedArrayType>
4937 template <
typename CLASS,
typename OutVector,
typename InVector>
4940 void (CLASS::*function_pointer)(
4944 const std::pair<unsigned int, unsigned int> &),
4945 CLASS * owning_class,
4947 const InVector &src,
4948 const bool zero_dst_vector)
const
4950 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
4968 template <
int dim,
typename Number,
typename VectorizedArrayType>
4969 template <
typename CLASS,
typename OutVector,
typename InVector>
4972 void (CLASS::*function_pointer)(
4976 const std::pair<unsigned int, unsigned int> &),
4977 CLASS * owning_class,
4979 const InVector &src,
4980 const std::function<
void(
const unsigned int,
const unsigned int)>
4981 &operation_before_loop,
4982 const std::function<
void(
const unsigned int,
const unsigned int)>
4983 & operation_after_loop,
4984 const unsigned int dof_handler_index_pre_post)
const
4986 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5001 operation_before_loop,
5002 operation_after_loop,
5003 dof_handler_index_pre_post);
5009 template <
int dim,
typename Number,
typename VectorizedArrayType>
5010 template <
typename CLASS,
typename OutVector,
typename InVector>
5013 void (CLASS::*cell_operation)(
5017 const std::pair<unsigned int, unsigned int> &),
5018 void (CLASS::*face_operation)(
5022 const std::pair<unsigned int, unsigned int> &),
5023 void (CLASS::*boundary_operation)(
5027 const std::pair<unsigned int, unsigned int> &),
5028 CLASS * owning_class,
5030 const InVector & src,
5031 const bool zero_dst_vector,
5032 const DataAccessOnFaces dst_vector_face_access,
5033 const DataAccessOnFaces src_vector_face_access)
const
5035 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5048 src_vector_face_access,
5049 dst_vector_face_access);
5055 template <
int dim,
typename Number,
typename VectorizedArrayType>
5056 template <
typename CLASS,
typename OutVector,
typename InVector>
5059 void (CLASS::*function_pointer)(
5063 const std::pair<unsigned int, unsigned int> &)
const,
5064 const CLASS * owning_class,
5066 const InVector & src,
5067 const bool zero_dst_vector,
5068 const DataAccessOnFaces src_vector_face_access)
const
5070 auto src_vector_face_access_temp = src_vector_face_access;
5076 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5089 src_vector_face_access_temp,
5096 template <
int dim,
typename Number,
typename VectorizedArrayType>
5097 template <
typename CLASS,
typename OutVector,
typename InVector>
5100 void (CLASS::*function_pointer)(
5104 const std::pair<unsigned int, unsigned int> &),
5105 CLASS * owning_class,
5107 const InVector & src,
5108 const bool zero_dst_vector,
5109 const DataAccessOnFaces src_vector_face_access)
const
5111 auto src_vector_face_access_temp = src_vector_face_access;
5117 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5130 src_vector_face_access_temp,
5137 template <
int dim,
typename Number,
typename VectorizedArrayType>
5138 template <
typename OutVector,
typename InVector>
5144 const std::pair<unsigned int, unsigned int> &)>
5147 const InVector & src,
5148 const bool zero_dst_vector,
5149 const DataAccessOnFaces src_vector_face_access)
const
5151 auto src_vector_face_access_temp = src_vector_face_access;
5158 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5161 Wrapper wrap(cell_operation,
nullptr,
nullptr);
5163 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5173 &Wrapper::cell_integrator,
5174 &Wrapper::face_integrator,
5175 &Wrapper::boundary_integrator,
5176 src_vector_face_access_temp,
void resize(const size_type new_size)
Abstract base class for mapping classes.
unsigned int n_active_fe_indices() const
void loop_cell_centric(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
const Table< 3, unsigned int > & get_cell_and_face_to_plain_faces() const
internal::MatrixFreeFunctions::MappingInfo< dim, Number, VectorizedArrayType > mapping_info
unsigned int get_cell_active_fe_index(const std::pair< unsigned int, unsigned int > range) const
AlignedVector< Number > * acquire_scratch_data_non_threadsafe() const
unsigned int n_ghost_cell_batches() const
void loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &face_operation, const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &boundary_operation, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void initialize_dof_handlers(const std::vector< const DoFHandler< dim, dim > * > &dof_handlers, const AdditionalData &additional_data)
types::boundary_id get_boundary_id(const unsigned int face_batch_index) const
Table< 4, internal::MatrixFreeFunctions::ShapeInfo< VectorizedArrayType > > shape_info
const Quadrature< dim > & get_quadrature(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
std::pair< unsigned int, unsigned int > create_cell_subrange_hp(const std::pair< unsigned int, unsigned int > &range, const unsigned int fe_degree, const unsigned int dof_handler_index=0) const
const IndexSet & get_ghost_set(const unsigned int dof_handler_index=0) const
void print(std::ostream &out) const
const Quadrature< dim - 1 > & get_face_quadrature(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
unsigned int n_inner_face_batches() const
void internal_reinit(const std::shared_ptr< hp::MappingCollection< dim >> &mapping, const std::vector< const DoFHandler< dim, dim > * > &dof_handlers, const std::vector< const AffineConstraints< number2 > * > &constraint, const std::vector< IndexSet > &locally_owned_set, const std::vector< hp::QCollection< q_dim >> &quad, const AdditionalData &additional_data)
unsigned int n_active_entries_per_cell_batch(const unsigned int cell_batch_index) const
void update_mapping(const Mapping< dim > &mapping)
unsigned int get_mg_level() const
void print_memory_consumption(StreamType &out) const
void update_mapping(const std::shared_ptr< hp::MappingCollection< dim >> &mapping)
bool mapping_initialized() const
void cell_loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0) const
const Number * constraint_pool_begin(const unsigned int pool_index) const
unsigned int get_n_q_points_face(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
const internal::MatrixFreeFunctions::ShapeInfo< VectorizedArrayType > & get_shape_info(const unsigned int dof_handler_index_component=0, const unsigned int quad_index=0, const unsigned int fe_base_element=0, const unsigned int hp_active_fe_index=0, const unsigned int hp_active_quad_index=0) const
std::pair< unsigned int, unsigned int > create_cell_subrange_hp_by_index(const std::pair< unsigned int, unsigned int > &range, const unsigned int fe_index, const unsigned int dof_handler_index=0) const
bool at_irregular_cell(const unsigned int cell_batch_index) const
~MatrixFree() override=default
void copy_from(const MatrixFree< dim, Number, VectorizedArrayType > &matrix_free_base)
void initialize_face_data_vector(AlignedVector< T > &vec) const
unsigned int get_dofs_per_cell(const unsigned int dof_handler_index=0, const unsigned int hp_active_fe_index=0) const
unsigned int n_constraint_pool_entries() const
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0) const
std::pair< int, int > get_cell_level_and_index(const unsigned int cell_batch_index, const unsigned int lane_index) const
std::pair< unsigned int, unsigned int > get_face_range_category(const std::pair< unsigned int, unsigned int > face_batch_range) const
const std::vector< unsigned int > & get_constrained_dofs(const unsigned int dof_handler_index=0) const
std::pair< unsigned int, unsigned int > get_face_category(const unsigned int face_batch_index) const
const internal::MatrixFreeFunctions::DoFInfo & get_dof_info(const unsigned int dof_handler_index_component=0) const
unsigned int get_dofs_per_face(const unsigned int dof_handler_index=0, const unsigned int hp_active_fe_index=0) const
const Number * constraint_pool_end(const unsigned int pool_index) const
internal::MatrixFreeFunctions::TaskInfo task_info
void release_scratch_data(const AlignedVector< VectorizedArrayType > *memory) const
void loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, void(CLASS::*face_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, void(CLASS::*boundary_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
std::pair< typename DoFHandler< dim >::cell_iterator, unsigned int > get_face_iterator(const unsigned int face_batch_index, const unsigned int lane_index, const bool interior=true, const unsigned int fe_component=0) const
bool mapping_is_initialized
void loop_cell_centric(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void initialize_dof_vector(LinearAlgebra::distributed::Vector< Number2 > &vec, const unsigned int dof_handler_index=0) const
MatrixFree(const MatrixFree< dim, Number, VectorizedArrayType > &other)
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const std::function< void(const unsigned int, const unsigned int)> &operation_before_loop, const std::function< void(const unsigned int, const unsigned int)> &operation_after_loop, const unsigned int dof_handler_index_pre_post=0) const
std::vector< std::pair< unsigned int, unsigned int > > cell_level_index
std::array< types::boundary_id, VectorizedArrayType::size()> get_faces_by_cells_boundary_id(const unsigned int cell_batch_index, const unsigned int face_number) const
std::vector< Number > constraint_pool_data
VectorizedArrayType vectorized_value_type
unsigned int n_cell_batches() const
bool indices_initialized() const
const internal::MatrixFreeFunctions::MappingInfo< dim, Number, VectorizedArrayType > & get_mapping_info() const
unsigned int get_cell_category(const unsigned int cell_batch_index) const
Threads::ThreadLocalStorage< std::list< std::pair< bool, AlignedVector< VectorizedArrayType > > > > scratch_pad
const internal::MatrixFreeFunctions::FaceToCellTopology< VectorizedArrayType::size()> & get_face_info(const unsigned int face_batch_index) const
std::size_t memory_consumption() const
unsigned int n_boundary_face_batches() const
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &) const, const CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false) const
std::list< std::pair< bool, AlignedVector< Number > > > scratch_pad_non_threadsafe
DoFHandler< dim >::cell_iterator get_cell_iterator(const unsigned int cell_batch_index, const unsigned int lane_index, const unsigned int dof_handler_index=0) const
void loop_cell_centric(const std::function< void(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
std::vector< SmartPointer< const DoFHandler< dim > > > dof_handlers
void initialize_indices(const std::vector< const AffineConstraints< number2 > * > &constraint, const std::vector< IndexSet > &locally_owned_set, const AdditionalData &additional_data)
unsigned int n_components() const
unsigned int n_physical_cells() const
void initialize_dof_vector(VectorType &vec, const unsigned int dof_handler_index=0) const
unsigned int get_n_q_points(const unsigned int quad_index=0, const unsigned int hp_active_fe_index=0) const
void reinit(const MappingType &mapping, const std::vector< const DoFHandler< dim > * > &dof_handler, const std::vector< const AffineConstraints< number2 > * > &constraint, const QuadratureType &quad, const AdditionalData &additional_data=AdditionalData())
void cell_loop(const std::function< void(const MatrixFree< dim, Number, VectorizedArrayType > &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &)> &cell_operation, OutVector &dst, const InVector &src, const bool zero_dst_vector=false) const
void cell_loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false) const
std::vector< internal::MatrixFreeFunctions::DoFInfo > dof_info
void initialize_cell_data_vector(AlignedVector< T > &vec) const
unsigned int n_ghost_inner_face_batches() const
AlignedVector< VectorizedArrayType > * acquire_scratch_data() const
void release_scratch_data_non_threadsafe(const AlignedVector< Number > *memory) const
void renumber_dofs(std::vector< types::global_dof_index > &renumbering, const unsigned int dof_handler_index=0)
unsigned int get_face_active_fe_index(const std::pair< unsigned int, unsigned int > range, const bool is_interior_face=true) const
unsigned int n_active_entries_per_face_batch(const unsigned int face_batch_index) const
bool indices_are_initialized
internal::MatrixFreeFunctions::FaceInfo< VectorizedArrayType::size()> face_info
void reinit(const MappingType &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< number2 > &constraint, const QuadratureType &quad, const AdditionalData &additional_data=AdditionalData())
unsigned int get_cell_range_category(const std::pair< unsigned int, unsigned int > cell_batch_range) const
void loop(void(CLASS::*cell_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), void(CLASS::*face_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), void(CLASS::*boundary_operation)(const MatrixFree &, OutVector &, const InVector &, const std::pair< unsigned int, unsigned int > &), CLASS *owning_class, OutVector &dst, const InVector &src, const bool zero_dst_vector=false, const DataAccessOnFaces dst_vector_face_access=DataAccessOnFaces::unspecified, const DataAccessOnFaces src_vector_face_access=DataAccessOnFaces::unspecified) const
void reinit(const MappingType &mapping, const std::vector< const DoFHandler< dim > * > &dof_handler, const std::vector< const AffineConstraints< number2 > * > &constraint, const std::vector< QuadratureType > &quad, const AdditionalData &additional_data=AdditionalData())
std::vector< unsigned int > constraint_pool_row_index
const std::shared_ptr< const Utilities::MPI::Partitioner > & get_vector_partitioner(const unsigned int dof_handler_index=0) const
unsigned int cell_level_index_end_local
unsigned int n_base_elements(const unsigned int dof_handler_index) const
const DoFHandler< dim > & get_dof_handler(const unsigned int dof_handler_index=0) const
const internal::MatrixFreeFunctions::TaskInfo & get_task_info() const
const IndexSet & get_locally_owned_set(const unsigned int dof_handler_index=0) const
static bool is_supported(const FiniteElement< dim, spacedim > &fe)
static constexpr unsigned int dimension
A class that provides a separate storage location on each thread that accesses the object.
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
@ update_JxW_values
Transformed quadrature weights.
@ update_gradients
Shape function gradients.
@ update_default
No update.
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcNotInitialized()
#define Assert(cond, exc)
static ::ExceptionBase & ExcNotImplemented()
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcIndexRange(std::size_t arg1, std::size_t arg2, std::size_t arg3)
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
typename ActiveSelector::cell_iterator cell_iterator
Number local_element(const size_type local_index) const
void reinit(const size_type size, const bool omit_zeroing_entries=false)
VectorType::value_type * begin(VectorType &V)
unsigned int minimum_parallel_grain_size
const types::boundary_id invalid_boundary_id
static const unsigned int invalid_unsigned_int
void apply_to_subranges(const tbb::blocked_range< RangeType > &range, const Function &f)
TasksParallelScheme tasks_parallel_scheme
bool hold_all_faces_to_owned_cells
UpdateFlags mapping_update_flags_inner_faces
AdditionalData(const TasksParallelScheme tasks_parallel_scheme=partition_partition, const unsigned int tasks_block_size=0, const UpdateFlags mapping_update_flags=update_gradients|update_JxW_values, const UpdateFlags mapping_update_flags_boundary_faces=update_default, const UpdateFlags mapping_update_flags_inner_faces=update_default, const UpdateFlags mapping_update_flags_faces_by_cells=update_default, const unsigned int mg_level=numbers::invalid_unsigned_int, const bool store_plain_indices=true, const bool initialize_indices=true, const bool initialize_mapping=true, const bool overlap_communication_computation=true, const bool hold_all_faces_to_owned_cells=false, const bool cell_vectorization_categories_strict=false, const bool allow_ghosted_vectors_in_loops=true)
AdditionalData & operator=(const AdditionalData &other)
std::vector< unsigned int > cell_vectorization_category
bool cell_vectorization_categories_strict
UpdateFlags mapping_update_flags_boundary_faces
UpdateFlags mapping_update_flags
UpdateFlags mapping_update_flags_faces_by_cells
AdditionalData(const AdditionalData &other)
bool allow_ghosted_vectors_in_loops
unsigned int tasks_block_size
bool overlap_communication_computation
static bool equal(const T *p1, const T *p2)
@ dof_access_face_interior
std::vector< unsigned int > cell_loop_pre_list_index
std::vector< unsigned int > cell_loop_post_list_index
std::vector< std::pair< unsigned int, unsigned int > > vector_zero_range_list
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
std::vector< std::pair< unsigned int, unsigned int > > cell_loop_pre_list
std::vector< unsigned int > vector_zero_range_list_index
std::vector< std::pair< unsigned int, unsigned int > > cell_loop_post_list
::Table< 3, unsigned int > cell_and_face_to_plain_faces
std::vector< FaceToCellTopology< vectorization_width > > faces
::Table< 3, types::boundary_id > cell_and_face_boundary_id
std::vector< unsigned int > boundary_partition_data
void loop(MFWorkerInterface &worker) const
unsigned int n_active_cells
std::vector< unsigned int > cell_partition_data
std::vector< unsigned int > face_partition_data