24 #include <deal.II/matrix_free/dof_info.templates.h>
32 namespace MatrixFreeFunctions
36 static_assert(std::is_same<compressed_constraint_kind, std::uint8_t>::value,
37 "Unexpected type for compressed hanging node indicators!");
66 for (
unsigned int i = 0; i < 3; ++i)
83 const unsigned int cell,
84 const bool apply_constraints)
const
95 const unsigned int n_vectorization_actual =
101 my_rows.reserve(n_vectorization * dofs_this_cell);
103 unsigned int total_size = 0;
104 for (
unsigned int v = 0; v < n_vectorization_actual; ++v)
106 const unsigned int ib =
107 (cell * n_vectorization + v) * n_fe_components;
108 const unsigned int ie =
109 (cell * n_vectorization + v + 1) * n_fe_components;
114 const bool has_constraints =
122 auto do_copy = [&](
const unsigned int *
begin,
123 const unsigned int *
end) {
124 const unsigned int shift = total_size;
126 my_rows.resize(total_size);
130 if (!has_constraints || apply_constraints)
132 const unsigned int *
begin =
134 const unsigned int *
end =
143 const unsigned int *
begin =
146 const unsigned int *
end =
begin + dofs_this_cell;
156 const MPI_Comm communicator_sm,
157 const bool use_vector_data_exchanger_full)
164 const std::size_t n_ghosts =
ghost_dofs.size();
171 std::vector<unsigned int> ghost_numbering(n_ghosts);
175 unsigned int n_unique_ghosts = 0;
179 std::vector<std::pair<types::global_dof_index, unsigned int>>
180 ghost_origin(n_ghosts);
181 for (std::size_t i = 0; i < n_ghosts; ++i)
184 ghost_origin[i].second = i;
186 std::sort(ghost_origin.begin(), ghost_origin.end());
189 ghost_numbering[ghost_origin[0].second] = 0;
190 for (std::size_t i = 1; i < n_ghosts; ++i)
192 if (ghost_origin[i].
first > ghost_origin[i - 1].
first + 1)
194 ghost_indices.
add_range(last_contiguous_start,
195 ghost_origin[i - 1].
first + 1);
196 last_contiguous_start = ghost_origin[i].first;
198 if (ghost_origin[i].
first > ghost_origin[i - 1].
first)
200 ghost_numbering[ghost_origin[i].second] = n_unique_ghosts;
203 ghost_indices.
add_range(last_contiguous_start,
204 ghost_origin.back().first + 1);
211 for (std::size_t i = 0; i < n_ghosts; ++i)
212 Assert(ghost_numbering[i] ==
221 const unsigned int n_boundary_cells = boundary_cells.size();
222 for (
unsigned int i = 0; i < n_boundary_cells; ++i)
224 unsigned int *data_ptr =
227 const unsigned int *row_end =
230 for (; data_ptr != row_end; ++data_ptr)
231 *data_ptr = ((*data_ptr < n_owned) ?
233 n_owned + ghost_numbering[*data_ptr - n_owned]);
238 bool has_hanging_nodes =
false;
251 for (
unsigned int comp = 0; comp <
n_components; ++comp)
255 if (has_hanging_nodes ||
260 unsigned int *data_ptr =
263 const unsigned int *row_end =
265 for (; data_ptr != row_end; ++data_ptr)
267 ((*data_ptr < n_owned) ?
269 n_owned + ghost_numbering[*data_ptr - n_owned]);
275 std::vector<types::global_dof_index> empty;
285 if (use_vector_data_exchanger_full ==
false)
291 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
300 const std::vector<unsigned int> & renumbering,
301 const std::vector<unsigned int> & constraint_pool_row_index,
302 const std::vector<unsigned char> &irregular_cells)
304 (void)constraint_pool_row_index;
310 std::vector<unsigned int> new_active_fe_index;
312 unsigned int position_cell = 0;
313 for (
unsigned int cell = 0;
317 const unsigned int n_comp =
318 (irregular_cells[cell] > 0 ? irregular_cells[cell] :
325 for (
unsigned int j = 1; j < n_comp; ++j)
330 new_active_fe_index.push_back(
fe_index);
331 position_cell += n_comp;
341 std::vector<std::pair<unsigned int, unsigned int>> new_row_starts(
345 std::vector<unsigned int> new_dof_indices;
346 std::vector<std::pair<unsigned short, unsigned short>>
347 new_constraint_indicator;
348 std::vector<unsigned int> new_plain_indices, new_rowstart_plain;
349 unsigned int position_cell = 0;
353 std::vector<compressed_constraint_kind> new_hanging_node_constraint_masks;
354 new_hanging_node_constraint_masks.reserve(
374 const unsigned int n_vect =
375 (irregular_cells[i] > 0 ? irregular_cells[i] :
379 this->dofs_per_cell[0];
381 for (
unsigned int j = 0; j < n_vect; ++j)
383 const unsigned int cell_no =
386 bool has_hanging_nodes =
false;
394 new_hanging_node_constraint_masks.push_back(
mask);
397 for (
unsigned int comp = 0; comp <
n_components; ++comp)
402 for (
unsigned int comp = 0; comp <
n_components; ++comp)
406 .
first = new_dof_indices.size();
409 .
second = new_constraint_indicator.size();
411 new_dof_indices.insert(
412 new_dof_indices.end(),
418 new_constraint_indicator.push_back(
427 new_plain_indices.size();
428 new_plain_indices.insert(
429 new_plain_indices.end(),
438 for (
unsigned int comp = 0; comp <
n_components; ++comp)
442 .
first = new_dof_indices.size();
445 .
second = new_constraint_indicator.size();
450 new_hanging_node_constraint_masks.push_back(
453 position_cell += n_vect;
460 .first = new_dof_indices.size();
463 .second = new_constraint_indicator.size();
466 new_constraint_indicator.size());
478 const unsigned int index_range =
492 const unsigned int row_length_ind =
498 const std::pair<unsigned short, unsigned short> *
505 for (; con_it != end_con; ++con_it)
509 constraint_pool_row_index.size() - 1);
517 if (irregular_cells[c] > 0)
531 const std::vector<unsigned char> &irregular_cells)
541 irregular_cells.size());
545 irregular_cells.size());
546 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
547 if (irregular_cells[i] > 0)
562 std::vector<unsigned int> index_kinds(
563 static_cast<unsigned int>(
567 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
569 const unsigned int ndofs =
571 const unsigned int n_comp =
575 bool has_constraints =
false;
576 for (
unsigned int j = 0; j < n_comp; ++j)
582 has_constraints =
true;
591 bool indices_are_contiguous =
true;
592 for (
unsigned int j = 0; j < n_comp; ++j)
596 this->dof_indices.data() +
602 for (
unsigned int i = 1; i < ndofs; ++i)
605 indices_are_contiguous =
false;
610 bool indices_are_interleaved_and_contiguous =
615 this->dof_indices.data() +
617 for (
unsigned int k = 0;
618 k < ndofs && indices_are_interleaved_and_contiguous;
620 for (
unsigned int j = 0; j < n_comp; ++j)
624 indices_are_interleaved_and_contiguous =
false;
629 if (indices_are_contiguous ||
630 indices_are_interleaved_and_contiguous)
632 for (
unsigned int j = 0; j < n_comp; ++j)
643 if (indices_are_interleaved_and_contiguous)
648 for (
unsigned int j = 0; j < n_comp; ++j)
652 else if (indices_are_contiguous)
656 for (
unsigned int j = 0; j < n_comp; ++j)
662 int indices_are_interleaved_and_mixed = 2;
667 for (
unsigned int j = 0; j < n_comp; ++j)
670 for (
unsigned int k = 0;
671 k < ndofs && indices_are_interleaved_and_mixed != 0;
673 for (
unsigned int j = 0; j < n_comp; ++j)
680 indices_are_interleaved_and_mixed = 0;
683 if (indices_are_interleaved_and_mixed == 2)
685 for (
unsigned int j = 0; j < n_comp; ++j)
689 for (
unsigned int j = 0; j < n_comp; ++j)
693 for (
unsigned int j = 0; j < n_comp; ++j)
696 indices_are_interleaved_and_mixed = 1;
699 if (indices_are_interleaved_and_mixed == 1 ||
711 this->dof_indices.data() +
728 bool is_sorted =
true;
729 unsigned int previous = indices[0];
730 for (
unsigned int l = 1;
l < n_comp; ++
l)
732 const unsigned int current = indices[
l * ndofs];
733 if (current <= previous)
740 for (
unsigned int j = 0; j <
l; ++j)
741 if (indices[j * ndofs] == current)
754 index_kinds[
static_cast<unsigned int>(
764 auto fix_single_interleaved_indices =
766 if (index_kinds[
static_cast<unsigned int>(
769 index_kinds[
static_cast<unsigned int>(variant)] > 0)
770 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
774 interleaved_contiguous_mixed_strides &&
782 index_kinds[
static_cast<unsigned int>(
785 index_kinds[
static_cast<unsigned int>(variant)]++;
794 unsigned int n_interleaved =
795 index_kinds[
static_cast<unsigned int>(
797 index_kinds[
static_cast<unsigned int>(
799 index_kinds[
static_cast<unsigned int>(
804 if (n_interleaved > 0 && index_kinds[
static_cast<unsigned int>(
806 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
812 index_kinds[
static_cast<unsigned int>(
814 index_kinds[
static_cast<unsigned int>(
820 if (n_interleaved > 0 &&
822 index_kinds[
static_cast<unsigned int>(
825 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
829 index_kinds[
static_cast<unsigned int>(
838 index_kinds[
static_cast<unsigned int>(
843 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
854 const unsigned int ndofs =
859 unsigned int *interleaved_dof_indices =
863 this->dof_indices_interleaved.size());
873 for (
unsigned int k = 0; k < ndofs; ++k)
875 const unsigned int *my_dof_indices =
dof_indices + k;
876 const unsigned int *
end =
878 for (; interleaved_dof_indices !=
end;
879 ++interleaved_dof_indices, my_dof_indices += ndofs)
880 *interleaved_dof_indices = *my_dof_indices;
890 const unsigned int n_owned_cells,
891 const unsigned int n_lanes,
894 const bool fill_cell_centric,
895 const MPI_Comm communicator_sm,
896 const bool use_vector_data_exchanger_full)
902 std::vector<types::global_dof_index> ghost_indices;
905 for (
unsigned int cell = 0; cell < n_owned_cells; ++cell)
922 ghost_indices.push_back(
925 std::sort(ghost_indices.begin(), ghost_indices.end());
926 ghost_indices.erase(std::unique(ghost_indices.begin(),
927 ghost_indices.end()),
928 ghost_indices.end());
930 compressed_set.
add_indices(ghost_indices.begin(), ghost_indices.end());
932 const bool all_ghosts_equal =
933 Utilities::MPI::min<int>(
static_cast<int>(
938 std::shared_ptr<const Utilities::MPI::Partitioner> temp_0;
940 if (all_ghosts_equal)
944 temp_0 = std::make_shared<Utilities::MPI::Partitioner>(
950 if (use_vector_data_exchanger_full ==
false)
956 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
957 temp_0, communicator_sm);
961 std::vector<FaceToCellTopology<1>> all_faces(inner_faces);
962 all_faces.insert(all_faces.end(),
963 ghosted_faces.begin(),
964 ghosted_faces.end());
967 2 * shape_info(0, 0).n_dimensions);
969 for (
unsigned int f = 0; f < all_faces.size(); ++f)
971 cell_and_face_to_faces(all_faces[f].cells_interior[0],
972 all_faces[f].interior_face_no) = f;
973 Assert(all_faces[f].cells_exterior[0] !=
976 cell_and_face_to_faces(all_faces[f].cells_exterior[0],
977 all_faces[f].exterior_face_no) = f;
981 const auto loop_over_faces =
982 [&](
const std::function<
983 void(
const unsigned int,
const unsigned int,
const bool)> &fu) {
984 for (
const auto &face : inner_faces)
987 fu(face.cells_exterior[0], face.exterior_face_no,
false );
991 const auto loop_over_all_faces =
992 [&](
const std::function<
993 void(
const unsigned int,
const unsigned int,
const bool)> &fu) {
994 for (
unsigned int c = 0; c < cell_and_face_to_faces.size(0); ++c)
995 for (
unsigned int d = 0;
d < cell_and_face_to_faces.size(1); ++
d)
997 const unsigned int f = cell_and_face_to_faces(c,
d);
1001 const unsigned int cell_m = all_faces[f].cells_interior[0];
1002 const unsigned int cell_p = all_faces[f].cells_exterior[0];
1004 const bool ext = c == cell_m;
1009 const unsigned int p = ext ? cell_p : cell_m;
1010 const unsigned int face_no = ext ?
1011 all_faces[f].exterior_face_no :
1012 all_faces[f].interior_face_no;
1014 fu(p, face_no,
true);
1018 const auto process_values =
1020 std::shared_ptr<const Utilities::MPI::Partitioner>
1021 &vector_partitioner_values,
1022 const std::function<void(
1023 const std::function<
void(
1024 const unsigned int,
const unsigned int,
const bool)> &)> &
loop) {
1025 bool all_nodal_and_tensorial = shape_info.size(1) == 1;
1027 if (all_nodal_and_tensorial)
1032 if (!si.nodal_at_cell_boundaries ||
1035 all_nodal_and_tensorial =
false;
1038 if (all_nodal_and_tensorial ==
false)
1042 bool has_noncontiguous_cell =
false;
1044 loop([&](
const unsigned int cell_no,
1045 const unsigned int face_no,
1047 const unsigned int index =
1052 const unsigned int stride =
1053 dof_indices_interleave_strides[dof_access_cell][cell_no];
1055 for (unsigned int e = 0; e < n_base_elements; ++e)
1056 for (unsigned int c = 0; c < n_components[e]; ++c)
1058 const ShapeInfo<double> &shape =
1059 shape_info(global_base_element_offset + e, 0);
1060 for (unsigned int j = 0;
1061 j < shape.dofs_per_component_on_face;
1063 ghost_indices.push_back(part.local_to_global(
1065 shape.face_to_cell_index_nodal(face_no, j) *
1067 i += shape.dofs_per_component_on_cell * stride;
1072 has_noncontiguous_cell =
true;
1074 has_noncontiguous_cell =
1075 Utilities::MPI::min<int>(
static_cast<int>(
1076 has_noncontiguous_cell),
1079 std::sort(ghost_indices.begin(), ghost_indices.end());
1080 ghost_indices.erase(std::unique(ghost_indices.begin(),
1081 ghost_indices.end()),
1082 ghost_indices.end());
1085 ghost_indices.end());
1087 const bool all_ghosts_equal =
1088 Utilities::MPI::min<int>(
static_cast<int>(
1092 if (all_ghosts_equal || has_noncontiguous_cell)
1096 vector_partitioner_values =
1097 std::make_shared<Utilities::MPI::Partitioner>(
1100 vector_partitioner_values.get())
1107 const auto process_gradients =
1109 const std::shared_ptr<const Utilities::MPI::Partitioner>
1110 &vector_partitoner_values,
1111 std::shared_ptr<const Utilities::MPI::Partitioner>
1112 &vector_partitioner_gradients,
1113 const std::function<void(
1114 const std::function<
void(
1115 const unsigned int,
const unsigned int,
const bool)> &)> &
loop) {
1116 bool all_hermite = shape_info.
size(1) == 1;
1119 for (
unsigned int c = 0; c < n_base_elements; ++c)
1120 if (shape_info(global_base_element_offset + c, 0).element_type !=
1122 all_hermite =
false;
1123 if (all_hermite ==
false ||
1124 vector_partitoner_values.get() == vector_partitioner.get())
1125 vector_partitioner_gradients = vector_partitioner;
1128 loop([&](
const unsigned int cell_no,
1129 const unsigned int face_no,
1131 const unsigned int index =
1132 dof_indices_contiguous[dof_access_cell][cell_no];
1134 index >= part.locally_owned_size()))
1136 const unsigned int stride =
1137 dof_indices_interleave_strides[dof_access_cell][cell_no];
1139 for (unsigned int e = 0; e < n_base_elements; ++e)
1140 for (unsigned int c = 0; c < n_components[e]; ++c)
1142 const ShapeInfo<double> &shape =
1143 shape_info(global_base_element_offset + e, 0);
1144 for (unsigned int j = 0;
1145 j < 2 * shape.dofs_per_component_on_face;
1147 ghost_indices.push_back(part.local_to_global(
1149 shape.face_to_cell_index_hermite(face_no, j) *
1151 i += shape.dofs_per_component_on_cell * stride;
1156 std::sort(ghost_indices.begin(), ghost_indices.end());
1157 ghost_indices.erase(std::unique(ghost_indices.begin(),
1158 ghost_indices.end()),
1159 ghost_indices.end());
1160 IndexSet compressed_set(part.size());
1161 compressed_set.add_indices(ghost_indices.begin(),
1162 ghost_indices.end());
1163 compressed_set.subtract_set(part.locally_owned_range());
1164 const bool all_ghosts_equal =
1165 Utilities::MPI::min<int>(
static_cast<int>(
1166 compressed_set.n_elements() ==
1167 part.ghost_indices().n_elements()),
1168 part.get_mpi_communicator()) != 0;
1169 if (all_ghosts_equal)
1170 vector_partitioner_gradients = vector_partitioner;
1173 vector_partitioner_gradients =
1174 std::make_shared<Utilities::MPI::Partitioner>(
1175 part.locally_owned_range(), part.get_mpi_communicator());
1177 vector_partitioner_gradients.get())
1178 ->set_ghost_indices(compressed_set, part.ghost_indices());
1183 std::shared_ptr<const Utilities::MPI::Partitioner> temp_1, temp_2, temp_3,
1187 process_values(temp_1, loop_over_faces);
1190 process_gradients(temp_1, temp_2, loop_over_faces);
1192 if (fill_cell_centric)
1194 ghost_indices.clear();
1196 process_values(temp_3, loop_over_all_faces);
1198 process_gradients(temp_3, temp_4, loop_over_all_faces);
1202 temp_3 = std::make_shared<Utilities::MPI::Partitioner>(
1203 part.locally_owned_range(), part.get_mpi_communicator());
1204 temp_4 = std::make_shared<Utilities::MPI::Partitioner>(
1205 part.locally_owned_range(), part.get_mpi_communicator());
1208 if (use_vector_data_exchanger_full ==
false)
1210 vector_exchanger_face_variants[1] = std::make_shared<
1211 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1213 vector_exchanger_face_variants[2] = std::make_shared<
1214 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1216 vector_exchanger_face_variants[3] = std::make_shared<
1217 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1219 vector_exchanger_face_variants[4] = std::make_shared<
1220 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1225 vector_exchanger_face_variants[1] =
1226 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1227 temp_1, communicator_sm);
1228 vector_exchanger_face_variants[2] =
1229 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1230 temp_2, communicator_sm);
1231 vector_exchanger_face_variants[3] =
1232 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1233 temp_3, communicator_sm);
1234 vector_exchanger_face_variants[4] =
1235 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1236 temp_4, communicator_sm);
1243 DoFInfo::compute_shared_memory_contiguous_indices(
1244 std::array<std::vector<std::pair<unsigned int, unsigned int>>, 3>
1245 &cell_indices_contiguous_sm)
1249 for (
unsigned int i = 0; i < 3; ++i)
1251 dof_indices_contiguous_sm[i].resize(
1252 cell_indices_contiguous_sm[i].size());
1254 for (
unsigned int j = 0; j < cell_indices_contiguous_sm[i].size();
1256 if (cell_indices_contiguous_sm[i][j].
first !=
1258 dof_indices_contiguous_sm[i][j] = {
1259 cell_indices_contiguous_sm[i][j].first,
1260 cell_indices_contiguous_sm[i][j].second * dofs_per_cell[0]};
1280 const unsigned int end,
1282 std::vector<std::mutex> & mutexes,
1283 std::vector<unsigned int> &row_lengths)
1285 std::vector<unsigned int> scratch;
1287 for (
unsigned int block =
begin; block <
end; ++block)
1293 dof_info.
row_starts[block * n_components].first,
1295 dof_info.
row_starts[(block + 1) * n_components].first);
1296 std::sort(scratch.begin(), scratch.end());
1297 std::vector<unsigned int>::const_iterator end_unique =
1298 std::unique(scratch.begin(), scratch.end());
1299 std::vector<unsigned int>::const_iterator it = scratch.begin();
1300 while (it != end_unique)
1304 const unsigned int next_bucket =
1306 std::lock_guard<std::mutex> lock(
1308 for (; it != end_unique && *it < next_bucket; ++it)
1319 const unsigned int end,
1321 const std::vector<unsigned int> &row_lengths,
1322 std::vector<std::mutex> & mutexes,
1325 std::vector<unsigned int> scratch;
1327 for (
unsigned int block =
begin; block <
end; ++block)
1333 dof_info.
row_starts[block * n_components].first,
1335 dof_info.
row_starts[(block + 1) * n_components].first);
1336 std::sort(scratch.begin(), scratch.end());
1337 std::vector<unsigned int>::const_iterator end_unique =
1338 std::unique(scratch.begin(), scratch.end());
1339 std::vector<unsigned int>::const_iterator it = scratch.begin();
1340 while (it != end_unique)
1342 const unsigned int next_bucket =
1344 std::lock_guard<std::mutex> lock(
1346 for (; it != end_unique && *it < next_bucket; ++it)
1347 if (row_lengths[*it] > 0)
1348 connectivity_dof.
add(*it, block);
1357 const unsigned int end,
1359 const std::vector<unsigned int> &renumbering,
1360 const ::SparsityPattern & connectivity_dof,
1363 ordered_vector row_entries;
1365 for (
unsigned int block =
begin; block <
end; ++block)
1367 row_entries.clear();
1371 dof_info.
row_starts[block * n_components].first,
1374 for (; it != end_cell; ++it)
1377 std::vector<types::global_dof_index>::iterator insert_pos =
1378 row_entries.begin();
1379 for (; sp != connectivity_dof.end(*it); ++sp)
1380 if (sp->column() != block)
1381 row_entries.insert(renumbering[sp->column()], insert_pos);
1384 row_entries.begin(),
1392 DoFInfo::make_connectivity_graph(
1394 const std::vector<unsigned int> &renumbering,
1397 unsigned int n_rows = (vector_partitioner->local_range().second -
1398 vector_partitioner->local_range().first) +
1399 vector_partitioner->ghost_indices().n_elements();
1406 std::vector<unsigned int> row_lengths(n_rows);
1412 [
this, &mutexes, &row_lengths](
const unsigned int begin,
1413 const unsigned int end) {
1414 internal::compute_row_lengths(
1415 begin, end, *this, mutexes, row_lengths);
1421 for (
unsigned int row = 0; row < n_rows; ++row)
1422 if (row_lengths[row] <= 1)
1423 row_lengths[row] = 0;
1434 [
this, &row_lengths, &mutexes, &connectivity_dof](
1435 const unsigned int begin,
const unsigned int end) {
1436 internal::fill_connectivity_dofs(
1437 begin, end, *this, row_lengths, mutexes, connectivity_dof);
1444 std::vector<unsigned int> reverse_numbering(task_info.
n_active_cells);
1454 [
this, &reverse_numbering, &connectivity_dof, &connectivity](
1455 const unsigned int begin,
const unsigned int end) {
1456 internal::fill_connectivity(begin,
1469 DoFInfo::compute_dof_renumbering(
1470 std::vector<types::global_dof_index> &renumbering)
1472 const unsigned int locally_owned_size =
1473 vector_partitioner->locally_owned_size();
1474 renumbering.resize(0);
1478 const unsigned int n_components = start_components.back();
1479 const unsigned int n_cell_batches =
1480 n_vectorization_lanes_filled[dof_access_cell].size();
1482 (row_starts.size() - 1) / vectorization_length / n_components,
1484 for (
unsigned int cell_no = 0; cell_no < n_cell_batches; ++cell_no)
1487 if (row_starts[cell_no * n_components * vectorization_length]
1489 row_starts[(cell_no + 1) * n_components * vectorization_length]
1492 const unsigned int ndofs =
1493 dofs_per_cell.size() == 1 ?
1495 (dofs_per_cell[cell_active_fe_index.size() > 0 ?
1496 cell_active_fe_index[cell_no] :
1498 const unsigned int *dof_ind =
1499 dof_indices.data() +
1500 row_starts[cell_no * n_components * vectorization_length].first;
1501 for (
unsigned int i = 0; i < ndofs; ++i)
1502 for (
unsigned int j = 0;
1503 j < n_vectorization_lanes_filled[dof_access_cell][cell_no];
1505 if (dof_ind[j * ndofs + i] < locally_owned_size)
1506 if (renumbering[dof_ind[j * ndofs + i]] ==
1508 renumbering[dof_ind[j * ndofs + i]] = counter++;
1515 dof_index = counter++;
1519 dof_index = vector_partitioner->local_to_global(dof_index);
1529 std::size_t memory =
sizeof(*this);
1531 (row_starts.capacity() *
sizeof(std::pair<unsigned int, unsigned int>));
1546 namespace MatrixFreeFunctions
1549 DoFInfo::read_dof_indices<double>(
1550 const std::vector<types::global_dof_index> &,
1551 const std::vector<types::global_dof_index> &,
1553 const ::AffineConstraints<double> &,
1555 ConstraintValues<double> &,
1559 DoFInfo::read_dof_indices<float>(
1560 const std::vector<types::global_dof_index> &,
1561 const std::vector<types::global_dof_index> &,
1563 const ::AffineConstraints<float> &,
1565 ConstraintValues<double> &,
1569 DoFInfo::process_hanging_node_constraints<1>(
1570 const HangingNodes<1> &,
1571 const std::vector<std::vector<unsigned int>> &,
1574 std::vector<types::global_dof_index> &);
1576 DoFInfo::process_hanging_node_constraints<2>(
1577 const HangingNodes<2> &,
1578 const std::vector<std::vector<unsigned int>> &,
1581 std::vector<types::global_dof_index> &);
1583 DoFInfo::process_hanging_node_constraints<3>(
1584 const HangingNodes<3> &,
1585 const std::vector<std::vector<unsigned int>> &,
1588 std::vector<types::global_dof_index> &);
1591 DoFInfo::compute_face_index_compression<1>(
1592 const std::vector<FaceToCellTopology<1>> &);
1594 DoFInfo::compute_face_index_compression<2>(
1595 const std::vector<FaceToCellTopology<2>> &);
1597 DoFInfo::compute_face_index_compression<4>(
1598 const std::vector<FaceToCellTopology<4>> &);
1600 DoFInfo::compute_face_index_compression<8>(
1601 const std::vector<FaceToCellTopology<8>> &);
1603 DoFInfo::compute_face_index_compression<16>(
1604 const std::vector<FaceToCellTopology<16>> &);
1607 DoFInfo::compute_vector_zero_access_pattern<1>(
1609 const std::vector<FaceToCellTopology<1>> &);
1611 DoFInfo::compute_vector_zero_access_pattern<2>(
1613 const std::vector<FaceToCellTopology<2>> &);
1615 DoFInfo::compute_vector_zero_access_pattern<4>(
1617 const std::vector<FaceToCellTopology<4>> &);
1619 DoFInfo::compute_vector_zero_access_pattern<8>(
1621 const std::vector<FaceToCellTopology<8>> &);
1623 DoFInfo::compute_vector_zero_access_pattern<16>(
1625 const std::vector<FaceToCellTopology<16>> &);
1628 DoFInfo::print_memory_consumption<std::ostream>(std::ostream &,
1631 DoFInfo::print_memory_consumption<ConditionalOStream>(
1636 DoFInfo::print<double>(
const std::vector<double> &,
1637 const std::vector<unsigned int> &,
1638 std::ostream &)
const;
1641 DoFInfo::print<float>(
const std::vector<float> &,
1642 const std::vector<unsigned int> &,
1643 std::ostream &)
const;
void add_entries(const size_type row, ForwardIterator begin, ForwardIterator end, const bool indices_are_unique_and_sorted=false)
size_type index_within_set(const size_type global_index) const
size_type n_elements() const
void subtract_set(const IndexSet &other)
void add_range(const size_type begin, const size_type end)
void add_indices(const ForwardIterator &begin, const ForwardIterator &end)
void add(const size_type i, const size_type j)
unsigned int locally_owned_size() const
types::global_dof_index local_to_global(const unsigned int local_index) const
virtual MPI_Comm get_mpi_communicator() const override
void set_ghost_indices(const IndexSet &ghost_indices, const IndexSet &larger_ghost_index_set=IndexSet())
types::global_dof_index size() const
const IndexSet & locally_owned_range() const
const IndexSet & ghost_indices() const
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcNotInitialized()
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
void loop(ITERATOR begin, std_cxx20::type_identity_t< ITERATOR > end, DOFINFO &dinfo, INFOBOX &info, const std::function< void(DOFINFO &, typename INFOBOX::CellInfo &)> &cell_worker, const std::function< void(DOFINFO &, typename INFOBOX::CellInfo &)> &boundary_worker, const std::function< void(DOFINFO &, DOFINFO &, typename INFOBOX::CellInfo &, typename INFOBOX::CellInfo &)> &face_worker, ASSEMBLER &assembler, const LoopControl &lctrl=LoopControl())
@ tensor_symmetric_hermite
std::enable_if_t< std::is_fundamental< T >::value, std::size_t > memory_consumption(const T &t)
void swap(MemorySpaceData< T, MemorySpace > &u, MemorySpaceData< T, MemorySpace > &v)
SymmetricTensor< 2, dim, Number > d(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
Tensor< 2, dim, Number > l(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::vector< Integer > invert_permutation(const std::vector< Integer > &permutation)
void fill_connectivity_dofs(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, const std::vector< unsigned int > &row_lengths, std::vector< std::mutex > &mutexes, ::SparsityPattern &connectivity_dof)
void fill_connectivity(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, const std::vector< unsigned int > &renumbering, const ::SparsityPattern &connectivity_dof, DynamicSparsityPattern &connectivity)
void compute_row_lengths(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, std::vector< std::mutex > &mutexes, std::vector< unsigned int > &row_lengths)
static constexpr unsigned int bucket_size_threading
constexpr compressed_constraint_kind unconstrained_compressed_constraint_kind
unsigned int n_active_cells(const internal::TriangulationImplementation::NumberCache< 1 > &c)
void copy(const T *begin, const T *end, U *dest)
static const unsigned int invalid_unsigned_int
const types::global_dof_index invalid_dof_index
void apply_to_subranges(const tbb::blocked_range< Iterator > &range, const Function &f)
unsigned int global_dof_index
unsigned short int fe_index
void reorder_cells(const TaskInfo &task_info, const std::vector< unsigned int > &renumbering, const std::vector< unsigned int > &constraint_pool_row_index, const std::vector< unsigned char > &irregular_cells)
std::vector< std::pair< unsigned short, unsigned short > > constraint_indicator
void compute_tight_partitioners(const Table< 2, ShapeInfo< double >> &shape_info, const unsigned int n_owned_cells, const unsigned int n_lanes, const std::vector< FaceToCellTopology< 1 >> &inner_faces, const std::vector< FaceToCellTopology< 1 >> &ghosted_faces, const bool fill_cell_centric, const MPI_Comm communicator_sm, const bool use_vector_data_exchanger_full)
std::vector< std::pair< unsigned int, unsigned int > > row_starts
unsigned int n_base_elements
std::vector< std::vector< bool > > hanging_node_constraint_masks_comp
unsigned int global_base_element_offset
unsigned int max_fe_index
void get_dof_indices_on_cell_batch(std::vector< unsigned int > &local_indices, const unsigned int cell_batch, const bool with_constraints=true) const
void compute_cell_index_compression(const std::vector< unsigned char > &irregular_cells)
std::vector< unsigned int > dofs_per_cell
void assign_ghosts(const std::vector< unsigned int > &boundary_cells, const MPI_Comm communicator_sm, const bool use_vector_data_exchanger_full)
unsigned int vectorization_length
std::shared_ptr< const internal::MatrixFreeFunctions::VectorDataExchange::Base > vector_exchanger
@ interleaved_contiguous_strided
@ interleaved_contiguous_mixed_strides
std::vector< std::vector< unsigned int > > fe_index_conversion
std::vector< unsigned int > dof_indices
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
std::vector< compressed_constraint_kind > hanging_node_constraint_masks
std::array< std::vector< unsigned int >, 3 > dof_indices_interleave_strides
std::vector< unsigned int > row_starts_plain_indices
std::vector< unsigned int > dofs_per_face
std::vector< unsigned int > n_components
std::vector< types::global_dof_index > ghost_dofs
std::array< std::vector< unsigned int >, 3 > dof_indices_contiguous
std::vector< unsigned int > cell_active_fe_index
std::array< std::shared_ptr< const internal::MatrixFreeFunctions::VectorDataExchange::Base >, 5 > vector_exchanger_face_variants
std::vector< unsigned int > plain_dof_indices
std::array< std::vector< unsigned char >, 3 > n_vectorization_lanes_filled
std::vector< unsigned int > start_components
std::vector< unsigned int > dof_indices_interleaved
std::array< std::vector< IndexStorageVariants >, 3 > index_storage_variants
unsigned int n_active_cells
std::vector< unsigned int > cell_partition_data