21 #include <deal.II/matrix_free/dof_info.templates.h>
29 namespace MatrixFreeFunctions
33 static_assert(std::is_same<compressed_constraint_kind, std::uint8_t>::value,
34 "Unexpected type for compressed hanging node indicators!");
64 for (
unsigned int i = 0; i < 3; ++i)
81 const unsigned int cell,
82 const bool apply_constraints)
const
85 const unsigned int fe_index =
93 const unsigned int n_vectorization_actual =
99 my_rows.reserve(n_vectorization * dofs_this_cell);
101 unsigned int total_size = 0;
102 for (
unsigned int v = 0; v < n_vectorization_actual; ++v)
104 const unsigned int ib =
105 (cell * n_vectorization + v) * n_fe_components;
106 const unsigned int ie =
107 (cell * n_vectorization + v + 1) * n_fe_components;
112 const bool has_constraints =
120 auto do_copy = [&](
const unsigned int *
begin,
121 const unsigned int *
end) {
122 const unsigned int shift = total_size;
124 my_rows.resize(total_size);
128 if (!has_constraints || apply_constraints)
130 const unsigned int *
begin =
132 const unsigned int *
end =
141 const unsigned int *
begin =
144 const unsigned int *
end =
begin + dofs_this_cell;
154 const MPI_Comm & communicator_sm,
155 const bool use_vector_data_exchanger_full)
162 const std::size_t n_ghosts =
ghost_dofs.size();
169 std::vector<unsigned int> ghost_numbering(n_ghosts);
173 unsigned int n_unique_ghosts = 0;
177 std::vector<std::pair<types::global_dof_index, unsigned int>>
178 ghost_origin(n_ghosts);
179 for (std::size_t i = 0; i < n_ghosts; ++i)
182 ghost_origin[i].second = i;
184 std::sort(ghost_origin.begin(), ghost_origin.end());
187 ghost_numbering[ghost_origin[0].second] = 0;
188 for (std::size_t i = 1; i < n_ghosts; ++i)
190 if (ghost_origin[i].
first > ghost_origin[i - 1].
first + 1)
192 ghost_indices.
add_range(last_contiguous_start,
193 ghost_origin[i - 1].
first + 1);
194 last_contiguous_start = ghost_origin[i].first;
196 if (ghost_origin[i].
first > ghost_origin[i - 1].
first)
198 ghost_numbering[ghost_origin[i].second] = n_unique_ghosts;
201 ghost_indices.
add_range(last_contiguous_start,
202 ghost_origin.back().first + 1);
209 for (std::size_t i = 0; i < n_ghosts; ++i)
210 Assert(ghost_numbering[i] ==
219 const unsigned int n_boundary_cells = boundary_cells.size();
220 for (
unsigned int i = 0; i < n_boundary_cells; ++i)
222 unsigned int *data_ptr =
225 const unsigned int *row_end =
228 for (; data_ptr != row_end; ++data_ptr)
229 *data_ptr = ((*data_ptr < n_owned) ?
231 n_owned + ghost_numbering[*data_ptr - n_owned]);
236 bool has_hanging_nodes =
false;
238 const unsigned int fe_index =
249 for (
unsigned int comp = 0; comp <
n_components; ++comp)
253 if (has_hanging_nodes ||
258 unsigned int *data_ptr =
261 const unsigned int *row_end =
263 for (; data_ptr != row_end; ++data_ptr)
265 ((*data_ptr < n_owned) ?
267 n_owned + ghost_numbering[*data_ptr - n_owned]);
273 std::vector<types::global_dof_index> empty;
283 if (use_vector_data_exchanger_full ==
false)
289 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
298 const std::vector<unsigned int> & renumbering,
299 const std::vector<unsigned int> & constraint_pool_row_index,
300 const std::vector<unsigned char> &irregular_cells)
302 (void)constraint_pool_row_index;
308 std::vector<unsigned int> new_active_fe_index;
310 unsigned int position_cell = 0;
311 for (
unsigned int cell = 0;
315 const unsigned int n_comp =
316 (irregular_cells[cell] > 0 ? irregular_cells[cell] :
321 unsigned int fe_index =
323 for (
unsigned int j = 1; j < n_comp; ++j)
328 new_active_fe_index.push_back(fe_index);
329 position_cell += n_comp;
339 std::vector<std::pair<unsigned int, unsigned int>> new_row_starts(
343 std::vector<unsigned int> new_dof_indices;
344 std::vector<std::pair<unsigned short, unsigned short>>
345 new_constraint_indicator;
346 std::vector<unsigned int> new_plain_indices, new_rowstart_plain;
347 unsigned int position_cell = 0;
351 std::vector<compressed_constraint_kind> new_hanging_node_constraint_masks;
352 new_hanging_node_constraint_masks.reserve(
372 const unsigned int n_vect =
373 (irregular_cells[i] > 0 ? irregular_cells[i] :
377 this->dofs_per_cell[0];
379 for (
unsigned int j = 0; j < n_vect; ++j)
381 const unsigned int cell_no =
384 bool has_hanging_nodes =
false;
392 new_hanging_node_constraint_masks.push_back(
mask);
395 for (
unsigned int comp = 0; comp <
n_components; ++comp)
400 for (
unsigned int comp = 0; comp <
n_components; ++comp)
404 .
first = new_dof_indices.size();
407 .
second = new_constraint_indicator.size();
409 new_dof_indices.insert(
410 new_dof_indices.end(),
416 new_constraint_indicator.push_back(
425 new_plain_indices.size();
426 new_plain_indices.insert(
427 new_plain_indices.end(),
436 for (
unsigned int comp = 0; comp <
n_components; ++comp)
440 .
first = new_dof_indices.size();
443 .
second = new_constraint_indicator.size();
448 new_hanging_node_constraint_masks.push_back(
451 position_cell += n_vect;
458 .first = new_dof_indices.size();
461 .second = new_constraint_indicator.size();
464 new_constraint_indicator.size());
476 const unsigned int index_range =
490 const unsigned int row_length_ind =
496 const std::pair<unsigned short, unsigned short> *
503 for (; con_it != end_con; ++con_it)
507 constraint_pool_row_index.size() - 1);
515 if (irregular_cells[c] > 0)
529 const std::vector<unsigned char> &irregular_cells)
539 irregular_cells.size());
543 irregular_cells.size());
544 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
545 if (irregular_cells[i] > 0)
560 std::vector<unsigned int> index_kinds(
561 static_cast<unsigned int>(
565 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
567 const unsigned int ndofs =
569 const unsigned int n_comp =
573 bool has_constraints =
false;
574 for (
unsigned int j = 0; j < n_comp; ++j)
580 has_constraints =
true;
589 bool indices_are_contiguous =
true;
590 for (
unsigned int j = 0; j < n_comp; ++j)
594 this->dof_indices.data() +
600 for (
unsigned int i = 1; i < ndofs; ++i)
603 indices_are_contiguous =
false;
608 bool indices_are_interleaved_and_contiguous =
613 this->dof_indices.data() +
615 for (
unsigned int k = 0;
616 k < ndofs && indices_are_interleaved_and_contiguous;
618 for (
unsigned int j = 0; j < n_comp; ++j)
622 indices_are_interleaved_and_contiguous =
false;
627 if (indices_are_contiguous ||
628 indices_are_interleaved_and_contiguous)
630 for (
unsigned int j = 0; j < n_comp; ++j)
641 if (indices_are_interleaved_and_contiguous)
646 for (
unsigned int j = 0; j < n_comp; ++j)
650 else if (indices_are_contiguous)
654 for (
unsigned int j = 0; j < n_comp; ++j)
660 int indices_are_interleaved_and_mixed = 2;
665 for (
unsigned int j = 0; j < n_comp; ++j)
668 for (
unsigned int k = 0;
669 k < ndofs && indices_are_interleaved_and_mixed != 0;
671 for (
unsigned int j = 0; j < n_comp; ++j)
678 indices_are_interleaved_and_mixed = 0;
681 if (indices_are_interleaved_and_mixed == 2)
683 for (
unsigned int j = 0; j < n_comp; ++j)
687 for (
unsigned int j = 0; j < n_comp; ++j)
691 for (
unsigned int j = 0; j < n_comp; ++j)
694 indices_are_interleaved_and_mixed = 1;
697 if (indices_are_interleaved_and_mixed == 1 ||
709 this->dof_indices.data() +
726 bool is_sorted =
true;
727 unsigned int previous = indices[0];
728 for (
unsigned int l = 1;
l < n_comp; ++
l)
730 const unsigned int current = indices[
l * ndofs];
731 if (current <= previous)
738 for (
unsigned int j = 0; j <
l; ++j)
739 if (indices[j * ndofs] == current)
752 index_kinds[
static_cast<unsigned int>(
762 auto fix_single_interleaved_indices =
764 if (index_kinds[
static_cast<unsigned int>(
767 index_kinds[
static_cast<unsigned int>(variant)] > 0)
768 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
772 interleaved_contiguous_mixed_strides &&
780 index_kinds[
static_cast<unsigned int>(
783 index_kinds[
static_cast<unsigned int>(variant)]++;
792 unsigned int n_interleaved =
793 index_kinds[
static_cast<unsigned int>(
795 index_kinds[
static_cast<unsigned int>(
797 index_kinds[
static_cast<unsigned int>(
802 if (n_interleaved > 0 && index_kinds[
static_cast<unsigned int>(
804 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
810 index_kinds[
static_cast<unsigned int>(
812 index_kinds[
static_cast<unsigned int>(
818 if (n_interleaved > 0 &&
820 index_kinds[
static_cast<unsigned int>(
823 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
827 index_kinds[
static_cast<unsigned int>(
836 index_kinds[
static_cast<unsigned int>(
841 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
852 const unsigned int ndofs =
857 unsigned int *interleaved_dof_indices =
861 this->dof_indices_interleaved.size());
871 for (
unsigned int k = 0; k < ndofs; ++k)
873 const unsigned int *my_dof_indices =
dof_indices + k;
874 const unsigned int *
end =
876 for (; interleaved_dof_indices !=
end;
877 ++interleaved_dof_indices, my_dof_indices += ndofs)
878 *interleaved_dof_indices = *my_dof_indices;
888 const unsigned int n_owned_cells,
889 const unsigned int n_lanes,
892 const bool fill_cell_centric,
893 const MPI_Comm & communicator_sm,
894 const bool use_vector_data_exchanger_full)
900 std::vector<types::global_dof_index> ghost_indices;
903 for (
unsigned int cell = 0; cell < n_owned_cells; ++cell)
911 const unsigned int fe_index =
920 ghost_indices.push_back(
923 std::sort(ghost_indices.begin(), ghost_indices.end());
924 ghost_indices.erase(std::unique(ghost_indices.begin(),
925 ghost_indices.end()),
926 ghost_indices.end());
928 compressed_set.
add_indices(ghost_indices.begin(), ghost_indices.end());
930 const bool all_ghosts_equal =
931 Utilities::MPI::min<int>(
static_cast<int>(
936 std::shared_ptr<const Utilities::MPI::Partitioner> temp_0;
938 if (all_ghosts_equal)
942 temp_0 = std::make_shared<Utilities::MPI::Partitioner>(
948 if (use_vector_data_exchanger_full ==
false)
954 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
955 temp_0, communicator_sm);
959 std::vector<FaceToCellTopology<1>> all_faces(inner_faces);
960 all_faces.insert(all_faces.end(),
961 ghosted_faces.begin(),
962 ghosted_faces.end());
965 2 * shape_info(0, 0).n_dimensions);
967 for (
unsigned int f = 0; f < all_faces.size(); ++f)
969 cell_and_face_to_faces(all_faces[f].cells_interior[0],
970 all_faces[f].interior_face_no) = f;
971 Assert(all_faces[f].cells_exterior[0] !=
974 cell_and_face_to_faces(all_faces[f].cells_exterior[0],
975 all_faces[f].exterior_face_no) = f;
979 const auto loop_over_faces =
980 [&](
const std::function<
981 void(
const unsigned int,
const unsigned int,
const bool)> &fu) {
982 for (
const auto &face : inner_faces)
985 fu(face.cells_exterior[0], face.exterior_face_no,
false );
989 const auto loop_over_all_faces =
990 [&](
const std::function<
991 void(
const unsigned int,
const unsigned int,
const bool)> &fu) {
992 for (
unsigned int c = 0; c < cell_and_face_to_faces.size(0); ++c)
993 for (
unsigned int d = 0;
d < cell_and_face_to_faces.size(1); ++
d)
995 const unsigned int f = cell_and_face_to_faces(c,
d);
999 const unsigned int cell_m = all_faces[f].cells_interior[0];
1000 const unsigned int cell_p = all_faces[f].cells_exterior[0];
1002 const bool ext = c == cell_m;
1007 const unsigned int p = ext ? cell_p : cell_m;
1008 const unsigned int face_no = ext ?
1009 all_faces[f].exterior_face_no :
1010 all_faces[f].interior_face_no;
1012 fu(p, face_no,
true);
1016 const auto process_values =
1018 std::shared_ptr<const Utilities::MPI::Partitioner>
1019 &vector_partitioner_values,
1020 const std::function<void(
1021 const std::function<
void(
1022 const unsigned int,
const unsigned int,
const bool)> &)> &
loop) {
1023 bool all_nodal_and_tensorial = shape_info.size(1) == 1;
1025 if (all_nodal_and_tensorial)
1030 if (!si.nodal_at_cell_boundaries ||
1033 all_nodal_and_tensorial =
false;
1036 if (all_nodal_and_tensorial ==
false)
1040 bool has_noncontiguous_cell =
false;
1042 loop([&](
const unsigned int cell_no,
1043 const unsigned int face_no,
1045 const unsigned int index =
1050 const unsigned int stride =
1051 dof_indices_interleave_strides[dof_access_cell][cell_no];
1053 for (unsigned int e = 0; e < n_base_elements; ++e)
1054 for (unsigned int c = 0; c < n_components[e]; ++c)
1056 const ShapeInfo<double> &shape =
1057 shape_info(global_base_element_offset + e, 0);
1058 for (unsigned int j = 0;
1059 j < shape.dofs_per_component_on_face;
1061 ghost_indices.push_back(part.local_to_global(
1063 shape.face_to_cell_index_nodal(face_no, j) *
1065 i += shape.dofs_per_component_on_cell * stride;
1070 has_noncontiguous_cell =
true;
1072 has_noncontiguous_cell =
1073 Utilities::MPI::min<int>(
static_cast<int>(
1074 has_noncontiguous_cell),
1077 std::sort(ghost_indices.begin(), ghost_indices.end());
1078 ghost_indices.erase(std::unique(ghost_indices.begin(),
1079 ghost_indices.end()),
1080 ghost_indices.end());
1083 ghost_indices.end());
1085 const bool all_ghosts_equal =
1086 Utilities::MPI::min<int>(
static_cast<int>(
1090 if (all_ghosts_equal || has_noncontiguous_cell)
1094 vector_partitioner_values =
1095 std::make_shared<Utilities::MPI::Partitioner>(
1098 vector_partitioner_values.get())
1105 const auto process_gradients =
1107 const std::shared_ptr<const Utilities::MPI::Partitioner>
1108 &vector_partitoner_values,
1109 std::shared_ptr<const Utilities::MPI::Partitioner>
1110 &vector_partitioner_gradients,
1111 const std::function<void(
1112 const std::function<
void(
1113 const unsigned int,
const unsigned int,
const bool)> &)> &
loop) {
1114 bool all_hermite = shape_info.
size(1) == 1;
1117 for (
unsigned int c = 0; c < n_base_elements; ++c)
1118 if (shape_info(global_base_element_offset + c, 0).element_type !=
1120 all_hermite =
false;
1121 if (all_hermite ==
false ||
1122 vector_partitoner_values.get() == vector_partitioner.get())
1123 vector_partitioner_gradients = vector_partitioner;
1126 loop([&](
const unsigned int cell_no,
1127 const unsigned int face_no,
1129 const unsigned int index =
1130 dof_indices_contiguous[dof_access_cell][cell_no];
1132 index >= part.local_size()))
1134 const unsigned int stride =
1135 dof_indices_interleave_strides[dof_access_cell][cell_no];
1137 for (unsigned int e = 0; e < n_base_elements; ++e)
1138 for (unsigned int c = 0; c < n_components[e]; ++c)
1140 const ShapeInfo<double> &shape =
1141 shape_info(global_base_element_offset + e, 0);
1142 for (unsigned int j = 0;
1143 j < 2 * shape.dofs_per_component_on_face;
1145 ghost_indices.push_back(part.local_to_global(
1147 shape.face_to_cell_index_hermite(face_no, j) *
1149 i += shape.dofs_per_component_on_cell * stride;
1154 std::sort(ghost_indices.begin(), ghost_indices.end());
1155 ghost_indices.erase(std::unique(ghost_indices.begin(),
1156 ghost_indices.end()),
1157 ghost_indices.end());
1158 IndexSet compressed_set(part.size());
1159 compressed_set.add_indices(ghost_indices.begin(),
1160 ghost_indices.end());
1161 compressed_set.subtract_set(part.locally_owned_range());
1162 const bool all_ghosts_equal =
1163 Utilities::MPI::min<int>(
static_cast<int>(
1164 compressed_set.n_elements() ==
1165 part.ghost_indices().n_elements()),
1166 part.get_mpi_communicator()) != 0;
1167 if (all_ghosts_equal)
1168 vector_partitioner_gradients = vector_partitioner;
1171 vector_partitioner_gradients =
1172 std::make_shared<Utilities::MPI::Partitioner>(
1173 part.locally_owned_range(), part.get_mpi_communicator());
1175 vector_partitioner_gradients.get())
1176 ->set_ghost_indices(compressed_set, part.ghost_indices());
1181 std::shared_ptr<const Utilities::MPI::Partitioner> temp_1, temp_2, temp_3,
1185 process_values(temp_1, loop_over_faces);
1188 process_gradients(temp_1, temp_2, loop_over_faces);
1190 if (fill_cell_centric)
1192 ghost_indices.clear();
1194 process_values(temp_3, loop_over_all_faces);
1196 process_gradients(temp_3, temp_4, loop_over_all_faces);
1200 temp_3 = std::make_shared<Utilities::MPI::Partitioner>(
1201 part.locally_owned_range(), part.get_mpi_communicator());
1202 temp_4 = std::make_shared<Utilities::MPI::Partitioner>(
1203 part.locally_owned_range(), part.get_mpi_communicator());
1206 if (use_vector_data_exchanger_full ==
false)
1208 vector_exchanger_face_variants[1] = std::make_shared<
1209 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1211 vector_exchanger_face_variants[2] = std::make_shared<
1212 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1214 vector_exchanger_face_variants[3] = std::make_shared<
1215 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1217 vector_exchanger_face_variants[4] = std::make_shared<
1218 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1223 vector_exchanger_face_variants[1] =
1224 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1225 temp_1, communicator_sm);
1226 vector_exchanger_face_variants[2] =
1227 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1228 temp_2, communicator_sm);
1229 vector_exchanger_face_variants[3] =
1230 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1231 temp_3, communicator_sm);
1232 vector_exchanger_face_variants[4] =
1233 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1234 temp_4, communicator_sm);
1241 DoFInfo::compute_shared_memory_contiguous_indices(
1242 std::array<std::vector<std::pair<unsigned int, unsigned int>>, 3>
1243 &cell_indices_contiguous_sm)
1247 for (
unsigned int i = 0; i < 3; ++i)
1249 dof_indices_contiguous_sm[i].resize(
1250 cell_indices_contiguous_sm[i].size());
1252 for (
unsigned int j = 0; j < cell_indices_contiguous_sm[i].size();
1254 if (cell_indices_contiguous_sm[i][j].
first !=
1256 dof_indices_contiguous_sm[i][j] = {
1257 cell_indices_contiguous_sm[i][j].first,
1258 cell_indices_contiguous_sm[i][j].second * dofs_per_cell[0]};
1278 const unsigned int end,
1280 std::vector<std::mutex> & mutexes,
1281 std::vector<unsigned int> &row_lengths)
1283 std::vector<unsigned int> scratch;
1285 for (
unsigned int block =
begin; block <
end; ++block)
1291 dof_info.
row_starts[block * n_components].first,
1293 dof_info.
row_starts[(block + 1) * n_components].first);
1294 std::sort(scratch.begin(), scratch.end());
1295 std::vector<unsigned int>::const_iterator end_unique =
1296 std::unique(scratch.begin(), scratch.end());
1297 std::vector<unsigned int>::const_iterator it = scratch.begin();
1298 while (it != end_unique)
1302 const unsigned int next_bucket =
1304 std::lock_guard<std::mutex> lock(
1306 for (; it != end_unique && *it < next_bucket; ++it)
1317 const unsigned int end,
1319 const std::vector<unsigned int> &row_lengths,
1320 std::vector<std::mutex> & mutexes,
1323 std::vector<unsigned int> scratch;
1325 for (
unsigned int block =
begin; block <
end; ++block)
1331 dof_info.
row_starts[block * n_components].first,
1333 dof_info.
row_starts[(block + 1) * n_components].first);
1334 std::sort(scratch.begin(), scratch.end());
1335 std::vector<unsigned int>::const_iterator end_unique =
1336 std::unique(scratch.begin(), scratch.end());
1337 std::vector<unsigned int>::const_iterator it = scratch.begin();
1338 while (it != end_unique)
1340 const unsigned int next_bucket =
1342 std::lock_guard<std::mutex> lock(
1344 for (; it != end_unique && *it < next_bucket; ++it)
1345 if (row_lengths[*it] > 0)
1346 connectivity_dof.
add(*it, block);
1355 const unsigned int end,
1357 const std::vector<unsigned int> &renumbering,
1358 const ::SparsityPattern & connectivity_dof,
1361 ordered_vector row_entries;
1363 for (
unsigned int block =
begin; block <
end; ++block)
1365 row_entries.clear();
1369 dof_info.
row_starts[block * n_components].first,
1372 for (; it != end_cell; ++it)
1375 std::vector<types::global_dof_index>::iterator insert_pos =
1376 row_entries.begin();
1377 for (; sp != connectivity_dof.end(*it); ++sp)
1378 if (sp->column() != block)
1379 row_entries.insert(renumbering[sp->column()], insert_pos);
1382 row_entries.begin(),
1390 DoFInfo::make_connectivity_graph(
1392 const std::vector<unsigned int> &renumbering,
1395 unsigned int n_rows = (vector_partitioner->local_range().second -
1396 vector_partitioner->local_range().first) +
1397 vector_partitioner->ghost_indices().n_elements();
1404 std::vector<unsigned int> row_lengths(n_rows);
1410 [
this, &mutexes, &row_lengths](
const unsigned int begin,
1411 const unsigned int end) {
1412 internal::compute_row_lengths(
1413 begin, end, *this, mutexes, row_lengths);
1419 for (
unsigned int row = 0; row < n_rows; ++row)
1420 if (row_lengths[row] <= 1)
1421 row_lengths[row] = 0;
1432 [
this, &row_lengths, &mutexes, &connectivity_dof](
1433 const unsigned int begin,
const unsigned int end) {
1434 internal::fill_connectivity_dofs(
1435 begin, end, *this, row_lengths, mutexes, connectivity_dof);
1442 std::vector<unsigned int> reverse_numbering(task_info.
n_active_cells);
1452 [
this, &reverse_numbering, &connectivity_dof, &connectivity](
1453 const unsigned int begin,
const unsigned int end) {
1454 internal::fill_connectivity(begin,
1467 DoFInfo::compute_dof_renumbering(
1468 std::vector<types::global_dof_index> &renumbering)
1470 const unsigned int locally_owned_size =
1471 vector_partitioner->locally_owned_size();
1472 renumbering.resize(0);
1476 const unsigned int n_components = start_components.back();
1477 const unsigned int n_cell_batches =
1478 n_vectorization_lanes_filled[dof_access_cell].size();
1480 (row_starts.size() - 1) / vectorization_length / n_components,
1482 for (
unsigned int cell_no = 0; cell_no < n_cell_batches; ++cell_no)
1485 if (row_starts[cell_no * n_components * vectorization_length]
1487 row_starts[(cell_no + 1) * n_components * vectorization_length]
1490 const unsigned int ndofs =
1491 dofs_per_cell.size() == 1 ?
1493 (dofs_per_cell[cell_active_fe_index.size() > 0 ?
1494 cell_active_fe_index[cell_no] :
1496 const unsigned int *dof_ind =
1497 dof_indices.data() +
1498 row_starts[cell_no * n_components * vectorization_length].first;
1499 for (
unsigned int i = 0; i < ndofs; ++i)
1500 for (
unsigned int j = 0;
1501 j < n_vectorization_lanes_filled[dof_access_cell][cell_no];
1503 if (dof_ind[j * ndofs + i] < locally_owned_size)
1504 if (renumbering[dof_ind[j * ndofs + i]] ==
1506 renumbering[dof_ind[j * ndofs + i]] = counter++;
1513 dof_index = counter++;
1517 dof_index = vector_partitioner->local_to_global(dof_index);
1527 std::size_t memory =
sizeof(*this);
1529 (row_starts.capacity() *
sizeof(std::pair<unsigned int, unsigned int>));
1544 namespace MatrixFreeFunctions
1546 template struct ConstraintValues<double>;
1548 template unsigned short
1550 const std::vector<std::pair<types::global_dof_index, float>> &);
1551 template unsigned short
1553 const std::vector<std::pair<types::global_dof_index, double>> &);
1557 DoFInfo::read_dof_indices<double>(
1558 const std::vector<types::global_dof_index> &,
1559 const std::vector<types::global_dof_index> &,
1561 const ::AffineConstraints<double> &,
1563 ConstraintValues<double> &,
1567 DoFInfo::read_dof_indices<float>(
1568 const std::vector<types::global_dof_index> &,
1569 const std::vector<types::global_dof_index> &,
1571 const ::AffineConstraints<float> &,
1573 ConstraintValues<double> &,
1577 DoFInfo::process_hanging_node_constraints<1>(
1578 const HangingNodes<1> &,
1579 const std::vector<std::vector<unsigned int>> &,
1582 std::vector<types::global_dof_index> &);
1584 DoFInfo::process_hanging_node_constraints<2>(
1585 const HangingNodes<2> &,
1586 const std::vector<std::vector<unsigned int>> &,
1589 std::vector<types::global_dof_index> &);
1591 DoFInfo::process_hanging_node_constraints<3>(
1592 const HangingNodes<3> &,
1593 const std::vector<std::vector<unsigned int>> &,
1596 std::vector<types::global_dof_index> &);
1599 DoFInfo::compute_face_index_compression<1>(
1600 const std::vector<FaceToCellTopology<1>> &);
1602 DoFInfo::compute_face_index_compression<2>(
1603 const std::vector<FaceToCellTopology<2>> &);
1605 DoFInfo::compute_face_index_compression<4>(
1606 const std::vector<FaceToCellTopology<4>> &);
1608 DoFInfo::compute_face_index_compression<8>(
1609 const std::vector<FaceToCellTopology<8>> &);
1611 DoFInfo::compute_face_index_compression<16>(
1612 const std::vector<FaceToCellTopology<16>> &);
1615 DoFInfo::compute_vector_zero_access_pattern<1>(
1617 const std::vector<FaceToCellTopology<1>> &);
1619 DoFInfo::compute_vector_zero_access_pattern<2>(
1621 const std::vector<FaceToCellTopology<2>> &);
1623 DoFInfo::compute_vector_zero_access_pattern<4>(
1625 const std::vector<FaceToCellTopology<4>> &);
1627 DoFInfo::compute_vector_zero_access_pattern<8>(
1629 const std::vector<FaceToCellTopology<8>> &);
1631 DoFInfo::compute_vector_zero_access_pattern<16>(
1633 const std::vector<FaceToCellTopology<16>> &);
1636 DoFInfo::print_memory_consumption<std::ostream>(std::ostream &,
1639 DoFInfo::print_memory_consumption<ConditionalOStream>(
1644 DoFInfo::print<double>(
const std::vector<double> &,
1645 const std::vector<unsigned int> &,
1646 std::ostream &)
const;
1649 DoFInfo::print<float>(
const std::vector<float> &,
1650 const std::vector<unsigned int> &,
1651 std::ostream &)
const;
void add_entries(const size_type row, ForwardIterator begin, ForwardIterator end, const bool indices_are_unique_and_sorted=false)
size_type index_within_set(const size_type global_index) const
size_type n_elements() const
void subtract_set(const IndexSet &other)
void add_range(const size_type begin, const size_type end)
void add_indices(const ForwardIterator &begin, const ForwardIterator &end)
void add(const size_type i, const size_type j)
unsigned int local_size() const
types::global_dof_index local_to_global(const unsigned int local_index) const
void set_ghost_indices(const IndexSet &ghost_indices, const IndexSet &larger_ghost_index_set=IndexSet())
virtual const MPI_Comm & get_mpi_communicator() const override
types::global_dof_index size() const
const IndexSet & locally_owned_range() const
const IndexSet & ghost_indices() const
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcNotInitialized()
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
void loop(ITERATOR begin, typename identity< ITERATOR >::type end, DOFINFO &dinfo, INFOBOX &info, const std::function< void(DOFINFO &, typename INFOBOX::CellInfo &)> &cell_worker, const std::function< void(DOFINFO &, typename INFOBOX::CellInfo &)> &boundary_worker, const std::function< void(DOFINFO &, DOFINFO &, typename INFOBOX::CellInfo &, typename INFOBOX::CellInfo &)> &face_worker, ASSEMBLER &assembler, const LoopControl &lctrl=LoopControl())
@ tensor_symmetric_hermite
std::enable_if_t< std::is_fundamental< T >::value, std::size_t > memory_consumption(const T &t)
void swap(MemorySpaceData< Number, MemorySpace > &, MemorySpaceData< Number, MemorySpace > &)
SymmetricTensor< 2, dim, Number > d(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
Tensor< 2, dim, Number > l(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::vector< Integer > invert_permutation(const std::vector< Integer > &permutation)
void fill_connectivity_dofs(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, const std::vector< unsigned int > &row_lengths, std::vector< std::mutex > &mutexes, ::SparsityPattern &connectivity_dof)
void fill_connectivity(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, const std::vector< unsigned int > &renumbering, const ::SparsityPattern &connectivity_dof, DynamicSparsityPattern &connectivity)
void compute_row_lengths(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, std::vector< std::mutex > &mutexes, std::vector< unsigned int > &row_lengths)
static constexpr unsigned int bucket_size_threading
constexpr compressed_constraint_kind unconstrained_compressed_constraint_kind
unsigned int n_active_cells(const internal::TriangulationImplementation::NumberCache< 1 > &c)
void copy(const T *begin, const T *end, U *dest)
static const unsigned int invalid_unsigned_int
const types::global_dof_index invalid_dof_index
void apply_to_subranges(const tbb::blocked_range< RangeType > &range, const Function &f)
unsigned int global_dof_index
unsigned short insert_entries(const std::vector< std::pair< types::global_dof_index, number2 >> &entries)
void reorder_cells(const TaskInfo &task_info, const std::vector< unsigned int > &renumbering, const std::vector< unsigned int > &constraint_pool_row_index, const std::vector< unsigned char > &irregular_cells)
std::vector< std::pair< unsigned short, unsigned short > > constraint_indicator
std::vector< std::pair< unsigned int, unsigned int > > row_starts
unsigned int n_base_elements
std::vector< std::vector< bool > > hanging_node_constraint_masks_comp
unsigned int global_base_element_offset
unsigned int max_fe_index
void get_dof_indices_on_cell_batch(std::vector< unsigned int > &local_indices, const unsigned int cell_batch, const bool with_constraints=true) const
void compute_cell_index_compression(const std::vector< unsigned char > &irregular_cells)
std::vector< unsigned int > dofs_per_cell
unsigned int vectorization_length
std::shared_ptr< const internal::MatrixFreeFunctions::VectorDataExchange::Base > vector_exchanger
@ interleaved_contiguous_strided
@ interleaved_contiguous_mixed_strides
std::vector< std::vector< unsigned int > > fe_index_conversion
std::vector< unsigned int > dof_indices
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
std::vector< compressed_constraint_kind > hanging_node_constraint_masks
std::array< std::vector< unsigned int >, 3 > dof_indices_interleave_strides
std::vector< unsigned int > row_starts_plain_indices
std::vector< unsigned int > dofs_per_face
std::vector< unsigned int > n_components
void assign_ghosts(const std::vector< unsigned int > &boundary_cells, const MPI_Comm &communicator_sm, const bool use_vector_data_exchanger_full)
std::vector< types::global_dof_index > ghost_dofs
std::array< std::vector< unsigned int >, 3 > dof_indices_contiguous
void compute_tight_partitioners(const Table< 2, ShapeInfo< double >> &shape_info, const unsigned int n_owned_cells, const unsigned int n_lanes, const std::vector< FaceToCellTopology< 1 >> &inner_faces, const std::vector< FaceToCellTopology< 1 >> &ghosted_faces, const bool fill_cell_centric, const MPI_Comm &communicator_sm, const bool use_vector_data_exchanger_full)
std::vector< unsigned int > cell_active_fe_index
std::array< std::shared_ptr< const internal::MatrixFreeFunctions::VectorDataExchange::Base >, 5 > vector_exchanger_face_variants
std::vector< unsigned int > plain_dof_indices
std::array< std::vector< unsigned char >, 3 > n_vectorization_lanes_filled
std::vector< unsigned int > start_components
std::vector< unsigned int > dof_indices_interleaved
std::array< std::vector< IndexStorageVariants >, 3 > index_storage_variants
unsigned int n_active_cells
std::vector< unsigned int > cell_partition_data