24 #include <deal.II/matrix_free/dof_info.templates.h>
32 namespace MatrixFreeFunctions
36 static_assert(std::is_same<compressed_constraint_kind, std::uint8_t>::value,
37 "Unexpected type for compressed hanging node indicators!");
67 for (
unsigned int i = 0; i < 3; ++i)
84 const unsigned int cell,
85 const bool apply_constraints)
const
96 const unsigned int n_vectorization_actual =
102 my_rows.reserve(n_vectorization * dofs_this_cell);
104 unsigned int total_size = 0;
105 for (
unsigned int v = 0; v < n_vectorization_actual; ++v)
107 const unsigned int ib =
108 (cell * n_vectorization + v) * n_fe_components;
109 const unsigned int ie =
110 (cell * n_vectorization + v + 1) * n_fe_components;
115 const bool has_constraints =
123 auto do_copy = [&](
const unsigned int *
begin,
124 const unsigned int *
end) {
125 const unsigned int shift = total_size;
127 my_rows.resize(total_size);
131 if (!has_constraints || apply_constraints)
133 const unsigned int *
begin =
135 const unsigned int *
end =
144 const unsigned int *
begin =
147 const unsigned int *
end =
begin + dofs_this_cell;
157 const MPI_Comm & communicator_sm,
158 const bool use_vector_data_exchanger_full)
165 const std::size_t n_ghosts =
ghost_dofs.size();
172 std::vector<unsigned int> ghost_numbering(n_ghosts);
176 unsigned int n_unique_ghosts = 0;
180 std::vector<std::pair<types::global_dof_index, unsigned int>>
181 ghost_origin(n_ghosts);
182 for (std::size_t i = 0; i < n_ghosts; ++i)
185 ghost_origin[i].second = i;
187 std::sort(ghost_origin.begin(), ghost_origin.end());
190 ghost_numbering[ghost_origin[0].second] = 0;
191 for (std::size_t i = 1; i < n_ghosts; ++i)
193 if (ghost_origin[i].
first > ghost_origin[i - 1].
first + 1)
195 ghost_indices.
add_range(last_contiguous_start,
196 ghost_origin[i - 1].
first + 1);
197 last_contiguous_start = ghost_origin[i].first;
199 if (ghost_origin[i].
first > ghost_origin[i - 1].
first)
201 ghost_numbering[ghost_origin[i].second] = n_unique_ghosts;
204 ghost_indices.
add_range(last_contiguous_start,
205 ghost_origin.back().first + 1);
212 for (std::size_t i = 0; i < n_ghosts; ++i)
213 Assert(ghost_numbering[i] ==
222 const unsigned int n_boundary_cells = boundary_cells.size();
223 for (
unsigned int i = 0; i < n_boundary_cells; ++i)
225 unsigned int *data_ptr =
228 const unsigned int *row_end =
231 for (; data_ptr != row_end; ++data_ptr)
232 *data_ptr = ((*data_ptr < n_owned) ?
234 n_owned + ghost_numbering[*data_ptr - n_owned]);
239 bool has_hanging_nodes =
false;
252 for (
unsigned int comp = 0; comp <
n_components; ++comp)
256 if (has_hanging_nodes ||
261 unsigned int *data_ptr =
264 const unsigned int *row_end =
266 for (; data_ptr != row_end; ++data_ptr)
268 ((*data_ptr < n_owned) ?
270 n_owned + ghost_numbering[*data_ptr - n_owned]);
276 std::vector<types::global_dof_index> empty;
286 if (use_vector_data_exchanger_full ==
false)
292 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
301 const std::vector<unsigned int> & renumbering,
302 const std::vector<unsigned int> & constraint_pool_row_index,
303 const std::vector<unsigned char> &irregular_cells)
305 (void)constraint_pool_row_index;
311 std::vector<unsigned int> new_active_fe_index;
313 unsigned int position_cell = 0;
314 for (
unsigned int cell = 0;
318 const unsigned int n_comp =
319 (irregular_cells[cell] > 0 ? irregular_cells[cell] :
326 for (
unsigned int j = 1; j < n_comp; ++j)
331 new_active_fe_index.push_back(
fe_index);
332 position_cell += n_comp;
342 std::vector<std::pair<unsigned int, unsigned int>> new_row_starts(
346 std::vector<unsigned int> new_dof_indices;
347 std::vector<std::pair<unsigned short, unsigned short>>
348 new_constraint_indicator;
349 std::vector<unsigned int> new_plain_indices, new_rowstart_plain;
350 unsigned int position_cell = 0;
354 std::vector<compressed_constraint_kind> new_hanging_node_constraint_masks;
355 new_hanging_node_constraint_masks.reserve(
375 const unsigned int n_vect =
376 (irregular_cells[i] > 0 ? irregular_cells[i] :
380 this->dofs_per_cell[0];
382 for (
unsigned int j = 0; j < n_vect; ++j)
384 const unsigned int cell_no =
387 bool has_hanging_nodes =
false;
395 new_hanging_node_constraint_masks.push_back(
mask);
398 for (
unsigned int comp = 0; comp <
n_components; ++comp)
403 for (
unsigned int comp = 0; comp <
n_components; ++comp)
407 .
first = new_dof_indices.size();
410 .
second = new_constraint_indicator.size();
412 new_dof_indices.insert(
413 new_dof_indices.end(),
419 new_constraint_indicator.push_back(
428 new_plain_indices.size();
429 new_plain_indices.insert(
430 new_plain_indices.end(),
439 for (
unsigned int comp = 0; comp <
n_components; ++comp)
443 .
first = new_dof_indices.size();
446 .
second = new_constraint_indicator.size();
451 new_hanging_node_constraint_masks.push_back(
454 position_cell += n_vect;
461 .first = new_dof_indices.size();
464 .second = new_constraint_indicator.size();
467 new_constraint_indicator.size());
479 const unsigned int index_range =
493 const unsigned int row_length_ind =
499 const std::pair<unsigned short, unsigned short> *
506 for (; con_it != end_con; ++con_it)
510 constraint_pool_row_index.size() - 1);
518 if (irregular_cells[c] > 0)
532 const std::vector<unsigned char> &irregular_cells)
542 irregular_cells.size());
546 irregular_cells.size());
547 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
548 if (irregular_cells[i] > 0)
563 std::vector<unsigned int> index_kinds(
564 static_cast<unsigned int>(
568 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
570 const unsigned int ndofs =
572 const unsigned int n_comp =
576 bool has_constraints =
false;
577 for (
unsigned int j = 0; j < n_comp; ++j)
583 has_constraints =
true;
592 bool indices_are_contiguous =
true;
593 for (
unsigned int j = 0; j < n_comp; ++j)
597 this->dof_indices.data() +
603 for (
unsigned int i = 1; i < ndofs; ++i)
606 indices_are_contiguous =
false;
611 bool indices_are_interleaved_and_contiguous =
616 this->dof_indices.data() +
618 for (
unsigned int k = 0;
619 k < ndofs && indices_are_interleaved_and_contiguous;
621 for (
unsigned int j = 0; j < n_comp; ++j)
625 indices_are_interleaved_and_contiguous =
false;
630 if (indices_are_contiguous ||
631 indices_are_interleaved_and_contiguous)
633 for (
unsigned int j = 0; j < n_comp; ++j)
644 if (indices_are_interleaved_and_contiguous)
649 for (
unsigned int j = 0; j < n_comp; ++j)
653 else if (indices_are_contiguous)
657 for (
unsigned int j = 0; j < n_comp; ++j)
663 int indices_are_interleaved_and_mixed = 2;
668 for (
unsigned int j = 0; j < n_comp; ++j)
671 for (
unsigned int k = 0;
672 k < ndofs && indices_are_interleaved_and_mixed != 0;
674 for (
unsigned int j = 0; j < n_comp; ++j)
681 indices_are_interleaved_and_mixed = 0;
684 if (indices_are_interleaved_and_mixed == 2)
686 for (
unsigned int j = 0; j < n_comp; ++j)
690 for (
unsigned int j = 0; j < n_comp; ++j)
694 for (
unsigned int j = 0; j < n_comp; ++j)
697 indices_are_interleaved_and_mixed = 1;
700 if (indices_are_interleaved_and_mixed == 1 ||
712 this->dof_indices.data() +
729 bool is_sorted =
true;
730 unsigned int previous = indices[0];
731 for (
unsigned int l = 1;
l < n_comp; ++
l)
733 const unsigned int current = indices[
l * ndofs];
734 if (current <= previous)
741 for (
unsigned int j = 0; j <
l; ++j)
742 if (indices[j * ndofs] == current)
755 index_kinds[
static_cast<unsigned int>(
765 auto fix_single_interleaved_indices =
767 if (index_kinds[
static_cast<unsigned int>(
770 index_kinds[
static_cast<unsigned int>(variant)] > 0)
771 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
775 interleaved_contiguous_mixed_strides &&
783 index_kinds[
static_cast<unsigned int>(
786 index_kinds[
static_cast<unsigned int>(variant)]++;
795 unsigned int n_interleaved =
796 index_kinds[
static_cast<unsigned int>(
798 index_kinds[
static_cast<unsigned int>(
800 index_kinds[
static_cast<unsigned int>(
805 if (n_interleaved > 0 && index_kinds[
static_cast<unsigned int>(
807 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
813 index_kinds[
static_cast<unsigned int>(
815 index_kinds[
static_cast<unsigned int>(
821 if (n_interleaved > 0 &&
823 index_kinds[
static_cast<unsigned int>(
826 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
830 index_kinds[
static_cast<unsigned int>(
839 index_kinds[
static_cast<unsigned int>(
844 for (
unsigned int i = 0; i < irregular_cells.size(); ++i)
855 const unsigned int ndofs =
860 unsigned int *interleaved_dof_indices =
864 this->dof_indices_interleaved.size());
874 for (
unsigned int k = 0; k < ndofs; ++k)
876 const unsigned int *my_dof_indices =
dof_indices + k;
877 const unsigned int *
end =
879 for (; interleaved_dof_indices !=
end;
880 ++interleaved_dof_indices, my_dof_indices += ndofs)
881 *interleaved_dof_indices = *my_dof_indices;
891 const unsigned int n_owned_cells,
892 const unsigned int n_lanes,
895 const bool fill_cell_centric,
896 const MPI_Comm & communicator_sm,
897 const bool use_vector_data_exchanger_full)
903 std::vector<types::global_dof_index> ghost_indices;
906 for (
unsigned int cell = 0; cell < n_owned_cells; ++cell)
923 ghost_indices.push_back(
926 std::sort(ghost_indices.begin(), ghost_indices.end());
927 ghost_indices.erase(std::unique(ghost_indices.begin(),
928 ghost_indices.end()),
929 ghost_indices.end());
931 compressed_set.
add_indices(ghost_indices.begin(), ghost_indices.end());
933 const bool all_ghosts_equal =
934 Utilities::MPI::min<int>(
static_cast<int>(
939 std::shared_ptr<const Utilities::MPI::Partitioner> temp_0;
941 if (all_ghosts_equal)
945 temp_0 = std::make_shared<Utilities::MPI::Partitioner>(
951 if (use_vector_data_exchanger_full ==
false)
957 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
958 temp_0, communicator_sm);
962 std::vector<FaceToCellTopology<1>> all_faces(inner_faces);
963 all_faces.insert(all_faces.end(),
964 ghosted_faces.begin(),
965 ghosted_faces.end());
968 2 * shape_info(0, 0).n_dimensions);
970 for (
unsigned int f = 0; f < all_faces.size(); ++f)
972 cell_and_face_to_faces(all_faces[f].cells_interior[0],
973 all_faces[f].interior_face_no) = f;
974 Assert(all_faces[f].cells_exterior[0] !=
977 cell_and_face_to_faces(all_faces[f].cells_exterior[0],
978 all_faces[f].exterior_face_no) = f;
982 const auto loop_over_faces =
983 [&](
const std::function<
984 void(
const unsigned int,
const unsigned int,
const bool)> &fu) {
985 for (
const auto &face : inner_faces)
988 fu(face.cells_exterior[0], face.exterior_face_no,
false );
992 const auto loop_over_all_faces =
993 [&](
const std::function<
994 void(
const unsigned int,
const unsigned int,
const bool)> &fu) {
995 for (
unsigned int c = 0; c < cell_and_face_to_faces.size(0); ++c)
996 for (
unsigned int d = 0;
d < cell_and_face_to_faces.size(1); ++
d)
998 const unsigned int f = cell_and_face_to_faces(c,
d);
1002 const unsigned int cell_m = all_faces[f].cells_interior[0];
1003 const unsigned int cell_p = all_faces[f].cells_exterior[0];
1005 const bool ext = c == cell_m;
1010 const unsigned int p = ext ? cell_p : cell_m;
1011 const unsigned int face_no = ext ?
1012 all_faces[f].exterior_face_no :
1013 all_faces[f].interior_face_no;
1015 fu(p, face_no,
true);
1019 const auto process_values =
1021 std::shared_ptr<const Utilities::MPI::Partitioner>
1022 &vector_partitioner_values,
1023 const std::function<void(
1024 const std::function<
void(
1025 const unsigned int,
const unsigned int,
const bool)> &)> &
loop) {
1026 bool all_nodal_and_tensorial = shape_info.size(1) == 1;
1028 if (all_nodal_and_tensorial)
1033 if (!si.nodal_at_cell_boundaries ||
1036 all_nodal_and_tensorial =
false;
1039 if (all_nodal_and_tensorial ==
false)
1043 bool has_noncontiguous_cell =
false;
1045 loop([&](
const unsigned int cell_no,
1046 const unsigned int face_no,
1048 const unsigned int index =
1053 const unsigned int stride =
1054 dof_indices_interleave_strides[dof_access_cell][cell_no];
1056 for (unsigned int e = 0; e < n_base_elements; ++e)
1057 for (unsigned int c = 0; c < n_components[e]; ++c)
1059 const ShapeInfo<double> &shape =
1060 shape_info(global_base_element_offset + e, 0);
1061 for (unsigned int j = 0;
1062 j < shape.dofs_per_component_on_face;
1064 ghost_indices.push_back(part.local_to_global(
1066 shape.face_to_cell_index_nodal(face_no, j) *
1068 i += shape.dofs_per_component_on_cell * stride;
1073 has_noncontiguous_cell =
true;
1075 has_noncontiguous_cell =
1076 Utilities::MPI::min<int>(
static_cast<int>(
1077 has_noncontiguous_cell),
1080 std::sort(ghost_indices.begin(), ghost_indices.end());
1081 ghost_indices.erase(std::unique(ghost_indices.begin(),
1082 ghost_indices.end()),
1083 ghost_indices.end());
1086 ghost_indices.end());
1088 const bool all_ghosts_equal =
1089 Utilities::MPI::min<int>(
static_cast<int>(
1093 if (all_ghosts_equal || has_noncontiguous_cell)
1097 vector_partitioner_values =
1098 std::make_shared<Utilities::MPI::Partitioner>(
1101 vector_partitioner_values.get())
1108 const auto process_gradients =
1110 const std::shared_ptr<const Utilities::MPI::Partitioner>
1111 &vector_partitoner_values,
1112 std::shared_ptr<const Utilities::MPI::Partitioner>
1113 &vector_partitioner_gradients,
1114 const std::function<void(
1115 const std::function<
void(
1116 const unsigned int,
const unsigned int,
const bool)> &)> &
loop) {
1117 bool all_hermite = shape_info.
size(1) == 1;
1120 for (
unsigned int c = 0; c < n_base_elements; ++c)
1121 if (shape_info(global_base_element_offset + c, 0).element_type !=
1123 all_hermite =
false;
1124 if (all_hermite ==
false ||
1125 vector_partitoner_values.get() == vector_partitioner.get())
1126 vector_partitioner_gradients = vector_partitioner;
1129 loop([&](
const unsigned int cell_no,
1130 const unsigned int face_no,
1132 const unsigned int index =
1133 dof_indices_contiguous[dof_access_cell][cell_no];
1135 index >= part.locally_owned_size()))
1137 const unsigned int stride =
1138 dof_indices_interleave_strides[dof_access_cell][cell_no];
1140 for (unsigned int e = 0; e < n_base_elements; ++e)
1141 for (unsigned int c = 0; c < n_components[e]; ++c)
1143 const ShapeInfo<double> &shape =
1144 shape_info(global_base_element_offset + e, 0);
1145 for (unsigned int j = 0;
1146 j < 2 * shape.dofs_per_component_on_face;
1148 ghost_indices.push_back(part.local_to_global(
1150 shape.face_to_cell_index_hermite(face_no, j) *
1152 i += shape.dofs_per_component_on_cell * stride;
1157 std::sort(ghost_indices.begin(), ghost_indices.end());
1158 ghost_indices.erase(std::unique(ghost_indices.begin(),
1159 ghost_indices.end()),
1160 ghost_indices.end());
1161 IndexSet compressed_set(part.size());
1162 compressed_set.add_indices(ghost_indices.begin(),
1163 ghost_indices.end());
1164 compressed_set.subtract_set(part.locally_owned_range());
1165 const bool all_ghosts_equal =
1166 Utilities::MPI::min<int>(
static_cast<int>(
1167 compressed_set.n_elements() ==
1168 part.ghost_indices().n_elements()),
1169 part.get_mpi_communicator()) != 0;
1170 if (all_ghosts_equal)
1171 vector_partitioner_gradients = vector_partitioner;
1174 vector_partitioner_gradients =
1175 std::make_shared<Utilities::MPI::Partitioner>(
1176 part.locally_owned_range(), part.get_mpi_communicator());
1178 vector_partitioner_gradients.get())
1179 ->set_ghost_indices(compressed_set, part.ghost_indices());
1184 std::shared_ptr<const Utilities::MPI::Partitioner> temp_1, temp_2, temp_3,
1188 process_values(temp_1, loop_over_faces);
1191 process_gradients(temp_1, temp_2, loop_over_faces);
1193 if (fill_cell_centric)
1195 ghost_indices.clear();
1197 process_values(temp_3, loop_over_all_faces);
1199 process_gradients(temp_3, temp_4, loop_over_all_faces);
1203 temp_3 = std::make_shared<Utilities::MPI::Partitioner>(
1204 part.locally_owned_range(), part.get_mpi_communicator());
1205 temp_4 = std::make_shared<Utilities::MPI::Partitioner>(
1206 part.locally_owned_range(), part.get_mpi_communicator());
1209 if (use_vector_data_exchanger_full ==
false)
1211 vector_exchanger_face_variants[1] = std::make_shared<
1212 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1214 vector_exchanger_face_variants[2] = std::make_shared<
1215 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1217 vector_exchanger_face_variants[3] = std::make_shared<
1218 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1220 vector_exchanger_face_variants[4] = std::make_shared<
1221 MatrixFreeFunctions::VectorDataExchange::PartitionerWrapper>(
1226 vector_exchanger_face_variants[1] =
1227 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1228 temp_1, communicator_sm);
1229 vector_exchanger_face_variants[2] =
1230 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1231 temp_2, communicator_sm);
1232 vector_exchanger_face_variants[3] =
1233 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1234 temp_3, communicator_sm);
1235 vector_exchanger_face_variants[4] =
1236 std::make_shared<MatrixFreeFunctions::VectorDataExchange::Full>(
1237 temp_4, communicator_sm);
1244 DoFInfo::compute_shared_memory_contiguous_indices(
1245 std::array<std::vector<std::pair<unsigned int, unsigned int>>, 3>
1246 &cell_indices_contiguous_sm)
1250 for (
unsigned int i = 0; i < 3; ++i)
1252 dof_indices_contiguous_sm[i].resize(
1253 cell_indices_contiguous_sm[i].size());
1255 for (
unsigned int j = 0; j < cell_indices_contiguous_sm[i].size();
1257 if (cell_indices_contiguous_sm[i][j].
first !=
1259 dof_indices_contiguous_sm[i][j] = {
1260 cell_indices_contiguous_sm[i][j].first,
1261 cell_indices_contiguous_sm[i][j].second * dofs_per_cell[0]};
1281 const unsigned int end,
1283 std::vector<std::mutex> & mutexes,
1284 std::vector<unsigned int> &row_lengths)
1286 std::vector<unsigned int> scratch;
1288 for (
unsigned int block =
begin; block <
end; ++block)
1294 dof_info.
row_starts[block * n_components].first,
1296 dof_info.
row_starts[(block + 1) * n_components].first);
1297 std::sort(scratch.begin(), scratch.end());
1298 std::vector<unsigned int>::const_iterator end_unique =
1299 std::unique(scratch.begin(), scratch.end());
1300 std::vector<unsigned int>::const_iterator it = scratch.begin();
1301 while (it != end_unique)
1305 const unsigned int next_bucket =
1307 std::lock_guard<std::mutex> lock(
1309 for (; it != end_unique && *it < next_bucket; ++it)
1320 const unsigned int end,
1322 const std::vector<unsigned int> &row_lengths,
1323 std::vector<std::mutex> & mutexes,
1326 std::vector<unsigned int> scratch;
1328 for (
unsigned int block =
begin; block <
end; ++block)
1334 dof_info.
row_starts[block * n_components].first,
1336 dof_info.
row_starts[(block + 1) * n_components].first);
1337 std::sort(scratch.begin(), scratch.end());
1338 std::vector<unsigned int>::const_iterator end_unique =
1339 std::unique(scratch.begin(), scratch.end());
1340 std::vector<unsigned int>::const_iterator it = scratch.begin();
1341 while (it != end_unique)
1343 const unsigned int next_bucket =
1345 std::lock_guard<std::mutex> lock(
1347 for (; it != end_unique && *it < next_bucket; ++it)
1348 if (row_lengths[*it] > 0)
1349 connectivity_dof.
add(*it, block);
1358 const unsigned int end,
1360 const std::vector<unsigned int> &renumbering,
1361 const ::SparsityPattern & connectivity_dof,
1364 ordered_vector row_entries;
1366 for (
unsigned int block =
begin; block <
end; ++block)
1368 row_entries.clear();
1372 dof_info.
row_starts[block * n_components].first,
1375 for (; it != end_cell; ++it)
1378 std::vector<types::global_dof_index>::iterator insert_pos =
1379 row_entries.begin();
1380 for (; sp != connectivity_dof.end(*it); ++sp)
1381 if (sp->column() != block)
1382 row_entries.insert(renumbering[sp->column()], insert_pos);
1385 row_entries.begin(),
1393 DoFInfo::make_connectivity_graph(
1395 const std::vector<unsigned int> &renumbering,
1398 unsigned int n_rows = (vector_partitioner->local_range().second -
1399 vector_partitioner->local_range().first) +
1400 vector_partitioner->ghost_indices().n_elements();
1407 std::vector<unsigned int> row_lengths(n_rows);
1413 [
this, &mutexes, &row_lengths](
const unsigned int begin,
1414 const unsigned int end) {
1415 internal::compute_row_lengths(
1416 begin, end, *this, mutexes, row_lengths);
1422 for (
unsigned int row = 0; row < n_rows; ++row)
1423 if (row_lengths[row] <= 1)
1424 row_lengths[row] = 0;
1435 [
this, &row_lengths, &mutexes, &connectivity_dof](
1436 const unsigned int begin,
const unsigned int end) {
1437 internal::fill_connectivity_dofs(
1438 begin, end, *this, row_lengths, mutexes, connectivity_dof);
1445 std::vector<unsigned int> reverse_numbering(task_info.
n_active_cells);
1455 [
this, &reverse_numbering, &connectivity_dof, &connectivity](
1456 const unsigned int begin,
const unsigned int end) {
1457 internal::fill_connectivity(begin,
1470 DoFInfo::compute_dof_renumbering(
1471 std::vector<types::global_dof_index> &renumbering)
1473 const unsigned int locally_owned_size =
1474 vector_partitioner->locally_owned_size();
1475 renumbering.resize(0);
1479 const unsigned int n_components = start_components.back();
1480 const unsigned int n_cell_batches =
1481 n_vectorization_lanes_filled[dof_access_cell].size();
1483 (row_starts.size() - 1) / vectorization_length / n_components,
1485 for (
unsigned int cell_no = 0; cell_no < n_cell_batches; ++cell_no)
1488 if (row_starts[cell_no * n_components * vectorization_length]
1490 row_starts[(cell_no + 1) * n_components * vectorization_length]
1493 const unsigned int ndofs =
1494 dofs_per_cell.size() == 1 ?
1496 (dofs_per_cell[cell_active_fe_index.size() > 0 ?
1497 cell_active_fe_index[cell_no] :
1499 const unsigned int *dof_ind =
1500 dof_indices.data() +
1501 row_starts[cell_no * n_components * vectorization_length].first;
1502 for (
unsigned int i = 0; i < ndofs; ++i)
1503 for (
unsigned int j = 0;
1504 j < n_vectorization_lanes_filled[dof_access_cell][cell_no];
1506 if (dof_ind[j * ndofs + i] < locally_owned_size)
1507 if (renumbering[dof_ind[j * ndofs + i]] ==
1509 renumbering[dof_ind[j * ndofs + i]] = counter++;
1516 dof_index = counter++;
1520 dof_index = vector_partitioner->local_to_global(dof_index);
1530 std::size_t memory =
sizeof(*this);
1532 (row_starts.capacity() *
sizeof(std::pair<unsigned int, unsigned int>));
1547 namespace MatrixFreeFunctions
1550 DoFInfo::read_dof_indices<double>(
1551 const std::vector<types::global_dof_index> &,
1552 const std::vector<types::global_dof_index> &,
1554 const ::AffineConstraints<double> &,
1556 ConstraintValues<double> &,
1560 DoFInfo::read_dof_indices<float>(
1561 const std::vector<types::global_dof_index> &,
1562 const std::vector<types::global_dof_index> &,
1564 const ::AffineConstraints<float> &,
1566 ConstraintValues<double> &,
1570 DoFInfo::process_hanging_node_constraints<1>(
1571 const HangingNodes<1> &,
1572 const std::vector<std::vector<unsigned int>> &,
1575 std::vector<types::global_dof_index> &);
1577 DoFInfo::process_hanging_node_constraints<2>(
1578 const HangingNodes<2> &,
1579 const std::vector<std::vector<unsigned int>> &,
1582 std::vector<types::global_dof_index> &);
1584 DoFInfo::process_hanging_node_constraints<3>(
1585 const HangingNodes<3> &,
1586 const std::vector<std::vector<unsigned int>> &,
1589 std::vector<types::global_dof_index> &);
1592 DoFInfo::compute_face_index_compression<1>(
1593 const std::vector<FaceToCellTopology<1>> &);
1595 DoFInfo::compute_face_index_compression<2>(
1596 const std::vector<FaceToCellTopology<2>> &);
1598 DoFInfo::compute_face_index_compression<4>(
1599 const std::vector<FaceToCellTopology<4>> &);
1601 DoFInfo::compute_face_index_compression<8>(
1602 const std::vector<FaceToCellTopology<8>> &);
1604 DoFInfo::compute_face_index_compression<16>(
1605 const std::vector<FaceToCellTopology<16>> &);
1608 DoFInfo::compute_vector_zero_access_pattern<1>(
1610 const std::vector<FaceToCellTopology<1>> &);
1612 DoFInfo::compute_vector_zero_access_pattern<2>(
1614 const std::vector<FaceToCellTopology<2>> &);
1616 DoFInfo::compute_vector_zero_access_pattern<4>(
1618 const std::vector<FaceToCellTopology<4>> &);
1620 DoFInfo::compute_vector_zero_access_pattern<8>(
1622 const std::vector<FaceToCellTopology<8>> &);
1624 DoFInfo::compute_vector_zero_access_pattern<16>(
1626 const std::vector<FaceToCellTopology<16>> &);
1629 DoFInfo::print_memory_consumption<std::ostream>(std::ostream &,
1632 DoFInfo::print_memory_consumption<ConditionalOStream>(
1637 DoFInfo::print<double>(
const std::vector<double> &,
1638 const std::vector<unsigned int> &,
1639 std::ostream &)
const;
1642 DoFInfo::print<float>(
const std::vector<float> &,
1643 const std::vector<unsigned int> &,
1644 std::ostream &)
const;
void add_entries(const size_type row, ForwardIterator begin, ForwardIterator end, const bool indices_are_unique_and_sorted=false)
size_type index_within_set(const size_type global_index) const
size_type n_elements() const
void subtract_set(const IndexSet &other)
void add_range(const size_type begin, const size_type end)
void add_indices(const ForwardIterator &begin, const ForwardIterator &end)
void add(const size_type i, const size_type j)
unsigned int locally_owned_size() const
types::global_dof_index local_to_global(const unsigned int local_index) const
void set_ghost_indices(const IndexSet &ghost_indices, const IndexSet &larger_ghost_index_set=IndexSet())
virtual const MPI_Comm & get_mpi_communicator() const override
types::global_dof_index size() const
const IndexSet & locally_owned_range() const
const IndexSet & ghost_indices() const
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcNotInitialized()
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
void loop(ITERATOR begin, typename identity< ITERATOR >::type end, DOFINFO &dinfo, INFOBOX &info, const std::function< void(DOFINFO &, typename INFOBOX::CellInfo &)> &cell_worker, const std::function< void(DOFINFO &, typename INFOBOX::CellInfo &)> &boundary_worker, const std::function< void(DOFINFO &, DOFINFO &, typename INFOBOX::CellInfo &, typename INFOBOX::CellInfo &)> &face_worker, ASSEMBLER &assembler, const LoopControl &lctrl=LoopControl())
@ tensor_symmetric_hermite
std::enable_if_t< std::is_fundamental< T >::value, std::size_t > memory_consumption(const T &t)
void swap(MemorySpaceData< T, MemorySpace > &u, MemorySpaceData< T, MemorySpace > &v)
SymmetricTensor< 2, dim, Number > d(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
Tensor< 2, dim, Number > l(const Tensor< 2, dim, Number > &F, const Tensor< 2, dim, Number > &dF_dt)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::vector< Integer > invert_permutation(const std::vector< Integer > &permutation)
void fill_connectivity_dofs(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, const std::vector< unsigned int > &row_lengths, std::vector< std::mutex > &mutexes, ::SparsityPattern &connectivity_dof)
void fill_connectivity(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, const std::vector< unsigned int > &renumbering, const ::SparsityPattern &connectivity_dof, DynamicSparsityPattern &connectivity)
void compute_row_lengths(const unsigned int begin, const unsigned int end, const DoFInfo &dof_info, std::vector< std::mutex > &mutexes, std::vector< unsigned int > &row_lengths)
static constexpr unsigned int bucket_size_threading
constexpr compressed_constraint_kind unconstrained_compressed_constraint_kind
unsigned int n_active_cells(const internal::TriangulationImplementation::NumberCache< 1 > &c)
void copy(const T *begin, const T *end, U *dest)
static const unsigned int invalid_unsigned_int
const types::global_dof_index invalid_dof_index
void apply_to_subranges(const tbb::blocked_range< RangeType > &range, const Function &f)
unsigned int global_dof_index
unsigned short int fe_index
void reorder_cells(const TaskInfo &task_info, const std::vector< unsigned int > &renumbering, const std::vector< unsigned int > &constraint_pool_row_index, const std::vector< unsigned char > &irregular_cells)
std::vector< std::pair< unsigned short, unsigned short > > constraint_indicator
std::vector< std::pair< unsigned int, unsigned int > > row_starts
unsigned int n_base_elements
std::vector< std::vector< bool > > hanging_node_constraint_masks_comp
unsigned int global_base_element_offset
unsigned int max_fe_index
void get_dof_indices_on_cell_batch(std::vector< unsigned int > &local_indices, const unsigned int cell_batch, const bool with_constraints=true) const
void compute_cell_index_compression(const std::vector< unsigned char > &irregular_cells)
std::vector< unsigned int > dofs_per_cell
unsigned int vectorization_length
std::shared_ptr< const internal::MatrixFreeFunctions::VectorDataExchange::Base > vector_exchanger
@ interleaved_contiguous_strided
@ interleaved_contiguous_mixed_strides
std::vector< std::vector< unsigned int > > fe_index_conversion
std::vector< unsigned int > dof_indices
std::shared_ptr< const Utilities::MPI::Partitioner > vector_partitioner
std::vector< compressed_constraint_kind > hanging_node_constraint_masks
std::array< std::vector< unsigned int >, 3 > dof_indices_interleave_strides
std::vector< unsigned int > row_starts_plain_indices
std::vector< unsigned int > dofs_per_face
std::vector< unsigned int > n_components
void assign_ghosts(const std::vector< unsigned int > &boundary_cells, const MPI_Comm &communicator_sm, const bool use_vector_data_exchanger_full)
std::vector< types::global_dof_index > ghost_dofs
std::array< std::vector< unsigned int >, 3 > dof_indices_contiguous
void compute_tight_partitioners(const Table< 2, ShapeInfo< double >> &shape_info, const unsigned int n_owned_cells, const unsigned int n_lanes, const std::vector< FaceToCellTopology< 1 >> &inner_faces, const std::vector< FaceToCellTopology< 1 >> &ghosted_faces, const bool fill_cell_centric, const MPI_Comm &communicator_sm, const bool use_vector_data_exchanger_full)
std::vector< unsigned int > cell_active_fe_index
std::array< std::shared_ptr< const internal::MatrixFreeFunctions::VectorDataExchange::Base >, 5 > vector_exchanger_face_variants
std::vector< unsigned int > plain_dof_indices
std::array< std::vector< unsigned char >, 3 > n_vectorization_lanes_filled
std::vector< unsigned int > start_components
std::vector< unsigned int > dof_indices_interleaved
std::array< std::vector< IndexStorageVariants >, 3 > index_storage_variants
unsigned int n_active_cells
std::vector< unsigned int > cell_partition_data