2305 const unsigned int n_components,
2307 typename Processor::Number2_ *global_vector_ptr,
2309 const EvaluationData &fe_eval,
2310 typename Processor::VectorizedArrayType_ *temp1)
2312 constexpr int dim = Processor::dim_;
2313 constexpr int fe_degree = Processor::fe_degree_;
2314 using VectorizedArrayType =
typename Processor::VectorizedArrayType_;
2315 constexpr int n_lanes = VectorizedArrayType::size();
2317 using Number =
typename Processor::Number_;
2318 using Number2_ =
typename Processor::Number2_;
2320 const auto &shape_data = fe_eval.get_shape_info().data.front();
2321 constexpr bool integrate = Processor::do_integrate;
2322 const unsigned int face_no = fe_eval.get_face_no();
2323 const auto &dof_info = fe_eval.get_dof_info();
2324 const unsigned int cell = fe_eval.get_cell_or_face_batch_id();
2326 fe_eval.get_dof_access_index();
2328 dof_info.index_storage_variants[dof_access_index].size());
2329 constexpr unsigned int dofs_per_face =
2331 const unsigned int subface_index = fe_eval.get_subface_index();
2333 const unsigned int n_filled_lanes =
2334 dof_info.n_vectorization_lanes_filled[dof_access_index][cell];
2336 bool all_faces_are_same = n_filled_lanes == n_lanes;
2337 if (n_face_orientations == n_lanes)
2338 for (
unsigned int v = 1; v < n_lanes; ++v)
2339 if (fe_eval.get_face_no(v) != fe_eval.get_face_no(0) ||
2340 fe_eval.get_face_orientation(v) != fe_eval.get_face_orientation(0))
2342 all_faces_are_same =
false;
2347 std::array<const unsigned int *, n_face_orientations> orientation = {};
2349 if (dim == 3 && n_face_orientations == n_lanes && !all_faces_are_same &&
2350 fe_eval.is_interior_face() == 0)
2351 for (
unsigned int v = 0; v < n_lanes; ++v)
2359 if (shape_data.nodal_at_cell_boundaries &&
2360 fe_eval.get_face_orientation(v) != 0)
2365 check_face_orientations ==
false)
2379 orientation[v] = &fe_eval.get_shape_info().face_orientations_dofs(
2380 fe_eval.get_face_orientation(v), 0);
2383 else if (dim == 3 && fe_eval.get_face_orientation() != 0)
2387 check_face_orientations ==
false)
2401 for (
unsigned int v = 0; v < n_face_orientations; ++v)
2402 orientation[v] = &fe_eval.get_shape_info().face_orientations_dofs(
2403 fe_eval.get_face_orientation(), 0);
2409 VectorizedArrayType grad_weight =
2411 .shape_data_on_face[0][fe_degree + (integrate ? (2 - face_no % 2) :
2412 (1 + face_no % 2))];
2415 std::array<const unsigned int *, n_face_orientations> index_array_hermite =
2419 if (n_face_orientations == 1)
2420 index_array_hermite[0] =
2421 &fe_eval.get_shape_info().face_to_cell_index_hermite(face_no, 0);
2424 for (
unsigned int v = 0; v < n_lanes; ++v)
2429 const auto face_no = fe_eval.get_face_no(v);
2432 shape_data.shape_data_on_face[0][fe_degree +
2434 (2 - (face_no % 2)) :
2435 (1 + (face_no % 2)))][0];
2437 index_array_hermite[v] =
2438 &fe_eval.get_shape_info().face_to_cell_index_hermite(face_no,
2445 std::array<const unsigned int *, n_face_orientations> index_array_nodal =
2447 if (shape_data.nodal_at_cell_boundaries ==
true)
2449 if (n_face_orientations == 1)
2450 index_array_nodal[0] =
2451 &fe_eval.get_shape_info().face_to_cell_index_nodal(face_no, 0);
2454 for (
unsigned int v = 0; v < n_lanes; ++v)
2459 const auto face_no = fe_eval.get_face_no(v);
2461 index_array_nodal[v] =
2462 &fe_eval.get_shape_info().face_to_cell_index_nodal(face_no,
2469 const auto reorientate = [&](
const unsigned int v,
const unsigned int i) {
2470 return (!check_face_orientations || orientation[v] ==
nullptr) ?
2477 fe_eval.get_cell_ids()[0] :
2479 const unsigned int *dof_indices =
2480 &dof_info.dof_indices_contiguous[dof_access_index][
cell_index];
2482 for (
unsigned int comp = 0; comp < n_components; ++comp)
2484 const std::size_t index_offset =
2485 dof_info.component_dof_indices_offset
2486 [fe_eval.get_active_fe_index()]
2487 [fe_eval.get_first_selected_component()] +
2491 if (n_face_orientations == 1 &&
2492 dof_info.index_storage_variants[dof_access_index][cell] ==
2494 interleaved_contiguous)
2497 dof_info.n_vectorization_lanes_filled[dof_access_index][cell],
2499 Number2_ *vector_ptr =
2500 global_vector_ptr + dof_indices[0] + index_offset * n_lanes;
2504 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2508 const unsigned int ind1 = index_array_hermite[0][2 * i];
2509 const unsigned int ind2 = index_array_hermite[0][2 * i + 1];
2510 const unsigned int i_ = reorientate(0, i);
2511 proc.hermite_grad_vectorized(temp1[i_],
2512 temp1[i_ + dofs_per_face],
2513 vector_ptr + ind1 * n_lanes,
2514 vector_ptr + ind2 * n_lanes,
2520 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2524 const unsigned int i_ = reorientate(0, i);
2525 const unsigned int ind = index_array_nodal[0][i];
2526 proc.value_vectorized(temp1[i_],
2527 vector_ptr + ind * n_lanes);
2533 else if (n_face_orientations == 1 &&
2534 dof_info.index_storage_variants[dof_access_index][cell] ==
2536 interleaved_contiguous_strided)
2539 dof_info.n_vectorization_lanes_filled[dof_access_index][cell],
2541 Number2_ *vector_ptr = global_vector_ptr + index_offset * n_lanes;
2544 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2548 const unsigned int i_ = reorientate(0, i);
2549 const unsigned int ind1 =
2550 index_array_hermite[0][2 * i] * n_lanes;
2551 const unsigned int ind2 =
2552 index_array_hermite[0][2 * i + 1] * n_lanes;
2553 proc.hermite_grad_vectorized_indexed(
2555 temp1[i_ + dofs_per_face],
2565 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2569 const unsigned int i_ = reorientate(0, i);
2570 const unsigned int ind = index_array_nodal[0][i] * n_lanes;
2571 proc.value_vectorized_indexed(temp1[i_],
2579 else if (n_face_orientations == 1 &&
2580 dof_info.index_storage_variants[dof_access_index][cell] ==
2582 interleaved_contiguous_mixed_strides)
2584 const unsigned int *strides =
2585 &dof_info.dof_indices_interleave_strides[dof_access_index]
2587 unsigned int indices[n_lanes];
2588 for (
unsigned int v = 0; v < n_lanes; ++v)
2589 indices[v] = dof_indices[v] + index_offset * strides[v];
2590 const unsigned int n_filled_lanes =
2591 dof_info.n_vectorization_lanes_filled[dof_access_index][cell];
2595 if (n_filled_lanes == n_lanes)
2596 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2600 const unsigned int i_ = reorientate(0, i);
2601 unsigned int ind1[n_lanes];
2603 for (
unsigned int v = 0; v < n_lanes; ++v)
2604 ind1[v] = indices[v] +
2605 index_array_hermite[0][2 * i] * strides[v];
2606 unsigned int ind2[n_lanes];
2608 for (
unsigned int v = 0; v < n_lanes; ++v)
2612 index_array_hermite[0][2 * i + 1] * strides[v];
2613 proc.hermite_grad_vectorized_indexed(
2615 temp1[i_ + dofs_per_face],
2624 if (integrate ==
false)
2625 for (
unsigned int i = 0; i < 2 * dofs_per_face; ++i)
2626 temp1[i] = VectorizedArrayType();
2628 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2629 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2631 const unsigned int i_ =
2632 reorientate(n_face_orientations == 1 ? 0 : v, i);
2635 temp1[i_ + dofs_per_face][v],
2639 [n_face_orientations == 1 ? 0 : v][2 * i] *
2643 index_array_hermite[n_face_orientations == 1 ?
2647 grad_weight[n_face_orientations == 1 ? 0 : v]);
2653 if (n_filled_lanes == n_lanes)
2654 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2657 unsigned int ind[n_lanes];
2659 for (
unsigned int v = 0; v < n_lanes; ++v)
2661 indices[v] + index_array_nodal[0][i] * strides[v];
2662 const unsigned int i_ = reorientate(0, i);
2663 proc.value_vectorized_indexed(temp1[i_],
2669 if (integrate ==
false)
2670 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2671 temp1[i] = VectorizedArrayType();
2673 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2674 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2676 temp1[reorientate(n_face_orientations == 1 ? 0 : v,
2680 index_array_nodal[n_face_orientations == 1 ? 0 : v]
2688 else if (n_face_orientations > 1 ||
2689 dof_info.index_storage_variants[dof_access_index][cell] ==
2693 Number2_ *vector_ptr = global_vector_ptr + index_offset;
2695 const bool vectorization_possible =
2696 all_faces_are_same && (sm_ptr ==
nullptr);
2698 std::array<Number2_ *, n_lanes> vector_ptrs;
2699 std::array<unsigned int, n_lanes> reordered_indices;
2701 if (vectorization_possible ==
false)
2704 if (n_face_orientations == 1)
2706 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2707 if (sm_ptr ==
nullptr)
2709 vector_ptrs[v] = vector_ptr + dof_indices[v];
2715 .dof_indices_contiguous_sm[dof_access_index]
2716 [cell * n_lanes + v];
2717 vector_ptrs[v] =
const_cast<Number2_ *
>(
2718 sm_ptr->operator[](temp.first).data() +
2719 temp.second + index_offset);
2722 else if (n_face_orientations == n_lanes)
2724 const auto &cells = fe_eval.get_cell_ids();
2725 for (
unsigned int v = 0; v < n_lanes; ++v)
2728 if (sm_ptr ==
nullptr)
2733 .dof_indices_contiguous[dof_access_index]
2740 .dof_indices_contiguous_sm[dof_access_index]
2742 vector_ptrs[v] =
const_cast<Number2_ *
>(
2743 sm_ptr->operator[](temp.first).data() +
2744 temp.second + index_offset);
2753 else if (n_face_orientations == n_lanes)
2755 for (
unsigned int v = 0; v < n_lanes; ++v)
2756 reordered_indices[v] =
2757 dof_info.dof_indices_contiguous[dof_access_index]
2758 [fe_eval.get_cell_ids()[v]];
2759 dof_indices = reordered_indices.data();
2764 if (vectorization_possible)
2765 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2767 const unsigned int ind1 = index_array_hermite[0][2 * i];
2768 const unsigned int ind2 =
2769 index_array_hermite[0][2 * i + 1];
2770 const unsigned int i_ = reorientate(0, i);
2772 proc.hermite_grad_vectorized_indexed(
2774 temp1[i_ + dofs_per_face],
2781 else if (n_face_orientations == 1)
2782 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2784 const unsigned int ind1 = index_array_hermite[0][2 * i];
2785 const unsigned int ind2 =
2786 index_array_hermite[0][2 * i + 1];
2787 const unsigned int i_ = reorientate(0, i);
2789 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2790 proc.hermite_grad(temp1[i_][v],
2791 temp1[i_ + dofs_per_face][v],
2792 vector_ptrs[v][ind1],
2793 vector_ptrs[v][ind2],
2796 if (integrate ==
false)
2797 for (
unsigned int v = n_filled_lanes; v < n_lanes; ++v)
2800 temp1[i + dofs_per_face][v] = 0.0;
2805 if (integrate ==
false && n_filled_lanes < n_lanes)
2806 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2807 temp1[i] = temp1[i + dofs_per_face] = Number();
2809 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2810 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2812 temp1[reorientate(v, i)][v],
2813 temp1[reorientate(v, i) + dofs_per_face][v],
2814 vector_ptrs[v][index_array_hermite[v][2 * i]],
2815 vector_ptrs[v][index_array_hermite[v][2 * i + 1]],
2821 if (vectorization_possible)
2822 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2824 const unsigned int ind = index_array_nodal[0][i];
2825 const unsigned int i_ = reorientate(0, i);
2827 proc.value_vectorized_indexed(temp1[i_],
2831 else if (n_face_orientations == 1)
2832 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2834 const unsigned int ind = index_array_nodal[0][i];
2835 const unsigned int i_ = reorientate(0, i);
2837 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2838 proc.value(temp1[i_][v], vector_ptrs[v][ind]);
2840 if (integrate ==
false)
2841 for (
unsigned int v = n_filled_lanes; v < n_lanes; ++v)
2846 if (integrate ==
false && n_filled_lanes < n_lanes)
2847 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2848 temp1[i] = Number();
2850 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2851 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2852 proc.value(temp1[reorientate(v, i)][v],
2853 vector_ptrs[v][index_array_nodal[v][i]]);
2863 temp1 += 3 * dofs_per_face;