2490 const unsigned int n_components,
2492 typename Processor::Number2_ *global_vector_ptr,
2494 const EvaluationData &fe_eval,
2495 typename Processor::VectorizedArrayType_ *temp1)
2497 constexpr int dim = Processor::dim_;
2498 constexpr int fe_degree = Processor::fe_degree_;
2499 using VectorizedArrayType =
typename Processor::VectorizedArrayType_;
2500 constexpr int n_lanes = VectorizedArrayType::size();
2502 using Number =
typename Processor::Number_;
2503 using Number2_ =
typename Processor::Number2_;
2505 const auto &shape_data = fe_eval.get_shape_info().data.front();
2506 constexpr bool integrate = Processor::do_integrate;
2507 const unsigned int face_no = fe_eval.get_face_no();
2508 const auto &dof_info = fe_eval.get_dof_info();
2509 const unsigned int cell = fe_eval.get_cell_or_face_batch_id();
2511 fe_eval.get_dof_access_index();
2513 dof_info.index_storage_variants[dof_access_index].size());
2514 constexpr unsigned int dofs_per_face =
2516 const unsigned int subface_index = fe_eval.get_subface_index();
2518 const unsigned int n_filled_lanes =
2519 dof_info.n_vectorization_lanes_filled[dof_access_index][cell];
2521 bool all_faces_are_same = n_filled_lanes == n_lanes;
2522 if (n_face_orientations == n_lanes)
2523 for (
unsigned int v = 1; v < n_lanes; ++v)
2524 if (fe_eval.get_face_no(v) != fe_eval.get_face_no(0) ||
2525 fe_eval.get_face_orientation(v) != fe_eval.get_face_orientation(0))
2527 all_faces_are_same =
false;
2532 std::array<const unsigned int *, n_face_orientations> orientation = {};
2534 if (dim == 3 && n_face_orientations == n_lanes && !all_faces_are_same &&
2535 fe_eval.is_interior_face() == 0)
2536 for (
unsigned int v = 0; v < n_lanes; ++v)
2544 if (shape_data.nodal_at_cell_boundaries &&
2545 fe_eval.get_face_orientation(v) != 0)
2550 check_face_orientations ==
false)
2564 orientation[v] = &fe_eval.get_shape_info().face_orientations_dofs(
2565 fe_eval.get_face_orientation(v), 0);
2568 else if (dim == 3 && fe_eval.get_face_orientation() != 0)
2572 check_face_orientations ==
false)
2586 for (
unsigned int v = 0; v < n_face_orientations; ++v)
2587 orientation[v] = &fe_eval.get_shape_info().face_orientations_dofs(
2588 fe_eval.get_face_orientation(), 0);
2594 VectorizedArrayType grad_weight =
2596 .shape_data_on_face[0][fe_degree + (integrate ? (2 - face_no % 2) :
2597 (1 + face_no % 2))];
2600 std::array<const unsigned int *, n_face_orientations> index_array_hermite =
2604 if (n_face_orientations == 1)
2605 index_array_hermite[0] =
2606 &fe_eval.get_shape_info().face_to_cell_index_hermite(face_no, 0);
2609 for (
unsigned int v = 0; v < n_lanes; ++v)
2614 const auto face_no = fe_eval.get_face_no(v);
2618 .shape_data_on_face[0][fe_degree + (integrate ?
2619 (2 - (face_no % 2)) :
2620 (1 + (face_no % 2)))];
2622 index_array_hermite[v] =
2623 &fe_eval.get_shape_info().face_to_cell_index_hermite(face_no,
2630 std::array<const unsigned int *, n_face_orientations> index_array_nodal =
2632 if (shape_data.nodal_at_cell_boundaries ==
true)
2634 if (n_face_orientations == 1)
2635 index_array_nodal[0] =
2636 &fe_eval.get_shape_info().face_to_cell_index_nodal(face_no, 0);
2639 for (
unsigned int v = 0; v < n_lanes; ++v)
2644 const auto face_no = fe_eval.get_face_no(v);
2646 index_array_nodal[v] =
2647 &fe_eval.get_shape_info().face_to_cell_index_nodal(face_no,
2654 const auto reorientate = [&](
const unsigned int v,
const unsigned int i) {
2655 return (!check_face_orientations || orientation[v] ==
nullptr) ?
2662 fe_eval.get_cell_ids()[0] :
2664 const unsigned int *dof_indices =
2665 &dof_info.dof_indices_contiguous[dof_access_index][
cell_index];
2667 for (
unsigned int comp = 0; comp < n_components; ++comp)
2669 const std::size_t index_offset =
2670 dof_info.component_dof_indices_offset
2671 [fe_eval.get_active_fe_index()]
2672 [fe_eval.get_first_selected_component()] +
2676 if (n_face_orientations == 1 &&
2677 dof_info.index_storage_variants[dof_access_index][cell] ==
2679 interleaved_contiguous)
2682 dof_info.n_vectorization_lanes_filled[dof_access_index][cell],
2684 Number2_ *vector_ptr =
2685 global_vector_ptr + dof_indices[0] + index_offset * n_lanes;
2689 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2693 const unsigned int ind1 = index_array_hermite[0][2 * i];
2694 const unsigned int ind2 = index_array_hermite[0][2 * i + 1];
2695 const unsigned int i_ = reorientate(0, i);
2696 proc.hermite_grad_vectorized(temp1[i_],
2697 temp1[i_ + dofs_per_face],
2698 vector_ptr + ind1 * n_lanes,
2699 vector_ptr + ind2 * n_lanes,
2705 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2709 const unsigned int i_ = reorientate(0, i);
2710 const unsigned int ind = index_array_nodal[0][i];
2711 proc.value_vectorized(temp1[i_],
2712 vector_ptr + ind * n_lanes);
2718 else if (n_face_orientations == 1 &&
2719 dof_info.index_storage_variants[dof_access_index][cell] ==
2721 interleaved_contiguous_strided)
2724 dof_info.n_vectorization_lanes_filled[dof_access_index][cell],
2726 Number2_ *vector_ptr = global_vector_ptr + index_offset * n_lanes;
2729 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2733 const unsigned int i_ = reorientate(0, i);
2734 const unsigned int ind1 =
2735 index_array_hermite[0][2 * i] * n_lanes;
2736 const unsigned int ind2 =
2737 index_array_hermite[0][2 * i + 1] * n_lanes;
2738 proc.hermite_grad_vectorized_indexed(
2740 temp1[i_ + dofs_per_face],
2750 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2754 const unsigned int i_ = reorientate(0, i);
2755 const unsigned int ind = index_array_nodal[0][i] * n_lanes;
2756 proc.value_vectorized_indexed(temp1[i_],
2764 else if (n_face_orientations == 1 &&
2765 dof_info.index_storage_variants[dof_access_index][cell] ==
2767 interleaved_contiguous_mixed_strides)
2769 const unsigned int *strides =
2770 &dof_info.dof_indices_interleave_strides[dof_access_index]
2772 unsigned int indices[n_lanes];
2773 for (
unsigned int v = 0; v < n_lanes; ++v)
2774 indices[v] = dof_indices[v] + index_offset * strides[v];
2775 const unsigned int n_filled_lanes =
2776 dof_info.n_vectorization_lanes_filled[dof_access_index][cell];
2780 if (n_filled_lanes == n_lanes)
2781 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2785 const unsigned int i_ = reorientate(0, i);
2786 unsigned int ind1[n_lanes];
2788 for (
unsigned int v = 0; v < n_lanes; ++v)
2789 ind1[v] = indices[v] +
2790 index_array_hermite[0][2 * i] * strides[v];
2791 unsigned int ind2[n_lanes];
2793 for (
unsigned int v = 0; v < n_lanes; ++v)
2797 index_array_hermite[0][2 * i + 1] * strides[v];
2798 proc.hermite_grad_vectorized_indexed(
2800 temp1[i_ + dofs_per_face],
2809 if (integrate ==
false)
2810 for (
unsigned int i = 0; i < 2 * dofs_per_face; ++i)
2811 temp1[i] = VectorizedArrayType();
2813 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2814 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2816 const unsigned int i_ =
2817 reorientate(n_face_orientations == 1 ? 0 : v, i);
2820 temp1[i_ + dofs_per_face][v],
2824 [n_face_orientations == 1 ? 0 : v][2 * i] *
2828 index_array_hermite[n_face_orientations == 1 ?
2832 grad_weight[n_face_orientations == 1 ? 0 : v]);
2838 if (n_filled_lanes == n_lanes)
2839 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2842 unsigned int ind[n_lanes];
2844 for (
unsigned int v = 0; v < n_lanes; ++v)
2846 indices[v] + index_array_nodal[0][i] * strides[v];
2847 const unsigned int i_ = reorientate(0, i);
2848 proc.value_vectorized_indexed(temp1[i_],
2854 if (integrate ==
false)
2855 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2856 temp1[i] = VectorizedArrayType();
2858 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2859 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2861 temp1[reorientate(n_face_orientations == 1 ? 0 : v,
2865 index_array_nodal[n_face_orientations == 1 ? 0 : v]
2873 else if (n_face_orientations > 1 ||
2874 dof_info.index_storage_variants[dof_access_index][cell] ==
2878 Number2_ *vector_ptr = global_vector_ptr + index_offset;
2880 const bool vectorization_possible =
2881 all_faces_are_same && (sm_ptr ==
nullptr);
2883 std::array<Number2_ *, n_lanes> vector_ptrs{{
nullptr}};
2884 std::array<unsigned int, n_lanes> reordered_indices{
2887 if (vectorization_possible ==
false)
2889 if (n_face_orientations == 1)
2891 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2892 if (sm_ptr ==
nullptr)
2894 vector_ptrs[v] = vector_ptr + dof_indices[v];
2900 .dof_indices_contiguous_sm[dof_access_index]
2901 [cell * n_lanes + v];
2902 vector_ptrs[v] =
const_cast<Number2_ *
>(
2903 sm_ptr->operator[](temp.first).
data() +
2904 temp.second + index_offset);
2907 else if (n_face_orientations == n_lanes)
2909 const auto &cells = fe_eval.get_cell_ids();
2910 for (
unsigned int v = 0; v < n_lanes; ++v)
2913 if (sm_ptr ==
nullptr)
2918 .dof_indices_contiguous[dof_access_index]
2925 .dof_indices_contiguous_sm[dof_access_index]
2927 vector_ptrs[v] =
const_cast<Number2_ *
>(
2928 sm_ptr->operator[](temp.first).
data() +
2929 temp.second + index_offset);
2938 else if (n_face_orientations == n_lanes)
2940 for (
unsigned int v = 0; v < n_lanes; ++v)
2941 reordered_indices[v] =
2942 dof_info.dof_indices_contiguous[dof_access_index]
2943 [fe_eval.get_cell_ids()[v]];
2944 dof_indices = reordered_indices.data();
2949 if (vectorization_possible)
2950 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2952 const unsigned int ind1 = index_array_hermite[0][2 * i];
2953 const unsigned int ind2 =
2954 index_array_hermite[0][2 * i + 1];
2955 const unsigned int i_ = reorientate(0, i);
2957 proc.hermite_grad_vectorized_indexed(
2959 temp1[i_ + dofs_per_face],
2966 else if (n_face_orientations == 1)
2967 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2969 const unsigned int ind1 = index_array_hermite[0][2 * i];
2970 const unsigned int ind2 =
2971 index_array_hermite[0][2 * i + 1];
2972 const unsigned int i_ = reorientate(0, i);
2974 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2975 proc.hermite_grad(temp1[i_][v],
2976 temp1[i_ + dofs_per_face][v],
2977 vector_ptrs[v][ind1],
2978 vector_ptrs[v][ind2],
2981 if (integrate ==
false)
2982 for (
unsigned int v = n_filled_lanes; v < n_lanes; ++v)
2985 temp1[i + dofs_per_face][v] = 0.0;
2990 if (integrate ==
false && n_filled_lanes < n_lanes)
2991 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2992 temp1[i] = temp1[i + dofs_per_face] = Number();
2994 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
2995 for (
unsigned int i = 0; i < dofs_per_face; ++i)
2997 temp1[reorientate(v, i)][v],
2998 temp1[reorientate(v, i) + dofs_per_face][v],
2999 vector_ptrs[v][index_array_hermite[v][2 * i]],
3000 vector_ptrs[v][index_array_hermite[v][2 * i + 1]],
3006 if (vectorization_possible)
3007 for (
unsigned int i = 0; i < dofs_per_face; ++i)
3009 const unsigned int ind = index_array_nodal[0][i];
3010 const unsigned int i_ = reorientate(0, i);
3012 proc.value_vectorized_indexed(temp1[i_],
3018 if constexpr (n_face_orientations == 1)
3019 for (
unsigned int i = 0; i < dofs_per_face; ++i)
3021 const unsigned int ind = index_array_nodal[0][i];
3022 const unsigned int i_ = reorientate(0, i);
3024 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
3025 proc.value(temp1[i_][v], vector_ptrs[v][ind]);
3027 if constexpr (integrate ==
false)
3028 for (
unsigned int v = n_filled_lanes; v < n_lanes;
3034 if (integrate ==
false && n_filled_lanes < n_lanes)
3035 for (
unsigned int i = 0; i < dofs_per_face; ++i)
3036 temp1[i] = Number();
3038 for (
unsigned int v = 0; v < n_filled_lanes; ++v)
3039 for (
unsigned int i = 0; i < dofs_per_face; ++i)
3040 proc.value(temp1[reorientate(v, i)][v],
3041 vector_ptrs[v][index_array_nodal[v][i]]);
3052 temp1 += 3 * dofs_per_face;