16#ifndef dealii__tensor_product_kernels_h
17#define dealii__tensor_product_kernels_h
23#include <deal.II/matrix_free/cuda_matrix_free.templates.h>
49#if KOKKOS_VERSION >= 40000
53 template <
int n_q_points_1d,
63 MemorySpace::Default::kokkos_space::execution_space>
::member_type
65 const Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
70 Number t[n_q_points_1d];
71 Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, n_q_points_1d),
79 for (
int k = 0;
k < n_q_points_1d; ++
k)
83 (
k +
q * n_q_points_1d);
90 team_member.team_barrier();
92 Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, n_q_points_1d),
107 template <
int n_q_points_1d,
117 MemorySpace::Default::kokkos_space::execution_space>
::member_type
119 const Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
124 using TeamType = Kokkos::TeamPolicy<
125 MemorySpace::Default::kokkos_space::execution_space>
::member_type;
126 constexpr unsigned int n_q_points =
Utilities::pow(n_q_points_1d, 2);
128 Number t[n_q_points];
130 Kokkos::TeamThreadMDRange<Kokkos::Rank<2>,
TeamType>(team_member,
133 Kokkos::parallel_for(
thread_policy, [&](
const int i,
const int j) {
134 int q_point = i +
j * n_q_points_1d;
141 const int base_in = (direction == 0) ? (n_q_points_1d * i) : i;
144 for (
int k = 1;
k < n_q_points_1d; ++
k)
153 team_member.team_barrier();
155 Kokkos::parallel_for(
thread_policy, [&](
const int i,
const int j) {
156 const int q_point = i +
j * n_q_points_1d;
158 (direction == 0) ? (
j + n_q_points_1d * i) : (i + n_q_points_1d *
j);
172 template <
int n_q_points_1d,
182 MemorySpace::Default::kokkos_space::execution_space>
::member_type
184 const Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
189 using TeamType = Kokkos::TeamPolicy<
190 MemorySpace::Default::kokkos_space::execution_space>
::member_type;
191 constexpr unsigned int n_q_points =
Utilities::pow(n_q_points_1d, 3);
193 Number t[n_q_points];
195 team_member, n_q_points_1d, n_q_points_1d, n_q_points_1d);
196 Kokkos::parallel_for(
199 i +
j * n_q_points_1d +
q * n_q_points_1d * n_q_points_1d;
208 (n_q_points_1d * (i + n_q_points_1d *
j)) :
209 (direction == 1 ? (i + n_q_points_1d * n_q_points_1d *
j) :
210 (i + n_q_points_1d *
j)));
213 for (
int k = 1;
k < n_q_points_1d; ++
k)
222 team_member.team_barrier();
224 Kokkos::parallel_for(
227 i +
j * n_q_points_1d +
q * n_q_points_1d * n_q_points_1d;
229 (direction == 0) ? (
q + n_q_points_1d * (i + n_q_points_1d *
j)) :
230 (direction == 1) ? (i + n_q_points_1d * (
q + n_q_points_1d *
j)) :
231 (i + n_q_points_1d * (
j + n_q_points_1d *
q));
257 MemorySpace::Default::kokkos_space::execution_space>
::member_type
259 const Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
264#if KOKKOS_VERSION >= 40000
265 if constexpr (dim == 1)
268 if constexpr (dim == 2)
271 if constexpr (dim == 3)
275 constexpr unsigned int n_q_points =
Utilities::pow(n_q_points_1d, dim);
277 Number t[n_q_points];
278 Kokkos::parallel_for(
279 Kokkos::TeamThreadRange(team_member, n_q_points),
281 const unsigned int i = (dim == 1) ? 0 :
q_point % n_q_points_1d;
282 const unsigned int j =
283 (dim == 3) ? (
q_point / n_q_points_1d) % n_q_points_1d : 0;
284 const unsigned int q =
286 (dim == 2) ? (
q_point / n_q_points_1d) % n_q_points_1d :
287 q_point / (n_q_points_1d * n_q_points_1d);
295 (direction == 0) ? (n_q_points_1d * (i + n_q_points_1d *
j)) :
296 (direction == 1) ? (i + n_q_points_1d * (n_q_points_1d *
j)) :
297 (i + n_q_points_1d *
j);
300 for (
int k = 1;
k < n_q_points_1d; ++
k)
304 (
in_place ? out(base +
k * stride) : in(base +
k * stride));
310 team_member.team_barrier();
312 Kokkos::parallel_for(
313 Kokkos::TeamThreadRange(team_member, n_q_points),
315 const unsigned int i = (dim == 1) ? 0 :
q_point % n_q_points_1d;
316 const unsigned int j =
317 (dim == 3) ? (
q_point / n_q_points_1d) % n_q_points_1d : 0;
318 const unsigned int q =
320 (dim == 2) ? (
q_point / n_q_points_1d) % n_q_points_1d :
321 q_point / (n_q_points_1d * n_q_points_1d);
324 (direction == 0) ? (
q + n_q_points_1d * (i + n_q_points_1d *
j)) :
325 (direction == 1) ? (i + n_q_points_1d * (
q + n_q_points_1d *
j)) :
326 (i + n_q_points_1d * (
j + n_q_points_1d *
q));
360 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
369 MemorySpace::Default::kokkos_space::execution_space>
::member_type;
374 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values,
375 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
377 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
383 template <
typename ViewType>
385 evaluate_values(ViewType
u);
391 template <
typename ViewTypeIn,
typename ViewTypeOut>
399 template <
typename ViewType1,
typename ViewType2>
406 template <
typename ViewType>
408 integrate_values(ViewType
u);
414 template <
bool add,
typename ViewType1,
typename ViewType2>
422 template <
typename ViewType1,
typename ViewType2>
430 template <
int direction,
443 template <
int direction,
457 template <
int direction,
474 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
shape_values;
479 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
485 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
491 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
500 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values,
501 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
503 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
505 : team_member(team_member)
506 , shape_values(shape_values)
507 , shape_gradients(shape_gradients)
508 , co_shape_gradients(co_shape_gradients)
513 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
514 template <
int direction,
529 team_member, shape_values, in, out);
534 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
535 template <
int direction,
550 team_member, shape_gradients, in, out);
555 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
556 template <
int direction,
571 team_member, co_shape_gradients, in, out);
576 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
577 template <
typename ViewType>
583 Number>::evaluate_values(ViewType
u)
585 if constexpr (dim == 1)
587 else if constexpr (dim == 2)
590 team_member.team_barrier();
593 else if constexpr (dim == 3)
596 team_member.team_barrier();
598 team_member.team_barrier();
602 Kokkos::abort(
"dim must not exceed 3!");
607 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
608 template <
typename ViewType>
614 Number>::integrate_values(ViewType
u)
616 if constexpr (dim == 1)
618 else if constexpr (dim == 2)
621 team_member.team_barrier();
624 else if constexpr (dim == 3)
627 team_member.team_barrier();
629 team_member.team_barrier();
633 Kokkos::abort(
"dim must not exceed 3!");
638 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
639 template <
typename ViewTypeIn,
typename ViewTypeOut>
648 if constexpr (dim == 1)
651 u, Kokkos::subview(
grad_u, Kokkos::ALL, 0));
653 else if constexpr (dim == 2)
656 u, Kokkos::subview(
grad_u, Kokkos::ALL, 0));
658 u, Kokkos::subview(
grad_u, Kokkos::ALL, 1));
660 team_member.team_barrier();
663 Kokkos::subview(
grad_u, Kokkos::ALL, 0));
665 Kokkos::subview(
grad_u, Kokkos::ALL, 1),
666 Kokkos::subview(
grad_u, Kokkos::ALL, 1));
668 else if constexpr (dim == 3)
671 u, Kokkos::subview(
grad_u, Kokkos::ALL, 0));
673 u, Kokkos::subview(
grad_u, Kokkos::ALL, 1));
675 u, Kokkos::subview(
grad_u, Kokkos::ALL, 2));
677 team_member.team_barrier();
680 Kokkos::subview(
grad_u, Kokkos::ALL, 0));
682 Kokkos::subview(
grad_u, Kokkos::ALL, 1),
683 Kokkos::subview(
grad_u, Kokkos::ALL, 1));
685 Kokkos::subview(
grad_u, Kokkos::ALL, 2));
687 team_member.team_barrier();
690 Kokkos::subview(
grad_u, Kokkos::ALL, 0));
692 Kokkos::subview(
grad_u, Kokkos::ALL, 1));
694 Kokkos::subview(
grad_u, Kokkos::ALL, 2),
695 Kokkos::subview(
grad_u, Kokkos::ALL, 2));
698 Kokkos::abort(
"dim must not exceed 3!");
703 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
704 template <
typename ViewType1,
typename ViewType2>
714 if constexpr (dim == 1)
717 team_member.team_barrier();
720 u, Kokkos::subview(
grad_u, Kokkos::ALL, 0));
722 else if constexpr (dim == 2)
725 team_member.team_barrier();
727 team_member.team_barrier();
730 u, Kokkos::subview(
grad_u, Kokkos::ALL, 0));
732 u, Kokkos::subview(
grad_u, Kokkos::ALL, 1));
734 else if constexpr (dim == 3)
737 team_member.team_barrier();
739 team_member.team_barrier();
741 team_member.team_barrier();
744 u, Kokkos::subview(
grad_u, Kokkos::ALL, 0));
746 u, Kokkos::subview(
grad_u, Kokkos::ALL, 1));
748 u, Kokkos::subview(
grad_u, Kokkos::ALL, 2));
751 Kokkos::abort(
"dim must not exceed 3!");
756 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
757 template <
bool add,
typename ViewType1,
typename ViewType2>
766 if constexpr (dim == 1)
769 Kokkos::subview(
grad_u, Kokkos::ALL, dim),
u);
771 else if constexpr (dim == 2)
774 Kokkos::subview(
grad_u, Kokkos::ALL, 0),
775 Kokkos::subview(
grad_u, Kokkos::ALL, 0));
781 team_member.team_barrier();
785 team_member.team_barrier();
787 Kokkos::subview(
grad_u, Kokkos::ALL, 1),
u);
789 else if constexpr (dim == 3)
792 Kokkos::subview(
grad_u, Kokkos::ALL, 0),
793 Kokkos::subview(
grad_u, Kokkos::ALL, 0));
803 team_member.team_barrier();
810 Kokkos::subview(
grad_u, Kokkos::ALL, 1),
811 Kokkos::subview(
grad_u, Kokkos::ALL, 1));
817 team_member.team_barrier();
821 team_member.team_barrier();
824 team_member.team_barrier();
826 Kokkos::subview(
grad_u, Kokkos::ALL, 2),
u);
829 Kokkos::abort(
"dim must not exceed 3!");
834 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
835 template <
typename ViewType1,
typename ViewType2>
845 if constexpr (dim == 1)
848 Kokkos::subview(
grad_u, Kokkos::ALL, 0),
u);
849 team_member.team_barrier();
853 else if constexpr (dim == 2)
856 Kokkos::subview(
grad_u, Kokkos::ALL, 1),
u);
857 team_member.team_barrier();
859 Kokkos::subview(
grad_u, Kokkos::ALL, 0),
u);
860 team_member.team_barrier();
863 team_member.team_barrier();
865 team_member.team_barrier();
867 else if constexpr (dim == 3)
870 Kokkos::subview(
grad_u, Kokkos::ALL, 2),
u);
871 team_member.team_barrier();
873 Kokkos::subview(
grad_u, Kokkos::ALL, 1),
u);
874 team_member.team_barrier();
876 Kokkos::subview(
grad_u, Kokkos::ALL, 0),
u);
877 team_member.team_barrier();
880 team_member.team_barrier();
882 team_member.team_barrier();
884 team_member.team_barrier();
887 Kokkos::abort(
"dim must not exceed 3!");
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
void apply(const Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type &team_member, const Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_data, const ViewTypeIn in, ViewTypeOut out)
T sum(const T &t, const MPI_Comm mpi_communicator)
constexpr T pow(const T base, const int iexp)
const InputIterator OutputIterator out
#define DEAL_II_HOST_DEVICE
const TeamHandle & team_member
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type TeamHandle