17#ifndef dealii_cuda_tensor_product_kernels_h
18#define dealii_cuda_tensor_product_kernels_h
24#include <deal.II/matrix_free/cuda_matrix_free.templates.h>
61 MemorySpace::Default::kokkos_space::execution_space>::member_type
63 const Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
68 constexpr unsigned int n_q_points =
Utilities::pow(n_q_points_1d, dim);
72 Kokkos::TeamThreadRange(team_member, n_q_points),
73 [&](
const int &q_point) {
74 const unsigned int i = (dim == 1) ? 0 : q_point % n_q_points_1d;
75 const unsigned int j =
76 (dim == 3) ? (q_point / n_q_points_1d) % n_q_points_1d : 0;
77 const unsigned int q =
78 (dim == 1) ? q_point :
79 (dim == 2) ? (q_point / n_q_points_1d) % n_q_points_1d :
80 q_point / (n_q_points_1d * n_q_points_1d);
85 for (
int k = 0; k < n_q_points_1d; ++k)
87 const unsigned int shape_idx =
88 dof_to_quad ? (q + k * n_q_points_1d) : (k + q * n_q_points_1d);
89 const unsigned int source_idx =
91 (k + n_q_points_1d * (i + n_q_points_1d * j)) :
93 (i + n_q_points_1d * (k + n_q_points_1d * j)) :
94 (i + n_q_points_1d * (j + n_q_points_1d * k));
95 t[q_point] += shape_data[shape_idx] *
96 (in_place ? out(source_idx) : in(source_idx));
101 team_member.team_barrier();
103 Kokkos::parallel_for(
104 Kokkos::TeamThreadRange(team_member, n_q_points),
105 [&](
const int &q_point) {
106 const unsigned int i = (dim == 1) ? 0 : q_point % n_q_points_1d;
107 const unsigned int j =
108 (dim == 3) ? (q_point / n_q_points_1d) % n_q_points_1d : 0;
109 const unsigned int q =
110 (dim == 1) ? q_point :
111 (dim == 2) ? (q_point / n_q_points_1d) % n_q_points_1d :
112 q_point / (n_q_points_1d * n_q_points_1d);
114 const unsigned int destination_idx =
115 (direction == 0) ? (q + n_q_points_1d * (i + n_q_points_1d * j)) :
116 (direction == 1) ? (i + n_q_points_1d * (q + n_q_points_1d * j)) :
117 (i + n_q_points_1d * (j + n_q_points_1d * q));
120 Kokkos::atomic_add(&out(destination_idx), t[q_point]);
122 out(destination_idx) = t[q_point];
148 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
156 MemorySpace::Default::kokkos_space::execution_space>::member_type;
161 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values,
162 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
164 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
171 template <
int direction,
176 typename ViewTypeOut>
178 values(
const ViewTypeIn in, ViewTypeOut out)
const;
184 template <
int direction,
189 typename ViewTypeOut>
191 gradients(
const ViewTypeIn in, ViewTypeOut out)
const;
197 template <
int direction,
202 typename ViewTypeOut>
204 co_gradients(
const ViewTypeIn in, ViewTypeOut out)
const;
209 template <
typename ViewType>
211 value_at_quad_pts(ViewType u);
216 template <
typename ViewType>
218 integrate_value(ViewType u);
224 template <
typename ViewTypeIn,
typename ViewTypeOut>
226 gradient_at_quad_pts(
const ViewTypeIn u, ViewTypeOut grad_u);
232 template <
typename ViewType1,
typename ViewType2>
234 value_and_gradient_at_quad_pts(ViewType1 u, ViewType2 grad_u);
240 template <
bool add,
typename ViewType1,
typename ViewType2>
242 integrate_gradient(ViewType1 u, ViewType2 grad_u);
248 template <
typename ViewType1,
typename ViewType2>
250 integrate_value_and_gradient(ViewType1 u, ViewType2 grad_u);
260 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
shape_values;
265 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
271 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
277 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
284 EvaluatorTensorProduct(
286 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values,
287 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
289 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
291 : team_member(team_member)
292 , shape_values(shape_values)
293 , shape_gradients(shape_gradients)
294 , co_shape_gradients(co_shape_gradients)
299 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
300 template <
int direction,
305 typename ViewTypeOut>
311 Number>::values(
const ViewTypeIn in,
312 ViewTypeOut out)
const
314 apply<dim, n_q_points_1d, Number, direction, dof_to_quad, add, in_place>(
315 team_member, shape_values, in, out);
320 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
321 template <
int direction,
326 typename ViewTypeOut>
332 Number>::gradients(
const ViewTypeIn in,
333 ViewTypeOut out)
const
335 apply<dim, n_q_points_1d, Number, direction, dof_to_quad, add, in_place>(
336 team_member, shape_gradients, in, out);
341 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
342 template <
int direction,
347 typename ViewTypeOut>
353 Number>::co_gradients(
const ViewTypeIn in,
354 ViewTypeOut out)
const
356 apply<dim, n_q_points_1d, Number, direction, dof_to_quad, add, in_place>(
357 team_member, co_shape_gradients, in, out);
362 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
363 template <
typename ViewType>
369 Number>::value_at_quad_pts(ViewType u)
375 values<0, true, false, true>(u, u);
381 values<0, true, false, true>(u, u);
382 team_member.team_barrier();
383 values<1, true, false, true>(u, u);
389 values<0, true, false, true>(u, u);
390 team_member.team_barrier();
391 values<1, true, false, true>(u, u);
392 team_member.team_barrier();
393 values<2, true, false, true>(u, u);
407 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
408 template <
typename ViewType>
414 Number>::integrate_value(ViewType u)
420 values<0, false, false, true>(u, u);
426 values<0, false, false, true>(u, u);
427 team_member.team_barrier();
428 values<1, false, false, true>(u, u);
434 values<0, false, false, true>(u, u);
435 team_member.team_barrier();
436 values<1, false, false, true>(u, u);
437 team_member.team_barrier();
438 values<2, false, false, true>(u, u);
452 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
453 template <
typename ViewTypeIn,
typename ViewTypeOut>
459 Number>::gradient_at_quad_pts(
const ViewTypeIn u,
466 gradients<0, true, false, false>(
467 u, Kokkos::subview(grad_u, Kokkos::ALL, 0));
473 gradients<0, true, false, false>(
474 u, Kokkos::subview(grad_u, Kokkos::ALL, 0));
475 values<0, true, false, false>(
476 u, Kokkos::subview(grad_u, Kokkos::ALL, 1));
478 team_member.team_barrier();
480 values<1, true, false, true>(
481 Kokkos::subview(grad_u, Kokkos::ALL, 0),
482 Kokkos::subview(grad_u, Kokkos::ALL, 0));
483 gradients<1, true, false, true>(
484 Kokkos::subview(grad_u, Kokkos::ALL, 1),
485 Kokkos::subview(grad_u, Kokkos::ALL, 1));
491 gradients<0, true, false, false>(
492 u, Kokkos::subview(grad_u, Kokkos::ALL, 0));
493 values<0, true, false, false>(
494 u, Kokkos::subview(grad_u, Kokkos::ALL, 1));
495 values<0, true, false, false>(
496 u, Kokkos::subview(grad_u, Kokkos::ALL, 2));
498 team_member.team_barrier();
500 values<1, true, false, true>(
501 Kokkos::subview(grad_u, Kokkos::ALL, 0),
502 Kokkos::subview(grad_u, Kokkos::ALL, 0));
503 gradients<1, true, false, true>(
504 Kokkos::subview(grad_u, Kokkos::ALL, 1),
505 Kokkos::subview(grad_u, Kokkos::ALL, 1));
506 values<1, true, false, true>(
507 Kokkos::subview(grad_u, Kokkos::ALL, 2),
508 Kokkos::subview(grad_u, Kokkos::ALL, 2));
510 team_member.team_barrier();
512 values<2, true, false, true>(
513 Kokkos::subview(grad_u, Kokkos::ALL, 0),
514 Kokkos::subview(grad_u, Kokkos::ALL, 0));
515 values<2, true, false, true>(
516 Kokkos::subview(grad_u, Kokkos::ALL, 1),
517 Kokkos::subview(grad_u, Kokkos::ALL, 1));
518 gradients<2, true, false, true>(
519 Kokkos::subview(grad_u, Kokkos::ALL, 2),
520 Kokkos::subview(grad_u, Kokkos::ALL, 2));
534 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
535 template <
typename ViewType1,
typename ViewType2>
541 Number>::value_and_gradient_at_quad_pts(ViewType1 u,
549 values<0, true, false, true>(u, u);
550 team_member.team_barrier();
552 co_gradients<0, true, false, false>(
553 u, Kokkos::subview(grad_u, Kokkos::ALL, 0));
559 values<0, true, false, true>(u, u);
560 team_member.team_barrier();
561 values<1, true, false, true>(u, u);
562 team_member.team_barrier();
564 co_gradients<0, true, false, false>(
565 u, Kokkos::subview(grad_u, Kokkos::ALL, 0));
566 co_gradients<1, true, false, false>(
567 u, Kokkos::subview(grad_u, Kokkos::ALL, 1));
573 values<0, true, false, true>(u, u);
574 team_member.team_barrier();
575 values<1, true, false, true>(u, u);
576 team_member.team_barrier();
577 values<2, true, false, true>(u, u);
578 team_member.team_barrier();
580 co_gradients<0, true, false, false>(
581 u, Kokkos::subview(grad_u, Kokkos::ALL, 0));
582 co_gradients<1, true, false, false>(
583 u, Kokkos::subview(grad_u, Kokkos::ALL, 1));
584 co_gradients<2, true, false, false>(
585 u, Kokkos::subview(grad_u, Kokkos::ALL, 2));
599 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
600 template <
bool add,
typename ViewType1,
typename ViewType2>
606 Number>::integrate_gradient(ViewType1 u,
613 gradients<0, false, add, false>(
614 Kokkos::subview(grad_u, Kokkos::ALL, dim), u);
620 gradients<0, false, false, true>(
621 Kokkos::subview(grad_u, Kokkos::ALL, 0),
622 Kokkos::subview(grad_u, Kokkos::ALL, 0));
623 values<0, false, false, true>(
624 Kokkos::subview(grad_u, Kokkos::ALL, 1),
625 Kokkos::subview(grad_u, Kokkos::ALL, 1));
627 team_member.team_barrier();
629 values<1, false, add, false>(
630 Kokkos::subview(grad_u, Kokkos::ALL, 0), u);
631 team_member.team_barrier();
632 gradients<1, false, true, false>(
633 Kokkos::subview(grad_u, Kokkos::ALL, 1), u);
639 gradients<0, false, false, true>(
640 Kokkos::subview(grad_u, Kokkos::ALL, 0),
641 Kokkos::subview(grad_u, Kokkos::ALL, 0));
642 values<0, false, false, true>(
643 Kokkos::subview(grad_u, Kokkos::ALL, 1),
644 Kokkos::subview(grad_u, Kokkos::ALL, 1));
645 values<0, false, false, true>(
646 Kokkos::subview(grad_u, Kokkos::ALL, 2),
647 Kokkos::subview(grad_u, Kokkos::ALL, 2));
649 team_member.team_barrier();
651 values<1, false, false, true>(
652 Kokkos::subview(grad_u, Kokkos::ALL, 0),
653 Kokkos::subview(grad_u, Kokkos::ALL, 0));
654 gradients<1, false, false, true>(
655 Kokkos::subview(grad_u, Kokkos::ALL, 1),
656 Kokkos::subview(grad_u, Kokkos::ALL, 1));
657 values<1, false, false, true>(
658 Kokkos::subview(grad_u, Kokkos::ALL, 2),
659 Kokkos::subview(grad_u, Kokkos::ALL, 2));
661 team_member.team_barrier();
663 values<2, false, add, false>(
664 Kokkos::subview(grad_u, Kokkos::ALL, 0), u);
665 team_member.team_barrier();
666 values<2, false, true, false>(
667 Kokkos::subview(grad_u, Kokkos::ALL, 1), u);
668 team_member.team_barrier();
669 gradients<2, false, true, false>(
670 Kokkos::subview(grad_u, Kokkos::ALL, 2), u);
684 template <
int dim,
int fe_degree,
int n_q_po
ints_1d,
typename Number>
685 template <
typename ViewType1,
typename ViewType2>
691 Number>::integrate_value_and_gradient(ViewType1 u,
699 co_gradients<0, false, true, false>(
700 Kokkos::subview(grad_u, Kokkos::ALL, 0), u);
701 team_member.team_barrier();
703 values<0, false, false, true>(u, u);
709 co_gradients<1, false, true, false>(
710 Kokkos::subview(grad_u, Kokkos::ALL, 1), u);
711 team_member.team_barrier();
712 co_gradients<0, false, true, false>(
713 Kokkos::subview(grad_u, Kokkos::ALL, 0), u);
714 team_member.team_barrier();
716 values<1, false, false, true>(u, u);
717 team_member.team_barrier();
718 values<0, false, false, true>(u, u);
719 team_member.team_barrier();
725 co_gradients<2, false, true, false>(
726 Kokkos::subview(grad_u, Kokkos::ALL, 2), u);
727 team_member.team_barrier();
728 co_gradients<1, false, true, false>(
729 Kokkos::subview(grad_u, Kokkos::ALL, 1), u);
730 team_member.team_barrier();
731 co_gradients<0, false, true, false>(
732 Kokkos::subview(grad_u, Kokkos::ALL, 0), u);
733 team_member.team_barrier();
735 values<2, false, false, true>(u, u);
736 team_member.team_barrier();
737 values<1, false, false, true>(u, u);
738 team_member.team_barrier();
739 values<0, false, false, true>(u, u);
740 team_member.team_barrier();
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
void apply(const Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type &team_member, const Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_data, const ViewTypeIn in, ViewTypeOut out)
constexpr T pow(const T base, const int iexp)
#define DEAL_II_HOST_DEVICE
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type TeamHandle
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
const TeamHandle & team_member
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients