Reference documentation for deal.II version 9.6.0
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
portable_matrix_free.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2017 - 2024 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15
16#ifndef dealii_portable_matrix_free_h
17#define dealii_portable_matrix_free_h
18
19#include <deal.II/base/config.h>
20
26#include <deal.II/base/tensor.h>
28
30
32#include <deal.II/fe/mapping.h>
33
35
39
40#include <Kokkos_Core.hpp>
41
42
43
45
46// Forward declaration
47namespace internal
48{
49 namespace MatrixFreeFunctions
50 {
51 enum class ConstraintKinds : std::uint16_t;
52 }
53} // namespace internal
54
55namespace Portable
56{
57 // forward declaration
58#ifndef DOXYGEN
59 namespace internal
60 {
61 template <int dim, typename Number>
62 class ReinitHelper;
63 }
64#endif
65
92 template <int dim, typename Number = double>
93 class MatrixFree : public Subscriptor
94 {
95 public:
98 using CellFilter =
100
105 {
112 const bool use_coloring = false,
113 const bool overlap_communication_computation = false)
117 {
118#ifndef DEAL_II_MPI_WITH_DEVICE_SUPPORT
122 "Overlapping communication and computation requires CUDA-aware MPI."));
123#endif
128 "Overlapping communication and coloring are incompatible options. Only one of them can be enabled."));
129 }
140
147
153 };
154
159 struct Data
160 {
164 Kokkos::View<point_type **, MemorySpace::Default::kokkos_space> q_points;
165
170 Kokkos::View<types::global_dof_index **,
173
177 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>
179
183 Kokkos::View<Number **, MemorySpace::Default::kokkos_space> JxW;
184
191
195 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
196
200 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
202
206 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
208
212 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
214
218 unsigned int n_cells;
219
223 unsigned int padding_length;
224
228 unsigned int row_start;
229
235
240 DEAL_II_HOST_DEVICE unsigned int
241 local_q_point_id(const unsigned int cell,
242 const unsigned int n_q_points,
243 const unsigned int q_point) const
244 {
245 return (row_start / padding_length + cell) * n_q_points + q_point;
246 }
247
248
254 get_quadrature_point(const unsigned int cell,
255 const unsigned int q_point) const
256 {
257 return q_points(cell, q_point);
258 }
259 };
260
265
269 unsigned int
271
282 template <typename IteratorFiltersType>
283 void
284 reinit(const Mapping<dim> &mapping,
286 const AffineConstraints<Number> &constraints,
287 const Quadrature<1> &quad,
288 const IteratorFiltersType &iterator_filter,
289 const AdditionalData &additional_data = AdditionalData());
290
294 void
295 reinit(const Mapping<dim> &mapping,
297 const AffineConstraints<Number> &constraints,
298 const Quadrature<1> &quad,
299 const AdditionalData &additional_data = AdditionalData());
300
304 void
306 const AffineConstraints<Number> &constraints,
307 const Quadrature<1> &quad,
308 const AdditionalData &additional_data = AdditionalData());
309
313 Data
314 get_data(unsigned int color) const;
315
316 // clang-format off
334 // clang-format on
335 template <typename Functor, typename VectorType>
336 void
337 cell_loop(const Functor &func,
338 const VectorType &src,
339 VectorType &dst) const;
340
356 template <typename Functor>
357 void
358 evaluate_coefficients(Functor func) const;
359
364 template <typename VectorType>
365 void
366 copy_constrained_values(const VectorType &src, VectorType &dst) const;
367
373 template <typename VectorType>
374 void
375 set_constrained_values(const Number val, VectorType &dst) const;
376
377#ifdef DEAL_II_WITH_CUDA
382 void
385#endif
386
392 void
395 const;
396
400 const std::vector<std::vector<CellFilter>> &
402
413 const std::shared_ptr<const Utilities::MPI::Partitioner> &
415
419 const DoFHandler<dim> &
421
425 std::size_t
427
428 private:
432 template <typename IteratorFiltersType>
433 void
436 const AffineConstraints<Number> &constraints,
437 const Quadrature<1> &quad,
438 const IteratorFiltersType &iterator_filter,
439 const std::shared_ptr<const MPI_Comm> &comm,
440 const AdditionalData additional_data);
441
446 template <typename Functor, typename VectorType>
447 void
448 serial_cell_loop(const Functor &func,
449 const VectorType &src,
450 VectorType &dst) const;
451
456 template <typename Functor>
457 void
459 const Functor &func,
461 &src,
463 const;
464
465#ifdef DEAL_II_WITH_CUDA
471 template <typename Functor>
472 void
474 const Functor &func,
477#endif
478
482 int my_id;
483
490
496
501
505 unsigned int fe_degree;
506
510 unsigned int dofs_per_cell;
511
515 unsigned int n_constrained_dofs;
516
520 unsigned int q_points_per_cell;
521
525 unsigned int n_colors;
526
530 std::vector<unsigned int> n_cells;
531
536 std::vector<Kokkos::View<point_type **, MemorySpace::Default::kokkos_space>>
538
543 std::vector<Kokkos::View<types::global_dof_index **,
546
551 std::vector<
552 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>>
554
559 std::vector<Kokkos::View<Number **, MemorySpace::Default::kokkos_space>>
561
565 Kokkos::View<types::global_dof_index *, MemorySpace::Default::kokkos_space>
567
571 std::vector<
575
579 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
580
584 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_gradients;
585
589 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
591
595 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
597
602 std::shared_ptr<const Utilities::MPI::Partitioner> partitioner;
603
604
609 unsigned int padding_length;
610
614 std::vector<unsigned int> row_start;
615
620
624 std::vector<std::vector<CellFilter>> graph;
625
626 friend class internal::ReinitHelper<dim, Number>;
627 };
628
629
630
631 template <int dim, typename Number>
633 {
634 using TeamHandle = Kokkos::TeamPolicy<
635 MemorySpace::Default::kokkos_space::execution_space>::member_type;
636
637 using SharedView1D = Kokkos::View<
638 Number *,
639 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
640 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
641 using SharedView2D = Kokkos::View<
642 Number *[dim],
643 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
644 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
645
648 const SharedView1D &values,
649 const SharedView2D &gradients)
651 , values(values)
653 {}
654
659
664
669 };
670
671
672
677 template <int dim, typename Number>
678 struct DataHost
679 {
683 typename Kokkos::View<Point<dim, Number> **,
686
691 typename Kokkos::View<types::global_dof_index **,
694
698 typename Kokkos::View<Number **[dim][dim],
701
705 typename Kokkos::View<Number **,
707
711 unsigned int n_cells;
712
716 unsigned int padding_length;
717
721 unsigned int row_start;
722
726 typename Kokkos::View<
729
735
736
737
741 unsigned int
742 local_q_point_id(const unsigned int cell,
743 const unsigned int n_q_points,
744 const unsigned int q_point) const
745 {
746 return (row_start / padding_length + cell) * n_q_points + q_point;
747 }
748
749
750
755 get_quadrature_point(const unsigned int cell,
756 const unsigned int q_point) const
757 {
758 return q_points(cell, q_point);
759 }
760 };
761
762
763
770 template <int dim, typename Number>
771 DataHost<dim, Number>
773 const typename ::Portable::MatrixFree<dim, Number>::Data &data,
774 const UpdateFlags &update_flags)
775 {
776 DataHost<dim, Number> data_host;
777
778 data_host.n_cells = data.n_cells;
779 data_host.padding_length = data.padding_length;
780 data_host.row_start = data.row_start;
781 data_host.use_coloring = data.use_coloring;
782
783 if (update_flags & update_quadrature_points)
784 {
785 data_host.q_points = Kokkos::create_mirror(data.q_points);
786 Kokkos::deep_copy(data_host.q_points, data.q_points);
787 }
788
789 data_host.local_to_global = Kokkos::create_mirror(data.local_to_global);
790 Kokkos::deep_copy(data_host.local_to_global, data.local_to_global);
791
792 if (update_flags & update_gradients)
793 {
794 data_host.inv_jacobian = Kokkos::create_mirror(data.inv_jacobian);
795 Kokkos::deep_copy(data_host.inv_jacobian, data.inv_jacobian);
796 }
797
798 if (update_flags & update_JxW_values)
799 {
800 data_host.JxW = Kokkos::create_mirror(data.JxW);
801 Kokkos::deep_copy(data_host.JxW, data.JxW);
802 }
803
804 data_host.constraint_mask = Kokkos::create_mirror(data.constraint_mask);
805 Kokkos::deep_copy(data_host.constraint_mask, data.constraint_mask);
806
807 return data_host;
808 }
809
810
811 /*----------------------- Inline functions ---------------------------------*/
812
813#ifndef DOXYGEN
814
815 template <int dim, typename Number>
816 inline const std::vector<std::vector<
819 {
820 return graph;
821 }
822
823
824
825 template <int dim, typename Number>
826 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
828 {
829 return partitioner;
830 }
831
832
833
834 template <int dim, typename Number>
835 inline const DoFHandler<dim> &
837 {
838 Assert(dof_handler != nullptr, ExcNotInitialized());
839
840 return *dof_handler;
841 }
842
843#endif
844
845} // namespace Portable
846
848
849#endif
Abstract base class for mapping classes.
Definition mapping.h:318
const std::vector< std::vector< CellFilter > > & get_colored_graph() const
void initialize_dof_vector(LinearAlgebra::CUDAWrappers::Vector< Number > &vec) const
void distributed_cell_loop(const Functor &func, const LinearAlgebra::CUDAWrappers::Vector< Number > &src, LinearAlgebra::CUDAWrappers::Vector< Number > &dst) const
std::vector< std::vector< CellFilter > > graph
void reinit(const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const AdditionalData &additional_data=AdditionalData())
void internal_reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const IteratorFiltersType &iterator_filter, const std::shared_ptr< const MPI_Comm > &comm, const AdditionalData additional_data)
std::vector< Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space > > constraint_mask
unsigned int get_padding_length() const
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > constraint_weights
const DoFHandler< dim > * dof_handler
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
void reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const IteratorFiltersType &iterator_filter, const AdditionalData &additional_data=AdditionalData())
std::vector< Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space > > inv_jacobian
Kokkos::View< types::global_dof_index *, MemorySpace::Default::kokkos_space > constrained_dofs
std::vector< Kokkos::View< point_type **, MemorySpace::Default::kokkos_space > > q_points
types::global_dof_index n_dofs
void distributed_cell_loop(const Functor &func, const LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &src, LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &dst) const
std::vector< unsigned int > row_start
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
std::vector< Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space > > local_to_global
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
std::size_t memory_consumption() const
void set_constrained_values(const Number val, VectorType &dst) const
void initialize_dof_vector(LinearAlgebra::distributed::Vector< Number, MemorySpace::Default > &vec) const
void copy_constrained_values(const VectorType &src, VectorType &dst) const
std::vector< Kokkos::View< Number **, MemorySpace::Default::kokkos_space > > JxW
Data get_data(unsigned int color) const
const DoFHandler< dim > & get_dof_handler() const
std::shared_ptr< const Utilities::MPI::Partitioner > partitioner
void serial_cell_loop(const Functor &func, const VectorType &src, VectorType &dst) const
void evaluate_coefficients(Functor func) const
void reinit(const Mapping< dim > &mapping, const DoFHandler< dim > &dof_handler, const AffineConstraints< Number > &constraints, const Quadrature< 1 > &quad, const AdditionalData &additional_data=AdditionalData())
const std::shared_ptr< const Utilities::MPI::Partitioner > & get_vector_partitioner() const
DataHost< dim, Number > copy_mf_data_to_host(const typename::Portable::MatrixFree< dim, Number >::Data &data, const UpdateFlags &update_flags)
std::vector< unsigned int > n_cells
void cell_loop(const Functor &func, const VectorType &src, VectorType &dst) const
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:503
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:504
#define Assert(cond, exc)
static ::ExceptionBase & ExcNotInitialized()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
UpdateFlags
@ update_JxW_values
Transformed quadrature weights.
@ update_gradients
Shape function gradients.
@ update_quadrature_points
Transformed quadrature points.
#define DEAL_II_HOST_DEVICE
Definition numbers.h:34
*braid_SplitCommworld & comm
::Kokkos::DefaultExecutionSpace::memory_space kokkos_space
Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space >::HostMirror local_to_global
Kokkos::View< Number **, MemorySpace::Default::kokkos_space >::HostMirror JxW
Point< dim, Number > get_quadrature_point(const unsigned int cell, const unsigned int q_point) const
Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space >::HostMirror constraint_mask
Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space >::HostMirror inv_jacobian
Kokkos::View< Point< dim, Number > **, MemorySpace::Default::kokkos_space >::HostMirror q_points
unsigned int local_q_point_id(const unsigned int cell, const unsigned int n_q_points, const unsigned int q_point) const
AdditionalData(const UpdateFlags mapping_update_flags=update_gradients|update_JxW_values|update_quadrature_points, const bool use_coloring=false, const bool overlap_communication_computation=false)
Kokkos::View< Number **, MemorySpace::Default::kokkos_space > JxW
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > constraint_weights
unsigned int local_q_point_id(const unsigned int cell, const unsigned int n_q_points, const unsigned int q_point) const
Kokkos::View<::internal::MatrixFreeFunctions::ConstraintKinds *, MemorySpace::Default::kokkos_space > constraint_mask
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_values
Kokkos::View< Number **[dim][dim], MemorySpace::Default::kokkos_space > inv_jacobian
Kokkos::View< point_type **, MemorySpace::Default::kokkos_space > q_points
Kokkos::View< types::global_dof_index **, MemorySpace::Default::kokkos_space > local_to_global
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > shape_gradients
Kokkos::View< Number *, MemorySpace::Default::kokkos_space > co_shape_gradients
Portable::MatrixFree< dim, Number >::point_type & get_quadrature_point(const unsigned int cell, const unsigned int q_point) const
Kokkos::View< Number *[dim], MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedView2D
Kokkos::View< Number *, MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space, Kokkos::MemoryTraits< Kokkos::Unmanaged > > SharedView1D
Kokkos::TeamPolicy< MemorySpace::Default::kokkos_space::execution_space >::member_type TeamHandle
SharedData(const TeamHandle &team_member, const SharedView1D &values, const SharedView2D &gradients)