Reference documentation for deal.II version GIT f8d87ffe95 2022-08-04 17:30:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
vector_data_exchange.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2020 - 2021 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 
17 #ifndef dealii_matrix_free_vector_data_exchange_h
18 #define dealii_matrix_free_vector_data_exchange_h
19 
20 
21 #include <deal.II/base/config.h>
22 
24 #include <deal.II/base/mpi.h>
25 
27 
28 
30 
31 #ifndef DOXYGEN
32 
33 // forward declaration
34 namespace Utilities
35 {
36  namespace MPI
37  {
38  class Partitioner;
39  }
40 } // namespace Utilities
41 
42 #endif
43 
44 
45 namespace internal
46 {
47  namespace MatrixFreeFunctions
48  {
53  namespace VectorDataExchange
54  {
58  class Base
59  {
60  public:
61  virtual ~Base() = default;
62 
63  virtual unsigned int
64  locally_owned_size() const = 0;
65 
66  virtual unsigned int
67  n_ghost_indices() const = 0;
68 
69  virtual unsigned int
70  n_import_indices() const = 0;
71 
72  virtual unsigned int
73  n_import_sm_procs() const = 0;
74 
76  size() const = 0;
77 
78  virtual void
80  const unsigned int communication_channel,
81  const ArrayView<const double> & locally_owned_array,
82  const std::vector<ArrayView<const double>> &shared_arrays,
83  const ArrayView<double> & ghost_array,
84  const ArrayView<double> & temporary_storage,
85  std::vector<MPI_Request> & requests) const = 0;
86 
87  virtual void
89  const ArrayView<const double> & locally_owned_array,
90  const std::vector<ArrayView<const double>> &shared_arrays,
91  const ArrayView<double> & ghost_array,
92  std::vector<MPI_Request> & requests) const = 0;
93 
94  virtual void
96  const VectorOperation::values vector_operation,
97  const unsigned int communication_channel,
98  const ArrayView<const double> & locally_owned_array,
99  const std::vector<ArrayView<const double>> &shared_arrays,
100  const ArrayView<double> & ghost_array,
101  const ArrayView<double> & temporary_storage,
102  std::vector<MPI_Request> & requests) const = 0;
103 
104  virtual void
106  const VectorOperation::values vector_operation,
107  const ArrayView<double> & locally_owned_storage,
108  const std::vector<ArrayView<const double>> &shared_arrays,
109  const ArrayView<double> & ghost_array,
110  const ArrayView<const double> & temporary_storage,
111  std::vector<MPI_Request> & requests) const = 0;
112 
113  virtual void
114  reset_ghost_values(const ArrayView<double> &ghost_array) const = 0;
115 
116  virtual void
118  const unsigned int communication_channel,
119  const ArrayView<const float> & locally_owned_array,
120  const std::vector<ArrayView<const float>> &shared_arrays,
121  const ArrayView<float> & ghost_array,
122  const ArrayView<float> & temporary_storage,
123  std::vector<MPI_Request> & requests) const = 0;
124 
125  virtual void
127  const ArrayView<const float> & locally_owned_array,
128  const std::vector<ArrayView<const float>> &shared_arrays,
129  const ArrayView<float> & ghost_array,
130  std::vector<MPI_Request> & requests) const = 0;
131 
132  virtual void
134  const VectorOperation::values vector_operation,
135  const unsigned int communication_channel,
136  const ArrayView<const float> & locally_owned_array,
137  const std::vector<ArrayView<const float>> &shared_arrays,
138  const ArrayView<float> & ghost_array,
139  const ArrayView<float> & temporary_storage,
140  std::vector<MPI_Request> & requests) const = 0;
141 
142  virtual void
144  const VectorOperation::values vector_operation,
145  const ArrayView<float> & locally_owned_storage,
146  const std::vector<ArrayView<const float>> &shared_arrays,
147  const ArrayView<float> & ghost_array,
148  const ArrayView<const float> & temporary_storage,
149  std::vector<MPI_Request> & requests) const = 0;
150 
151  virtual void
152  reset_ghost_values(const ArrayView<float> &ghost_array) const = 0;
153  };
154 
155 
159  class PartitionerWrapper : public Base
160  {
161  public:
163  const std::shared_ptr<const Utilities::MPI::Partitioner>
164  &partitioner);
165 
166  virtual ~PartitionerWrapper() = default;
167 
168  unsigned int
169  locally_owned_size() const override;
170 
171  unsigned int
172  n_ghost_indices() const override;
173 
174  unsigned int
175  n_import_indices() const override;
176 
177  unsigned int
178  n_import_sm_procs() const override;
179 
181  size() const override;
182 
183  void
185  const unsigned int communication_channel,
186  const ArrayView<const double> & locally_owned_array,
187  const std::vector<ArrayView<const double>> &shared_arrays,
188  const ArrayView<double> & ghost_array,
189  const ArrayView<double> & temporary_storage,
190  std::vector<MPI_Request> & requests) const override;
191 
192  void
194  const ArrayView<const double> & locally_owned_array,
195  const std::vector<ArrayView<const double>> &shared_arrays,
196  const ArrayView<double> & ghost_array,
197  std::vector<MPI_Request> & requests) const override;
198 
199  void
201  const VectorOperation::values vector_operation,
202  const unsigned int communication_channel,
203  const ArrayView<const double> & locally_owned_array,
204  const std::vector<ArrayView<const double>> &shared_arrays,
205  const ArrayView<double> & ghost_array,
206  const ArrayView<double> & temporary_storage,
207  std::vector<MPI_Request> & requests) const override;
208 
209  void
211  const VectorOperation::values vector_operation,
212  const ArrayView<double> & locally_owned_storage,
213  const std::vector<ArrayView<const double>> &shared_arrays,
214  const ArrayView<double> & ghost_array,
215  const ArrayView<const double> & temporary_storage,
216  std::vector<MPI_Request> & requests) const override;
217 
218  void
219  reset_ghost_values(const ArrayView<double> &ghost_array) const override;
220 
221  void
223  const unsigned int communication_channel,
224  const ArrayView<const float> & locally_owned_array,
225  const std::vector<ArrayView<const float>> &shared_arrays,
226  const ArrayView<float> & ghost_array,
227  const ArrayView<float> & temporary_storage,
228  std::vector<MPI_Request> & requests) const override;
229 
230  void
232  const ArrayView<const float> & locally_owned_array,
233  const std::vector<ArrayView<const float>> &shared_arrays,
234  const ArrayView<float> & ghost_array,
235  std::vector<MPI_Request> & requests) const override;
236 
237  void
239  const VectorOperation::values vector_operation,
240  const unsigned int communication_channel,
241  const ArrayView<const float> & locally_owned_array,
242  const std::vector<ArrayView<const float>> &shared_arrays,
243  const ArrayView<float> & ghost_array,
244  const ArrayView<float> & temporary_storage,
245  std::vector<MPI_Request> & requests) const override;
246 
247  void
249  const VectorOperation::values vector_operation,
250  const ArrayView<float> & locally_owned_storage,
251  const std::vector<ArrayView<const float>> &shared_arrays,
252  const ArrayView<float> & ghost_array,
253  const ArrayView<const float> & temporary_storage,
254  std::vector<MPI_Request> & requests) const override;
255 
256  void
257  reset_ghost_values(const ArrayView<float> &ghost_array) const override;
258 
259  private:
260  template <typename Number>
261  void
262  reset_ghost_values_impl(const ArrayView<Number> &ghost_array) const;
263 
264  const std::shared_ptr<const Utilities::MPI::Partitioner> partitioner;
265  };
266 
267 
268 
274  class Full : public Base
275  {
276  public:
277  Full(
278  const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner,
279  const MPI_Comm &communicator_sm);
280 
281  unsigned int
282  locally_owned_size() const override;
283 
284  unsigned int
285  n_ghost_indices() const override;
286 
287  unsigned int
288  n_import_indices() const override;
289 
290  virtual unsigned int
291  n_import_sm_procs() const override;
292 
294  size() const override;
295 
296  const MPI_Comm &
297  get_sm_mpi_communicator() const;
298 
299  void
301  const unsigned int communication_channel,
302  const ArrayView<const double> & locally_owned_array,
303  const std::vector<ArrayView<const double>> &shared_arrays,
304  const ArrayView<double> & ghost_array,
305  const ArrayView<double> & temporary_storage,
306  std::vector<MPI_Request> & requests) const override;
307 
308  void
310  const ArrayView<const double> & locally_owned_array,
311  const std::vector<ArrayView<const double>> &shared_arrays,
312  const ArrayView<double> & ghost_array,
313  std::vector<MPI_Request> & requests) const override;
314 
315  void
317  const VectorOperation::values vector_operation,
318  const unsigned int communication_channel,
319  const ArrayView<const double> & locally_owned_array,
320  const std::vector<ArrayView<const double>> &shared_arrays,
321  const ArrayView<double> & ghost_array,
322  const ArrayView<double> & temporary_storage,
323  std::vector<MPI_Request> & requests) const override;
324 
325  void
327  const VectorOperation::values vector_operation,
328  const ArrayView<double> & locally_owned_storage,
329  const std::vector<ArrayView<const double>> &shared_arrays,
330  const ArrayView<double> & ghost_array,
331  const ArrayView<const double> & temporary_storage,
332  std::vector<MPI_Request> & requests) const override;
333 
334  void
335  reset_ghost_values(const ArrayView<double> &ghost_array) const override;
336 
337  void
339  const unsigned int communication_channel,
340  const ArrayView<const float> & locally_owned_array,
341  const std::vector<ArrayView<const float>> &shared_arrays,
342  const ArrayView<float> & ghost_array,
343  const ArrayView<float> & temporary_storage,
344  std::vector<MPI_Request> & requests) const override;
345 
346  void
348  const ArrayView<const float> & locally_owned_array,
349  const std::vector<ArrayView<const float>> &shared_arrays,
350  const ArrayView<float> & ghost_array,
351  std::vector<MPI_Request> & requests) const override;
352 
353  void
355  const VectorOperation::values vector_operation,
356  const unsigned int communication_channel,
357  const ArrayView<const float> & locally_owned_array,
358  const std::vector<ArrayView<const float>> &shared_arrays,
359  const ArrayView<float> & ghost_array,
360  const ArrayView<float> & temporary_storage,
361  std::vector<MPI_Request> & requests) const override;
362 
363  void
365  const VectorOperation::values vector_operation,
366  const ArrayView<float> & locally_owned_storage,
367  const std::vector<ArrayView<const float>> &shared_arrays,
368  const ArrayView<float> & ghost_array,
369  const ArrayView<const float> & temporary_storage,
370  std::vector<MPI_Request> & requests) const override;
371 
372  void
373  reset_ghost_values(const ArrayView<float> &ghost_array) const override;
374 
375  private:
376  template <typename Number>
377  void
379  const unsigned int communication_channel,
380  const ArrayView<const Number> & locally_owned_array,
381  const std::vector<ArrayView<const Number>> &shared_arrays,
382  const ArrayView<Number> & ghost_array,
383  const ArrayView<Number> & temporary_storage,
384  std::vector<MPI_Request> & requests) const;
385 
386  template <typename Number>
387  void
389  const ArrayView<const Number> & locally_owned_array,
390  const std::vector<ArrayView<const Number>> &shared_arrays,
391  const ArrayView<Number> & ghost_array,
392  std::vector<MPI_Request> & requests) const;
393 
394  template <typename Number>
395  void
397  const VectorOperation::values vector_operation,
398  const unsigned int communication_channel,
399  const ArrayView<const Number> & locally_owned_array,
400  const std::vector<ArrayView<const Number>> &shared_arrays,
401  const ArrayView<Number> & ghost_array,
402  const ArrayView<Number> & temporary_storage,
403  std::vector<MPI_Request> & requests) const;
404 
405  template <typename Number>
406  void
408  const VectorOperation::values vector_operation,
409  const ArrayView<Number> & locally_owned_storage,
410  const std::vector<ArrayView<const Number>> &shared_arrays,
411  const ArrayView<Number> & ghost_array,
412  const ArrayView<const Number> & temporary_storage,
413  std::vector<MPI_Request> & requests) const;
414 
415  template <typename Number>
416  void
417  reset_ghost_values_impl(const ArrayView<Number> &ghost_array) const;
418 
419  private:
423  const MPI_Comm comm;
424 
428  const MPI_Comm comm_sm;
429 
433  const unsigned int n_local_elements;
434 
438  const unsigned int n_ghost_elements;
439 
444 
450 
455  std::pair<std::vector<unsigned int>,
456  std::vector<std::pair<unsigned int, unsigned int>>>
458 
463  std::vector<std::array<unsigned int, 3>> ghost_targets_data;
464 
471  std::vector<std::array<unsigned int, 3>> import_targets_data;
472 
481  std::pair<std::vector<unsigned int>,
482  std::vector<std::pair<unsigned int, unsigned int>>>
484 
489  std::vector<unsigned int> sm_ghost_ranks;
490 
495  std::pair<std::vector<unsigned int>,
496  std::vector<std::pair<unsigned int, unsigned int>>>
498 
503  std::pair<std::vector<unsigned int>,
504  std::vector<std::pair<unsigned int, unsigned int>>>
506 
511  std::vector<unsigned int> sm_import_ranks;
512 
517  std::pair<std::vector<unsigned int>,
518  std::vector<std::pair<unsigned int, unsigned int>>>
520 
525  std::pair<std::vector<unsigned int>,
526  std::vector<std::pair<unsigned int, unsigned int>>>
528  };
529 
530  } // namespace VectorDataExchange
531  } // end of namespace MatrixFreeFunctions
532 } // end of namespace internal
533 
535 
536 #endif
virtual void export_to_ghosted_array_finish(const ArrayView< const float > &locally_owned_array, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, std::vector< MPI_Request > &requests) const =0
virtual void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const =0
virtual void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< float > &locally_owned_storage, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, const ArrayView< const float > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const float > &locally_owned_array, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, const ArrayView< float > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void reset_ghost_values(const ArrayView< float > &ghost_array) const =0
virtual types::global_dof_index size() const =0
virtual unsigned int locally_owned_size() const =0
virtual void reset_ghost_values(const ArrayView< double > &ghost_array) const =0
virtual void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const float > &locally_owned_array, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, const ArrayView< float > &temporary_storage, std::vector< MPI_Request > &requests) const =0
void export_to_ghosted_array_finish_impl(const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_start_impl(const unsigned int communication_channel, const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values(const ArrayView< double > &ghost_array) const override
std::vector< std::array< unsigned int, 3 > > ghost_targets_data
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_export_data_this
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_export_data
std::vector< std::array< unsigned int, 3 > > import_targets_data
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > ghost_indices_subset_data
Full(const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner, const MPI_Comm &communicator_sm)
void import_from_ghosted_array_finish_impl(const VectorOperation::values vector_operation, const ArrayView< Number > &locally_owned_storage, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< const Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const override
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_import_data
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values_impl(const ArrayView< Number > &ghost_array) const
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_import_data_this
void import_from_ghosted_array_start_impl(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const override
virtual types::global_dof_index size() const override
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > import_indices_data
const std::shared_ptr< const Utilities::MPI::Partitioner > partitioner
PartitionerWrapper(const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner)
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values_impl(const ArrayView< Number > &ghost_array) const
void reset_ghost_values(const ArrayView< double > &ghost_array) const override
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const override
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:442
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:443
unsigned int global_dof_index
Definition: types.h:76