Reference documentation for deal.II version GIT 921d917bf4 2023-02-06 18:40:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
vector_data_exchange.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2020 - 2021 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 
17 #ifndef dealii_matrix_free_vector_data_exchange_h
18 #define dealii_matrix_free_vector_data_exchange_h
19 
20 
21 #include <deal.II/base/config.h>
22 
24 #include <deal.II/base/mpi_stub.h>
26 
28 
29 #include <memory>
30 
32 
33 namespace internal
34 {
35  namespace MatrixFreeFunctions
36  {
41  namespace VectorDataExchange
42  {
46  class Base
47  {
48  public:
49  virtual ~Base() = default;
50 
51  virtual unsigned int
52  locally_owned_size() const = 0;
53 
54  virtual unsigned int
55  n_ghost_indices() const = 0;
56 
57  virtual unsigned int
58  n_import_indices() const = 0;
59 
60  virtual unsigned int
61  n_import_sm_procs() const = 0;
62 
64  size() const = 0;
65 
66  virtual void
68  const unsigned int communication_channel,
69  const ArrayView<const double> & locally_owned_array,
70  const std::vector<ArrayView<const double>> &shared_arrays,
71  const ArrayView<double> & ghost_array,
72  const ArrayView<double> & temporary_storage,
73  std::vector<MPI_Request> & requests) const = 0;
74 
75  virtual void
77  const ArrayView<const double> & locally_owned_array,
78  const std::vector<ArrayView<const double>> &shared_arrays,
79  const ArrayView<double> & ghost_array,
80  std::vector<MPI_Request> & requests) const = 0;
81 
82  virtual void
84  const VectorOperation::values vector_operation,
85  const unsigned int communication_channel,
86  const ArrayView<const double> & locally_owned_array,
87  const std::vector<ArrayView<const double>> &shared_arrays,
88  const ArrayView<double> & ghost_array,
89  const ArrayView<double> & temporary_storage,
90  std::vector<MPI_Request> & requests) const = 0;
91 
92  virtual void
94  const VectorOperation::values vector_operation,
95  const ArrayView<double> & locally_owned_storage,
96  const std::vector<ArrayView<const double>> &shared_arrays,
97  const ArrayView<double> & ghost_array,
98  const ArrayView<const double> & temporary_storage,
99  std::vector<MPI_Request> & requests) const = 0;
100 
101  virtual void
102  reset_ghost_values(const ArrayView<double> &ghost_array) const = 0;
103 
104  virtual void
106  const unsigned int communication_channel,
107  const ArrayView<const float> & locally_owned_array,
108  const std::vector<ArrayView<const float>> &shared_arrays,
109  const ArrayView<float> & ghost_array,
110  const ArrayView<float> & temporary_storage,
111  std::vector<MPI_Request> & requests) const = 0;
112 
113  virtual void
115  const ArrayView<const float> & locally_owned_array,
116  const std::vector<ArrayView<const float>> &shared_arrays,
117  const ArrayView<float> & ghost_array,
118  std::vector<MPI_Request> & requests) const = 0;
119 
120  virtual void
122  const VectorOperation::values vector_operation,
123  const unsigned int communication_channel,
124  const ArrayView<const float> & locally_owned_array,
125  const std::vector<ArrayView<const float>> &shared_arrays,
126  const ArrayView<float> & ghost_array,
127  const ArrayView<float> & temporary_storage,
128  std::vector<MPI_Request> & requests) const = 0;
129 
130  virtual void
132  const VectorOperation::values vector_operation,
133  const ArrayView<float> & locally_owned_storage,
134  const std::vector<ArrayView<const float>> &shared_arrays,
135  const ArrayView<float> & ghost_array,
136  const ArrayView<const float> & temporary_storage,
137  std::vector<MPI_Request> & requests) const = 0;
138 
139  virtual void
140  reset_ghost_values(const ArrayView<float> &ghost_array) const = 0;
141  };
142 
143 
147  class PartitionerWrapper : public Base
148  {
149  public:
151  const std::shared_ptr<const Utilities::MPI::Partitioner>
152  &partitioner);
153 
154  virtual ~PartitionerWrapper() = default;
155 
156  unsigned int
157  locally_owned_size() const override;
158 
159  unsigned int
160  n_ghost_indices() const override;
161 
162  unsigned int
163  n_import_indices() const override;
164 
165  unsigned int
166  n_import_sm_procs() const override;
167 
169  size() const override;
170 
171  void
173  const unsigned int communication_channel,
174  const ArrayView<const double> & locally_owned_array,
175  const std::vector<ArrayView<const double>> &shared_arrays,
176  const ArrayView<double> & ghost_array,
177  const ArrayView<double> & temporary_storage,
178  std::vector<MPI_Request> & requests) const override;
179 
180  void
182  const ArrayView<const double> & locally_owned_array,
183  const std::vector<ArrayView<const double>> &shared_arrays,
184  const ArrayView<double> & ghost_array,
185  std::vector<MPI_Request> & requests) const override;
186 
187  void
189  const VectorOperation::values vector_operation,
190  const unsigned int communication_channel,
191  const ArrayView<const double> & locally_owned_array,
192  const std::vector<ArrayView<const double>> &shared_arrays,
193  const ArrayView<double> & ghost_array,
194  const ArrayView<double> & temporary_storage,
195  std::vector<MPI_Request> & requests) const override;
196 
197  void
199  const VectorOperation::values vector_operation,
200  const ArrayView<double> & locally_owned_storage,
201  const std::vector<ArrayView<const double>> &shared_arrays,
202  const ArrayView<double> & ghost_array,
203  const ArrayView<const double> & temporary_storage,
204  std::vector<MPI_Request> & requests) const override;
205 
206  void
207  reset_ghost_values(const ArrayView<double> &ghost_array) const override;
208 
209  void
211  const unsigned int communication_channel,
212  const ArrayView<const float> & locally_owned_array,
213  const std::vector<ArrayView<const float>> &shared_arrays,
214  const ArrayView<float> & ghost_array,
215  const ArrayView<float> & temporary_storage,
216  std::vector<MPI_Request> & requests) const override;
217 
218  void
220  const ArrayView<const float> & locally_owned_array,
221  const std::vector<ArrayView<const float>> &shared_arrays,
222  const ArrayView<float> & ghost_array,
223  std::vector<MPI_Request> & requests) const override;
224 
225  void
227  const VectorOperation::values vector_operation,
228  const unsigned int communication_channel,
229  const ArrayView<const float> & locally_owned_array,
230  const std::vector<ArrayView<const float>> &shared_arrays,
231  const ArrayView<float> & ghost_array,
232  const ArrayView<float> & temporary_storage,
233  std::vector<MPI_Request> & requests) const override;
234 
235  void
237  const VectorOperation::values vector_operation,
238  const ArrayView<float> & locally_owned_storage,
239  const std::vector<ArrayView<const float>> &shared_arrays,
240  const ArrayView<float> & ghost_array,
241  const ArrayView<const float> & temporary_storage,
242  std::vector<MPI_Request> & requests) const override;
243 
244  void
245  reset_ghost_values(const ArrayView<float> &ghost_array) const override;
246 
247  private:
248  template <typename Number>
249  void
250  reset_ghost_values_impl(const ArrayView<Number> &ghost_array) const;
251 
252  const std::shared_ptr<const Utilities::MPI::Partitioner> partitioner;
253  };
254 
255 
256 
262  class Full : public Base
263  {
264  public:
265  Full(
266  const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner,
267  const MPI_Comm &communicator_sm);
268 
269  unsigned int
270  locally_owned_size() const override;
271 
272  unsigned int
273  n_ghost_indices() const override;
274 
275  unsigned int
276  n_import_indices() const override;
277 
278  virtual unsigned int
279  n_import_sm_procs() const override;
280 
282  size() const override;
283 
284  const MPI_Comm &
285  get_sm_mpi_communicator() const;
286 
287  void
289  const unsigned int communication_channel,
290  const ArrayView<const double> & locally_owned_array,
291  const std::vector<ArrayView<const double>> &shared_arrays,
292  const ArrayView<double> & ghost_array,
293  const ArrayView<double> & temporary_storage,
294  std::vector<MPI_Request> & requests) const override;
295 
296  void
298  const ArrayView<const double> & locally_owned_array,
299  const std::vector<ArrayView<const double>> &shared_arrays,
300  const ArrayView<double> & ghost_array,
301  std::vector<MPI_Request> & requests) const override;
302 
303  void
305  const VectorOperation::values vector_operation,
306  const unsigned int communication_channel,
307  const ArrayView<const double> & locally_owned_array,
308  const std::vector<ArrayView<const double>> &shared_arrays,
309  const ArrayView<double> & ghost_array,
310  const ArrayView<double> & temporary_storage,
311  std::vector<MPI_Request> & requests) const override;
312 
313  void
315  const VectorOperation::values vector_operation,
316  const ArrayView<double> & locally_owned_storage,
317  const std::vector<ArrayView<const double>> &shared_arrays,
318  const ArrayView<double> & ghost_array,
319  const ArrayView<const double> & temporary_storage,
320  std::vector<MPI_Request> & requests) const override;
321 
322  void
323  reset_ghost_values(const ArrayView<double> &ghost_array) const override;
324 
325  void
327  const unsigned int communication_channel,
328  const ArrayView<const float> & locally_owned_array,
329  const std::vector<ArrayView<const float>> &shared_arrays,
330  const ArrayView<float> & ghost_array,
331  const ArrayView<float> & temporary_storage,
332  std::vector<MPI_Request> & requests) const override;
333 
334  void
336  const ArrayView<const float> & locally_owned_array,
337  const std::vector<ArrayView<const float>> &shared_arrays,
338  const ArrayView<float> & ghost_array,
339  std::vector<MPI_Request> & requests) const override;
340 
341  void
343  const VectorOperation::values vector_operation,
344  const unsigned int communication_channel,
345  const ArrayView<const float> & locally_owned_array,
346  const std::vector<ArrayView<const float>> &shared_arrays,
347  const ArrayView<float> & ghost_array,
348  const ArrayView<float> & temporary_storage,
349  std::vector<MPI_Request> & requests) const override;
350 
351  void
353  const VectorOperation::values vector_operation,
354  const ArrayView<float> & locally_owned_storage,
355  const std::vector<ArrayView<const float>> &shared_arrays,
356  const ArrayView<float> & ghost_array,
357  const ArrayView<const float> & temporary_storage,
358  std::vector<MPI_Request> & requests) const override;
359 
360  void
361  reset_ghost_values(const ArrayView<float> &ghost_array) const override;
362 
363  private:
364  template <typename Number>
365  void
367  const unsigned int communication_channel,
368  const ArrayView<const Number> & locally_owned_array,
369  const std::vector<ArrayView<const Number>> &shared_arrays,
370  const ArrayView<Number> & ghost_array,
371  const ArrayView<Number> & temporary_storage,
372  std::vector<MPI_Request> & requests) const;
373 
374  template <typename Number>
375  void
377  const ArrayView<const Number> & locally_owned_array,
378  const std::vector<ArrayView<const Number>> &shared_arrays,
379  const ArrayView<Number> & ghost_array,
380  std::vector<MPI_Request> & requests) const;
381 
382  template <typename Number>
383  void
385  const VectorOperation::values vector_operation,
386  const unsigned int communication_channel,
387  const ArrayView<const Number> & locally_owned_array,
388  const std::vector<ArrayView<const Number>> &shared_arrays,
389  const ArrayView<Number> & ghost_array,
390  const ArrayView<Number> & temporary_storage,
391  std::vector<MPI_Request> & requests) const;
392 
393  template <typename Number>
394  void
396  const VectorOperation::values vector_operation,
397  const ArrayView<Number> & locally_owned_storage,
398  const std::vector<ArrayView<const Number>> &shared_arrays,
399  const ArrayView<Number> & ghost_array,
400  const ArrayView<const Number> & temporary_storage,
401  std::vector<MPI_Request> & requests) const;
402 
403  template <typename Number>
404  void
405  reset_ghost_values_impl(const ArrayView<Number> &ghost_array) const;
406 
407  private:
411  const MPI_Comm comm;
412 
416  const MPI_Comm comm_sm;
417 
421  const unsigned int n_local_elements;
422 
426  const unsigned int n_ghost_elements;
427 
432 
438 
443  std::pair<std::vector<unsigned int>,
444  std::vector<std::pair<unsigned int, unsigned int>>>
446 
451  std::vector<std::array<unsigned int, 3>> ghost_targets_data;
452 
459  std::vector<std::array<unsigned int, 3>> import_targets_data;
460 
469  std::pair<std::vector<unsigned int>,
470  std::vector<std::pair<unsigned int, unsigned int>>>
472 
477  std::vector<unsigned int> sm_ghost_ranks;
478 
483  std::pair<std::vector<unsigned int>,
484  std::vector<std::pair<unsigned int, unsigned int>>>
486 
491  std::pair<std::vector<unsigned int>,
492  std::vector<std::pair<unsigned int, unsigned int>>>
494 
499  std::vector<unsigned int> sm_import_ranks;
500 
505  std::pair<std::vector<unsigned int>,
506  std::vector<std::pair<unsigned int, unsigned int>>>
508 
513  std::pair<std::vector<unsigned int>,
514  std::vector<std::pair<unsigned int, unsigned int>>>
516  };
517 
518  } // namespace VectorDataExchange
519  } // end of namespace MatrixFreeFunctions
520 } // end of namespace internal
521 
523 
524 #endif
virtual void export_to_ghosted_array_finish(const ArrayView< const float > &locally_owned_array, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, std::vector< MPI_Request > &requests) const =0
virtual void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const =0
virtual void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< float > &locally_owned_storage, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, const ArrayView< const float > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const float > &locally_owned_array, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, const ArrayView< float > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const =0
virtual void reset_ghost_values(const ArrayView< float > &ghost_array) const =0
virtual types::global_dof_index size() const =0
virtual unsigned int locally_owned_size() const =0
virtual void reset_ghost_values(const ArrayView< double > &ghost_array) const =0
virtual void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const float > &locally_owned_array, const std::vector< ArrayView< const float >> &shared_arrays, const ArrayView< float > &ghost_array, const ArrayView< float > &temporary_storage, std::vector< MPI_Request > &requests) const =0
void export_to_ghosted_array_finish_impl(const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_start_impl(const unsigned int communication_channel, const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values(const ArrayView< double > &ghost_array) const override
std::vector< std::array< unsigned int, 3 > > ghost_targets_data
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_export_data_this
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_export_data
std::vector< std::array< unsigned int, 3 > > import_targets_data
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > ghost_indices_subset_data
Full(const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner, const MPI_Comm &communicator_sm)
void import_from_ghosted_array_finish_impl(const VectorOperation::values vector_operation, const ArrayView< Number > &locally_owned_storage, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< const Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const override
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_import_data
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values_impl(const ArrayView< Number > &ghost_array) const
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > sm_import_data_this
void import_from_ghosted_array_start_impl(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const Number > &locally_owned_array, const std::vector< ArrayView< const Number >> &shared_arrays, const ArrayView< Number > &ghost_array, const ArrayView< Number > &temporary_storage, std::vector< MPI_Request > &requests) const
void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const override
virtual types::global_dof_index size() const override
std::pair< std::vector< unsigned int >, std::vector< std::pair< unsigned int, unsigned int > > > import_indices_data
const std::shared_ptr< const Utilities::MPI::Partitioner > partitioner
PartitionerWrapper(const std::shared_ptr< const Utilities::MPI::Partitioner > &partitioner)
void export_to_ghosted_array_start(const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void import_from_ghosted_array_finish(const VectorOperation::values vector_operation, const ArrayView< double > &locally_owned_storage, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< const double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void reset_ghost_values_impl(const ArrayView< Number > &ghost_array) const
void reset_ghost_values(const ArrayView< double > &ghost_array) const override
void import_from_ghosted_array_start(const VectorOperation::values vector_operation, const unsigned int communication_channel, const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, const ArrayView< double > &temporary_storage, std::vector< MPI_Request > &requests) const override
void export_to_ghosted_array_finish(const ArrayView< const double > &locally_owned_array, const std::vector< ArrayView< const double >> &shared_arrays, const ArrayView< double > &ghost_array, std::vector< MPI_Request > &requests) const override
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:461
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:462
unsigned int global_dof_index
Definition: types.h:81