Reference documentation for deal.II version 9.4.1
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
mpi_large_count.h
Go to the documentation of this file.
1// ---------------------------------------------------------------------
2//
3// Copyright (C) 2022 - 2022 by the deal.II authors
4//
5// This file is part of the deal.II library.
6//
7// The deal.II library is free software; you can use it, redistribute
8// it, and/or modify it under the terms of the GNU Lesser General
9// Public License as published by the Free Software Foundation; either
10// version 2.1 of the License, or (at your option) any later version.
11// The full text of the license can be found in the file LICENSE.md at
12// the top level directory of deal.II.
13//
14// ---------------------------------------------------------------------
15
16//
17// The content of this file is a slightly modified version of the
18// header-only library BigMPICompat - a tiny MPI 4.x compatibility
19// library released under MIT license at
20// https://github.com/tjhei/BigMPICompat and relicensed and included
21// here with permission.
22//
23
24#ifndef BIG_MPI_COMPAT_H
25#define BIG_MPI_COMPAT_H
26
27#include <deal.II/base/config.h>
28
29#ifdef DEAL_II_WITH_MPI
30# include <mpi.h>
31// required for std::numeric_limits used below.
32# include <limits>
33# ifndef MPI_VERSION
34# error "Your MPI implementation does not define MPI_VERSION!"
35# endif
36
37# if MPI_VERSION < 3
38# error "BigMPICompat requires at least MPI 3.0"
39# endif
40
41DEAL_II_NAMESPACE_OPEN
42
43namespace Utilities
44{
45 namespace MPI
46 {
52 namespace LargeCount
53 {
58 static constexpr MPI_Count mpi_max_int_count =
59 std::numeric_limits<int>::max();
60
66 inline int
67 Type_contiguous_c(MPI_Count count,
68 MPI_Datatype oldtype,
69 MPI_Datatype *newtype)
70 {
71# if MPI_VERSION >= 4
72 return MPI_Type_contiguous_c(count, oldtype, newtype);
73# else
74 if (count <= LargeCount::mpi_max_int_count)
75 return MPI_Type_contiguous(count, oldtype, newtype);
76 else
77 {
78 int ierr;
79 const MPI_Count max_signed_int = LargeCount::mpi_max_int_count;
80
81 MPI_Count size_old;
82 ierr = MPI_Type_size_x(oldtype, &size_old);
83
84 MPI_Count n_chunks = count / max_signed_int;
85 MPI_Count n_remaining_elements = count % max_signed_int;
86
87 MPI_Datatype chunks;
88 ierr = MPI_Type_vector(
89 n_chunks, max_signed_int, max_signed_int, oldtype, &chunks);
90 if (ierr != MPI_SUCCESS)
91 return ierr;
92
93 MPI_Datatype remainder;
94 ierr =
95 MPI_Type_contiguous(n_remaining_elements, oldtype, &remainder);
96 if (ierr != MPI_SUCCESS)
97 return ierr;
98
99 int blocklengths[2] = {1, 1};
100 MPI_Aint displacements[2] = {0,
101 static_cast<MPI_Aint>(n_chunks) *
102 size_old * max_signed_int};
103 MPI_Datatype types[2] = {chunks, remainder};
104 ierr = MPI_Type_create_struct(
105 2, blocklengths, displacements, types, newtype);
106 if (ierr != MPI_SUCCESS)
107 return ierr;
108
109 ierr = MPI_Type_commit(newtype);
110 if (ierr != MPI_SUCCESS)
111 return ierr;
112
113 ierr = MPI_Type_free(&chunks);
114 if (ierr != MPI_SUCCESS)
115 return ierr;
116
117 ierr = MPI_Type_free(&remainder);
118 if (ierr != MPI_SUCCESS)
119 return ierr;
120
121# ifndef MPI_COMPAT_SKIP_SIZE_CHECK
122 MPI_Count size_new;
123 ierr = MPI_Type_size_x(*newtype, &size_new);
124 if (ierr != MPI_SUCCESS)
125 return ierr;
126
127 if (size_old * count != size_new)
128 {
129 // This error can happen when you are using a very old and
130 // buggy MPI implementation. There is nothing we can do
131 // here, unfortunately. Please update your installation.
132 // std::cerr
133 //<< "MPI_Type_contiguous_c() produced an invalid result.
134 // Expected =
135 //"
136 //<< size_old << " * " << count << " = " << size_old * count
137 //<< " but received " << size_new << std::endl;
138 return MPI_ERR_INTERN;
139 }
140# endif
141
142 return MPI_SUCCESS;
143 }
144# endif
145 }
146
147
153 inline int
154 Send_c(const void * buf,
155 MPI_Count count,
156 MPI_Datatype datatype,
157 int dest,
158 int tag,
159 MPI_Comm comm)
160 {
161# if MPI_VERSION >= 4
162 return MPI_Send_c(buf, count, datatype, dest, tag, comm);
163# else
164 if (count <= LargeCount::mpi_max_int_count)
165 return MPI_Send(buf, count, datatype, dest, tag, comm);
166
167 MPI_Datatype bigtype;
168 int ierr;
169 ierr = Type_contiguous_c(count, datatype, &bigtype);
170 if (ierr != MPI_SUCCESS)
171 return ierr;
172 ierr = MPI_Type_commit(&bigtype);
173 if (ierr != MPI_SUCCESS)
174 return ierr;
175
176 ierr = MPI_Send(buf, 1, bigtype, dest, tag, comm);
177 if (ierr != MPI_SUCCESS)
178 return ierr;
179
180 ierr = MPI_Type_free(&bigtype);
181 if (ierr != MPI_SUCCESS)
182 return ierr;
183 return MPI_SUCCESS;
184# endif
185 }
186
192 inline int
193 Recv_c(void * buf,
194 MPI_Count count,
195 MPI_Datatype datatype,
196 int source,
197 int tag,
198 MPI_Comm comm,
199 MPI_Status * status)
200 {
201# if MPI_VERSION >= 4
202 return MPI_Recv_c(buf, count, datatype, source, tag, comm, status);
203# else
204 if (count <= LargeCount::mpi_max_int_count)
205 return MPI_Recv(buf, count, datatype, source, tag, comm, status);
206
207 MPI_Datatype bigtype;
208 int ierr;
209 ierr = Type_contiguous_c(count, datatype, &bigtype);
210 if (ierr != MPI_SUCCESS)
211 return ierr;
212
213 ierr = MPI_Type_commit(&bigtype);
214 if (ierr != MPI_SUCCESS)
215 return ierr;
216
217 ierr = MPI_Recv(buf, 1, bigtype, source, tag, comm, status);
218 if (ierr != MPI_SUCCESS)
219 return ierr;
220
221 ierr = MPI_Type_free(&bigtype);
222 if (ierr != MPI_SUCCESS)
223 return ierr;
224 return MPI_SUCCESS;
225# endif
226 }
227
234 inline int
235 Bcast_c(void * buf,
236 MPI_Count count,
237 MPI_Datatype datatype,
238 unsigned int root_mpi_rank,
239 MPI_Comm comm)
240 {
241# if MPI_VERSION >= 4
242 return MPI_Bcast_c(buf, count, datatype, root_mpi_rank, comm);
243# else
244 if (count <= LargeCount::mpi_max_int_count)
245 return MPI_Bcast(buf, count, datatype, root_mpi_rank, comm);
246
247 MPI_Datatype bigtype;
248 int ierr;
249 ierr = Type_contiguous_c(count, datatype, &bigtype);
250 if (ierr != MPI_SUCCESS)
251 return ierr;
252 ierr = MPI_Type_commit(&bigtype);
253 if (ierr != MPI_SUCCESS)
254 return ierr;
255
256 ierr = MPI_Bcast(buf, 1, bigtype, root_mpi_rank, comm);
257 if (ierr != MPI_SUCCESS)
258 return ierr;
259
260 ierr = MPI_Type_free(&bigtype);
261 if (ierr != MPI_SUCCESS)
262 return ierr;
263 return MPI_SUCCESS;
264# endif
265 }
266
272 inline int
273 File_write_at_c(MPI_File fh,
274 MPI_Offset offset,
275 const void * buf,
276 MPI_Count count,
277 MPI_Datatype datatype,
278 MPI_Status * status)
279 {
280 if (count <= LargeCount::mpi_max_int_count)
281 return MPI_File_write_at(fh, offset, buf, count, datatype, status);
282
283 MPI_Datatype bigtype;
284 int ierr;
285 ierr = Type_contiguous_c(count, datatype, &bigtype);
286 if (ierr != MPI_SUCCESS)
287 return ierr;
288 ierr = MPI_Type_commit(&bigtype);
289 if (ierr != MPI_SUCCESS)
290 return ierr;
291
292 ierr = MPI_File_write_at(fh, offset, buf, 1, bigtype, status);
293 if (ierr != MPI_SUCCESS)
294 return ierr;
295
296 ierr = MPI_Type_free(&bigtype);
297 if (ierr != MPI_SUCCESS)
298 return ierr;
299 return MPI_SUCCESS;
300 }
301
308 inline int
309 File_write_at_all_c(MPI_File fh,
310 MPI_Offset offset,
311 const void * buf,
312 MPI_Count count,
313 MPI_Datatype datatype,
314 MPI_Status * status)
315 {
316 if (count <= LargeCount::mpi_max_int_count)
317 return MPI_File_write_at_all(
318 fh, offset, buf, count, datatype, status);
319
320 MPI_Datatype bigtype;
321 int ierr;
322 ierr = Type_contiguous_c(count, datatype, &bigtype);
323 if (ierr != MPI_SUCCESS)
324 return ierr;
325 ierr = MPI_Type_commit(&bigtype);
326 if (ierr != MPI_SUCCESS)
327 return ierr;
328
329 ierr = MPI_File_write_at_all(fh, offset, buf, 1, bigtype, status);
330 if (ierr != MPI_SUCCESS)
331 return ierr;
332
333 ierr = MPI_Type_free(&bigtype);
334 if (ierr != MPI_SUCCESS)
335 return ierr;
336 return MPI_SUCCESS;
337 }
338
344 inline int
345 File_write_ordered_c(MPI_File fh,
346 const void * buf,
347 MPI_Count count,
348 MPI_Datatype datatype,
349 MPI_Status * status)
350 {
351 if (count <= LargeCount::mpi_max_int_count)
352 return MPI_File_write_ordered(fh, buf, count, datatype, status);
353
354 MPI_Datatype bigtype;
355 int ierr;
356 ierr = Type_contiguous_c(count, datatype, &bigtype);
357 if (ierr != MPI_SUCCESS)
358 return ierr;
359 ierr = MPI_Type_commit(&bigtype);
360 if (ierr != MPI_SUCCESS)
361 return ierr;
362
363 ierr = MPI_File_write_ordered(fh, buf, 1, bigtype, status);
364 if (ierr != MPI_SUCCESS)
365 return ierr;
366
367 ierr = MPI_Type_free(&bigtype);
368 if (ierr != MPI_SUCCESS)
369 return ierr;
370 return MPI_SUCCESS;
371 }
372
379 inline int
380 File_read_at_c(MPI_File fh,
381 MPI_Offset offset,
382 void * buf,
383 MPI_Count count,
384 MPI_Datatype datatype,
385 MPI_Status * status)
386 {
387 if (count <= LargeCount::mpi_max_int_count)
388 return MPI_File_read_at(fh, offset, buf, count, datatype, status);
389
390 MPI_Datatype bigtype;
391 int ierr;
392 ierr = Type_contiguous_c(count, datatype, &bigtype);
393 if (ierr != MPI_SUCCESS)
394 return ierr;
395 ierr = MPI_Type_commit(&bigtype);
396 if (ierr != MPI_SUCCESS)
397 return ierr;
398
399 ierr = MPI_File_read_at(fh, offset, buf, 1, bigtype, status);
400 if (ierr != MPI_SUCCESS)
401 return ierr;
402
403 ierr = MPI_Type_free(&bigtype);
404 if (ierr != MPI_SUCCESS)
405 return ierr;
406 return MPI_SUCCESS;
407 }
408
415 inline int
416 File_read_at_all_c(MPI_File fh,
417 MPI_Offset offset,
418 void * buf,
419 MPI_Count count,
420 MPI_Datatype datatype,
421 MPI_Status * status)
422 {
423 if (count <= LargeCount::mpi_max_int_count)
424 return MPI_File_read_at_all(fh, offset, buf, count, datatype, status);
425
426 MPI_Datatype bigtype;
427 int ierr;
428 ierr = Type_contiguous_c(count, datatype, &bigtype);
429 if (ierr != MPI_SUCCESS)
430 return ierr;
431 ierr = MPI_Type_commit(&bigtype);
432 if (ierr != MPI_SUCCESS)
433 return ierr;
434
435 ierr = MPI_File_read_at_all(fh, offset, buf, 1, bigtype, status);
436 if (ierr != MPI_SUCCESS)
437 return ierr;
438
439 ierr = MPI_Type_free(&bigtype);
440 if (ierr != MPI_SUCCESS)
441 return ierr;
442 return MPI_SUCCESS;
443 }
444
445 } // namespace LargeCount
446 } // namespace MPI
447} // namespace Utilities
448
449DEAL_II_NAMESPACE_CLOSE
450
451#endif
452#endif
MPI_Comm
DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:442
DEAL_II_NAMESPACE_CLOSE
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:443
Utilities::MPI::LargeCount::File_write_at_c
int File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition: mpi_large_count.h:273
Utilities::MPI::LargeCount::File_write_ordered_c
int File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition: mpi_large_count.h:345
Utilities::MPI::LargeCount::File_write_at_all_c
int File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition: mpi_large_count.h:309
Utilities::MPI::LargeCount::Recv_c
int Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
Definition: mpi_large_count.h:193
Utilities::MPI::LargeCount::File_read_at_c
int File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition: mpi_large_count.h:380
Utilities::MPI::LargeCount::Type_contiguous_c
int Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype, MPI_Datatype *newtype)
Definition: mpi_large_count.h:67
Utilities::MPI::LargeCount::Bcast_c
int Bcast_c(void *buf, MPI_Count count, MPI_Datatype datatype, unsigned int root_mpi_rank, MPI_Comm comm)
Definition: mpi_large_count.h:235
Utilities::MPI::LargeCount::Send_c
int Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
Definition: mpi_large_count.h:154
Utilities::MPI::LargeCount::File_read_at_all_c
int File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition: mpi_large_count.h:416
Utilities::MPI::LargeCount::mpi_max_int_count
static constexpr MPI_Count mpi_max_int_count
Definition: mpi_large_count.h:58
types
Definition: types.h:32
comm
const MPI_Comm & comm
Definition: tria_description.cc:725
Generated by doxygen 1.9.5