Reference documentation for deal.II version 9.6.0
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
mpi_large_count.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2022 - 2023 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15//
16// The content of this file is a slightly modified version of the
17// header-only library BigMPICompat - a tiny MPI 4.x compatibility
18// library released under MIT license at
19// https://github.com/tjhei/BigMPICompat and relicensed and included
20// here with permission.
21//
22
23#ifndef BIG_MPI_COMPAT_H
24#define BIG_MPI_COMPAT_H
25
26#include <deal.II/base/config.h>
27
28#ifdef DEAL_II_WITH_MPI
29# include <mpi.h>
30// required for std::numeric_limits used below.
31# include <limits>
32# ifndef MPI_VERSION
33# error "Your MPI implementation does not define MPI_VERSION!"
34# endif
35
36# if MPI_VERSION < 3
37# error "BigMPICompat requires at least MPI 3.0"
38# endif
39
40DEAL_II_NAMESPACE_OPEN
41
42namespace Utilities
43{
44 namespace MPI
45 {
51 namespace LargeCount
52 {
57 static constexpr MPI_Count mpi_max_int_count =
58 std::numeric_limits<int>::max();
59
65 inline int
66 Type_contiguous_c(MPI_Count count,
67 MPI_Datatype oldtype,
68 MPI_Datatype *newtype)
69 {
70# if MPI_VERSION >= 4
71 return MPI_Type_contiguous_c(count, oldtype, newtype);
72# else
73 if (count <= LargeCount::mpi_max_int_count)
74 return MPI_Type_contiguous(count, oldtype, newtype);
75 else
76 {
77 int ierr;
78 const MPI_Count max_signed_int = LargeCount::mpi_max_int_count;
79
80 MPI_Count size_old;
81 ierr = MPI_Type_size_x(oldtype, &size_old);
82
83 MPI_Count n_chunks = count / max_signed_int;
84 MPI_Count n_remaining_elements = count % max_signed_int;
85
86 MPI_Datatype chunks;
87 ierr = MPI_Type_vector(
88 n_chunks, max_signed_int, max_signed_int, oldtype, &chunks);
89 if (ierr != MPI_SUCCESS)
90 return ierr;
91
92 MPI_Datatype remainder;
93 ierr =
94 MPI_Type_contiguous(n_remaining_elements, oldtype, &remainder);
95 if (ierr != MPI_SUCCESS)
96 return ierr;
97
98 int blocklengths[2] = {1, 1};
99 MPI_Aint displacements[2] = {0,
100 static_cast<MPI_Aint>(n_chunks) *
101 size_old * max_signed_int};
102 MPI_Datatype types[2] = {chunks, remainder};
103 ierr = MPI_Type_create_struct(
104 2, blocklengths, displacements, types, newtype);
105 if (ierr != MPI_SUCCESS)
106 return ierr;
107
108 ierr = MPI_Type_commit(newtype);
109 if (ierr != MPI_SUCCESS)
110 return ierr;
111
112 ierr = MPI_Type_free(&chunks);
113 if (ierr != MPI_SUCCESS)
114 return ierr;
115
116 ierr = MPI_Type_free(&remainder);
117 if (ierr != MPI_SUCCESS)
118 return ierr;
119
120# ifndef MPI_COMPAT_SKIP_SIZE_CHECK
121 MPI_Count size_new;
122 ierr = MPI_Type_size_x(*newtype, &size_new);
123 if (ierr != MPI_SUCCESS)
124 return ierr;
125
126 if (size_old * count != size_new)
127 {
128 // This error can happen when you are using a very old and
129 // buggy MPI implementation. There is nothing we can do
130 // here, unfortunately. Please update your installation.
131 // std::cerr
132 //<< "MPI_Type_contiguous_c() produced an invalid result.
133 // Expected =
134 //"
135 //<< size_old << " * " << count << " = " << size_old * count
136 //<< " but received " << size_new << std::endl;
137 return MPI_ERR_INTERN;
138 }
139# endif
140
141 return MPI_SUCCESS;
142 }
143# endif
144 }
145
146
152 inline int
153 Send_c(const void *buf,
154 MPI_Count count,
155 MPI_Datatype datatype,
156 int dest,
157 int tag,
158 MPI_Comm comm)
159 {
160# if MPI_VERSION >= 4
161 return MPI_Send_c(buf, count, datatype, dest, tag, comm);
162# else
163 if (count <= LargeCount::mpi_max_int_count)
164 return MPI_Send(buf, count, datatype, dest, tag, comm);
165
166 MPI_Datatype bigtype;
167 int ierr;
168 ierr = Type_contiguous_c(count, datatype, &bigtype);
169 if (ierr != MPI_SUCCESS)
170 return ierr;
171 ierr = MPI_Type_commit(&bigtype);
172 if (ierr != MPI_SUCCESS)
173 return ierr;
174
175 ierr = MPI_Send(buf, 1, bigtype, dest, tag, comm);
176 if (ierr != MPI_SUCCESS)
177 return ierr;
178
179 ierr = MPI_Type_free(&bigtype);
180 if (ierr != MPI_SUCCESS)
181 return ierr;
182 return MPI_SUCCESS;
183# endif
184 }
185
191 inline int
192 Recv_c(void *buf,
193 MPI_Count count,
194 MPI_Datatype datatype,
195 int source,
196 int tag,
197 MPI_Comm comm,
198 MPI_Status *status)
199 {
200# if MPI_VERSION >= 4
201 return MPI_Recv_c(buf, count, datatype, source, tag, comm, status);
202# else
203 if (count <= LargeCount::mpi_max_int_count)
204 return MPI_Recv(buf, count, datatype, source, tag, comm, status);
205
206 MPI_Datatype bigtype;
207 int ierr;
208 ierr = Type_contiguous_c(count, datatype, &bigtype);
209 if (ierr != MPI_SUCCESS)
210 return ierr;
211
212 ierr = MPI_Type_commit(&bigtype);
213 if (ierr != MPI_SUCCESS)
214 return ierr;
215
216 ierr = MPI_Recv(buf, 1, bigtype, source, tag, comm, status);
217 if (ierr != MPI_SUCCESS)
218 return ierr;
219
220 ierr = MPI_Type_free(&bigtype);
221 if (ierr != MPI_SUCCESS)
222 return ierr;
223 return MPI_SUCCESS;
224# endif
225 }
226
233 inline int
234 Bcast_c(void *buf,
235 MPI_Count count,
236 MPI_Datatype datatype,
237 unsigned int root_mpi_rank,
238 MPI_Comm comm)
239 {
240# if MPI_VERSION >= 4
241 return MPI_Bcast_c(buf, count, datatype, root_mpi_rank, comm);
242# else
243 if (count <= LargeCount::mpi_max_int_count)
244 return MPI_Bcast(buf, count, datatype, root_mpi_rank, comm);
245
246 MPI_Datatype bigtype;
247 int ierr;
248 ierr = Type_contiguous_c(count, datatype, &bigtype);
249 if (ierr != MPI_SUCCESS)
250 return ierr;
251 ierr = MPI_Type_commit(&bigtype);
252 if (ierr != MPI_SUCCESS)
253 return ierr;
254
255 ierr = MPI_Bcast(buf, 1, bigtype, root_mpi_rank, comm);
256 if (ierr != MPI_SUCCESS)
257 return ierr;
258
259 ierr = MPI_Type_free(&bigtype);
260 if (ierr != MPI_SUCCESS)
261 return ierr;
262 return MPI_SUCCESS;
263# endif
264 }
265
271 inline int
272 File_write_at_c(MPI_File fh,
273 MPI_Offset offset,
274 const void *buf,
275 MPI_Count count,
276 MPI_Datatype datatype,
277 MPI_Status *status)
278 {
279 if (count <= LargeCount::mpi_max_int_count)
280 return MPI_File_write_at(fh, offset, buf, count, datatype, status);
281
282 MPI_Datatype bigtype;
283 int ierr;
284 ierr = Type_contiguous_c(count, datatype, &bigtype);
285 if (ierr != MPI_SUCCESS)
286 return ierr;
287 ierr = MPI_Type_commit(&bigtype);
288 if (ierr != MPI_SUCCESS)
289 return ierr;
290
291 ierr = MPI_File_write_at(fh, offset, buf, 1, bigtype, status);
292 if (ierr != MPI_SUCCESS)
293 return ierr;
294
295 ierr = MPI_Type_free(&bigtype);
296 if (ierr != MPI_SUCCESS)
297 return ierr;
298 return MPI_SUCCESS;
299 }
300
307 inline int
308 File_write_at_all_c(MPI_File fh,
309 MPI_Offset offset,
310 const void *buf,
311 MPI_Count count,
312 MPI_Datatype datatype,
313 MPI_Status *status)
314 {
315 if (count <= LargeCount::mpi_max_int_count)
316 return MPI_File_write_at_all(
317 fh, offset, buf, count, datatype, status);
318
319 MPI_Datatype bigtype;
320 int ierr;
321 ierr = Type_contiguous_c(count, datatype, &bigtype);
322 if (ierr != MPI_SUCCESS)
323 return ierr;
324 ierr = MPI_Type_commit(&bigtype);
325 if (ierr != MPI_SUCCESS)
326 return ierr;
327
328 ierr = MPI_File_write_at_all(fh, offset, buf, 1, bigtype, status);
329 if (ierr != MPI_SUCCESS)
330 return ierr;
331
332 ierr = MPI_Type_free(&bigtype);
333 if (ierr != MPI_SUCCESS)
334 return ierr;
335 return MPI_SUCCESS;
336 }
337
343 inline int
344 File_write_ordered_c(MPI_File fh,
345 const void *buf,
346 MPI_Count count,
347 MPI_Datatype datatype,
348 MPI_Status *status)
349 {
350 if (count <= LargeCount::mpi_max_int_count)
351 return MPI_File_write_ordered(fh, buf, count, datatype, status);
352
353 MPI_Datatype bigtype;
354 int ierr;
355 ierr = Type_contiguous_c(count, datatype, &bigtype);
356 if (ierr != MPI_SUCCESS)
357 return ierr;
358 ierr = MPI_Type_commit(&bigtype);
359 if (ierr != MPI_SUCCESS)
360 return ierr;
361
362 ierr = MPI_File_write_ordered(fh, buf, 1, bigtype, status);
363 if (ierr != MPI_SUCCESS)
364 return ierr;
365
366 ierr = MPI_Type_free(&bigtype);
367 if (ierr != MPI_SUCCESS)
368 return ierr;
369 return MPI_SUCCESS;
370 }
371
378 inline int
379 File_read_at_c(MPI_File fh,
380 MPI_Offset offset,
381 void *buf,
382 MPI_Count count,
383 MPI_Datatype datatype,
384 MPI_Status *status)
385 {
386 if (count <= LargeCount::mpi_max_int_count)
387 return MPI_File_read_at(fh, offset, buf, count, datatype, status);
388
389 MPI_Datatype bigtype;
390 int ierr;
391 ierr = Type_contiguous_c(count, datatype, &bigtype);
392 if (ierr != MPI_SUCCESS)
393 return ierr;
394 ierr = MPI_Type_commit(&bigtype);
395 if (ierr != MPI_SUCCESS)
396 return ierr;
397
398 ierr = MPI_File_read_at(fh, offset, buf, 1, bigtype, status);
399 if (ierr != MPI_SUCCESS)
400 return ierr;
401
402 ierr = MPI_Type_free(&bigtype);
403 if (ierr != MPI_SUCCESS)
404 return ierr;
405 return MPI_SUCCESS;
406 }
407
414 inline int
415 File_read_at_all_c(MPI_File fh,
416 MPI_Offset offset,
417 void *buf,
418 MPI_Count count,
419 MPI_Datatype datatype,
420 MPI_Status *status)
421 {
422 if (count <= LargeCount::mpi_max_int_count)
423 return MPI_File_read_at_all(fh, offset, buf, count, datatype, status);
424
425 MPI_Datatype bigtype;
426 int ierr;
427 ierr = Type_contiguous_c(count, datatype, &bigtype);
428 if (ierr != MPI_SUCCESS)
429 return ierr;
430 ierr = MPI_Type_commit(&bigtype);
431 if (ierr != MPI_SUCCESS)
432 return ierr;
433
434 ierr = MPI_File_read_at_all(fh, offset, buf, 1, bigtype, status);
435 if (ierr != MPI_SUCCESS)
436 return ierr;
437
438 ierr = MPI_Type_free(&bigtype);
439 if (ierr != MPI_SUCCESS)
440 return ierr;
441 return MPI_SUCCESS;
442 }
443
444 } // namespace LargeCount
445 } // namespace MPI
446} // namespace Utilities
447
448DEAL_II_NAMESPACE_CLOSE
449
450#endif
451#endif
MPI_Comm
DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:503
DEAL_II_NAMESPACE_CLOSE
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:504
InitializeLibrary::MPI
@ MPI
Utilities::MPI::LargeCount::File_write_at_c
int File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:272
Utilities::MPI::LargeCount::File_write_ordered_c
int File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:344
Utilities::MPI::LargeCount::File_write_at_all_c
int File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:308
Utilities::MPI::LargeCount::Recv_c
int Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
Definition mpi_large_count.h:192
Utilities::MPI::LargeCount::File_read_at_c
int File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:379
Utilities::MPI::LargeCount::Type_contiguous_c
int Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype, MPI_Datatype *newtype)
Definition mpi_large_count.h:66
Utilities::MPI::LargeCount::Bcast_c
int Bcast_c(void *buf, MPI_Count count, MPI_Datatype datatype, unsigned int root_mpi_rank, MPI_Comm comm)
Definition mpi_large_count.h:234
Utilities::MPI::LargeCount::Send_c
int Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
Definition mpi_large_count.h:153
Utilities::MPI::LargeCount::File_read_at_all_c
int File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:415
Utilities::MPI::LargeCount::mpi_max_int_count
static constexpr MPI_Count mpi_max_int_count
Definition mpi_large_count.h:57
types
Definition types.h:32
comm
*braid_SplitCommworld & comm
Definition parallel_in_time.h:1981
Generated by doxygen 1.11.0