deal.II version GIT relicensing-7040-g409d9ea789 2026-11-06 04:40:01+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
mpi_large_count.h
Go to the documentation of this file.
1// -----------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception OR LGPL-2.1-or-later
4// Copyright (C) 2022 - 2026 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Detailed license information governing the source code and contributions
9// can be found in LICENSE.md and CONTRIBUTING.md at the top level directory.
10//
11// -----------------------------------------------------------------------------
12
13//
14// The content of this file is a slightly modified version of the
15// header-only library BigMPICompat - a tiny MPI 4.x compatibility
16// library released under MIT license at
17// https://github.com/tjhei/BigMPICompat and relicensed and included
18// here with permission.
19//
20
21#ifndef dealii_base_mpi_large_count_h
22#define dealii_base_mpi_large_count_h
23
24#include <deal.II/base/config.h>
25
26#ifdef DEAL_II_WITH_MPI
27
28DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
29# include <mpi.h>
30DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
31
32// required for std::numeric_limits used below.
33# include <limits>
34
35#endif
36
37DEAL_II_NAMESPACE_OPEN
38
39#ifdef DEAL_II_WITH_MPI
40namespace Utilities
41{
42 namespace MPI
43 {
49 namespace LargeCount
50 {
55 inline constexpr MPI_Count mpi_max_int_count =
56 std::numeric_limits<int>::max();
57
63 inline int
64 Type_contiguous_c(MPI_Count count,
65 MPI_Datatype oldtype,
66 MPI_Datatype *newtype)
67 {
68# if MPI_VERSION >= 4
69 return MPI_Type_contiguous_c(count, oldtype, newtype);
70# else
71 if (count <= LargeCount::mpi_max_int_count)
72 return MPI_Type_contiguous(count, oldtype, newtype);
73 else
74 {
75 int ierr;
76 const MPI_Count max_signed_int = LargeCount::mpi_max_int_count;
77
78 MPI_Count size_old;
79 ierr = MPI_Type_size_x(oldtype, &size_old);
80
81 MPI_Count n_chunks = count / max_signed_int;
82 MPI_Count n_remaining_elements = count % max_signed_int;
83
84 MPI_Datatype chunks;
85 ierr = MPI_Type_vector(
86 n_chunks, max_signed_int, max_signed_int, oldtype, &chunks);
87 if (ierr != MPI_SUCCESS)
88 return ierr;
89
90 MPI_Datatype remainder;
91 ierr =
92 MPI_Type_contiguous(n_remaining_elements, oldtype, &remainder);
93 if (ierr != MPI_SUCCESS)
94 return ierr;
95
96 int blocklengths[2] = {1, 1};
97 MPI_Aint displacements[2] = {0,
98 static_cast<MPI_Aint>(n_chunks) *
99 size_old * max_signed_int};
100 MPI_Datatype types[2] = {chunks, remainder};
101 ierr = MPI_Type_create_struct(
102 2, blocklengths, displacements, types, newtype);
103 if (ierr != MPI_SUCCESS)
104 return ierr;
105
106 ierr = MPI_Type_commit(newtype);
107 if (ierr != MPI_SUCCESS)
108 return ierr;
109
110 ierr = MPI_Type_free(&chunks);
111 if (ierr != MPI_SUCCESS)
112 return ierr;
113
114 ierr = MPI_Type_free(&remainder);
115 if (ierr != MPI_SUCCESS)
116 return ierr;
117
118# ifndef MPI_COMPAT_SKIP_SIZE_CHECK
119 MPI_Count size_new;
120 ierr = MPI_Type_size_x(*newtype, &size_new);
121 if (ierr != MPI_SUCCESS)
122 return ierr;
123
124 if (size_old * count != size_new)
125 {
126 // This error can happen when you are using a very old and
127 // buggy MPI implementation. There is nothing we can do
128 // here, unfortunately. Please update your installation.
129 // std::cerr
130 //<< "MPI_Type_contiguous_c() produced an invalid result.
131 // Expected =
132 //"
133 //<< size_old << " * " << count << " = " << size_old * count
134 //<< " but received " << size_new << std::endl;
135 return MPI_ERR_INTERN;
136 }
137# endif
138
139 return MPI_SUCCESS;
140 }
141# endif
142 }
143
144
150 inline int
151 Send_c(const void *buf,
152 MPI_Count count,
153 MPI_Datatype datatype,
154 int dest,
155 int tag,
156 MPI_Comm comm)
157 {
158# if MPI_VERSION >= 4
159 return MPI_Send_c(buf, count, datatype, dest, tag, comm);
160# else
161 if (count <= LargeCount::mpi_max_int_count)
162 return MPI_Send(buf, count, datatype, dest, tag, comm);
163
164 MPI_Datatype bigtype;
165 int ierr;
166 ierr = Type_contiguous_c(count, datatype, &bigtype);
167 if (ierr != MPI_SUCCESS)
168 return ierr;
169 ierr = MPI_Type_commit(&bigtype);
170 if (ierr != MPI_SUCCESS)
171 return ierr;
172
173 ierr = MPI_Send(buf, 1, bigtype, dest, tag, comm);
174 if (ierr != MPI_SUCCESS)
175 return ierr;
176
177 ierr = MPI_Type_free(&bigtype);
178 if (ierr != MPI_SUCCESS)
179 return ierr;
180 return MPI_SUCCESS;
181# endif
182 }
183
189 inline int
190 Recv_c(void *buf,
191 MPI_Count count,
192 MPI_Datatype datatype,
193 int source,
194 int tag,
195 MPI_Comm comm,
196 MPI_Status *status)
197 {
198# if MPI_VERSION >= 4
199 return MPI_Recv_c(buf, count, datatype, source, tag, comm, status);
200# else
201 if (count <= LargeCount::mpi_max_int_count)
202 return MPI_Recv(buf, count, datatype, source, tag, comm, status);
203
204 MPI_Datatype bigtype;
205 int ierr;
206 ierr = Type_contiguous_c(count, datatype, &bigtype);
207 if (ierr != MPI_SUCCESS)
208 return ierr;
209
210 ierr = MPI_Type_commit(&bigtype);
211 if (ierr != MPI_SUCCESS)
212 return ierr;
213
214 ierr = MPI_Recv(buf, 1, bigtype, source, tag, comm, status);
215 if (ierr != MPI_SUCCESS)
216 return ierr;
217
218 ierr = MPI_Type_free(&bigtype);
219 if (ierr != MPI_SUCCESS)
220 return ierr;
221 return MPI_SUCCESS;
222# endif
223 }
224
232 inline int
233 Get_count_c(const MPI_Status *status,
234 MPI_Datatype datatype,
235 MPI_Count *count)
236 {
237# if MPI_VERSION >= 4
238 return MPI_Get_count_c(status, datatype, count);
239# else
240 // For a basic datatype the number of elements equals the count, and
241 // in contrast to MPI_Get_count() this also works if the message was
242 // sent with a large derived datatype as done in Send_c().
243 return MPI_Get_elements_x(status, datatype, count);
244# endif
245 }
246
253 inline int
254 Bcast_c(void *buf,
255 MPI_Count count,
256 MPI_Datatype datatype,
257 unsigned int root_mpi_rank,
258 MPI_Comm comm)
259 {
260# if MPI_VERSION >= 4
261 return MPI_Bcast_c(buf, count, datatype, root_mpi_rank, comm);
262# else
263 if (count <= LargeCount::mpi_max_int_count)
264 return MPI_Bcast(buf, count, datatype, root_mpi_rank, comm);
265
266 MPI_Datatype bigtype;
267 int ierr;
268 ierr = Type_contiguous_c(count, datatype, &bigtype);
269 if (ierr != MPI_SUCCESS)
270 return ierr;
271 ierr = MPI_Type_commit(&bigtype);
272 if (ierr != MPI_SUCCESS)
273 return ierr;
274
275 ierr = MPI_Bcast(buf, 1, bigtype, root_mpi_rank, comm);
276 if (ierr != MPI_SUCCESS)
277 return ierr;
278
279 ierr = MPI_Type_free(&bigtype);
280 if (ierr != MPI_SUCCESS)
281 return ierr;
282 return MPI_SUCCESS;
283# endif
284 }
285
291 inline int
292 File_write_at_c(MPI_File fh,
293 MPI_Offset offset,
294 const void *buf,
295 MPI_Count count,
296 MPI_Datatype datatype,
297 MPI_Status *status)
298 {
299 if (count <= LargeCount::mpi_max_int_count)
300 return MPI_File_write_at(fh, offset, buf, count, datatype, status);
301
302 MPI_Datatype bigtype;
303 int ierr;
304 ierr = Type_contiguous_c(count, datatype, &bigtype);
305 if (ierr != MPI_SUCCESS)
306 return ierr;
307 ierr = MPI_Type_commit(&bigtype);
308 if (ierr != MPI_SUCCESS)
309 return ierr;
310
311 ierr = MPI_File_write_at(fh, offset, buf, 1, bigtype, status);
312 if (ierr != MPI_SUCCESS)
313 return ierr;
314
315 ierr = MPI_Type_free(&bigtype);
316 if (ierr != MPI_SUCCESS)
317 return ierr;
318 return MPI_SUCCESS;
319 }
320
327 inline int
328 File_write_at_all_c(MPI_File fh,
329 MPI_Offset offset,
330 const void *buf,
331 MPI_Count count,
332 MPI_Datatype datatype,
333 MPI_Status *status)
334 {
335 if (count <= LargeCount::mpi_max_int_count)
336 return MPI_File_write_at_all(
337 fh, offset, buf, count, datatype, status);
338
339 MPI_Datatype bigtype;
340 int ierr;
341 ierr = Type_contiguous_c(count, datatype, &bigtype);
342 if (ierr != MPI_SUCCESS)
343 return ierr;
344 ierr = MPI_Type_commit(&bigtype);
345 if (ierr != MPI_SUCCESS)
346 return ierr;
347
348 ierr = MPI_File_write_at_all(fh, offset, buf, 1, bigtype, status);
349 if (ierr != MPI_SUCCESS)
350 return ierr;
351
352 ierr = MPI_Type_free(&bigtype);
353 if (ierr != MPI_SUCCESS)
354 return ierr;
355 return MPI_SUCCESS;
356 }
357
363 inline int
364 File_write_ordered_c(MPI_File fh,
365 const void *buf,
366 MPI_Count count,
367 MPI_Datatype datatype,
368 MPI_Status *status)
369 {
370 if (count <= LargeCount::mpi_max_int_count)
371 return MPI_File_write_ordered(fh, buf, count, datatype, status);
372
373 MPI_Datatype bigtype;
374 int ierr;
375 ierr = Type_contiguous_c(count, datatype, &bigtype);
376 if (ierr != MPI_SUCCESS)
377 return ierr;
378 ierr = MPI_Type_commit(&bigtype);
379 if (ierr != MPI_SUCCESS)
380 return ierr;
381
382 ierr = MPI_File_write_ordered(fh, buf, 1, bigtype, status);
383 if (ierr != MPI_SUCCESS)
384 return ierr;
385
386 ierr = MPI_Type_free(&bigtype);
387 if (ierr != MPI_SUCCESS)
388 return ierr;
389 return MPI_SUCCESS;
390 }
391
398 inline int
399 File_read_at_c(MPI_File fh,
400 MPI_Offset offset,
401 void *buf,
402 MPI_Count count,
403 MPI_Datatype datatype,
404 MPI_Status *status)
405 {
406 if (count <= LargeCount::mpi_max_int_count)
407 return MPI_File_read_at(fh, offset, buf, count, datatype, status);
408
409 MPI_Datatype bigtype;
410 int ierr;
411 ierr = Type_contiguous_c(count, datatype, &bigtype);
412 if (ierr != MPI_SUCCESS)
413 return ierr;
414 ierr = MPI_Type_commit(&bigtype);
415 if (ierr != MPI_SUCCESS)
416 return ierr;
417
418 ierr = MPI_File_read_at(fh, offset, buf, 1, bigtype, status);
419 if (ierr != MPI_SUCCESS)
420 return ierr;
421
422 ierr = MPI_Type_free(&bigtype);
423 if (ierr != MPI_SUCCESS)
424 return ierr;
425 return MPI_SUCCESS;
426 }
427
434 inline int
435 File_read_at_all_c(MPI_File fh,
436 MPI_Offset offset,
437 void *buf,
438 MPI_Count count,
439 MPI_Datatype datatype,
440 MPI_Status *status)
441 {
442 if (count <= LargeCount::mpi_max_int_count)
443 return MPI_File_read_at_all(fh, offset, buf, count, datatype, status);
444
445 MPI_Datatype bigtype;
446 int ierr;
447 ierr = Type_contiguous_c(count, datatype, &bigtype);
448 if (ierr != MPI_SUCCESS)
449 return ierr;
450 ierr = MPI_Type_commit(&bigtype);
451 if (ierr != MPI_SUCCESS)
452 return ierr;
453
454 ierr = MPI_File_read_at_all(fh, offset, buf, 1, bigtype, status);
455 if (ierr != MPI_SUCCESS)
456 return ierr;
457
458 ierr = MPI_Type_free(&bigtype);
459 if (ierr != MPI_SUCCESS)
460 return ierr;
461 return MPI_SUCCESS;
462 }
463
464 } // namespace LargeCount
465 } // namespace MPI
466} // namespace Utilities
467
468#endif
469
470DEAL_II_NAMESPACE_CLOSE
471#endif
MPI_Comm
DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:38
DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
#define DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
Definition config.h:636
DEAL_II_NAMESPACE_CLOSE
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:39
DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
#define DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
Definition config.h:680
InitializeLibrary::MPI
@ MPI
comm
const MPI_Comm comm
Definition mpi.cc:912
Utilities::MPI::LargeCount::File_write_at_c
int File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:292
Utilities::MPI::LargeCount::File_write_ordered_c
int File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:364
Utilities::MPI::LargeCount::File_write_at_all_c
int File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:328
Utilities::MPI::LargeCount::Recv_c
int Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
Definition mpi_large_count.h:190
Utilities::MPI::LargeCount::File_read_at_c
int File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:399
Utilities::MPI::LargeCount::Type_contiguous_c
int Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype, MPI_Datatype *newtype)
Definition mpi_large_count.h:64
Utilities::MPI::LargeCount::Bcast_c
int Bcast_c(void *buf, MPI_Count count, MPI_Datatype datatype, unsigned int root_mpi_rank, MPI_Comm comm)
Definition mpi_large_count.h:254
Utilities::MPI::LargeCount::Get_count_c
int Get_count_c(const MPI_Status *status, MPI_Datatype datatype, MPI_Count *count)
Definition mpi_large_count.h:233
Utilities::MPI::LargeCount::Send_c
int Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
Definition mpi_large_count.h:151
Utilities::MPI::LargeCount::File_read_at_all_c
int File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
Definition mpi_large_count.h:435
Utilities::MPI::LargeCount::mpi_max_int_count
constexpr MPI_Count mpi_max_int_count
Definition mpi_large_count.h:55
types
Definition types.h:30
Generated by doxygen 1.9.8