Loading [MathJax]/extensions/TeX/newcommand.js
 deal.II version GIT relicensing-3075-gc235bd4825 2025-04-15 08:10:00+00:00
\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}} \newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=} \newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]} \newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages Concepts
mpi_large_count.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2022 - 2023 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15//
16// The content of this file is a slightly modified version of the
17// header-only library BigMPICompat - a tiny MPI 4.x compatibility
18// library released under MIT license at
19// https://github.com/tjhei/BigMPICompat and relicensed and included
20// here with permission.
21//
22
23#ifndef BIG_MPI_COMPAT_H
24#define BIG_MPI_COMPAT_H
25
26#include <deal.II/base/config.h>
27
28#ifdef DEAL_II_WITH_MPI
29
31# include <mpi.h>
33
34# ifndef MPI_VERSION
35# error "Your MPI implementation does not define MPI_VERSION!"
36# endif
37
38# if MPI_VERSION < 3
39# error "BigMPICompat requires at least MPI 3.0"
40# endif
41
42
43// required for std::numeric_limits used below.
44# include <limits>
45
47
48namespace Utilities
49{
50 namespace MPI
51 {
57 namespace LargeCount
58 {
63 inline constexpr MPI_Count mpi_max_int_count =
64 std::numeric_limits<int>::max();
65
71 inline int
72 Type_contiguous_c(MPI_Count count,
73 MPI_Datatype oldtype,
74 MPI_Datatype *newtype)
75 {
76# if MPI_VERSION >= 4
77 return MPI_Type_contiguous_c(count, oldtype, newtype);
78# else
80 return MPI_Type_contiguous(count, oldtype, newtype);
81 else
82 {
83 int ierr;
84 const MPI_Count max_signed_int = LargeCount::mpi_max_int_count;
85
86 MPI_Count size_old;
87 ierr = MPI_Type_size_x(oldtype, &size_old);
88
89 MPI_Count n_chunks = count / max_signed_int;
90 MPI_Count n_remaining_elements = count % max_signed_int;
91
92 MPI_Datatype chunks;
93 ierr = MPI_Type_vector(
94 n_chunks, max_signed_int, max_signed_int, oldtype, &chunks);
95 if (ierr != MPI_SUCCESS)
96 return ierr;
97
98 MPI_Datatype remainder;
99 ierr =
100 MPI_Type_contiguous(n_remaining_elements, oldtype, &remainder);
101 if (ierr != MPI_SUCCESS)
102 return ierr;
103
104 int blocklengths[2] = {1, 1};
105 MPI_Aint displacements[2] = {0,
106 static_cast<MPI_Aint>(n_chunks) *
107 size_old * max_signed_int};
108 MPI_Datatype types[2] = {chunks, remainder};
109 ierr = MPI_Type_create_struct(
110 2, blocklengths, displacements, types, newtype);
111 if (ierr != MPI_SUCCESS)
112 return ierr;
113
114 ierr = MPI_Type_commit(newtype);
115 if (ierr != MPI_SUCCESS)
116 return ierr;
117
118 ierr = MPI_Type_free(&chunks);
119 if (ierr != MPI_SUCCESS)
120 return ierr;
121
122 ierr = MPI_Type_free(&remainder);
123 if (ierr != MPI_SUCCESS)
124 return ierr;
125
126# ifndef MPI_COMPAT_SKIP_SIZE_CHECK
127 MPI_Count size_new;
128 ierr = MPI_Type_size_x(*newtype, &size_new);
129 if (ierr != MPI_SUCCESS)
130 return ierr;
131
132 if (size_old * count != size_new)
133 {
134 // This error can happen when you are using a very old and
135 // buggy MPI implementation. There is nothing we can do
136 // here, unfortunately. Please update your installation.
137 // std::cerr
138 //<< "MPI_Type_contiguous_c() produced an invalid result.
139 // Expected =
140 //"
141 //<< size_old << " * " << count << " = " << size_old * count
142 //<< " but received " << size_new << std::endl;
143 return MPI_ERR_INTERN;
144 }
145# endif
146
147 return MPI_SUCCESS;
148 }
149# endif
150 }
151
152
158 inline int
159 Send_c(const void *buf,
160 MPI_Count count,
161 MPI_Datatype datatype,
162 int dest,
163 int tag,
165 {
166# if MPI_VERSION >= 4
167 return MPI_Send_c(buf, count, datatype, dest, tag, comm);
168# else
170 return MPI_Send(buf, count, datatype, dest, tag, comm);
171
172 MPI_Datatype bigtype;
173 int ierr;
174 ierr = Type_contiguous_c(count, datatype, &bigtype);
175 if (ierr != MPI_SUCCESS)
176 return ierr;
177 ierr = MPI_Type_commit(&bigtype);
178 if (ierr != MPI_SUCCESS)
179 return ierr;
180
181 ierr = MPI_Send(buf, 1, bigtype, dest, tag, comm);
182 if (ierr != MPI_SUCCESS)
183 return ierr;
184
185 ierr = MPI_Type_free(&bigtype);
186 if (ierr != MPI_SUCCESS)
187 return ierr;
188 return MPI_SUCCESS;
189# endif
190 }
191
197 inline int
198 Recv_c(void *buf,
199 MPI_Count count,
200 MPI_Datatype datatype,
201 int source,
202 int tag,
204 MPI_Status *status)
205 {
206# if MPI_VERSION >= 4
207 return MPI_Recv_c(buf, count, datatype, source, tag, comm, status);
208# else
210 return MPI_Recv(buf, count, datatype, source, tag, comm, status);
211
212 MPI_Datatype bigtype;
213 int ierr;
214 ierr = Type_contiguous_c(count, datatype, &bigtype);
215 if (ierr != MPI_SUCCESS)
216 return ierr;
217
218 ierr = MPI_Type_commit(&bigtype);
219 if (ierr != MPI_SUCCESS)
220 return ierr;
221
222 ierr = MPI_Recv(buf, 1, bigtype, source, tag, comm, status);
223 if (ierr != MPI_SUCCESS)
224 return ierr;
225
226 ierr = MPI_Type_free(&bigtype);
227 if (ierr != MPI_SUCCESS)
228 return ierr;
229 return MPI_SUCCESS;
230# endif
231 }
232
239 inline int
240 Bcast_c(void *buf,
241 MPI_Count count,
242 MPI_Datatype datatype,
243 unsigned int root_mpi_rank,
245 {
246# if MPI_VERSION >= 4
247 return MPI_Bcast_c(buf, count, datatype, root_mpi_rank, comm);
248# else
250 return MPI_Bcast(buf, count, datatype, root_mpi_rank, comm);
251
252 MPI_Datatype bigtype;
253 int ierr;
254 ierr = Type_contiguous_c(count, datatype, &bigtype);
255 if (ierr != MPI_SUCCESS)
256 return ierr;
257 ierr = MPI_Type_commit(&bigtype);
258 if (ierr != MPI_SUCCESS)
259 return ierr;
260
261 ierr = MPI_Bcast(buf, 1, bigtype, root_mpi_rank, comm);
262 if (ierr != MPI_SUCCESS)
263 return ierr;
264
265 ierr = MPI_Type_free(&bigtype);
266 if (ierr != MPI_SUCCESS)
267 return ierr;
268 return MPI_SUCCESS;
269# endif
270 }
271
277 inline int
278 File_write_at_c(MPI_File fh,
279 MPI_Offset offset,
280 const void *buf,
281 MPI_Count count,
282 MPI_Datatype datatype,
283 MPI_Status *status)
284 {
286 return MPI_File_write_at(fh, offset, buf, count, datatype, status);
287
288 MPI_Datatype bigtype;
289 int ierr;
290 ierr = Type_contiguous_c(count, datatype, &bigtype);
291 if (ierr != MPI_SUCCESS)
292 return ierr;
293 ierr = MPI_Type_commit(&bigtype);
294 if (ierr != MPI_SUCCESS)
295 return ierr;
296
297 ierr = MPI_File_write_at(fh, offset, buf, 1, bigtype, status);
298 if (ierr != MPI_SUCCESS)
299 return ierr;
300
301 ierr = MPI_Type_free(&bigtype);
302 if (ierr != MPI_SUCCESS)
303 return ierr;
304 return MPI_SUCCESS;
305 }
306
313 inline int
315 MPI_Offset offset,
316 const void *buf,
317 MPI_Count count,
318 MPI_Datatype datatype,
319 MPI_Status *status)
320 {
322 return MPI_File_write_at_all(
323 fh, offset, buf, count, datatype, status);
324
325 MPI_Datatype bigtype;
326 int ierr;
327 ierr = Type_contiguous_c(count, datatype, &bigtype);
328 if (ierr != MPI_SUCCESS)
329 return ierr;
330 ierr = MPI_Type_commit(&bigtype);
331 if (ierr != MPI_SUCCESS)
332 return ierr;
333
334 ierr = MPI_File_write_at_all(fh, offset, buf, 1, bigtype, status);
335 if (ierr != MPI_SUCCESS)
336 return ierr;
337
338 ierr = MPI_Type_free(&bigtype);
339 if (ierr != MPI_SUCCESS)
340 return ierr;
341 return MPI_SUCCESS;
342 }
343
349 inline int
351 const void *buf,
352 MPI_Count count,
353 MPI_Datatype datatype,
354 MPI_Status *status)
355 {
357 return MPI_File_write_ordered(fh, buf, count, datatype, status);
358
359 MPI_Datatype bigtype;
360 int ierr;
361 ierr = Type_contiguous_c(count, datatype, &bigtype);
362 if (ierr != MPI_SUCCESS)
363 return ierr;
364 ierr = MPI_Type_commit(&bigtype);
365 if (ierr != MPI_SUCCESS)
366 return ierr;
367
368 ierr = MPI_File_write_ordered(fh, buf, 1, bigtype, status);
369 if (ierr != MPI_SUCCESS)
370 return ierr;
371
372 ierr = MPI_Type_free(&bigtype);
373 if (ierr != MPI_SUCCESS)
374 return ierr;
375 return MPI_SUCCESS;
376 }
377
384 inline int
385 File_read_at_c(MPI_File fh,
386 MPI_Offset offset,
387 void *buf,
388 MPI_Count count,
389 MPI_Datatype datatype,
390 MPI_Status *status)
391 {
393 return MPI_File_read_at(fh, offset, buf, count, datatype, status);
394
395 MPI_Datatype bigtype;
396 int ierr;
397 ierr = Type_contiguous_c(count, datatype, &bigtype);
398 if (ierr != MPI_SUCCESS)
399 return ierr;
400 ierr = MPI_Type_commit(&bigtype);
401 if (ierr != MPI_SUCCESS)
402 return ierr;
403
404 ierr = MPI_File_read_at(fh, offset, buf, 1, bigtype, status);
405 if (ierr != MPI_SUCCESS)
406 return ierr;
407
408 ierr = MPI_Type_free(&bigtype);
409 if (ierr != MPI_SUCCESS)
410 return ierr;
411 return MPI_SUCCESS;
412 }
413
420 inline int
422 MPI_Offset offset,
423 void *buf,
424 MPI_Count count,
425 MPI_Datatype datatype,
426 MPI_Status *status)
427 {
429 return MPI_File_read_at_all(fh, offset, buf, count, datatype, status);
430
431 MPI_Datatype bigtype;
432 int ierr;
433 ierr = Type_contiguous_c(count, datatype, &bigtype);
434 if (ierr != MPI_SUCCESS)
435 return ierr;
436 ierr = MPI_Type_commit(&bigtype);
437 if (ierr != MPI_SUCCESS)
438 return ierr;
439
440 ierr = MPI_File_read_at_all(fh, offset, buf, 1, bigtype, status);
441 if (ierr != MPI_SUCCESS)
442 return ierr;
443
444 ierr = MPI_Type_free(&bigtype);
445 if (ierr != MPI_SUCCESS)
446 return ierr;
447 return MPI_SUCCESS;
448 }
449
450 } // namespace LargeCount
451 } // namespace MPI
452} // namespace Utilities
453
455
456#endif
457#endif
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:35
#define DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
Definition config.h:597
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:36
#define DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
Definition config.h:640
const MPI_Comm comm
Definition mpi.cc:924
int File_write_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
int File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
int Type_contiguous_c(MPI_Count count, MPI_Datatype oldtype, MPI_Datatype *newtype)
int Bcast_c(void *buf, MPI_Count count, MPI_Datatype datatype, unsigned int root_mpi_rank, MPI_Comm comm)
int Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
int File_read_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
constexpr MPI_Count mpi_max_int_count
Definition types.h:32