deal.II version GIT relicensing-2901-g19332422bd 2025-03-23 19:50:00+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
mpi_consensus_algorithms.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2020 - 2023 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15#ifndef dealii_mpi_consensus_algorithm_h
16#define dealii_mpi_consensus_algorithm_h
17
18#include <deal.II/base/config.h>
19
20#include <deal.II/base/mpi.h>
21#include <deal.II/base/mpi.templates.h>
23
24#include <set>
25#include <vector>
26
28
29
30namespace Utilities
31{
32 namespace MPI
33 {
132 namespace ConsensusAlgorithms
133 {
163 template <typename RequestType, typename AnswerType>
164 class DEAL_II_DEPRECATED_EARLY Process
165 {
166 public:
171 virtual ~Process() = default;
172
179 virtual std::vector<unsigned int>
181
191 virtual void
192 create_request(const unsigned int other_rank, RequestType &send_buffer);
193
206 virtual void
207 answer_request(const unsigned int other_rank,
208 const RequestType &buffer_recv,
209 AnswerType &request_buffer);
210
218 virtual void
219 read_answer(const unsigned int other_rank,
220 const AnswerType &recv_buffer);
221 };
222
223
224
238 template <typename RequestType, typename AnswerType>
240 {
241 public:
245 Interface() = default;
246
251 virtual ~Interface() = default;
252
266 DEAL_II_DEPRECATED_EARLY
267 std::vector<unsigned int>
269
293 virtual std::vector<unsigned int>
295 const std::vector<unsigned int> &targets,
296 const std::function<RequestType(const unsigned int)> &create_request,
297 const std::function<AnswerType(const unsigned int,
298 const RequestType &)> &answer_request,
299 const std::function<void(const unsigned int, const AnswerType &)>
300 &process_answer,
301 const MPI_Comm comm) = 0;
302 };
303
304
318 template <typename RequestType, typename AnswerType>
319 class NBX : public Interface<RequestType, AnswerType>
320 {
321 public:
325 NBX() = default;
326
330 virtual ~NBX() = default;
331
332 // Import the declarations from the base class.
333 using Interface<RequestType, AnswerType>::run;
334
338 virtual std::vector<unsigned int>
340 const std::vector<unsigned int> &targets,
341 const std::function<RequestType(const unsigned int)> &create_request,
342 const std::function<AnswerType(const unsigned int,
343 const RequestType &)> &answer_request,
344 const std::function<void(const unsigned int, const AnswerType &)>
345 &process_answer,
346 const MPI_Comm comm) override;
347
348 private:
349#ifdef DEAL_II_WITH_MPI
353 std::vector<std::vector<char>> send_buffers;
354
358 std::vector<MPI_Request> send_requests;
359
367 std::vector<std::unique_ptr<std::vector<char>>> request_buffers;
368
372 std::vector<std::unique_ptr<MPI_Request>> request_requests;
373
378
379 // request for barrier
380 MPI_Request barrier_request;
381#endif
382
386 std::set<unsigned int> requesting_processes;
387
393 bool
395 const std::function<void(const unsigned int, const AnswerType &)>
396 &process_answer,
397 const MPI_Comm comm);
398
403 void
405
411 bool
413
419 void
421 const std::function<AnswerType(const unsigned int,
422 const RequestType &)> &answer_request,
423 const MPI_Comm comm);
424
429 void
431 const std::vector<unsigned int> &targets,
432 const std::function<RequestType(const unsigned int)> &create_request,
433 const MPI_Comm comm);
434
439 void
441 };
442
443
488 template <typename RequestType, typename AnswerType>
489 std::vector<unsigned int>
490 nbx(const std::vector<unsigned int> &targets,
491 const std::function<RequestType(const unsigned int)> &create_request,
492 const std::function<AnswerType(const unsigned int,
493 const RequestType &)> &answer_request,
494 const std::function<void(const unsigned int, const AnswerType &)>
495 &process_answer,
496 const MPI_Comm comm);
497
535 template <typename RequestType>
536 std::vector<unsigned int>
537 nbx(const std::vector<unsigned int> &targets,
538 const std::function<RequestType(const unsigned int)> &create_request,
539 const std::function<void(const unsigned int, const RequestType &)>
540 &process_request,
541 const MPI_Comm comm);
542
568 template <typename RequestType, typename AnswerType>
569 class PEX : public Interface<RequestType, AnswerType>
570 {
571 public:
575 PEX() = default;
576
580 virtual ~PEX() = default;
581
582 // Import the declarations from the base class.
583 using Interface<RequestType, AnswerType>::run;
584
588 virtual std::vector<unsigned int>
590 const std::vector<unsigned int> &targets,
591 const std::function<RequestType(const unsigned int)> &create_request,
592 const std::function<AnswerType(const unsigned int,
593 const RequestType &)> &answer_request,
594 const std::function<void(const unsigned int, const AnswerType &)>
595 &process_answer,
596 const MPI_Comm comm) override;
597
598 private:
599#ifdef DEAL_II_WITH_MPI
603 std::vector<std::vector<char>> send_buffers;
604
608 std::vector<std::vector<char>> recv_buffers;
609
613 std::vector<MPI_Request> send_request_requests;
614
618 std::vector<std::vector<char>> requests_buffers;
619
623 std::vector<MPI_Request> send_answer_requests;
624#endif
628 std::set<unsigned int> requesting_processes;
629
634 unsigned int
636 const std::vector<unsigned int> &targets,
637 const std::function<RequestType(const unsigned int)> &create_request,
638 const MPI_Comm comm);
639
644 void
646 const unsigned int index,
647 const std::function<AnswerType(const unsigned int,
648 const RequestType &)> &answer_request,
649 const MPI_Comm comm);
650
655 void
657 const unsigned int n_targets,
658 const std::function<void(const unsigned int, const AnswerType &)>
659 &process_answer,
660 const MPI_Comm comm);
661
666 void
668 };
669
670
671
728 template <typename RequestType, typename AnswerType>
729 std::vector<unsigned int>
730 pex(const std::vector<unsigned int> &targets,
731 const std::function<RequestType(const unsigned int)> &create_request,
732 const std::function<AnswerType(const unsigned int,
733 const RequestType &)> &answer_request,
734 const std::function<void(const unsigned int, const AnswerType &)>
735 &process_answer,
736 const MPI_Comm comm);
737
775 template <typename RequestType>
776 std::vector<unsigned int>
777 pex(const std::vector<unsigned int> &targets,
778 const std::function<RequestType(const unsigned int)> &create_request,
779 const std::function<void(const unsigned int, const RequestType &)>
780 &process_request,
781 const MPI_Comm comm);
782
783
788 template <typename RequestType, typename AnswerType>
789 class Serial : public Interface<RequestType, AnswerType>
790 {
791 public:
795 Serial() = default;
796
797 // Import the declarations from the base class.
798 using Interface<RequestType, AnswerType>::run;
799
803 virtual std::vector<unsigned int>
805 const std::vector<unsigned int> &targets,
806 const std::function<RequestType(const unsigned int)> &create_request,
807 const std::function<AnswerType(const unsigned int,
808 const RequestType &)> &answer_request,
809 const std::function<void(const unsigned int, const AnswerType &)>
810 &process_answer,
811 const MPI_Comm comm) override;
812 };
813
814
815
848 template <typename RequestType, typename AnswerType>
849 std::vector<unsigned int>
851 const std::vector<unsigned int> &targets,
852 const std::function<RequestType(const unsigned int)> &create_request,
853 const std::function<AnswerType(const unsigned int, const RequestType &)>
854 &answer_request,
855 const std::function<void(const unsigned int, const AnswerType &)>
856 &process_answer,
857 const MPI_Comm comm);
858
888 template <typename RequestType>
889 std::vector<unsigned int>
891 const std::vector<unsigned int> &targets,
892 const std::function<RequestType(const unsigned int)> &create_request,
893 const std::function<void(const unsigned int, const RequestType &)>
894 &process_request,
895 const MPI_Comm comm);
896
897
898
911 template <typename RequestType, typename AnswerType>
912 class Selector : public Interface<RequestType, AnswerType>
913 {
914 public:
918 Selector() = default;
919
923 virtual ~Selector() = default;
924
925 // Import the declarations from the base class.
926 using Interface<RequestType, AnswerType>::run;
927
933 virtual std::vector<unsigned int>
935 const std::vector<unsigned int> &targets,
936 const std::function<RequestType(const unsigned int)> &create_request,
937 const std::function<AnswerType(const unsigned int,
938 const RequestType &)> &answer_request,
939 const std::function<void(const unsigned int, const AnswerType &)>
940 &process_answer,
941 const MPI_Comm comm) override;
942
943 private:
944 // Pointer to the actual ConsensusAlgorithms::Interface implementation.
945 std::shared_ptr<Interface<RequestType, AnswerType>> consensus_algo;
946 };
947
948
949
994 template <typename RequestType, typename AnswerType>
995 std::vector<unsigned int>
997 const std::vector<unsigned int> &targets,
998 const std::function<RequestType(const unsigned int)> &create_request,
999 const std::function<AnswerType(const unsigned int, const RequestType &)>
1000 &answer_request,
1001 const std::function<void(const unsigned int, const AnswerType &)>
1002 &process_answer,
1003 const MPI_Comm comm);
1004
1042 template <typename RequestType>
1043 std::vector<unsigned int>
1045 const std::vector<unsigned int> &targets,
1046 const std::function<RequestType(const unsigned int)> &create_request,
1047 const std::function<void(const unsigned int, const RequestType &)>
1048 &process_request,
1049 const MPI_Comm comm);
1050
1051
1052
1053#ifndef DOXYGEN
1054 // Implementation of the functions in this namespace.
1055
1056 template <typename RequestType, typename AnswerType>
1057 std::vector<unsigned int>
1058 nbx(const std::vector<unsigned int> &targets,
1059 const std::function<RequestType(const unsigned int)> &create_request,
1060 const std::function<AnswerType(const unsigned int,
1061 const RequestType &)> &answer_request,
1062 const std::function<void(const unsigned int, const AnswerType &)>
1063 &process_answer,
1064 const MPI_Comm comm)
1065 {
1067 targets, create_request, answer_request, process_answer, comm);
1068 }
1069
1070
1071
1072 template <typename RequestType>
1073 std::vector<unsigned int>
1074 nbx(const std::vector<unsigned int> &targets,
1075 const std::function<RequestType(const unsigned int)> &create_request,
1076 const std::function<void(const unsigned int, const RequestType &)>
1077 &process_request,
1078 const MPI_Comm comm)
1079 {
1080 // TODO: For the moment, simply implement this special case by
1081 // forwarding to the other function with rewritten function
1082 // objects and using an empty type as answer type. This way,
1083 // we have the interface in place and can provide a more
1084 // efficient implementation later on.
1085 using EmptyType = std::tuple<>;
1086
1087 return nbx<RequestType, EmptyType>(
1088 targets,
1089 create_request,
1090 // answer_request:
1091 [&process_request](const unsigned int source_rank,
1092 const RequestType &request) -> EmptyType {
1093 process_request(source_rank, request);
1094 // Return something. What it is is arbitrary here, except that
1095 // we want it to be as small an object as possible. Using
1096 // std::tuple<> is interpreted as an empty object that is packed
1097 // down to a zero-length char array.
1098 return {};
1099 },
1100 // process_answer:
1101 [](const unsigned int /*target_rank */,
1102 const EmptyType & /*answer*/) {},
1103 comm);
1104 }
1105
1106
1107
1108 template <typename RequestType, typename AnswerType>
1109 std::vector<unsigned int>
1110 pex(const std::vector<unsigned int> &targets,
1111 const std::function<RequestType(const unsigned int)> &create_request,
1112 const std::function<AnswerType(const unsigned int,
1113 const RequestType &)> &answer_request,
1114 const std::function<void(const unsigned int, const AnswerType &)>
1115 &process_answer,
1116 const MPI_Comm comm)
1117 {
1118 return PEX<RequestType, AnswerType>().run(
1119 targets, create_request, answer_request, process_answer, comm);
1120 }
1121
1122
1123
1124 template <typename RequestType>
1125 std::vector<unsigned int>
1126 pex(const std::vector<unsigned int> &targets,
1127 const std::function<RequestType(const unsigned int)> &create_request,
1128 const std::function<void(const unsigned int, const RequestType &)>
1129 &process_request,
1130 const MPI_Comm comm)
1131 {
1132 // TODO: For the moment, simply implement this special case by
1133 // forwarding to the other function with rewritten function
1134 // objects and using an empty type as answer type. This way,
1135 // we have the interface in place and can provide a more
1136 // efficient implementation later on.
1137 using EmptyType = std::tuple<>;
1138
1139 return pex<RequestType, EmptyType>(
1140 targets,
1141 create_request,
1142 // answer_request:
1143 [&process_request](const unsigned int source_rank,
1144 const RequestType &request) -> EmptyType {
1145 process_request(source_rank, request);
1146 // Return something. What it is is arbitrary here, except that
1147 // we want it to be as small an object as possible. Using
1148 // std::tuple<> is interpreted as an empty object that is packed
1149 // down to a zero-length char array.
1150 return {};
1151 },
1152 // process_answer:
1153 [](const unsigned int /*target_rank */,
1154 const EmptyType & /*answer*/) {},
1155 comm);
1156 }
1157
1158
1159
1160 template <typename RequestType, typename AnswerType>
1161 std::vector<unsigned int>
1162 serial(
1163 const std::vector<unsigned int> &targets,
1164 const std::function<RequestType(const unsigned int)> &create_request,
1165 const std::function<AnswerType(const unsigned int, const RequestType &)>
1166 &answer_request,
1167 const std::function<void(const unsigned int, const AnswerType &)>
1168 &process_answer,
1169 const MPI_Comm comm)
1170 {
1171 return Serial<RequestType, AnswerType>().run(
1172 targets, create_request, answer_request, process_answer, comm);
1173 }
1174
1175
1176
1177 template <typename RequestType>
1178 std::vector<unsigned int>
1179 serial(
1180 const std::vector<unsigned int> &targets,
1181 const std::function<RequestType(const unsigned int)> &create_request,
1182 const std::function<void(const unsigned int, const RequestType &)>
1183 &process_request,
1184 const MPI_Comm comm)
1185 {
1186 // TODO: For the moment, simply implement this special case by
1187 // forwarding to the other function with rewritten function
1188 // objects and using an empty type as answer type. This way,
1189 // we have the interface in place and can provide a more
1190 // efficient implementation later on.
1191 using EmptyType = std::tuple<>;
1192
1193 return serial<RequestType, EmptyType>(
1194 targets,
1195 create_request,
1196 // answer_request:
1197 [&process_request](const unsigned int source_rank,
1198 const RequestType &request) -> EmptyType {
1199 process_request(source_rank, request);
1200 // Return something. What it is is arbitrary here, except that
1201 // we want it to be as small an object as possible. Using
1202 // std::tuple<> is interpreted as an empty object that is packed
1203 // down to a zero-length char array.
1204 return {};
1205 },
1206 // process_answer:
1207 [](const unsigned int /*target_rank */,
1208 const EmptyType & /*answer*/) {},
1209 comm);
1210 }
1211
1212
1213
1214 template <typename RequestType, typename AnswerType>
1215 std::vector<unsigned int>
1216 selector(
1217 const std::vector<unsigned int> &targets,
1218 const std::function<RequestType(const unsigned int)> &create_request,
1219 const std::function<AnswerType(const unsigned int, const RequestType &)>
1220 &answer_request,
1221 const std::function<void(const unsigned int, const AnswerType &)>
1222 &process_answer,
1223 const MPI_Comm comm)
1224 {
1225 return Selector<RequestType, AnswerType>().run(
1226 targets, create_request, answer_request, process_answer, comm);
1227 }
1228
1229
1230
1231 template <typename RequestType>
1232 std::vector<unsigned int>
1233 selector(
1234 const std::vector<unsigned int> &targets,
1235 const std::function<RequestType(const unsigned int)> &create_request,
1236 const std::function<void(const unsigned int, const RequestType &)>
1237 &process_request,
1238 const MPI_Comm comm)
1239 {
1240 // TODO: For the moment, simply implement this special case by
1241 // forwarding to the other function with rewritten function
1242 // objects and using an empty type as answer type. This way,
1243 // we have the interface in place and can provide a more
1244 // efficient implementation later on.
1245 using EmptyType = std::tuple<>;
1246
1247 return selector<RequestType, EmptyType>(
1248 targets,
1249 create_request,
1250 // answer_request:
1251 [&process_request](const unsigned int source_rank,
1252 const RequestType &request) -> EmptyType {
1253 process_request(source_rank, request);
1254 // Return something. What it is is arbitrary here, except that
1255 // we want it to be as small an object as possible. Using
1256 // std::tuple<> is interpreted as an empty object that is packed
1257 // down to a zero-length char array.
1258 return {};
1259 },
1260 // process_answer:
1261 [](const unsigned int /*target_rank */,
1262 const EmptyType & /*answer*/) {},
1263 comm);
1264 }
1265
1266#endif
1267
1268
1269 } // namespace ConsensusAlgorithms
1270 } // end of namespace MPI
1271} // end of namespace Utilities
1272
1273
1274
1275#ifndef DOXYGEN
1276
1277// ----------------- Implementation of template functions
1278
1279namespace Utilities
1280{
1281 namespace MPI
1282 {
1283 namespace ConsensusAlgorithms
1284 {
1285 namespace internal
1286 {
1291 inline bool
1292 has_unique_elements(const std::vector<unsigned int> &targets)
1293 {
1294 std::vector<unsigned int> my_destinations = targets;
1295 std::sort(my_destinations.begin(), my_destinations.end());
1296 return (std::adjacent_find(my_destinations.begin(),
1297 my_destinations.end()) ==
1298 my_destinations.end());
1299 }
1300
1301
1302
1306 inline void
1307 handle_exception(std::exception_ptr &&exception, const MPI_Comm comm)
1308 {
1309# ifdef DEAL_II_WITH_MPI
1310 // an exception within a ConsensusAlgorithm likely causes an
1311 // MPI deadlock. Abort with a reasonable error message instead.
1312 try
1313 {
1314 std::rethrow_exception(exception);
1315 }
1316 catch (ExceptionBase &exc)
1317 {
1318 // report name of the deal.II exception:
1319 std::cerr
1320 << std::endl
1321 << std::endl
1322 << "----------------------------------------------------"
1323 << std::endl;
1324 std::cerr
1325 << "Exception '" << exc.get_exc_name() << "'"
1326 << " on rank " << Utilities::MPI::this_mpi_process(comm)
1327 << " on processing: " << std::endl
1328 << exc.what() << std::endl
1329 << "Aborting!" << std::endl
1330 << "----------------------------------------------------"
1331 << std::endl;
1332
1333 // Then bring down the whole MPI world
1334 MPI_Abort(comm, 255);
1335 }
1336 catch (std::exception &exc)
1337 {
1338 std::cerr
1339 << std::endl
1340 << std::endl
1341 << "----------------------------------------------------"
1342 << std::endl;
1343 std::cerr
1344 << "Exception within ConsensusAlgorithm"
1345 << " on rank " << Utilities::MPI::this_mpi_process(comm)
1346 << " on processing: " << std::endl
1347 << exc.what() << std::endl
1348 << "Aborting!" << std::endl
1349 << "----------------------------------------------------"
1350 << std::endl;
1351
1352 // Then bring down the whole MPI world
1353 MPI_Abort(comm, 255);
1354 }
1355 catch (...)
1356 {
1357 std::cerr
1358 << std::endl
1359 << std::endl
1360 << "----------------------------------------------------"
1361 << std::endl;
1362 std::cerr
1363 << "Unknown exception within ConsensusAlgorithm!" << std::endl
1364 << "Aborting!" << std::endl
1365 << "----------------------------------------------------"
1366 << std::endl;
1367
1368 // Then bring down the whole MPI world
1369 MPI_Abort(comm, 255);
1370 }
1371# else
1372 (void)comm;
1373
1374 // No need to be concerned about deadlocks without MPI.
1375 // Defer to exception handling further up the callstack.
1376 std::rethrow_exception(exception);
1377# endif
1378 }
1379 } // namespace internal
1380
1381
1382
1383 template <typename RequestType, typename AnswerType>
1384 void
1386 const RequestType &,
1387 AnswerType &)
1388 {
1389 // nothing to do
1390 }
1391
1392
1393
1394 template <typename RequestType, typename AnswerType>
1395 void
1397 RequestType &)
1398 {
1399 // nothing to do
1400 }
1401
1402
1403
1404 template <typename RequestType, typename AnswerType>
1405 void
1407 const AnswerType &)
1408 {
1409 // nothing to do
1410 }
1411
1412
1413
1414 template <typename RequestType, typename AnswerType>
1415 std::vector<unsigned int>
1417 Process<RequestType, AnswerType> &process,
1418 const MPI_Comm comm)
1419 {
1420 // Unpack the 'process' object and call the function that takes
1421 // function objects for all operations.
1422 return run(
1423 process.compute_targets(),
1424 /* create_request: */
1425 [&process](const unsigned int target) {
1426 RequestType request;
1427 process.create_request(target, request);
1428 return request;
1429 },
1430 /* answer_request: */
1431 [&process](const unsigned int source, const RequestType &request) {
1432 AnswerType answer;
1433 process.answer_request(source, request, answer);
1434 return answer;
1435 },
1436 /* process_answer: */
1437 [&process](const unsigned int target, const AnswerType &answer) {
1438 process.read_answer(target, answer);
1439 },
1440 comm);
1441 }
1442
1443
1444
1445 template <typename RequestType, typename AnswerType>
1446 std::vector<unsigned int>
1448 const std::vector<unsigned int> &targets,
1449 const std::function<RequestType(const unsigned int)> &create_request,
1450 const std::function<AnswerType(const unsigned int, const RequestType &)>
1451 &answer_request,
1452 const std::function<void(const unsigned int, const AnswerType &)>
1453 &process_answer,
1454 const MPI_Comm comm)
1455 {
1456 Assert(internal::has_unique_elements(targets),
1457 ExcMessage("The consensus algorithms expect that each process "
1458 "only sends a single message to another process, "
1459 "but the targets provided include duplicates."));
1460
1461 static CollectiveMutex mutex;
1462 CollectiveMutex::ScopedLock lock(mutex, comm);
1463
1464 try
1465 {
1466 // 1) Send data to identified targets and start receiving
1467 // the answers from these very same processes.
1468 start_communication(targets, create_request, comm);
1469
1470 // 2) Until all posted receive operations are known to have
1471 // completed, answer requests and keep checking whether all
1472 // requests of this process have been answered.
1473 //
1474 // The requests that we catch in the answer_requests()
1475 // function originate elsewhere, that is, they are not in
1476 // response to our own messages
1477 //
1478 // Note also that we may not catch all incoming requests in
1479 // the following two lines: our own requests may have been
1480 // satisfied before we've dealt with all incoming requests.
1481 // That's ok: We will get around to dealing with all
1482 // remaining message later. We just want to move on to the
1483 // next step as early as possible.
1484 while (all_locally_originated_receives_are_completed(process_answer,
1485 comm) == false)
1486 maybe_answer_one_request(answer_request, comm);
1487
1488 // 3) Signal to all other processes that all requests of this
1489 // process have been answered
1490 signal_finish(comm);
1491
1492 // 4) Nevertheless, this process has to keep on answering
1493 // (potential) incoming requests until all processes have
1494 // received the answer to all requests
1495 while (all_remotely_originated_receives_are_completed() == false)
1496 maybe_answer_one_request(answer_request, comm);
1497
1498 // 5) process the answer to all requests
1499 clean_up_and_end_communication(comm);
1500 }
1501 catch (...)
1502 {
1503 internal::handle_exception(std::current_exception(), comm);
1504 }
1505
1506 return std::vector<unsigned int>(requesting_processes.begin(),
1507 requesting_processes.end());
1508 }
1509
1510
1511
1512 template <typename RequestType, typename AnswerType>
1513 void
1515 const std::vector<unsigned int> &targets,
1516 const std::function<RequestType(const unsigned int)> &create_request,
1517 const MPI_Comm comm)
1518 {
1519# ifdef DEAL_II_WITH_MPI
1520 // 1)
1521 const auto n_targets = targets.size();
1522
1523 const int tag_request = Utilities::MPI::internal::Tags::
1525
1526 // 2) allocate memory
1527 send_requests.resize(n_targets);
1528 send_buffers.resize(n_targets);
1529
1530 {
1531 // 4) send and receive
1532 for (unsigned int index = 0; index < n_targets; ++index)
1533 {
1534 const unsigned int rank = targets[index];
1536
1537 auto &send_buffer = send_buffers[index];
1538 send_buffer =
1539 (create_request ? Utilities::pack(create_request(rank), false) :
1540 std::vector<char>());
1541
1542 // Post a request to send data
1543 auto ierr = MPI_Isend(send_buffer.data(),
1544 send_buffer.size(),
1545 MPI_CHAR,
1546 rank,
1547 tag_request,
1548 comm,
1549 &send_requests[index]);
1550 AssertThrowMPI(ierr);
1551 }
1552
1553 // Also record that we expect an answer from each target we sent
1554 // a request to:
1555 n_outstanding_answers = n_targets;
1556 }
1557# else
1558 (void)targets;
1559 (void)create_request;
1560 (void)comm;
1561# endif
1562 }
1563
1564
1565
1566 template <typename RequestType, typename AnswerType>
1567 bool
1570 const std::function<void(const unsigned int, const AnswerType &)>
1571 &process_answer,
1572 const MPI_Comm comm)
1573 {
1574# ifdef DEAL_II_WITH_MPI
1575 // We know that all requests have come in when we have pending
1576 // messages from all targets with the right tag (some of which we may
1577 // have already taken care of below, after discovering their existence).
1578 // We can check for pending messages with MPI_IProbe, which returns
1579 // immediately with a return code that indicates whether
1580 // it has found a message from any process with a given
1581 // tag.
1582 if (n_outstanding_answers == 0)
1583 return true;
1584 else
1585 {
1586 const int tag_deliver = Utilities::MPI::internal::Tags::
1588
1589 int request_is_pending;
1590 MPI_Status status;
1591 const auto ierr = MPI_Iprobe(
1592 MPI_ANY_SOURCE, tag_deliver, comm, &request_is_pending, &status);
1593 AssertThrowMPI(ierr);
1594
1595 // If there is no pending message with this tag,
1596 // then we are clearly not done receiving everything
1597 // yet -- so return false.
1598 if (request_is_pending == 0)
1599 return false;
1600 else
1601 {
1602 // OK, so we have gotten a reply to our request from
1603 // one rank. Let us process it.
1604 const auto target = status.MPI_SOURCE;
1605
1606 // Then query the size of the message, allocate enough memory,
1607 // receive the data, and process it.
1608 int message_size;
1609 {
1610 const int ierr =
1611 MPI_Get_count(&status, MPI_CHAR, &message_size);
1612 AssertThrowMPI(ierr);
1613 }
1614 std::vector<char> recv_buffer(message_size);
1615
1616 {
1617 const int tag_deliver = Utilities::MPI::internal::Tags::
1619
1620 const int ierr = MPI_Recv(recv_buffer.data(),
1621 recv_buffer.size(),
1622 MPI_CHAR,
1623 target,
1624 tag_deliver,
1625 comm,
1626 MPI_STATUS_IGNORE);
1627 AssertThrowMPI(ierr);
1628 }
1629
1630 if (process_answer)
1631 process_answer(target,
1632 Utilities::unpack<AnswerType>(recv_buffer,
1633 false));
1634
1635 // Finally, remove this rank from the list of outstanding
1636 // targets:
1637 --n_outstanding_answers;
1638
1639 // We could do another go-around from the top of this
1640 // else-branch to see whether there are actually other messages
1641 // that are currently pending. But that would mean spending
1642 // substantial time in receiving answers while we should also be
1643 // sending answers to requests we have received from other
1644 // places. So let it be enough for now. If there are outstanding
1645 // answers, we will get back to this function before long and
1646 // can take care of them then.
1647 return (n_outstanding_answers == 0);
1648 }
1649 }
1650
1651# else
1652 (void)process_answer;
1653 (void)comm;
1654
1655 return true;
1656# endif
1657 }
1658
1659
1660
1661 template <typename RequestType, typename AnswerType>
1662 void
1664 const std::function<AnswerType(const unsigned int, const RequestType &)>
1665 &answer_request,
1666 const MPI_Comm comm)
1667 {
1668# ifdef DEAL_II_WITH_MPI
1669
1670 const int tag_request = Utilities::MPI::internal::Tags::
1672 const int tag_deliver = Utilities::MPI::internal::Tags::
1674
1675 // Check if there is a request pending. By selecting the
1676 // tag_request tag, these are other processes asking for
1677 // our own replies, not these other processes' replies
1678 // to our own requests.
1679 //
1680 // There may be multiple such pending messages. We
1681 // only answer one.
1682 MPI_Status status;
1683 int request_is_pending;
1684 const auto ierr = MPI_Iprobe(
1685 MPI_ANY_SOURCE, tag_request, comm, &request_is_pending, &status);
1686 AssertThrowMPI(ierr);
1687
1688 if (request_is_pending != 0)
1689 {
1690 // Get the rank of the requesting process and add it to the
1691 // list of requesting processes (which may contain duplicates).
1692 const auto other_rank = status.MPI_SOURCE;
1693
1694 Assert(requesting_processes.find(other_rank) ==
1695 requesting_processes.end(),
1696 ExcMessage("Process is requesting a second time!"));
1697 requesting_processes.insert(other_rank);
1698
1699 // get size of incoming message
1700 int number_amount;
1701 auto ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1702 AssertThrowMPI(ierr);
1703
1704 // allocate memory for incoming message
1705 std::vector<char> buffer_recv(number_amount);
1706 ierr = MPI_Recv(buffer_recv.data(),
1707 number_amount,
1708 MPI_CHAR,
1709 other_rank,
1710 tag_request,
1711 comm,
1712 MPI_STATUS_IGNORE);
1713 AssertThrowMPI(ierr);
1714
1715 // Allocate memory for an answer message to the current request,
1716 // and ask the 'process' object to produce an answer:
1717 request_buffers.emplace_back(std::make_unique<std::vector<char>>());
1718 auto &request_buffer = *request_buffers.back();
1719 if (answer_request)
1720 request_buffer =
1721 Utilities::pack(answer_request(other_rank,
1722 Utilities::unpack<RequestType>(
1723 buffer_recv, false)),
1724 false);
1725
1726 // Then initiate sending the answer back to the requester.
1727 request_requests.emplace_back(std::make_unique<MPI_Request>());
1728 ierr = MPI_Isend(request_buffer.data(),
1729 request_buffer.size(),
1730 MPI_CHAR,
1731 other_rank,
1732 tag_deliver,
1733 comm,
1734 request_requests.back().get());
1735 AssertThrowMPI(ierr);
1736 }
1737# else
1738 (void)answer_request;
1739 (void)comm;
1740# endif
1741 }
1742
1743
1744
1745 template <typename RequestType, typename AnswerType>
1746 void
1748 {
1749# ifdef DEAL_II_WITH_MPI
1750 const auto ierr = MPI_Ibarrier(comm, &barrier_request);
1751 AssertThrowMPI(ierr);
1752# else
1753 (void)comm;
1754# endif
1755 }
1756
1757
1758
1759 template <typename RequestType, typename AnswerType>
1760 bool
1761 NBX<RequestType,
1762 AnswerType>::all_remotely_originated_receives_are_completed()
1763 {
1764# ifdef DEAL_II_WITH_MPI
1765 int all_ranks_reached_barrier;
1766 const auto ierr = MPI_Test(&barrier_request,
1767 &all_ranks_reached_barrier,
1768 MPI_STATUS_IGNORE);
1769 AssertThrowMPI(ierr);
1770 return all_ranks_reached_barrier != 0;
1771# else
1772 return true;
1773# endif
1774 }
1775
1776
1777
1778 template <typename RequestType, typename AnswerType>
1779 void
1781 const MPI_Comm comm)
1782 {
1783 (void)comm;
1784# ifdef DEAL_II_WITH_MPI
1785 // clean up
1786 {
1787 if (send_requests.size() > 0)
1788 {
1789 const int ierr = MPI_Waitall(send_requests.size(),
1790 send_requests.data(),
1791 MPI_STATUSES_IGNORE);
1792 AssertThrowMPI(ierr);
1793 }
1794
1795 int ierr = MPI_Wait(&barrier_request, MPI_STATUS_IGNORE);
1796 AssertThrowMPI(ierr);
1797
1798 for (auto &i : request_requests)
1799 {
1800 ierr = MPI_Wait(i.get(), MPI_STATUS_IGNORE);
1801 AssertThrowMPI(ierr);
1802 }
1803
1804 if constexpr (running_in_debug_mode())
1805 {
1806 // note: IBarrier seems to make problem during testing, this
1807 // additional Barrier seems to help
1808 ierr = MPI_Barrier(comm);
1809 AssertThrowMPI(ierr);
1810 }
1811 }
1812# endif
1813 }
1814
1815
1816
1817 template <typename RequestType, typename AnswerType>
1818 std::vector<unsigned int>
1820 const std::vector<unsigned int> &targets,
1821 const std::function<RequestType(const unsigned int)> &create_request,
1822 const std::function<AnswerType(const unsigned int, const RequestType &)>
1823 &answer_request,
1824 const std::function<void(const unsigned int, const AnswerType &)>
1825 &process_answer,
1826 const MPI_Comm comm)
1827 {
1828 Assert(internal::has_unique_elements(targets),
1829 ExcMessage("The consensus algorithms expect that each process "
1830 "only sends a single message to another process, "
1831 "but the targets provided include duplicates."));
1832
1833 static CollectiveMutex mutex;
1834 CollectiveMutex::ScopedLock lock(mutex, comm);
1835
1836 try
1837 {
1838 // 1) Send requests and start receiving the answers.
1839 // In particular, determine how many requests we should expect
1840 // on the current process.
1841 const unsigned int n_requests =
1842 start_communication(targets, create_request, comm);
1843
1844 // 2) Answer requests:
1845 for (unsigned int request = 0; request < n_requests; ++request)
1846 answer_one_request(request, answer_request, comm);
1847
1848 // 3) Process answers:
1849 process_incoming_answers(targets.size(), process_answer, comm);
1850
1851 // 4) Make sure all sends have successfully terminated:
1852 clean_up_and_end_communication();
1853 }
1854 catch (...)
1855 {
1856 internal::handle_exception(std::current_exception(), comm);
1857 }
1858
1859 return std::vector<unsigned int>(requesting_processes.begin(),
1860 requesting_processes.end());
1861 }
1862
1863
1864
1865 template <typename RequestType, typename AnswerType>
1866 unsigned int
1868 const std::vector<unsigned int> &targets,
1869 const std::function<RequestType(const unsigned int)> &create_request,
1870 const MPI_Comm comm)
1871 {
1872# ifdef DEAL_II_WITH_MPI
1873 const int tag_request = Utilities::MPI::internal::Tags::
1875
1876 // 1) determine with which processes this process wants to communicate
1877 // with
1878 const unsigned int n_targets = targets.size();
1879
1880 // 2) determine who wants to communicate with this process
1881 const unsigned int n_sources =
1883
1884 // 2) allocate memory
1885 recv_buffers.resize(n_targets);
1886 send_buffers.resize(n_targets);
1887 send_request_requests.resize(n_targets);
1888
1889 send_answer_requests.resize(n_sources);
1890 requests_buffers.resize(n_sources);
1891
1892 // 4) send and receive
1893 for (unsigned int i = 0; i < n_targets; ++i)
1894 {
1895 const unsigned int rank = targets[i];
1897
1898 // pack data which should be sent
1899 auto &send_buffer = send_buffers[i];
1900 if (create_request)
1901 send_buffer = Utilities::pack(create_request(rank), false);
1902
1903 // start to send data
1904 auto ierr = MPI_Isend(send_buffer.data(),
1905 send_buffer.size(),
1906 MPI_CHAR,
1907 rank,
1908 tag_request,
1909 comm,
1910 &send_request_requests[i]);
1911 AssertThrowMPI(ierr);
1912 }
1913
1914 return n_sources;
1915# else
1916 (void)targets;
1917 (void)create_request;
1918 (void)comm;
1919 return 0;
1920# endif
1921 }
1922
1923
1924
1925 template <typename RequestType, typename AnswerType>
1926 void
1928 const unsigned int index,
1929 const std::function<AnswerType(const unsigned int, const RequestType &)>
1930 &answer_request,
1931 const MPI_Comm comm)
1932 {
1933# ifdef DEAL_II_WITH_MPI
1934 const int tag_request = Utilities::MPI::internal::Tags::
1936 const int tag_deliver = Utilities::MPI::internal::Tags::
1938
1939 // Wait until we have a message ready for retrieval, though we don't
1940 // care which process it is from.
1941 MPI_Status status;
1942 int ierr = MPI_Probe(MPI_ANY_SOURCE, tag_request, comm, &status);
1943 AssertThrowMPI(ierr);
1944
1945 // Get rank of incoming message and verify that it makes sense
1946 const unsigned int other_rank = status.MPI_SOURCE;
1947
1948 Assert(requesting_processes.find(other_rank) ==
1949 requesting_processes.end(),
1950 ExcMessage(
1951 "A process is sending a request after a request from "
1952 "the same process has previously already been "
1953 "received. This algorithm does not expect this to happen."));
1954 requesting_processes.insert(other_rank);
1955
1956 // Actually get the incoming message:
1957 int number_amount;
1958 ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1959 AssertThrowMPI(ierr);
1960
1961 std::vector<char> buffer_recv(number_amount);
1962 ierr = MPI_Recv(buffer_recv.data(),
1963 number_amount,
1964 MPI_CHAR,
1965 other_rank,
1966 tag_request,
1967 comm,
1968 &status);
1969 AssertThrowMPI(ierr);
1970
1971 // Process request by asking the user-provided function for
1972 // the answer and post a send for it.
1973 auto &request_buffer = requests_buffers[index];
1974 request_buffer =
1975 (answer_request ?
1976 Utilities::pack(answer_request(other_rank,
1977 Utilities::unpack<RequestType>(
1978 buffer_recv, false)),
1979 false) :
1980 std::vector<char>());
1981
1982 ierr = MPI_Isend(request_buffer.data(),
1983 request_buffer.size(),
1984 MPI_CHAR,
1985 other_rank,
1986 tag_deliver,
1987 comm,
1988 &send_answer_requests[index]);
1989 AssertThrowMPI(ierr);
1990# else
1991 (void)answer_request;
1992 (void)comm;
1993 (void)index;
1994# endif
1995 }
1996
1997
1998
1999 template <typename RequestType, typename AnswerType>
2000 void
2002 const unsigned int n_targets,
2003 const std::function<void(const unsigned int, const AnswerType &)>
2004 &process_answer,
2005 const MPI_Comm comm)
2006 {
2007# ifdef DEAL_II_WITH_MPI
2008 const int tag_deliver = Utilities::MPI::internal::Tags::
2010
2011 // We know how many targets we have sent requests to. These
2012 // targets will all eventually send us their responses, but
2013 // we need not process them in order -- rather, just see what
2014 // comes in and then look at message originators' ranks and
2015 // message sizes
2016 for (unsigned int i = 0; i < n_targets; ++i)
2017 {
2018 MPI_Status status;
2019 {
2020 const int ierr =
2021 MPI_Probe(MPI_ANY_SOURCE, tag_deliver, comm, &status);
2022 AssertThrowMPI(ierr);
2023 }
2024
2025 const auto other_rank = status.MPI_SOURCE;
2026 int message_size;
2027 {
2028 const int ierr = MPI_Get_count(&status, MPI_CHAR, &message_size);
2029 AssertThrowMPI(ierr);
2030 }
2031 std::vector<char> recv_buffer(message_size);
2032
2033 // Now actually receive the answer. Because the MPI_Probe
2034 // above blocks until we have a message, we know that the
2035 // following MPI_Recv call will immediately succeed.
2036 {
2037 const int ierr = MPI_Recv(recv_buffer.data(),
2038 recv_buffer.size(),
2039 MPI_CHAR,
2040 other_rank,
2041 tag_deliver,
2042 comm,
2043 MPI_STATUS_IGNORE);
2044 AssertThrowMPI(ierr);
2045 }
2046
2047 if (process_answer)
2048 process_answer(other_rank,
2049 Utilities::unpack<AnswerType>(recv_buffer, false));
2050 }
2051# else
2052 (void)n_targets;
2053 (void)process_answer;
2054 (void)comm;
2055# endif
2056 }
2057
2058
2059
2060 template <typename RequestType, typename AnswerType>
2061 void
2063 {
2064# ifdef DEAL_II_WITH_MPI
2065 // Finalize all MPI_Request objects for both the
2066 // send-request and receive-answer operations.
2067 if (send_request_requests.size() > 0)
2068 {
2069 const int ierr = MPI_Waitall(send_request_requests.size(),
2070 send_request_requests.data(),
2071 MPI_STATUSES_IGNORE);
2072 AssertThrowMPI(ierr);
2073 }
2074
2075 // Then also check the send-answer requests.
2076 if (send_answer_requests.size() > 0)
2077 {
2078 const int ierr = MPI_Waitall(send_answer_requests.size(),
2079 send_answer_requests.data(),
2080 MPI_STATUSES_IGNORE);
2081 AssertThrowMPI(ierr);
2082 }
2083# endif
2084 }
2085
2086
2087
2088 template <typename RequestType, typename AnswerType>
2089 std::vector<unsigned int>
2091 const std::vector<unsigned int> &targets,
2092 const std::function<RequestType(const unsigned int)> &create_request,
2093 const std::function<AnswerType(const unsigned int, const RequestType &)>
2094 &answer_request,
2095 const std::function<void(const unsigned int, const AnswerType &)>
2096 &process_answer,
2097 const MPI_Comm comm)
2098 {
2099 (void)comm;
2101 ExcMessage("You shouldn't use the 'Serial' class on "
2102 "communicators that have more than one process "
2103 "associated with it."));
2104
2105 // The only valid target for a serial program is itself.
2106 if (targets.size() != 0)
2107 {
2108 Assert(targets.size() == 1,
2109 ExcMessage(
2110 "On a single process, the only valid target "
2111 "is process zero (the process itself), which can only be "
2112 "listed once."));
2113 AssertDimension(targets[0], 0);
2114
2115 // Since the caller indicates that there is a target, and since we
2116 // know that it is the current process, let the process send
2117 // something to itself.
2118 const RequestType request =
2119 (create_request ? create_request(0) : RequestType());
2120 const AnswerType answer =
2121 (answer_request ? answer_request(0, request) : AnswerType());
2122
2123 if (process_answer)
2124 process_answer(0, answer);
2125 }
2126
2127 return targets; // nothing to do
2128 }
2129
2130
2131
2132 template <typename RequestType, typename AnswerType>
2133 std::vector<unsigned int>
2135 const std::vector<unsigned int> &targets,
2136 const std::function<RequestType(const unsigned int)> &create_request,
2137 const std::function<AnswerType(const unsigned int, const RequestType &)>
2138 &answer_request,
2139 const std::function<void(const unsigned int, const AnswerType &)>
2140 &process_answer,
2141 const MPI_Comm comm)
2142 {
2143 // Depending on the number of processes we switch between
2144 // implementations. We reduce the threshold for debug mode to be
2145 // able to test also the non-blocking implementation. This feature
2146 // is tested by:
2147 // tests/multigrid/transfer_matrix_free_06.with_mpi=true.with_p4est=true.with_trilinos=true.mpirun=10.output
2148
2149 const unsigned int n_procs = (Utilities::MPI::job_supports_mpi() ?
2151 1);
2152# ifdef DEAL_II_WITH_MPI
2153# ifdef DEBUG
2154 if (n_procs > 10)
2155# else
2156 if (n_procs > 99)
2157# endif
2158 consensus_algo.reset(new NBX<RequestType, AnswerType>());
2159 else
2160# endif
2161 if (n_procs > 1)
2162 consensus_algo.reset(new PEX<RequestType, AnswerType>());
2163 else
2164 consensus_algo.reset(new Serial<RequestType, AnswerType>());
2165
2166 return consensus_algo->run(
2167 targets, create_request, answer_request, process_answer, comm);
2168 }
2169
2170
2171 } // namespace ConsensusAlgorithms
2172 } // end of namespace MPI
2173} // end of namespace Utilities
2174
2175#endif // DOXYGEN
2176
2177
2179
2180#endif
const char * get_exc_name() const
virtual const char * what() const noexcept override
std::vector< unsigned int > run(Process< RequestType, AnswerType > &process, const MPI_Comm comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)=0
void start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm comm)
void maybe_answer_one_request(const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm comm)
std::vector< std::unique_ptr< std::vector< char > > > request_buffers
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
void clean_up_and_end_communication(const MPI_Comm comm)
bool all_locally_originated_receives_are_completed(const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< std::unique_ptr< MPI_Request > > request_requests
std::vector< std::vector< char > > send_buffers
void signal_finish(const MPI_Comm comm)
unsigned int start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm comm)
std::vector< std::vector< char > > requests_buffers
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
std::vector< std::vector< char > > send_buffers
void answer_one_request(const unsigned int index, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm comm)
std::vector< std::vector< char > > recv_buffers
void process_incoming_answers(const unsigned int n_targets, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
virtual std::vector< unsigned int > compute_targets()=0
virtual void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer)
virtual void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer)
virtual void create_request(const unsigned int other_rank, RequestType &send_buffer)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
std::shared_ptr< Interface< RequestType, AnswerType > > consensus_algo
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm) override
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:40
constexpr bool running_in_debug_mode()
Definition config.h:78
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:41
#define Assert(cond, exc)
#define AssertDimension(dim1, dim2)
#define AssertThrowMPI(error_code)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcMessage(std::string arg1)
const MPI_Comm comm
Definition mpi.cc:924
const unsigned int n_procs
Definition mpi.cc:935
std::vector< unsigned int > selector(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< unsigned int > nbx(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
std::vector< unsigned int > pex(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
@ consensus_algorithm_nbx_process_deliver
ConsensusAlgorithms::NBX::process.
Definition mpi_tags.h:91
@ consensus_algorithm_pex_process_deliver
ConsensusAlgorithms::PEX::process.
Definition mpi_tags.h:96
@ consensus_algorithm_nbx_answer_request
ConsensusAlgorithms::NBX::process.
Definition mpi_tags.h:89
@ consensus_algorithm_pex_answer_request
ConsensusAlgorithms::PEX::process.
Definition mpi_tags.h:94
unsigned int n_mpi_processes(const MPI_Comm mpi_communicator)
Definition mpi.cc:99
unsigned int this_mpi_process(const MPI_Comm mpi_communicator)
Definition mpi.cc:114
bool job_supports_mpi()
Definition mpi.cc:692
unsigned int compute_n_point_to_point_communications(const MPI_Comm mpi_comm, const std::vector< unsigned int > &destinations)
Definition mpi.cc:371
std::size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
Definition utilities.h:1382
void run(const Iterator &begin, const std_cxx20::type_identity_t< Iterator > &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
STL namespace.