Reference documentation for deal.II version 9.4.0
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
mpi_consensus_algorithms.h
Go to the documentation of this file.
1// ---------------------------------------------------------------------
2//
3// Copyright (C) 2020 - 2022 by the deal.II authors
4//
5// This file is part of the deal.II library.
6//
7// The deal.II library is free software; you can use it, redistribute
8// it, and/or modify it under the terms of the GNU Lesser General
9// Public License as published by the Free Software Foundation; either
10// version 2.1 of the License, or (at your option) any later version.
11// The full text of the license can be found in the file LICENSE.md at
12// the top level directory of deal.II.
13//
14// ---------------------------------------------------------------------
15
16#ifndef dealii_mpi_consensus_algorithm_h
17#define dealii_mpi_consensus_algorithm_h
18
19#include <deal.II/base/config.h>
20
21#include <deal.II/base/mpi.h>
22#include <deal.II/base/mpi.templates.h>
23
25
26
27namespace Utilities
28{
29 namespace MPI
30 {
131 namespace ConsensusAlgorithms
132 {
157 template <typename RequestType, typename AnswerType>
159 {
160 public:
165 virtual ~Process() = default;
166
173 virtual std::vector<unsigned int>
175
185 virtual void
186 create_request(const unsigned int other_rank, RequestType &send_buffer);
187
200 virtual void
201 answer_request(const unsigned int other_rank,
202 const RequestType &buffer_recv,
203 AnswerType & request_buffer);
204
212 virtual void
213 read_answer(const unsigned int other_rank,
214 const AnswerType & recv_buffer);
215 };
216
217
218
232 template <typename RequestType, typename AnswerType>
234 {
235 public:
240
255 const MPI_Comm & comm);
256
261 virtual ~Interface() = default;
262
273 std::vector<unsigned int>
275
284 std::vector<unsigned int>
286
310 virtual std::vector<unsigned int>
312 const std::vector<unsigned int> & targets,
313 const std::function<RequestType(const unsigned int)> &create_request,
314 const std::function<AnswerType(const unsigned int,
315 const RequestType &)> &answer_request,
316 const std::function<void(const unsigned int, const AnswerType &)>
317 & process_answer,
318 const MPI_Comm &comm) = 0;
319
320 private:
330
339 };
340
341
355 template <typename RequestType, typename AnswerType>
356 class NBX : public Interface<RequestType, AnswerType>
357 {
358 public:
362 NBX() = default;
363
378
382 virtual ~NBX() = default;
383
384 // Import the declarations from the base class.
385 using Interface<RequestType, AnswerType>::run;
386
390 virtual std::vector<unsigned int>
392 const std::vector<unsigned int> & targets,
393 const std::function<RequestType(const unsigned int)> &create_request,
394 const std::function<AnswerType(const unsigned int,
395 const RequestType &)> &answer_request,
396 const std::function<void(const unsigned int, const AnswerType &)>
397 & process_answer,
398 const MPI_Comm &comm) override;
399
400 private:
401#ifdef DEAL_II_WITH_MPI
405 std::vector<std::vector<char>> send_buffers;
406
410 std::vector<MPI_Request> send_requests;
411
419 std::vector<std::unique_ptr<std::vector<char>>> request_buffers;
420
424 std::vector<std::unique_ptr<MPI_Request>> request_requests;
425
430
431 // request for barrier
432 MPI_Request barrier_request;
433#endif
434
438 std::set<unsigned int> requesting_processes;
439
445 bool
447 const std::function<void(const unsigned int, const AnswerType &)>
448 & process_answer,
449 const MPI_Comm &comm);
450
455 void
457
463 bool
465
471 void
473 const std::function<AnswerType(const unsigned int,
474 const RequestType &)> &answer_request,
475 const MPI_Comm & comm);
476
481 void
483 const std::vector<unsigned int> & targets,
484 const std::function<RequestType(const unsigned int)> &create_request,
485 const MPI_Comm & comm);
486
491 void
493 };
494
495
529 template <typename RequestType, typename AnswerType>
530 std::vector<unsigned int>
531 nbx(const std::vector<unsigned int> & targets,
532 const std::function<RequestType(const unsigned int)> &create_request,
533 const std::function<AnswerType(const unsigned int,
534 const RequestType &)> &answer_request,
535 const std::function<void(const unsigned int, const AnswerType &)>
536 & process_answer,
537 const MPI_Comm &comm);
538
565 template <typename RequestType>
566 std::vector<unsigned int>
567 nbx(const std::vector<unsigned int> & targets,
568 const std::function<RequestType(const unsigned int)> &create_request,
569 const std::function<void(const unsigned int, const RequestType &)>
570 & process_request,
571 const MPI_Comm &comm);
572
598 template <typename RequestType, typename AnswerType>
599 class PEX : public Interface<RequestType, AnswerType>
600 {
601 public:
605 PEX() = default;
606
607
622
626 virtual ~PEX() = default;
627
628 // Import the declarations from the base class.
629 using Interface<RequestType, AnswerType>::run;
630
634 virtual std::vector<unsigned int>
636 const std::vector<unsigned int> & targets,
637 const std::function<RequestType(const unsigned int)> &create_request,
638 const std::function<AnswerType(const unsigned int,
639 const RequestType &)> &answer_request,
640 const std::function<void(const unsigned int, const AnswerType &)>
641 & process_answer,
642 const MPI_Comm &comm) override;
643
644 private:
645#ifdef DEAL_II_WITH_MPI
649 std::vector<std::vector<char>> send_buffers;
650
654 std::vector<std::vector<char>> recv_buffers;
655
659 std::vector<MPI_Request> send_request_requests;
660
664 std::vector<std::vector<char>> requests_buffers;
665
669 std::vector<MPI_Request> send_answer_requests;
670#endif
674 std::set<unsigned int> requesting_processes;
675
680 unsigned int
682 const std::vector<unsigned int> & targets,
683 const std::function<RequestType(const unsigned int)> &create_request,
684 const MPI_Comm & comm);
685
690 void
692 const unsigned int index,
693 const std::function<AnswerType(const unsigned int,
694 const RequestType &)> &answer_request,
695 const MPI_Comm & comm);
696
701 void
703 const unsigned int n_targets,
704 const std::function<void(const unsigned int, const AnswerType &)>
705 & process_answer,
706 const MPI_Comm &comm);
707
712 void
714 };
715
716
717
763 template <typename RequestType, typename AnswerType>
764 std::vector<unsigned int>
765 pex(const std::vector<unsigned int> & targets,
766 const std::function<RequestType(const unsigned int)> &create_request,
767 const std::function<AnswerType(const unsigned int,
768 const RequestType &)> &answer_request,
769 const std::function<void(const unsigned int, const AnswerType &)>
770 & process_answer,
771 const MPI_Comm &comm);
772
799 template <typename RequestType>
800 std::vector<unsigned int>
801 pex(const std::vector<unsigned int> & targets,
802 const std::function<RequestType(const unsigned int)> &create_request,
803 const std::function<void(const unsigned int, const RequestType &)>
804 & process_request,
805 const MPI_Comm &comm);
806
807
812 template <typename RequestType, typename AnswerType>
813 class Serial : public Interface<RequestType, AnswerType>
814 {
815 public:
819 Serial() = default;
820
835
836 // Import the declarations from the base class.
837 using Interface<RequestType, AnswerType>::run;
838
842 virtual std::vector<unsigned int>
844 const std::vector<unsigned int> & targets,
845 const std::function<RequestType(const unsigned int)> &create_request,
846 const std::function<AnswerType(const unsigned int,
847 const RequestType &)> &answer_request,
848 const std::function<void(const unsigned int, const AnswerType &)>
849 & process_answer,
850 const MPI_Comm &comm) override;
851 };
852
853
854
884 template <typename RequestType, typename AnswerType>
885 std::vector<unsigned int>
887 const std::vector<unsigned int> & targets,
888 const std::function<RequestType(const unsigned int)> &create_request,
889 const std::function<AnswerType(const unsigned int, const RequestType &)>
890 &answer_request,
891 const std::function<void(const unsigned int, const AnswerType &)>
892 & process_answer,
893 const MPI_Comm &comm);
894
921 template <typename RequestType>
922 std::vector<unsigned int>
924 const std::vector<unsigned int> & targets,
925 const std::function<RequestType(const unsigned int)> &create_request,
926 const std::function<void(const unsigned int, const RequestType &)>
927 & process_request,
928 const MPI_Comm &comm);
929
930
931
944 template <typename RequestType, typename AnswerType>
945 class Selector : public Interface<RequestType, AnswerType>
946 {
947 public:
951 Selector() = default;
952
967 const MPI_Comm & comm);
968
972 virtual ~Selector() = default;
973
974 // Import the declarations from the base class.
975 using Interface<RequestType, AnswerType>::run;
976
982 virtual std::vector<unsigned int>
984 const std::vector<unsigned int> & targets,
985 const std::function<RequestType(const unsigned int)> &create_request,
986 const std::function<AnswerType(const unsigned int,
987 const RequestType &)> &answer_request,
988 const std::function<void(const unsigned int, const AnswerType &)>
989 & process_answer,
990 const MPI_Comm &comm) override;
991
992 private:
993 // Pointer to the actual ConsensusAlgorithms::Interface implementation.
994 std::shared_ptr<Interface<RequestType, AnswerType>> consensus_algo;
995 };
996
997
998
1032 template <typename RequestType, typename AnswerType>
1033 std::vector<unsigned int>
1035 const std::vector<unsigned int> & targets,
1036 const std::function<RequestType(const unsigned int)> &create_request,
1037 const std::function<AnswerType(const unsigned int, const RequestType &)>
1038 &answer_request,
1039 const std::function<void(const unsigned int, const AnswerType &)>
1040 & process_answer,
1041 const MPI_Comm &comm);
1042
1069 template <typename RequestType>
1070 std::vector<unsigned int>
1072 const std::vector<unsigned int> & targets,
1073 const std::function<RequestType(const unsigned int)> &create_request,
1074 const std::function<void(const unsigned int, const RequestType &)>
1075 & process_request,
1076 const MPI_Comm &comm);
1077
1078
1085 template <typename RequestType, typename AnswerType>
1087 : public Process<RequestType, AnswerType>
1088 {
1089 public:
1100 const std::function<std::vector<unsigned int>()>
1101 &function_compute_targets,
1102 const std::function<void(const unsigned int, RequestType &)>
1103 & function_create_request = {},
1104 const std::function<void(const unsigned int,
1105 const RequestType &,
1106 AnswerType &)> &function_answer_request = {},
1107 const std::function<void(const unsigned int, const AnswerType &)>
1108 &function_read_answer = {});
1109
1113 std::vector<unsigned int>
1115
1119 void
1120 create_request(const unsigned int other_rank,
1121 RequestType & send_buffer) override;
1122
1126 void
1127 answer_request(const unsigned int other_rank,
1128 const RequestType &buffer_recv,
1129 AnswerType & request_buffer) override;
1130
1134 void
1135 read_answer(const unsigned int other_rank,
1136 const AnswerType & recv_buffer) override;
1137
1138 private:
1139 const std::function<std::vector<unsigned int>()>
1141 const std::function<void(const int, RequestType &)>
1143 const std::function<
1144 void(const unsigned int, const RequestType &, AnswerType &)>
1146 const std::function<void(const int, const AnswerType &)>
1148 };
1149
1150
1151#ifndef DOXYGEN
1152 // Implementation of the functions in this namespace.
1153
1154 template <typename RequestType, typename AnswerType>
1155 std::vector<unsigned int>
1156 nbx(const std::vector<unsigned int> & targets,
1157 const std::function<RequestType(const unsigned int)> &create_request,
1158 const std::function<AnswerType(const unsigned int,
1159 const RequestType &)> &answer_request,
1160 const std::function<void(const unsigned int, const AnswerType &)>
1161 & process_answer,
1162 const MPI_Comm &comm)
1163 {
1165 targets, create_request, answer_request, process_answer, comm);
1166 }
1167
1168
1169
1170 template <typename RequestType>
1171 std::vector<unsigned int>
1172 nbx(const std::vector<unsigned int> & targets,
1173 const std::function<RequestType(const unsigned int)> &create_request,
1174 const std::function<void(const unsigned int, const RequestType &)>
1175 & process_request,
1176 const MPI_Comm &comm)
1177 {
1178 // TODO: For the moment, simply implement this special case by
1179 // forwarding to the other function with rewritten function
1180 // objects and using an empty type as answer type. This way,
1181 // we have the interface in place and can provide a more
1182 // efficient implementation later on.
1183 using EmptyType = std::tuple<>;
1184
1185 return nbx<RequestType, EmptyType>(
1186 targets,
1187 create_request,
1188 // answer_request:
1189 [&process_request](const unsigned int source_rank,
1190 const RequestType &request) -> EmptyType {
1191 process_request(source_rank, request);
1192 // Return something. What it is is arbitrary here, except that
1193 // we want it to be as small an object as possible. Using
1194 // std::tuple<> is interpreted as an empty object that is packed
1195 // down to a zero-length char array.
1196 return {};
1197 },
1198 // process_answer:
1199 [](const unsigned int /*target_rank */,
1200 const EmptyType & /*answer*/) {},
1201 comm);
1202 }
1203
1204
1205
1206 template <typename RequestType, typename AnswerType>
1207 std::vector<unsigned int>
1208 pex(const std::vector<unsigned int> & targets,
1209 const std::function<RequestType(const unsigned int)> &create_request,
1210 const std::function<AnswerType(const unsigned int,
1211 const RequestType &)> &answer_request,
1212 const std::function<void(const unsigned int, const AnswerType &)>
1213 & process_answer,
1214 const MPI_Comm &comm)
1215 {
1216 return PEX<RequestType, AnswerType>().run(
1217 targets, create_request, answer_request, process_answer, comm);
1218 }
1219
1220
1221
1222 template <typename RequestType>
1223 std::vector<unsigned int>
1224 pex(const std::vector<unsigned int> & targets,
1225 const std::function<RequestType(const unsigned int)> &create_request,
1226 const std::function<void(const unsigned int, const RequestType &)>
1227 & process_request,
1228 const MPI_Comm &comm)
1229 {
1230 // TODO: For the moment, simply implement this special case by
1231 // forwarding to the other function with rewritten function
1232 // objects and using an empty type as answer type. This way,
1233 // we have the interface in place and can provide a more
1234 // efficient implementation later on.
1235 using EmptyType = std::tuple<>;
1236
1237 return pex<RequestType, EmptyType>(
1238 targets,
1239 create_request,
1240 // answer_request:
1241 [&process_request](const unsigned int source_rank,
1242 const RequestType &request) -> EmptyType {
1243 process_request(source_rank, request);
1244 // Return something. What it is is arbitrary here, except that
1245 // we want it to be as small an object as possible. Using
1246 // std::tuple<> is interpreted as an empty object that is packed
1247 // down to a zero-length char array.
1248 return {};
1249 },
1250 // process_answer:
1251 [](const unsigned int /*target_rank */,
1252 const EmptyType & /*answer*/) {},
1253 comm);
1254 }
1255
1256
1257
1258 template <typename RequestType, typename AnswerType>
1259 std::vector<unsigned int>
1260 serial(
1261 const std::vector<unsigned int> & targets,
1262 const std::function<RequestType(const unsigned int)> &create_request,
1263 const std::function<AnswerType(const unsigned int, const RequestType &)>
1264 &answer_request,
1265 const std::function<void(const unsigned int, const AnswerType &)>
1266 & process_answer,
1267 const MPI_Comm &comm)
1268 {
1269 return Serial<RequestType, AnswerType>().run(
1270 targets, create_request, answer_request, process_answer, comm);
1271 }
1272
1273
1274
1275 template <typename RequestType>
1276 std::vector<unsigned int>
1277 serial(
1278 const std::vector<unsigned int> & targets,
1279 const std::function<RequestType(const unsigned int)> &create_request,
1280 const std::function<void(const unsigned int, const RequestType &)>
1281 & process_request,
1282 const MPI_Comm &comm)
1283 {
1284 // TODO: For the moment, simply implement this special case by
1285 // forwarding to the other function with rewritten function
1286 // objects and using an empty type as answer type. This way,
1287 // we have the interface in place and can provide a more
1288 // efficient implementation later on.
1289 using EmptyType = std::tuple<>;
1290
1291 return serial<RequestType, EmptyType>(
1292 targets,
1293 create_request,
1294 // answer_request:
1295 [&process_request](const unsigned int source_rank,
1296 const RequestType &request) -> EmptyType {
1297 process_request(source_rank, request);
1298 // Return something. What it is is arbitrary here, except that
1299 // we want it to be as small an object as possible. Using
1300 // std::tuple<> is interpreted as an empty object that is packed
1301 // down to a zero-length char array.
1302 return {};
1303 },
1304 // process_answer:
1305 [](const unsigned int /*target_rank */,
1306 const EmptyType & /*answer*/) {},
1307 comm);
1308 }
1309
1310
1311
1312 template <typename RequestType, typename AnswerType>
1313 std::vector<unsigned int>
1314 selector(
1315 const std::vector<unsigned int> & targets,
1316 const std::function<RequestType(const unsigned int)> &create_request,
1317 const std::function<AnswerType(const unsigned int, const RequestType &)>
1318 &answer_request,
1319 const std::function<void(const unsigned int, const AnswerType &)>
1320 & process_answer,
1321 const MPI_Comm &comm)
1322 {
1323 return Selector<RequestType, AnswerType>().run(
1324 targets, create_request, answer_request, process_answer, comm);
1325 }
1326
1327
1328
1329 template <typename RequestType>
1330 std::vector<unsigned int>
1331 selector(
1332 const std::vector<unsigned int> & targets,
1333 const std::function<RequestType(const unsigned int)> &create_request,
1334 const std::function<void(const unsigned int, const RequestType &)>
1335 & process_request,
1336 const MPI_Comm &comm)
1337 {
1338 // TODO: For the moment, simply implement this special case by
1339 // forwarding to the other function with rewritten function
1340 // objects and using an empty type as answer type. This way,
1341 // we have the interface in place and can provide a more
1342 // efficient implementation later on.
1343 using EmptyType = std::tuple<>;
1344
1345 return selector<RequestType, EmptyType>(
1346 targets,
1347 create_request,
1348 // answer_request:
1349 [&process_request](const unsigned int source_rank,
1350 const RequestType &request) -> EmptyType {
1351 process_request(source_rank, request);
1352 // Return something. What it is is arbitrary here, except that
1353 // we want it to be as small an object as possible. Using
1354 // std::tuple<> is interpreted as an empty object that is packed
1355 // down to a zero-length char array.
1356 return {};
1357 },
1358 // process_answer:
1359 [](const unsigned int /*target_rank */,
1360 const EmptyType & /*answer*/) {},
1361 comm);
1362 }
1363
1364
1365
1366 template <typename RequestType, typename AnswerType>
1368 const std::function<std::vector<unsigned int>()>
1369 &function_compute_targets,
1370 const std::function<void(const unsigned int, RequestType &)>
1371 & function_create_request,
1372 const std::function<void(const unsigned int,
1373 const RequestType &,
1374 AnswerType &)> &function_answer_request,
1375 const std::function<void(const unsigned int, const AnswerType &)>
1376 &function_read_answer)
1377 : function_compute_targets(function_compute_targets)
1378 , function_create_request(function_create_request)
1379 , function_answer_request(function_answer_request)
1380 , function_read_answer(function_read_answer)
1381 {}
1382
1383
1384
1385 template <typename RequestType, typename AnswerType>
1386 std::vector<unsigned int>
1388 {
1389 return function_compute_targets();
1390 }
1391
1392
1393
1394 template <typename RequestType, typename AnswerType>
1395 void
1397 const unsigned int other_rank,
1398 RequestType & send_buffer)
1399 {
1400 if (function_create_request)
1401 function_create_request(other_rank, send_buffer);
1402 }
1403
1404
1405
1406 template <typename RequestType, typename AnswerType>
1407 void
1409 const unsigned int other_rank,
1410 const RequestType &buffer_recv,
1411 AnswerType & request_buffer)
1412 {
1413 if (function_answer_request)
1414 function_answer_request(other_rank, buffer_recv, request_buffer);
1415 }
1416
1417
1418
1419 template <typename RequestType, typename AnswerType>
1420 void
1422 const unsigned int other_rank,
1423 const AnswerType & recv_buffer)
1424 {
1425 if (function_read_answer)
1426 function_read_answer(other_rank, recv_buffer);
1427 }
1428
1429#endif
1430
1431
1432 } // namespace ConsensusAlgorithms
1433 } // end of namespace MPI
1434} // end of namespace Utilities
1435
1436
1437
1438#ifndef DOXYGEN
1439
1440// ----------------- Implementation of template functions
1441
1442namespace Utilities
1443{
1444 namespace MPI
1445 {
1446 namespace ConsensusAlgorithms
1447 {
1448 namespace
1449 {
1465# ifndef DEAL_II_MSVC
1466 [[gnu::unused]]
1467# endif
1468 inline bool
1469 has_unique_elements(const std::vector<unsigned int> &targets)
1470 {
1471 std::vector<unsigned int> my_destinations = targets;
1472 std::sort(my_destinations.begin(), my_destinations.end());
1473 return (std::adjacent_find(my_destinations.begin(),
1474 my_destinations.end()) ==
1475 my_destinations.end());
1476 }
1477 } // namespace
1478
1479
1480
1481 template <typename RequestType, typename AnswerType>
1482 void
1484 const RequestType &,
1485 AnswerType &)
1486 {
1487 // nothing to do
1488 }
1489
1490
1491
1492 template <typename RequestType, typename AnswerType>
1493 void
1495 RequestType &)
1496 {
1497 // nothing to do
1498 }
1499
1500
1501
1502 template <typename RequestType, typename AnswerType>
1503 void
1505 const AnswerType &)
1506 {
1507 // nothing to do
1508 }
1509
1510
1511
1512 template <typename RequestType, typename AnswerType>
1514 Process<RequestType, AnswerType> &process,
1515 const MPI_Comm & comm)
1516 : process(&process)
1517 , comm(comm)
1518 {}
1519
1520
1521
1522 template <typename RequestType, typename AnswerType>
1524 : process(nullptr)
1525 , comm(MPI_COMM_NULL)
1526 {}
1527
1528
1529
1530 template <typename RequestType, typename AnswerType>
1531 std::vector<unsigned int>
1533 {
1534 Assert(process != nullptr,
1535 ExcMessage("This function can only be called if the "
1536 "deprecated non-default constructor of this class "
1537 "has previously been called to set the Process "
1538 "object and a communicator."));
1539 return run(*process, comm);
1540 }
1541
1542
1543
1544 template <typename RequestType, typename AnswerType>
1545 std::vector<unsigned int>
1547 Process<RequestType, AnswerType> &process,
1548 const MPI_Comm & comm)
1549 {
1550 // Unpack the 'process' object and call the function that takes
1551 // function objects for all operations.
1552 return run(
1553 process.compute_targets(),
1554 /* create_request: */
1555 [&process](const unsigned int target) {
1556 RequestType request;
1557 process.create_request(target, request);
1558 return request;
1559 },
1560 /* answer_request: */
1561 [&process](const unsigned int source, const RequestType &request) {
1562 AnswerType answer;
1563 process.answer_request(source, request, answer);
1564 return answer;
1565 },
1566 /* process_answer: */
1567 [&process](const unsigned int target, const AnswerType &answer) {
1568 process.read_answer(target, answer);
1569 },
1570 comm);
1571 }
1572
1573
1574
1575 template <typename RequestType, typename AnswerType>
1577 Process<RequestType, AnswerType> &process,
1578 const MPI_Comm & comm)
1579 : Interface<RequestType, AnswerType>(process, comm)
1580 {}
1581
1582
1583
1584 template <typename RequestType, typename AnswerType>
1585 std::vector<unsigned int>
1587 const std::vector<unsigned int> & targets,
1588 const std::function<RequestType(const unsigned int)> &create_request,
1589 const std::function<AnswerType(const unsigned int, const RequestType &)>
1590 &answer_request,
1591 const std::function<void(const unsigned int, const AnswerType &)>
1592 & process_answer,
1593 const MPI_Comm &comm)
1594 {
1595 Assert(has_unique_elements(targets),
1596 ExcMessage("The consensus algorithms expect that each process "
1597 "only sends a single message to another process, "
1598 "but the targets provided include duplicates."));
1599
1600 static CollectiveMutex mutex;
1601 CollectiveMutex::ScopedLock lock(mutex, comm);
1602
1603 // 1) Send data to identified targets and start receiving
1604 // the answers from these very same processes.
1605 start_communication(targets, create_request, comm);
1606
1607 // 2) Until all posted receive operations are known to have completed,
1608 // answer requests and keep checking whether all requests of
1609 // this process have been answered.
1610 //
1611 // The requests that we catch in the answer_requests() function
1612 // originate elsewhere, that is, they are not in response
1613 // to our own messages
1614 //
1615 // Note also that we may not catch all incoming requests in
1616 // the following two lines: our own requests may have been
1617 // satisfied before we've dealt with all incoming requests.
1618 // That's ok: We will get around to dealing with all remaining
1619 // message later. We just want to move on to the next step
1620 // as early as possible.
1621 while (all_locally_originated_receives_are_completed(process_answer,
1622 comm) == false)
1623 maybe_answer_one_request(answer_request, comm);
1624
1625 // 3) Signal to all other processes that all requests of this process
1626 // have been answered
1627 signal_finish(comm);
1628
1629 // 4) Nevertheless, this process has to keep on answering (potential)
1630 // incoming requests until all processes have received the
1631 // answer to all requests
1632 while (all_remotely_originated_receives_are_completed() == false)
1633 maybe_answer_one_request(answer_request, comm);
1634
1635 // 5) process the answer to all requests
1636 clean_up_and_end_communication(comm);
1637
1638 return std::vector<unsigned int>(requesting_processes.begin(),
1639 requesting_processes.end());
1640 }
1641
1642
1643
1644 template <typename RequestType, typename AnswerType>
1645 void
1647 const std::vector<unsigned int> & targets,
1648 const std::function<RequestType(const unsigned int)> &create_request,
1649 const MPI_Comm & comm)
1650 {
1651# ifdef DEAL_II_WITH_MPI
1652 // 1)
1653 const auto n_targets = targets.size();
1654
1655 const int tag_request = Utilities::MPI::internal::Tags::
1657
1658 // 2) allocate memory
1659 send_requests.resize(n_targets);
1660 send_buffers.resize(n_targets);
1661
1662 {
1663 // 4) send and receive
1664 for (unsigned int index = 0; index < n_targets; ++index)
1665 {
1666 const unsigned int rank = targets[index];
1668
1669 auto &send_buffer = send_buffers[index];
1670 send_buffer =
1671 (create_request ? Utilities::pack(create_request(rank), false) :
1672 std::vector<char>());
1673
1674 // Post a request to send data
1675 auto ierr = MPI_Isend(send_buffer.data(),
1676 send_buffer.size(),
1677 MPI_CHAR,
1678 rank,
1679 tag_request,
1680 comm,
1681 &send_requests[index]);
1682 AssertThrowMPI(ierr);
1683 }
1684
1685 // Also record that we expect an answer from each target we sent
1686 // a request to:
1687 n_outstanding_answers = n_targets;
1688 }
1689# else
1690 (void)targets;
1691 (void)create_request;
1692 (void)comm;
1693# endif
1694 }
1695
1696
1697
1698 template <typename RequestType, typename AnswerType>
1699 bool
1702 const std::function<void(const unsigned int, const AnswerType &)>
1703 & process_answer,
1704 const MPI_Comm &comm)
1705 {
1706# ifdef DEAL_II_WITH_MPI
1707 // We know that all requests have come in when we have pending
1708 // messages from all targets with the right tag (some of which we may
1709 // have already taken care of below, after discovering their existence).
1710 // We can check for pending messages with MPI_IProbe, which returns
1711 // immediately with a return code that indicates whether
1712 // it has found a message from any process with a given
1713 // tag.
1714 if (n_outstanding_answers == 0)
1715 return true;
1716 else
1717 {
1718 const int tag_deliver = Utilities::MPI::internal::Tags::
1720
1721 int request_is_pending;
1722 MPI_Status status;
1723 const auto ierr = MPI_Iprobe(
1724 MPI_ANY_SOURCE, tag_deliver, comm, &request_is_pending, &status);
1725 AssertThrowMPI(ierr);
1726
1727 // If there is no pending message with this tag,
1728 // then we are clearly not done receiving everything
1729 // yet -- so return false.
1730 if (request_is_pending == 0)
1731 return false;
1732 else
1733 {
1734 // OK, so we have gotten a reply to our answer from
1735 // one rank. Let us process it, after double checking
1736 // that it is indeed one we were still expecting:
1737 const auto target = status.MPI_SOURCE;
1738
1739 // Then query the size of the message, allocate enough memory,
1740 // receive the data, and process it.
1741 int message_size;
1742 {
1743 const int ierr =
1744 MPI_Get_count(&status, MPI_CHAR, &message_size);
1745 AssertThrowMPI(ierr);
1746 }
1747 std::vector<char> recv_buffer(message_size);
1748
1749 {
1750 const int tag_deliver = Utilities::MPI::internal::Tags::
1752
1753 const int ierr = MPI_Recv(recv_buffer.data(),
1754 recv_buffer.size(),
1755 MPI_CHAR,
1756 target,
1757 tag_deliver,
1758 comm,
1759 MPI_STATUS_IGNORE);
1760 AssertThrowMPI(ierr);
1761 }
1762
1763 if (process_answer)
1764 process_answer(target,
1765 Utilities::unpack<AnswerType>(recv_buffer,
1766 false));
1767
1768 // Finally, remove this rank from the list of outstanding
1769 // targets:
1770 --n_outstanding_answers;
1771
1772 // We could do another go-around from the top of this
1773 // else-branch to see whether there are actually other messages
1774 // that are currently pending. But that would mean spending
1775 // substantial time in receiving answers while we should also be
1776 // sending answers to requests we have received from other
1777 // places. So let it be enough for now. If there are outstanding
1778 // answers, we will get back to this function before long and
1779 // can take care of them then.
1780 return (n_outstanding_answers == 0);
1781 }
1782 }
1783
1784# else
1785 (void)process_answer;
1786 (void)comm;
1787
1788 return true;
1789# endif
1790 }
1791
1792
1793
1794 template <typename RequestType, typename AnswerType>
1795 void
1797 const std::function<AnswerType(const unsigned int, const RequestType &)>
1798 & answer_request,
1799 const MPI_Comm &comm)
1800 {
1801# ifdef DEAL_II_WITH_MPI
1802
1803 const int tag_request = Utilities::MPI::internal::Tags::
1805 const int tag_deliver = Utilities::MPI::internal::Tags::
1807
1808 // Check if there is a request pending. By selecting the
1809 // tag_request tag, these are other processes asking for
1810 // our own replies, not these other processes' replies
1811 // to our own requests.
1812 //
1813 // There may be multiple such pending messages. We
1814 // only answer one.
1815 MPI_Status status;
1816 int request_is_pending;
1817 const auto ierr = MPI_Iprobe(
1818 MPI_ANY_SOURCE, tag_request, comm, &request_is_pending, &status);
1819 AssertThrowMPI(ierr);
1820
1821 if (request_is_pending != 0)
1822 {
1823 // Get the rank of the requesting process and add it to the
1824 // list of requesting processes (which may contain duplicates).
1825 const auto other_rank = status.MPI_SOURCE;
1826
1827 Assert(requesting_processes.find(other_rank) ==
1828 requesting_processes.end(),
1829 ExcMessage("Process is requesting a second time!"));
1830 requesting_processes.insert(other_rank);
1831
1832 // get size of incoming message
1833 int number_amount;
1834 auto ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
1835 AssertThrowMPI(ierr);
1836
1837 // allocate memory for incoming message
1838 std::vector<char> buffer_recv(number_amount);
1839 ierr = MPI_Recv(buffer_recv.data(),
1840 number_amount,
1841 MPI_CHAR,
1842 other_rank,
1843 tag_request,
1844 comm,
1845 MPI_STATUS_IGNORE);
1846 AssertThrowMPI(ierr);
1847
1848 // Allocate memory for an answer message to the current request,
1849 // and ask the 'process' object to produce an answer:
1850 request_buffers.emplace_back(std::make_unique<std::vector<char>>());
1851 auto &request_buffer = *request_buffers.back();
1852 if (answer_request)
1853 request_buffer =
1854 Utilities::pack(answer_request(other_rank,
1855 Utilities::unpack<RequestType>(
1856 buffer_recv, false)),
1857 false);
1858
1859 // Then initiate sending the answer back to the requester.
1860 request_requests.emplace_back(std::make_unique<MPI_Request>());
1861 ierr = MPI_Isend(request_buffer.data(),
1862 request_buffer.size(),
1863 MPI_CHAR,
1864 other_rank,
1865 tag_deliver,
1866 comm,
1867 request_requests.back().get());
1868 AssertThrowMPI(ierr);
1869 }
1870# else
1871 (void)answer_request;
1872 (void)comm;
1873# endif
1874 }
1875
1876
1877
1878 template <typename RequestType, typename AnswerType>
1879 void
1881 {
1882# ifdef DEAL_II_WITH_MPI
1883 const auto ierr = MPI_Ibarrier(comm, &barrier_request);
1884 AssertThrowMPI(ierr);
1885# else
1886 (void)comm;
1887# endif
1888 }
1889
1890
1891
1892 template <typename RequestType, typename AnswerType>
1893 bool
1894 NBX<RequestType,
1895 AnswerType>::all_remotely_originated_receives_are_completed()
1896 {
1897# ifdef DEAL_II_WITH_MPI
1898 int all_ranks_reached_barrier;
1899 const auto ierr = MPI_Test(&barrier_request,
1900 &all_ranks_reached_barrier,
1901 MPI_STATUSES_IGNORE);
1902 AssertThrowMPI(ierr);
1903 return all_ranks_reached_barrier != 0;
1904# else
1905 return true;
1906# endif
1907 }
1908
1909
1910
1911 template <typename RequestType, typename AnswerType>
1912 void
1914 const MPI_Comm &comm)
1915 {
1916 (void)comm;
1917# ifdef DEAL_II_WITH_MPI
1918 // clean up
1919 {
1920 if (send_requests.size() > 0)
1921 {
1922 const int ierr = MPI_Waitall(send_requests.size(),
1923 send_requests.data(),
1924 MPI_STATUSES_IGNORE);
1925 AssertThrowMPI(ierr);
1926 }
1927
1928 int ierr = MPI_Wait(&barrier_request, MPI_STATUS_IGNORE);
1929 AssertThrowMPI(ierr);
1930
1931 for (auto &i : request_requests)
1932 {
1933 ierr = MPI_Wait(i.get(), MPI_STATUS_IGNORE);
1934 AssertThrowMPI(ierr);
1935 }
1936
1937# ifdef DEBUG
1938 // note: IBarrier seems to make problem during testing, this
1939 // additional Barrier seems to help
1940 ierr = MPI_Barrier(comm);
1941 AssertThrowMPI(ierr);
1942# endif
1943 }
1944# endif
1945 }
1946
1947
1948
1949 template <typename RequestType, typename AnswerType>
1951 Process<RequestType, AnswerType> &process,
1952 const MPI_Comm & comm)
1953 : Interface<RequestType, AnswerType>(process, comm)
1954 {}
1955
1956
1957
1958 template <typename RequestType, typename AnswerType>
1959 std::vector<unsigned int>
1961 const std::vector<unsigned int> & targets,
1962 const std::function<RequestType(const unsigned int)> &create_request,
1963 const std::function<AnswerType(const unsigned int, const RequestType &)>
1964 &answer_request,
1965 const std::function<void(const unsigned int, const AnswerType &)>
1966 & process_answer,
1967 const MPI_Comm &comm)
1968 {
1969 Assert(has_unique_elements(targets),
1970 ExcMessage("The consensus algorithms expect that each process "
1971 "only sends a single message to another process, "
1972 "but the targets provided include duplicates."));
1973
1974 static CollectiveMutex mutex;
1975 CollectiveMutex::ScopedLock lock(mutex, comm);
1976
1977 // 1) Send requests and start receiving the answers.
1978 // In particular, determine how many requests we should expect
1979 // on the current process.
1980 const unsigned int n_requests =
1981 start_communication(targets, create_request, comm);
1982
1983 // 2) Answer requests:
1984 for (unsigned int request = 0; request < n_requests; ++request)
1985 answer_one_request(request, answer_request, comm);
1986
1987 // 3) Process answers:
1988 process_incoming_answers(targets.size(), process_answer, comm);
1989
1990 // 4) Make sure all sends have successfully terminated:
1991 clean_up_and_end_communication();
1992
1993 return std::vector<unsigned int>(requesting_processes.begin(),
1994 requesting_processes.end());
1995 }
1996
1997
1998
1999 template <typename RequestType, typename AnswerType>
2000 unsigned int
2002 const std::vector<unsigned int> & targets,
2003 const std::function<RequestType(const unsigned int)> &create_request,
2004 const MPI_Comm & comm)
2005 {
2006# ifdef DEAL_II_WITH_MPI
2007 const int tag_request = Utilities::MPI::internal::Tags::
2009
2010 // 1) determine with which processes this process wants to communicate
2011 // with
2012 const unsigned int n_targets = targets.size();
2013
2014 // 2) determine who wants to communicate with this process
2015 const unsigned int n_sources =
2017
2018 // 2) allocate memory
2019 recv_buffers.resize(n_targets);
2020 send_buffers.resize(n_targets);
2021 send_request_requests.resize(n_targets);
2022
2023 send_answer_requests.resize(n_sources);
2024 requests_buffers.resize(n_sources);
2025
2026 // 4) send and receive
2027 for (unsigned int i = 0; i < n_targets; ++i)
2028 {
2029 const unsigned int rank = targets[i];
2031
2032 // pack data which should be sent
2033 auto &send_buffer = send_buffers[i];
2034 if (create_request)
2035 send_buffer = Utilities::pack(create_request(rank), false);
2036
2037 // start to send data
2038 auto ierr = MPI_Isend(send_buffer.data(),
2039 send_buffer.size(),
2040 MPI_CHAR,
2041 rank,
2042 tag_request,
2043 comm,
2044 &send_request_requests[i]);
2045 AssertThrowMPI(ierr);
2046 }
2047
2048 return n_sources;
2049# else
2050 (void)targets;
2051 (void)create_request;
2052 (void)comm;
2053 return 0;
2054# endif
2055 }
2056
2057
2058
2059 template <typename RequestType, typename AnswerType>
2060 void
2062 const unsigned int index,
2063 const std::function<AnswerType(const unsigned int, const RequestType &)>
2064 & answer_request,
2065 const MPI_Comm &comm)
2066 {
2067# ifdef DEAL_II_WITH_MPI
2068 const int tag_request = Utilities::MPI::internal::Tags::
2070 const int tag_deliver = Utilities::MPI::internal::Tags::
2072
2073 // Wait until we have a message ready for retrieval, though we don't
2074 // care which process it is from.
2075 MPI_Status status;
2076 int ierr = MPI_Probe(MPI_ANY_SOURCE, tag_request, comm, &status);
2077 AssertThrowMPI(ierr);
2078
2079 // Get rank of incoming message and verify that it makes sense
2080 const unsigned int other_rank = status.MPI_SOURCE;
2081
2082 Assert(requesting_processes.find(other_rank) ==
2083 requesting_processes.end(),
2084 ExcMessage(
2085 "A process is sending a request after a request from "
2086 "the same process has previously already been "
2087 "received. This algorithm does not expect this to happen."));
2088 requesting_processes.insert(other_rank);
2089
2090 // Actually get the incoming message:
2091 int number_amount;
2092 ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount);
2093 AssertThrowMPI(ierr);
2094
2095 std::vector<char> buffer_recv(number_amount);
2096 ierr = MPI_Recv(buffer_recv.data(),
2097 number_amount,
2098 MPI_CHAR,
2099 other_rank,
2100 tag_request,
2101 comm,
2102 &status);
2103 AssertThrowMPI(ierr);
2104
2105 // Process request by asking the user-provided function for
2106 // the answer and post a send for it.
2107 auto &request_buffer = requests_buffers[index];
2108 request_buffer =
2109 (answer_request ?
2110 Utilities::pack(answer_request(other_rank,
2111 Utilities::unpack<RequestType>(
2112 buffer_recv, false)),
2113 false) :
2114 std::vector<char>());
2115
2116 ierr = MPI_Isend(request_buffer.data(),
2117 request_buffer.size(),
2118 MPI_CHAR,
2119 other_rank,
2120 tag_deliver,
2121 comm,
2122 &send_answer_requests[index]);
2123 AssertThrowMPI(ierr);
2124# else
2125 (void)answer_request;
2126 (void)comm;
2127 (void)index;
2128# endif
2129 }
2130
2131
2132
2133 template <typename RequestType, typename AnswerType>
2134 void
2136 const unsigned int n_targets,
2137 const std::function<void(const unsigned int, const AnswerType &)>
2138 & process_answer,
2139 const MPI_Comm &comm)
2140 {
2141# ifdef DEAL_II_WITH_MPI
2142 const int tag_deliver = Utilities::MPI::internal::Tags::
2144
2145 // We know how many targets we have sent requests to. These
2146 // targets will all eventually send us their responses, but
2147 // we need not process them in order -- rather, just see what
2148 // comes in and then look at message originators' ranks and
2149 // message sizes
2150 for (unsigned int i = 0; i < n_targets; ++i)
2151 {
2152 MPI_Status status;
2153 {
2154 const int ierr =
2155 MPI_Probe(MPI_ANY_SOURCE, tag_deliver, comm, &status);
2156 AssertThrowMPI(ierr);
2157 }
2158
2159 const auto other_rank = status.MPI_SOURCE;
2160 int message_size;
2161 {
2162 const int ierr = MPI_Get_count(&status, MPI_CHAR, &message_size);
2163 AssertThrowMPI(ierr);
2164 }
2165 std::vector<char> recv_buffer(message_size);
2166
2167 // Now actually receive the answer. Because the MPI_Probe
2168 // above blocks until we have a message, we know that the
2169 // following MPI_Recv call will immediately succeed.
2170 {
2171 const int ierr = MPI_Recv(recv_buffer.data(),
2172 recv_buffer.size(),
2173 MPI_CHAR,
2174 other_rank,
2175 tag_deliver,
2176 comm,
2177 MPI_STATUS_IGNORE);
2178 AssertThrowMPI(ierr);
2179 }
2180
2181 if (process_answer)
2182 process_answer(other_rank,
2183 Utilities::unpack<AnswerType>(recv_buffer, false));
2184 }
2185# else
2186 (void)n_targets;
2187 (void)process_answer;
2188 (void)comm;
2189# endif
2190 }
2191
2192
2193
2194 template <typename RequestType, typename AnswerType>
2195 void
2197 {
2198# ifdef DEAL_II_WITH_MPI
2199 // Finalize all MPI_Request objects for both the
2200 // send-request and receive-answer operations.
2201 if (send_request_requests.size() > 0)
2202 {
2203 const int ierr = MPI_Waitall(send_request_requests.size(),
2204 send_request_requests.data(),
2205 MPI_STATUSES_IGNORE);
2206 AssertThrowMPI(ierr);
2207 }
2208
2209 // Then also check the send-answer requests.
2210 if (send_answer_requests.size() > 0)
2211 {
2212 const int ierr = MPI_Waitall(send_answer_requests.size(),
2213 send_answer_requests.data(),
2214 MPI_STATUSES_IGNORE);
2215 AssertThrowMPI(ierr);
2216 }
2217# endif
2218 }
2219
2220
2221
2222 template <typename RequestType, typename AnswerType>
2224 Process<RequestType, AnswerType> &process,
2225 const MPI_Comm & comm)
2226 : Interface<RequestType, AnswerType>(process, comm)
2227 {}
2228
2229
2230
2231 template <typename RequestType, typename AnswerType>
2232 std::vector<unsigned int>
2234 const std::vector<unsigned int> & targets,
2235 const std::function<RequestType(const unsigned int)> &create_request,
2236 const std::function<AnswerType(const unsigned int, const RequestType &)>
2237 &answer_request,
2238 const std::function<void(const unsigned int, const AnswerType &)>
2239 & process_answer,
2240 const MPI_Comm &comm)
2241 {
2242 (void)comm;
2245 ExcMessage("You shouldn't use the 'Serial' class on "
2246 "communicators that have more than one process "
2247 "associated with it."));
2248
2249 // The only valid target for a serial program is itself.
2250 if (targets.size() != 0)
2251 {
2252 Assert(targets.size() == 1,
2253 ExcMessage(
2254 "On a single process, the only valid target "
2255 "is process zero (the process itself), which can only be "
2256 "listed once."));
2257 AssertDimension(targets[0], 0);
2258
2259 // Since the caller indicates that there is a target, and since we
2260 // know that it is the current process, let the process send
2261 // something to itself.
2262 const RequestType request =
2263 (create_request ? create_request(0) : RequestType());
2264 const AnswerType answer =
2265 (answer_request ? answer_request(0, request) : AnswerType());
2266
2267 if (process_answer)
2268 process_answer(0, answer);
2269 }
2270
2271 return targets; // nothing to do
2272 }
2273
2274
2275
2276 template <typename RequestType, typename AnswerType>
2278 Process<RequestType, AnswerType> &process,
2279 const MPI_Comm & comm)
2280 : Interface<RequestType, AnswerType>(process, comm)
2281 {}
2282
2283
2284
2285 template <typename RequestType, typename AnswerType>
2286 std::vector<unsigned int>
2288 const std::vector<unsigned int> & targets,
2289 const std::function<RequestType(const unsigned int)> &create_request,
2290 const std::function<AnswerType(const unsigned int, const RequestType &)>
2291 &answer_request,
2292 const std::function<void(const unsigned int, const AnswerType &)>
2293 & process_answer,
2294 const MPI_Comm &comm)
2295 {
2296 // Depending on the number of processes we switch between
2297 // implementations. We reduce the threshold for debug mode to be
2298 // able to test also the non-blocking implementation. This feature
2299 // is tested by:
2300 // tests/multigrid/transfer_matrix_free_06.with_mpi=true.with_p4est=true.with_trilinos=true.mpirun=10.output
2301
2302 const unsigned int n_procs = (Utilities::MPI::job_supports_mpi() ?
2304 1);
2305# ifdef DEAL_II_WITH_MPI
2306# ifdef DEBUG
2307 if (n_procs > 10)
2308# else
2309 if (n_procs > 99)
2310# endif
2311 consensus_algo.reset(new NBX<RequestType, AnswerType>());
2312 else
2313# endif
2314 if (n_procs > 1)
2315 consensus_algo.reset(new PEX<RequestType, AnswerType>());
2316 else
2317 consensus_algo.reset(new Serial<RequestType, AnswerType>());
2318
2319 return consensus_algo->run(
2320 targets, create_request, answer_request, process_answer, comm);
2321 }
2322
2323
2324 } // namespace ConsensusAlgorithms
2325 } // end of namespace MPI
2326} // end of namespace Utilities
2327
2328#endif // DOXYGEN
2329
2330
2332
2333#endif
std::vector< unsigned int > compute_targets() override
const std::function< void(const unsigned int, const RequestType &, AnswerType &)> function_answer_request
const std::function< std::vector< unsigned int >()> function_compute_targets
void create_request(const unsigned int other_rank, RequestType &send_buffer) override
const std::function< void(const int, RequestType &)> function_create_request
void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer) override
void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer) override
const std::function< void(const int, const AnswerType &)> function_read_answer
AnonymousProcess(const std::function< std::vector< unsigned int >()> &function_compute_targets, const std::function< void(const unsigned int, RequestType &)> &function_create_request={}, const std::function< void(const unsigned int, const RequestType &, AnswerType &)> &function_answer_request={}, const std::function< void(const unsigned int, const AnswerType &)> &function_read_answer={})
Interface(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)=0
std::vector< unsigned int > run(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
bool all_locally_originated_receives_are_completed(const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
void maybe_answer_one_request(const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm &comm)
void signal_finish(const MPI_Comm &comm)
std::vector< std::unique_ptr< std::vector< char > > > request_buffers
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
NBX(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
std::vector< std::unique_ptr< MPI_Request > > request_requests
std::vector< std::vector< char > > send_buffers
void clean_up_and_end_communication(const MPI_Comm &comm)
void start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm &comm)
void answer_one_request(const unsigned int index, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const MPI_Comm &comm)
std::vector< std::vector< char > > requests_buffers
unsigned int start_communication(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const MPI_Comm &comm)
std::vector< std::vector< char > > send_buffers
PEX(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
void process_incoming_answers(const unsigned int n_targets, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< std::vector< char > > recv_buffers
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
virtual std::vector< unsigned int > compute_targets()=0
virtual void answer_request(const unsigned int other_rank, const RequestType &buffer_recv, AnswerType &request_buffer)
virtual void read_answer(const unsigned int other_rank, const AnswerType &recv_buffer)
virtual void create_request(const unsigned int other_rank, RequestType &send_buffer)
Selector(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
std::shared_ptr< Interface< RequestType, AnswerType > > consensus_algo
virtual std::vector< unsigned int > run(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm) override
Serial(Process< RequestType, AnswerType > &process, const MPI_Comm &comm)
#define DEAL_II_DEPRECATED
Definition: config.h:164
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:442
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:443
#define Assert(cond, exc)
Definition: exceptions.h:1473
#define AssertDimension(dim1, dim2)
Definition: exceptions.h:1667
#define AssertThrowMPI(error_code)
Definition: exceptions.h:1790
#define AssertIndexRange(index, range)
Definition: exceptions.h:1732
static ::ExceptionBase & ExcMessage(std::string arg1)
std::vector< unsigned int > nbx(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > selector(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
std::vector< unsigned int > pex(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm &comm)
@ consensus_algorithm_nbx_process_deliver
ConsensusAlgorithms::NBX::process.
Definition: mpi_tags.h:91
@ consensus_algorithm_pex_process_deliver
ConsensusAlgorithms::PEX::process.
Definition: mpi_tags.h:96
@ consensus_algorithm_nbx_answer_request
ConsensusAlgorithms::NBX::process.
Definition: mpi_tags.h:89
@ consensus_algorithm_pex_answer_request
ConsensusAlgorithms::PEX::process.
Definition: mpi_tags.h:94
unsigned int compute_n_point_to_point_communications(const MPI_Comm &mpi_comm, const std::vector< unsigned int > &destinations)
Definition: mpi.cc:413
bool job_supports_mpi()
Definition: mpi.cc:1014
unsigned int n_mpi_processes(const MPI_Comm &mpi_communicator)
Definition: mpi.cc:140
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
Definition: utilities.h:1483
void run(const Iterator &begin, const typename identity< Iterator >::type &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
Definition: work_stream.h:474
STL namespace.
const MPI_Comm & comm