Reference documentation for deal.II version GIT relicensing-515-g7835781687 2024-05-02 18:40:02+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
init_finalize.cc
Go to the documentation of this file.
1// ---------------------------------------------------------------------
2//
3// Copyright (C) 2023 by the deal.II authors
4//
5// This file is part of the deal.II library.
6//
7// The deal.II library is free software; you can use it, redistribute
8// it, and/or modify it under the terms of the GNU Lesser General
9// Public License as published by the Free Software Foundation; either
10// version 2.1 of the License, or (at your option) any later version.
11// The full text of the license can be found in the file LICENSE.md at
12// the top level directory of deal.II.
13//
14// ---------------------------------------------------------------------
15
17#include <deal.II/base/mpi.h>
19
23
24#include <Kokkos_Core.hpp>
25
26#ifdef DEAL_II_WITH_TRILINOS
27# ifdef DEAL_II_WITH_MPI
30
31# include <Epetra_MpiComm.h>
32# endif
33#endif
34
35#ifdef DEAL_II_WITH_PETSC
38
39# include <petscsys.h>
40#endif
41
42#ifdef DEAL_II_WITH_SLEPC
44
45# include <slepcsys.h>
46#endif
47
48#ifdef DEAL_II_WITH_P4EST
49# include <p4est_bits.h>
50#endif
51
52#ifdef DEAL_II_TRILINOS_WITH_ZOLTAN
53# include <zoltan_cpp.h>
54#endif
55
57
58
59/* Force initialization of static struct: */
61
62
64 char **&argv,
65 const InitializeLibrary &libraries,
66 const unsigned int max_num_threads)
67 : libraries(libraries)
68{
69 static bool constructor_has_already_run = false;
70 (void)constructor_has_already_run;
71 Assert(constructor_has_already_run == false,
72 ExcMessage("You can only create a single object of this class "
73 "in a program since it initializes the MPI system."));
74
75
76 int ierr = 0;
77#ifdef DEAL_II_WITH_MPI
78 if (static_cast<bool>(libraries & InitializeLibrary::MPI))
79 {
80 // if we have PETSc, we will initialize it and let it handle MPI.
81 // Otherwise, we will do it.
82 int MPI_has_been_started = 0;
83 ierr = MPI_Initialized(&MPI_has_been_started);
84 AssertThrowMPI(ierr);
85 AssertThrow(MPI_has_been_started == 0,
86 ExcMessage("MPI error. You can only start MPI once!"));
87
88 int provided;
89 // this works like ierr = MPI_Init (&argc, &argv); but tells MPI that
90 // we might use several threads but never call two MPI functions at the
91 // same time. For an explanation see on why we do this see
92 // http://www.open-mpi.org/community/lists/users/2010/03/12244.php
93 int wanted = MPI_THREAD_SERIALIZED;
94 ierr = MPI_Init_thread(&argc, &argv, wanted, &provided);
95 AssertThrowMPI(ierr);
96
97 // disable for now because at least some implementations always return
98 // MPI_THREAD_SINGLE.
99 // Assert(max_num_threads==1 || provided != MPI_THREAD_SINGLE,
100 // ExcMessage("MPI reports that we are not allowed to use multiple
101 // threads."));
102 }
103#else
104 // make sure the compiler doesn't warn about these variables
105 (void)argc;
106 (void)argv;
107 (void)ierr;
108#endif
109
110 // Initialize Kokkos
111 if (static_cast<bool>(libraries & InitializeLibrary::Kokkos))
112 {
113 // argv has argc+1 elements and the last one is a nullptr. For appending
114 // one element we thus create a new argv by copying the first argc
115 // elements, append the new option, and then a nullptr.
116 //
117 // We do get in trouble, though, if a user program is called with
118 // '--help' as a command line argument. This '--help' gets passed on to
119 // Kokkos, which promptly responds with a lengthy message that the user
120 // likely did not intend. As a consequence, filter out this specific
121 // flag.
122 std::vector<char *> argv_new;
123 for (auto *const arg : make_array_view(&argv[0], &argv[0] + argc))
124 if (strcmp(arg, "--help") != 0)
125 argv_new.push_back(arg);
126
127 std::stringstream threads_flag;
128#if KOKKOS_VERSION >= 30700
129 threads_flag << "--kokkos-num-threads=" << MultithreadInfo::n_threads();
130#else
131 threads_flag << "--kokkos-threads=" << MultithreadInfo::n_threads();
132#endif
133 const std::string threads_flag_string = threads_flag.str();
134 argv_new.push_back(const_cast<char *>(threads_flag_string.c_str()));
135 argv_new.push_back(nullptr);
136
137 // The first argument in Kokkos::initialize is of type int&. Hence, we
138 // need to define a new variable to pass to it (instead of using argc+1
139 // inline).
140 int argc_new = argv_new.size() - 1;
141 Kokkos::initialize(argc_new, argv_new.data());
142 }
143
144 // we are allowed to call MPI_Init ourselves and PETScInitialize will
145 // detect this. This allows us to use MPI_Init_thread instead.
146#ifdef DEAL_II_WITH_PETSC
147 PetscErrorCode pierr;
148# ifdef DEAL_II_WITH_SLEPC
149 // Initialize SLEPc (with PETSc):
150 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc))
151 {
152 finalize_petscslepc = SlepcInitializeCalled ? false : true;
153 pierr = SlepcInitialize(&argc, &argv, nullptr, nullptr);
155 }
156# else
157 // or just initialize PETSc alone:
158 if (static_cast<bool>(libraries & InitializeLibrary::PETSc))
159 {
160 finalize_petscslepc = PetscInitializeCalled ? false : true;
161 pierr = PetscInitialize(&argc, &argv, nullptr, nullptr);
162 AssertThrow(pierr == 0, ExcPETScError(pierr));
163 }
164# endif
165
166 // Disable PETSc exception handling. This just prints a large wall
167 // of text that is not particularly helpful for what we do:
168 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc) ||
169 static_cast<bool>(libraries & InitializeLibrary::PETSc))
170 {
171 pierr = PetscPopSignalHandler();
172 AssertThrow(pierr == 0, ExcPETScError(pierr));
173 }
174#endif
175
176 // Initialize zoltan
177#ifdef DEAL_II_TRILINOS_WITH_ZOLTAN
178 if (static_cast<bool>(libraries & InitializeLibrary::Zoltan))
179 {
180 float version;
181 Zoltan_Initialize(argc, argv, &version);
182 }
183#endif
184
185 // Initialize p4est and libsc components
186#ifdef DEAL_II_WITH_P4EST
187 if (static_cast<bool>(libraries & InitializeLibrary::P4EST))
188 {
189# if DEAL_II_P4EST_VERSION_GTE(2, 5, 0, 0)
190 // This feature is broken in version 2.0.0 for calls to
191 // MPI_Comm_create_group (see cburstedde/p4est#30).
192 // Disabling it leads to more verbose p4est error messages
193 // which should be fine.
194 sc_init(MPI_COMM_WORLD, 0, 0, nullptr, SC_LP_SILENT);
195# endif
196 p4est_init(nullptr, SC_LP_SILENT);
197 }
198#endif
199
200 constructor_has_already_run = true;
201
202
203 // Now also see how many threads we'd like to run
204 if (max_num_threads != numbers::invalid_unsigned_int)
205 {
206 // set maximum number of threads (also respecting the environment
207 // variable that the called function evaluates) based on what the
208 // user asked
209 MultithreadInfo::set_thread_limit(max_num_threads);
210 }
211 else
212 // user wants automatic choice
213 {
214 unsigned int n_threads = MultithreadInfo::n_cores();
215#ifdef DEAL_II_WITH_MPI
216 if (static_cast<bool>(libraries & InitializeLibrary::MPI))
217 {
218 int MPI_has_been_started = 0;
219 int ierr = MPI_Initialized(&MPI_has_been_started);
220 AssertThrowMPI(ierr);
221
222 // we need to figure out how many MPI processes there are on the
223 // current node, as well as how many CPU cores we have. for the
224 // first task, check what get_hostname() returns and then do an
225 // allgather so each processor gets the answer
226 //
227 // in calculating the length of the string, don't forget the
228 // terminating \0 on C-style strings
229 const std::string hostname = Utilities::System::get_hostname();
230
231 int my_hostname_size = hostname.size() + 1;
232 int max_hostname_size = -1;
233 ierr = MPI_Allreduce(&my_hostname_size,
234 &max_hostname_size,
235 1,
236 MPI_INT,
237 MPI_MAX,
238 MPI_COMM_WORLD);
239 AssertThrowMPI(ierr);
240 std::vector<char> hostname_array(max_hostname_size);
241 std::copy(hostname.c_str(),
242 hostname.c_str() + hostname.size() + 1,
243 hostname_array.begin());
244
245 int n_mpi_processes = 1;
246 if (MPI_has_been_started)
247 {
248 ierr = MPI_Comm_size(MPI_COMM_WORLD, &n_mpi_processes);
249 AssertThrowMPI(ierr);
250 }
251 std::vector<char> all_hostnames(max_hostname_size * n_mpi_processes);
252 ierr = MPI_Allgather(hostname_array.data(),
253 max_hostname_size,
254 MPI_CHAR,
255 all_hostnames.data(),
256 max_hostname_size,
257 MPI_CHAR,
258 MPI_COMM_WORLD);
259 AssertThrowMPI(ierr);
260
261 // search how often our own hostname appears and the how-manyth
262 // instance the current process represents
263 unsigned int n_local_processes = 0;
264 unsigned int nth_process_on_host = 0;
265 int rank = 0;
266 if (MPI_has_been_started)
267 {
268 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
269 AssertThrowMPI(ierr);
270 }
271 for (int i = 0; i < n_mpi_processes; ++i)
272 if (std::string(all_hostnames.data() + i * max_hostname_size) ==
273 hostname)
274 {
275 ++n_local_processes;
276 if (i <= rank)
277 ++nth_process_on_host;
278 }
279 Assert(nth_process_on_host > 0, ExcInternalError());
280
281
282 // compute how many cores each process gets. if the number does not
283 // divide evenly, then we get one more core if we are among the
284 // first few processes
285 //
286 // if the number would be zero, round up to one since every process
287 // needs to have at least one thread
288 n_threads =
289 std::max(MultithreadInfo::n_cores() / n_local_processes +
290 (nth_process_on_host <=
291 MultithreadInfo::n_cores() % n_local_processes ?
292 1 :
293 0),
294 1U);
295 }
296#endif
297
298 // finally set this number of threads
300 }
301
302 // As a final step call the at_mpi_init() signal handler.
304}
305
306
307
308void
310{
311 // insert if it is not in the set already:
312 requests.insert(&request);
313}
314
315
316
317void
319{
320 Assert(requests.find(&request) != requests.end(),
322 "You tried to call unregister_request() with an invalid request."));
323
324 requests.erase(&request);
325}
326
327
328
329std::set<MPI_Request *> InitFinalize::requests;
330
331
332
333void
335{
336 if (!is_finalized)
337 {
338 // First, call the at_mpi_finalize() signal handler.
340
341 // make memory pool release all PETSc/Trilinos/MPI-based vectors that
342 // are no longer used at this point. this is relevant because the static
343 // object destructors run for these vectors at the end of the program
344 // would run after MPI_Finalize is called, leading to errors
345
346#ifdef DEAL_II_WITH_MPI
347 // Before exiting, wait for nonblocking communication to complete:
348 for (auto *request : requests)
349 {
350 const int ierr = MPI_Wait(request, MPI_STATUS_IGNORE);
351 AssertThrowMPI(ierr);
352 }
353
354 // Start with deal.II MPI vectors and delete vectors from the pools:
356 LinearAlgebra::distributed::Vector<double>>::release_unused_memory();
358 release_unused_memory();
360 LinearAlgebra::distributed::Vector<float>>::release_unused_memory();
362 release_unused_memory();
363
364 // Next with Trilinos:
365# ifdef DEAL_II_WITH_TRILINOS
367 TrilinosWrappers::MPI::Vector>::release_unused_memory();
369 TrilinosWrappers::MPI::BlockVector>::release_unused_memory();
370# endif
371#endif
372
373
374 // Now deal with PETSc (with or without MPI). Only delete the vectors if
375 // finalize hasn't been called yet, otherwise this will lead to errors.
376#ifdef DEAL_II_WITH_PETSC
377 if (!PetscFinalizeCalled)
378 {
380 PETScWrappers::MPI::Vector>::release_unused_memory();
382 PETScWrappers::MPI::BlockVector>::release_unused_memory();
383 }
384# ifdef DEAL_II_WITH_SLEPC
385 // and now end SLEPc with PETSc if we did so
386 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc) &&
388 {
389 PetscErrorCode ierr = SlepcFinalize();
390 AssertThrow(ierr == 0,
392 }
393# else
394 // or just end PETSc if we did so
395 if (static_cast<bool>(libraries & InitializeLibrary::PETSc) &&
397 {
398 PetscErrorCode ierr = PetscFinalize();
399 AssertThrow(ierr == 0, ExcPETScError(ierr));
400 }
401# endif
402#endif
403
404#ifdef DEAL_II_WITH_P4EST
405 // now end p4est and libsc
406 // Note: p4est has no finalize function
407 if (static_cast<bool>(libraries & InitializeLibrary::P4EST))
408 sc_finalize();
409#endif
410
411
412 // Finalize Kokkos
413 if (static_cast<bool>(libraries & InitializeLibrary::Kokkos))
414 Kokkos::finalize();
415
416 // only MPI_Finalize if we are running with MPI. We also need to do this
417 // when running PETSc, because we initialize MPI ourselves before
418 // calling PetscInitialize
419#ifdef DEAL_II_WITH_MPI
420 int MPI_has_been_started = 0;
421 const int ierr = MPI_Initialized(&MPI_has_been_started);
422 AssertThrowMPI(ierr);
423 if (static_cast<bool>(libraries & InitializeLibrary::MPI) &&
424 (MPI_has_been_started))
425 {
426# if __cpp_lib_uncaught_exceptions >= 201411
427 // std::uncaught_exception() is deprecated in c++17
428 if (std::uncaught_exceptions() > 0)
429# else
430 if (std::uncaught_exception() == true)
431# endif
432 {
433 // do not try to call MPI_Finalize to avoid a deadlock.
434 }
435 else
436 {
437 const int ierr = MPI_Finalize();
438 (void)ierr;
439 AssertNothrow(ierr == MPI_SUCCESS, ::ExcMPI(ierr));
440 }
441 }
442#endif
443 is_finalized = true;
444 }
445}
446
447
448
453
454
ArrayView< typename std::remove_reference< typename std::iterator_traits< Iterator >::reference >::type, MemorySpaceType > make_array_view(const Iterator begin, const Iterator end)
Definition array_view.h:949
InitializeLibrary libraries
static void unregister_request(MPI_Request &request)
static void register_request(MPI_Request &request)
static std::set< MPI_Request * > requests
bool finalize_petscslepc
InitFinalize(int &argc, char **&argv, const InitializeLibrary &libraries, const unsigned int max_num_threads=numbers::invalid_unsigned_int)
static Signals signals
static unsigned int n_cores()
static unsigned int n_threads()
static void set_thread_limit(const unsigned int max_threads=numbers::invalid_unsigned_int)
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:502
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:503
static ::ExceptionBase & ExcSLEPcError(int arg1)
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertNothrow(cond, exc)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
InitializeLibrary
std::string get_hostname()
Definition utilities.cc:998
static const unsigned int invalid_unsigned_int
Definition types.h:220
::VectorizedArray< Number, width > max(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
****code *  *  MPI_Finalize()
boost::signals2::signal< void()> at_mpi_init
boost::signals2::signal< void()> at_mpi_finalize