Loading [MathJax]/extensions/TeX/newcommand.js
 deal.II version GIT relicensing-3083-g7b89508ac7 2025-04-18 12:50:00+00:00
\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}} \newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=} \newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]} \newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages Concepts
init_finalize.cc
Go to the documentation of this file.
1// ---------------------------------------------------------------------
2//
3// Copyright (C) 2023 - 2024 by the deal.II authors
4//
5// This file is part of the deal.II library.
6//
7// The deal.II library is free software; you can use it, redistribute
8// it, and/or modify it under the terms of the GNU Lesser General
9// Public License as published by the Free Software Foundation; either
10// version 2.1 of the License, or (at your option) any later version.
11// The full text of the license can be found in the file LICENSE.md at
12// the top level directory of deal.II.
13//
14// ---------------------------------------------------------------------
15
17#include <deal.II/base/mpi.h>
19
23
24#include <Kokkos_Core.hpp>
25
26#ifdef DEAL_II_WITH_TRILINOS
27# ifdef DEAL_II_WITH_MPI
30
31# include <Epetra_MpiComm.h>
32# endif
33#endif
34
35#ifdef DEAL_II_WITH_PETSC
38
39# include <petscsys.h>
40#endif
41
42#ifdef DEAL_II_WITH_SLEPC
44
45# include <slepcsys.h>
46#endif
47
48#ifdef DEAL_II_WITH_P4EST
49# include <p4est_bits.h>
50#endif
51
52#ifdef DEAL_II_TRILINOS_WITH_ZOLTAN
53# include <zoltan_cpp.h>
54#endif
55
56#include <set>
57#include <string>
58
59
61
62
63/* Force initialization of static struct: */
65
66
67InitFinalize::InitFinalize([[maybe_unused]] int &argc,
68 [[maybe_unused]] char **&argv,
69 const InitializeLibrary &libraries,
70 const unsigned int max_num_threads)
71 : libraries(libraries)
72{
73 [[maybe_unused]] static bool constructor_has_already_run = false;
74 Assert(constructor_has_already_run == false,
75 ExcMessage("You can only create a single object of this class "
76 "in a program since it initializes the MPI system."));
77
78
79 [[maybe_unused]] int ierr = 0;
80#ifdef DEAL_II_WITH_MPI
81 if (static_cast<bool>(libraries & InitializeLibrary::MPI))
82 {
83 // if we have PETSc, we will initialize it and let it handle MPI.
84 // Otherwise, we will do it.
85 int MPI_has_been_started = 0;
86 ierr = MPI_Initialized(&MPI_has_been_started);
87 AssertThrowMPI(ierr);
88 AssertThrow(MPI_has_been_started == 0,
89 ExcMessage("MPI error. You can only start MPI once!"));
90
91 int provided;
92 // this works like ierr = MPI_Init (&argc, &argv); but tells MPI that
93 // we might use several threads but never call two MPI functions at the
94 // same time. For an explanation see on why we do this see
95 // http://www.open-mpi.org/community/lists/users/2010/03/12244.php
96 int wanted = MPI_THREAD_SERIALIZED;
97 ierr = MPI_Init_thread(&argc, &argv, wanted, &provided);
98 AssertThrowMPI(ierr);
99
100 // disable for now because at least some implementations always return
101 // MPI_THREAD_SINGLE.
102 // Assert(max_num_threads==1 || provided != MPI_THREAD_SINGLE,
103 // ExcMessage("MPI reports that we are not allowed to use multiple
104 // threads."));
105 }
106#endif
107
108 // we are allowed to call MPI_Init ourselves and PETScInitialize will
109 // detect this. This allows us to use MPI_Init_thread instead.
110#ifdef DEAL_II_WITH_PETSC
111 PetscErrorCode pierr;
112# ifdef DEAL_II_WITH_SLEPC
113 // Initialize SLEPc (with PETSc):
114 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc))
115 {
116 finalize_petscslepc = SlepcInitializeCalled ? false : true;
117 pierr = SlepcInitialize(&argc, &argv, nullptr, nullptr);
119 }
120# else
121 // or just initialize PETSc alone:
122 if (static_cast<bool>(libraries & InitializeLibrary::PETSc))
123 {
124 finalize_petscslepc = PetscInitializeCalled ? false : true;
125 pierr = PetscInitialize(&argc, &argv, nullptr, nullptr);
126 AssertThrow(pierr == 0, ExcPETScError(pierr));
127 }
128# endif
129
130 // Disable PETSc exception handling. This just prints a large wall
131 // of text that is not particularly helpful for what we do:
132 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc) ||
133 static_cast<bool>(libraries & InitializeLibrary::PETSc))
134 {
135 pierr = PetscPopSignalHandler();
136 AssertThrow(pierr == 0, ExcPETScError(pierr));
137 }
138#endif
139
140 // Initialize zoltan
141#ifdef DEAL_II_TRILINOS_WITH_ZOLTAN
142 if (static_cast<bool>(libraries & InitializeLibrary::Zoltan))
143 {
144 float version;
145 Zoltan_Initialize(argc, argv, &version);
146 }
147#endif
148
149 // Initialize p4est and libsc components
150#ifdef DEAL_II_WITH_P4EST
151 if (static_cast<bool>(libraries & InitializeLibrary::P4EST))
152 {
153# if DEAL_II_P4EST_VERSION_GTE(2, 5, 0, 0)
154 // This feature is broken in version 2.0.0 for calls to
155 // MPI_Comm_create_group (see cburstedde/p4est#30).
156 // Disabling it leads to more verbose p4est error messages
157 // which should be fine.
158 sc_init(MPI_COMM_WORLD, 0, 0, nullptr, SC_LP_SILENT);
159# endif
160 p4est_init(nullptr, SC_LP_SILENT);
161 }
162#endif
163
164 constructor_has_already_run = true;
165
166
167 // Now also see how many threads we'd like to run
168 if (max_num_threads != numbers::invalid_unsigned_int)
169 {
170 // set maximum number of threads (also respecting the environment
171 // variable that the called function evaluates) based on what the
172 // user asked
173 MultithreadInfo::set_thread_limit(max_num_threads);
174 }
175 else
176 // user wants automatic choice
177 {
178 unsigned int n_threads = MultithreadInfo::n_cores();
179#ifdef DEAL_II_WITH_MPI
180 if (static_cast<bool>(libraries & InitializeLibrary::MPI))
181 {
182 int MPI_has_been_started = 0;
183 int ierr = MPI_Initialized(&MPI_has_been_started);
184 AssertThrowMPI(ierr);
185
186 // we need to figure out how many MPI processes there are on the
187 // current node, as well as how many CPU cores we have. for the
188 // first task, check what get_hostname() returns and then do an
189 // allgather so each processor gets the answer
190 //
191 // in calculating the length of the string, don't forget the
192 // terminating \0 on C-style strings
193 const std::string hostname = Utilities::System::get_hostname();
194
195 int my_hostname_size = hostname.size() + 1;
196 int max_hostname_size = -1;
197 ierr = MPI_Allreduce(&my_hostname_size,
198 &max_hostname_size,
199 1,
200 MPI_INT,
201 MPI_MAX,
202 MPI_COMM_WORLD);
203 AssertThrowMPI(ierr);
204 std::vector<char> hostname_array(max_hostname_size);
205 std::copy(hostname.c_str(),
206 hostname.c_str() + hostname.size() + 1,
207 hostname_array.begin());
208
209 int n_mpi_processes = 1;
210 if (MPI_has_been_started)
211 {
212 ierr = MPI_Comm_size(MPI_COMM_WORLD, &n_mpi_processes);
213 AssertThrowMPI(ierr);
214 }
215 std::vector<char> all_hostnames(max_hostname_size * n_mpi_processes);
216 ierr = MPI_Allgather(hostname_array.data(),
217 max_hostname_size,
218 MPI_CHAR,
219 all_hostnames.data(),
220 max_hostname_size,
221 MPI_CHAR,
222 MPI_COMM_WORLD);
223 AssertThrowMPI(ierr);
224
225 // search how often our own hostname appears and the how-manyth
226 // instance the current process represents
227 unsigned int n_local_processes = 0;
228 unsigned int nth_process_on_host = 0;
229 int rank = 0;
230 if (MPI_has_been_started)
231 {
232 ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
233 AssertThrowMPI(ierr);
234 }
235 for (int i = 0; i < n_mpi_processes; ++i)
236 if (std::string(all_hostnames.data() + i * max_hostname_size) ==
237 hostname)
238 {
239 ++n_local_processes;
240 if (i <= rank)
241 ++nth_process_on_host;
242 }
243 Assert(nth_process_on_host > 0, ExcInternalError());
244
245
246 // compute how many cores each process gets. if the number does not
247 // divide evenly, then we get one more core if we are among the
248 // first few processes
249 //
250 // if the number would be zero, round up to one since every process
251 // needs to have at least one thread
252 n_threads =
253 std::max(MultithreadInfo::n_cores() / n_local_processes +
254 (nth_process_on_host <=
255 MultithreadInfo::n_cores() % n_local_processes ?
256 1 :
257 0),
258 1U);
259 }
260#endif
261
262 // finally set this number of threads
264 }
265
266 // Initialize Kokkos
267 if (static_cast<bool>(libraries & InitializeLibrary::Kokkos))
268 {
269 // argv has argc+1 elements and the last one is a nullptr. For appending
270 // one element we thus create a new argv by copying the first argc
271 // elements, append the new option, and then a nullptr.
272 //
273 // We do get in trouble, though, if a user program is called with
274 // '--help' as a command line argument. This '--help' gets passed on to
275 // Kokkos, which promptly responds with a lengthy message that the user
276 // likely did not intend. As a consequence, filter out this specific
277 // flag.
278 std::vector<char *> argv_new;
279 for (auto *const arg : make_array_view(&argv[0], &argv[0] + argc))
280 if (std::strcmp(arg, "--help") != 0)
281 argv_new.push_back(arg);
282
283 std::stringstream threads_flag;
284#if DEAL_II_KOKKOS_VERSION_GTE(3, 7, 0)
285 threads_flag << "--kokkos-num-threads=" << MultithreadInfo::n_threads();
286#else
287 threads_flag << "--kokkos-threads=" << MultithreadInfo::n_threads();
288#endif
289 const std::string threads_flag_string = threads_flag.str();
290 argv_new.push_back(const_cast<char *>(threads_flag_string.c_str()));
291 argv_new.push_back(nullptr);
292
293 // The first argument in Kokkos::initialize is of type int&. Hence, we
294 // need to define a new variable to pass to it (instead of using argc+1
295 // inline).
296 int argc_new = argv_new.size() - 1;
297 Kokkos::initialize(argc_new, argv_new.data());
298 }
299
300 // As a final step call the at_mpi_init() signal handler.
302}
303
304
305
306void
308{
309 // insert if it is not in the set already:
310 requests.insert(&request);
311}
312
313
314
315void
317{
318 Assert(requests.find(&request) != requests.end(),
320 "You tried to call unregister_request() with an invalid request."));
321
322 requests.erase(&request);
323}
324
325
326
327std::set<MPI_Request *> InitFinalize::requests;
328
329
330
331void
333{
334 if (!is_finalized)
335 {
336 // First, call the at_mpi_finalize() signal handler.
338
339 // make memory pool release all PETSc/Trilinos/MPI-based vectors that
340 // are no longer used at this point. this is relevant because the static
341 // object destructors run for these vectors at the end of the program
342 // would run after MPI_Finalize is called, leading to errors
343
344#ifdef DEAL_II_WITH_MPI
345 // Before exiting, wait for nonblocking communication to complete:
346 for (auto *request : requests)
347 {
348 const int ierr = MPI_Wait(request, MPI_STATUS_IGNORE);
349 AssertThrowMPI(ierr);
350 }
351
352 // Start with deal.II MPI vectors and delete vectors from the pools:
354 LinearAlgebra::distributed::Vector<double>>::release_unused_memory();
356 release_unused_memory();
358 LinearAlgebra::distributed::Vector<float>>::release_unused_memory();
360 release_unused_memory();
361
362 // Next with Trilinos:
363# ifdef DEAL_II_WITH_TRILINOS
365 TrilinosWrappers::MPI::Vector>::release_unused_memory();
367 TrilinosWrappers::MPI::BlockVector>::release_unused_memory();
368# endif
369#endif
370
371
372 // Now deal with PETSc (with or without MPI). Only delete the vectors if
373 // finalize hasn't been called yet, otherwise this will lead to errors.
374#ifdef DEAL_II_WITH_PETSC
375 if (!PetscFinalizeCalled)
376 {
378 PETScWrappers::MPI::Vector>::release_unused_memory();
380 PETScWrappers::MPI::BlockVector>::release_unused_memory();
381 }
382# ifdef DEAL_II_WITH_SLEPC
383 // and now end SLEPc with PETSc if we did so
384 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc) &&
386 {
387 PetscErrorCode ierr = SlepcFinalize();
388 AssertThrow(ierr == 0,
390 }
391# else
392 // or just end PETSc if we did so
393 if (static_cast<bool>(libraries & InitializeLibrary::PETSc) &&
395 {
396 PetscErrorCode ierr = PetscFinalize();
397 AssertThrow(ierr == 0, ExcPETScError(ierr));
398 }
399# endif
400#endif
401
402#ifdef DEAL_II_WITH_P4EST
403 // now end p4est and libsc
404 // Note: p4est has no finalize function
405 if (static_cast<bool>(libraries & InitializeLibrary::P4EST))
406 sc_finalize();
407#endif
408
409
410 // Finalize Kokkos
411 if (static_cast<bool>(libraries & InitializeLibrary::Kokkos))
412 Kokkos::finalize();
413
414 // only MPI_Finalize if we are running with MPI. We also need to do this
415 // when running PETSc, because we initialize MPI ourselves before
416 // calling PetscInitialize
417#ifdef DEAL_II_WITH_MPI
418 int MPI_has_been_started = 0;
419 const int ierr = MPI_Initialized(&MPI_has_been_started);
420 AssertThrowMPI(ierr);
421 if (static_cast<bool>(libraries & InitializeLibrary::MPI) &&
422 (MPI_has_been_started))
423 {
424 if (std::uncaught_exceptions() > 0)
425 {
426 // do not try to call MPI_Finalize to avoid a deadlock.
427 }
428 else
429 {
430 [[maybe_unused]] const int ierr = MPI_Finalize();
431
432 AssertNothrow(ierr == MPI_SUCCESS, ::ExcMPI(ierr));
433 }
434 }
435#endif
436 is_finalized = true;
437 }
438}
439
440
441
446
447
ArrayView< std::remove_reference_t< typename std::iterator_traits< Iterator >::reference >, MemorySpaceType > make_array_view(const Iterator begin, const Iterator end)
Definition array_view.h:954
InitializeLibrary libraries
static void unregister_request(MPI_Request &request)
static void register_request(MPI_Request &request)
static std::set< MPI_Request * > requests
bool finalize_petscslepc
InitFinalize(int &argc, char **&argv, const InitializeLibrary &libraries, const unsigned int max_num_threads=numbers::invalid_unsigned_int)
static Signals signals
static unsigned int n_cores()
static unsigned int n_threads()
static void set_thread_limit(const unsigned int max_threads=numbers::invalid_unsigned_int)
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:35
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:36
static ::ExceptionBase & ExcSLEPcError(int arg1)
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertNothrow(cond, exc)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
InitializeLibrary
std::string get_hostname()
Definition utilities.cc:996
constexpr unsigned int invalid_unsigned_int
Definition types.h:238
::VectorizedArray< Number, width > max(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
****code *  *  MPI_Finalize()
boost::signals2::signal< void()> at_mpi_init
boost::signals2::signal< void()> at_mpi_finalize