deal.II version GIT relicensing-1884-g9d0a489066 2024-09-19 12:30:00+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
init_finalize.cc
Go to the documentation of this file.
1// ---------------------------------------------------------------------
2//
3// Copyright (C) 2023 - 2024 by the deal.II authors
4//
5// This file is part of the deal.II library.
6//
7// The deal.II library is free software; you can use it, redistribute
8// it, and/or modify it under the terms of the GNU Lesser General
9// Public License as published by the Free Software Foundation; either
10// version 2.1 of the License, or (at your option) any later version.
11// The full text of the license can be found in the file LICENSE.md at
12// the top level directory of deal.II.
13//
14// ---------------------------------------------------------------------
15
17#include <deal.II/base/mpi.h>
19
23
24#include <Kokkos_Core.hpp>
25
26#ifdef DEAL_II_WITH_TRILINOS
27# ifdef DEAL_II_WITH_MPI
30
31# include <Epetra_MpiComm.h>
32# endif
33#endif
34
35#ifdef DEAL_II_WITH_PETSC
38
39# include <petscsys.h>
40#endif
41
42#ifdef DEAL_II_WITH_SLEPC
44
45# include <slepcsys.h>
46#endif
47
48#ifdef DEAL_II_WITH_P4EST
49# include <p4est_bits.h>
50#endif
51
52#ifdef DEAL_II_TRILINOS_WITH_ZOLTAN
53# include <zoltan_cpp.h>
54#endif
55
57
58
59/* Force initialization of static struct: */
61
62
64 [[maybe_unused]] char **&argv,
65 const InitializeLibrary &libraries,
66 const unsigned int max_num_threads)
67 : libraries(libraries)
68{
69 [[maybe_unused]] static bool constructor_has_already_run = false;
71 ExcMessage("You can only create a single object of this class "
72 "in a program since it initializes the MPI system."));
73
74
75 [[maybe_unused]] int ierr = 0;
76#ifdef DEAL_II_WITH_MPI
77 if (static_cast<bool>(libraries & InitializeLibrary::MPI))
78 {
79 // if we have PETSc, we will initialize it and let it handle MPI.
80 // Otherwise, we will do it.
85 ExcMessage("MPI error. You can only start MPI once!"));
86
87 int provided;
88 // this works like ierr = MPI_Init (&argc, &argv); but tells MPI that
89 // we might use several threads but never call two MPI functions at the
90 // same time. For an explanation see on why we do this see
91 // http://www.open-mpi.org/community/lists/users/2010/03/12244.php
95
96 // disable for now because at least some implementations always return
97 // MPI_THREAD_SINGLE.
98 // Assert(max_num_threads==1 || provided != MPI_THREAD_SINGLE,
99 // ExcMessage("MPI reports that we are not allowed to use multiple
100 // threads."));
101 }
102#endif
103
104 // we are allowed to call MPI_Init ourselves and PETScInitialize will
105 // detect this. This allows us to use MPI_Init_thread instead.
106#ifdef DEAL_II_WITH_PETSC
108# ifdef DEAL_II_WITH_SLEPC
109 // Initialize SLEPc (with PETSc):
110 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc))
111 {
113 pierr = SlepcInitialize(&argc, &argv, nullptr, nullptr);
115 }
116# else
117 // or just initialize PETSc alone:
118 if (static_cast<bool>(libraries & InitializeLibrary::PETSc))
119 {
121 pierr = PetscInitialize(&argc, &argv, nullptr, nullptr);
123 }
124# endif
125
126 // Disable PETSc exception handling. This just prints a large wall
127 // of text that is not particularly helpful for what we do:
128 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc) ||
129 static_cast<bool>(libraries & InitializeLibrary::PETSc))
130 {
133 }
134#endif
135
136 // Initialize zoltan
137#ifdef DEAL_II_TRILINOS_WITH_ZOLTAN
138 if (static_cast<bool>(libraries & InitializeLibrary::Zoltan))
139 {
140 float version;
141 Zoltan_Initialize(argc, argv, &version);
142 }
143#endif
144
145 // Initialize p4est and libsc components
146#ifdef DEAL_II_WITH_P4EST
147 if (static_cast<bool>(libraries & InitializeLibrary::P4EST))
148 {
149# if DEAL_II_P4EST_VERSION_GTE(2, 5, 0, 0)
150 // This feature is broken in version 2.0.0 for calls to
151 // MPI_Comm_create_group (see cburstedde/p4est#30).
152 // Disabling it leads to more verbose p4est error messages
153 // which should be fine.
154 sc_init(MPI_COMM_WORLD, 0, 0, nullptr, SC_LP_SILENT);
155# endif
156 p4est_init(nullptr, SC_LP_SILENT);
157 }
158#endif
159
161
162
163 // Now also see how many threads we'd like to run
165 {
166 // set maximum number of threads (also respecting the environment
167 // variable that the called function evaluates) based on what the
168 // user asked
170 }
171 else
172 // user wants automatic choice
173 {
174 unsigned int n_threads = MultithreadInfo::n_cores();
175#ifdef DEAL_II_WITH_MPI
176 if (static_cast<bool>(libraries & InitializeLibrary::MPI))
177 {
178 int MPI_has_been_started = 0;
181
182 // we need to figure out how many MPI processes there are on the
183 // current node, as well as how many CPU cores we have. for the
184 // first task, check what get_hostname() returns and then do an
185 // allgather so each processor gets the answer
186 //
187 // in calculating the length of the string, don't forget the
188 // terminating \0 on C-style strings
189 const std::string hostname = Utilities::System::get_hostname();
190
191 int my_hostname_size = hostname.size() + 1;
192 int max_hostname_size = -1;
195 1,
196 MPI_INT,
197 MPI_MAX,
200 std::vector<char> hostname_array(max_hostname_size);
201 std::copy(hostname.c_str(),
202 hostname.c_str() + hostname.size() + 1,
203 hostname_array.begin());
204
205 int n_mpi_processes = 1;
207 {
208 ierr = MPI_Comm_size(MPI_COMM_WORLD, &n_mpi_processes);
210 }
211 std::vector<char> all_hostnames(max_hostname_size * n_mpi_processes);
214 MPI_CHAR,
215 all_hostnames.data(),
217 MPI_CHAR,
220
221 // search how often our own hostname appears and the how-manyth
222 // instance the current process represents
223 unsigned int n_local_processes = 0;
224 unsigned int nth_process_on_host = 0;
225 int rank = 0;
227 {
230 }
231 for (int i = 0; i < n_mpi_processes; ++i)
232 if (std::string(all_hostnames.data() + i * max_hostname_size) ==
233 hostname)
234 {
236 if (i <= rank)
238 }
240
241
242 // compute how many cores each process gets. if the number does not
243 // divide evenly, then we get one more core if we are among the
244 // first few processes
245 //
246 // if the number would be zero, round up to one since every process
247 // needs to have at least one thread
248 n_threads =
252 1 :
253 0),
254 1U);
255 }
256#endif
257
258 // finally set this number of threads
260 }
261
262 // Initialize Kokkos
263 if (static_cast<bool>(libraries & InitializeLibrary::Kokkos))
264 {
265 // argv has argc+1 elements and the last one is a nullptr. For appending
266 // one element we thus create a new argv by copying the first argc
267 // elements, append the new option, and then a nullptr.
268 //
269 // We do get in trouble, though, if a user program is called with
270 // '--help' as a command line argument. This '--help' gets passed on to
271 // Kokkos, which promptly responds with a lengthy message that the user
272 // likely did not intend. As a consequence, filter out this specific
273 // flag.
274 std::vector<char *> argv_new;
275 for (auto *const arg : make_array_view(&argv[0], &argv[0] + argc))
276 if (strcmp(arg, "--help") != 0)
277 argv_new.push_back(arg);
278
279 std::stringstream threads_flag;
280#if KOKKOS_VERSION >= 30700
281 threads_flag << "--kokkos-num-threads=" << MultithreadInfo::n_threads();
282#else
283 threads_flag << "--kokkos-threads=" << MultithreadInfo::n_threads();
284#endif
285 const std::string threads_flag_string = threads_flag.str();
286 argv_new.push_back(const_cast<char *>(threads_flag_string.c_str()));
287 argv_new.push_back(nullptr);
288
289 // The first argument in Kokkos::initialize is of type int&. Hence, we
290 // need to define a new variable to pass to it (instead of using argc+1
291 // inline).
292 int argc_new = argv_new.size() - 1;
293 Kokkos::initialize(argc_new, argv_new.data());
294 }
295
296 // As a final step call the at_mpi_init() signal handler.
297 signals.at_mpi_init();
298}
299
300
301
302void
304{
305 // insert if it is not in the set already:
306 requests.insert(&request);
307}
308
309
310
311void
313{
314 Assert(requests.find(&request) != requests.end(),
316 "You tried to call unregister_request() with an invalid request."));
317
318 requests.erase(&request);
319}
320
321
322
323std::set<MPI_Request *> InitFinalize::requests;
324
325
326
327void
329{
330 if (!is_finalized)
331 {
332 // First, call the at_mpi_finalize() signal handler.
333 signals.at_mpi_finalize();
334
335 // make memory pool release all PETSc/Trilinos/MPI-based vectors that
336 // are no longer used at this point. this is relevant because the static
337 // object destructors run for these vectors at the end of the program
338 // would run after MPI_Finalize is called, leading to errors
339
340#ifdef DEAL_II_WITH_MPI
341 // Before exiting, wait for nonblocking communication to complete:
342 for (auto *request : requests)
343 {
344 const int ierr = MPI_Wait(request, MPI_STATUS_IGNORE);
346 }
347
348 // Start with deal.II MPI vectors and delete vectors from the pools:
350 LinearAlgebra::distributed::Vector<double>>::release_unused_memory();
354 LinearAlgebra::distributed::Vector<float>>::release_unused_memory();
357
358 // Next with Trilinos:
359# ifdef DEAL_II_WITH_TRILINOS
361 TrilinosWrappers::MPI::Vector>::release_unused_memory();
363 TrilinosWrappers::MPI::BlockVector>::release_unused_memory();
364# endif
365#endif
366
367
368 // Now deal with PETSc (with or without MPI). Only delete the vectors if
369 // finalize hasn't been called yet, otherwise this will lead to errors.
370#ifdef DEAL_II_WITH_PETSC
372 {
374 PETScWrappers::MPI::Vector>::release_unused_memory();
376 PETScWrappers::MPI::BlockVector>::release_unused_memory();
377 }
378# ifdef DEAL_II_WITH_SLEPC
379 // and now end SLEPc with PETSc if we did so
380 if (static_cast<bool>(libraries & InitializeLibrary::SLEPc) &&
382 {
384 AssertThrow(ierr == 0,
386 }
387# else
388 // or just end PETSc if we did so
389 if (static_cast<bool>(libraries & InitializeLibrary::PETSc) &&
391 {
394 }
395# endif
396#endif
397
398#ifdef DEAL_II_WITH_P4EST
399 // now end p4est and libsc
400 // Note: p4est has no finalize function
401 if (static_cast<bool>(libraries & InitializeLibrary::P4EST))
402 sc_finalize();
403#endif
404
405
406 // Finalize Kokkos
407 if (static_cast<bool>(libraries & InitializeLibrary::Kokkos))
408 Kokkos::finalize();
409
410 // only MPI_Finalize if we are running with MPI. We also need to do this
411 // when running PETSc, because we initialize MPI ourselves before
412 // calling PetscInitialize
413#ifdef DEAL_II_WITH_MPI
414 int MPI_has_been_started = 0;
417 if (static_cast<bool>(libraries & InitializeLibrary::MPI) &&
419 {
420 if (std::uncaught_exceptions() > 0)
421 {
422 // do not try to call MPI_Finalize to avoid a deadlock.
423 }
424 else
425 {
426 [[maybe_unused]] const int ierr = MPI_Finalize();
427
429 }
430 }
431#endif
432 is_finalized = true;
433 }
434}
435
436
437
442
443
ArrayView< std::remove_reference_t< typename std::iterator_traits< Iterator >::reference >, MemorySpaceType > make_array_view(const Iterator begin, const Iterator end)
Definition array_view.h:949
InitializeLibrary libraries
static void unregister_request(MPI_Request &request)
static void register_request(MPI_Request &request)
static std::set< MPI_Request * > requests
bool finalize_petscslepc
InitFinalize(int &argc, char **&argv, const InitializeLibrary &libraries, const unsigned int max_num_threads=numbers::invalid_unsigned_int)
static Signals signals
static unsigned int n_cores()
static unsigned int n_threads()
static void set_thread_limit(const unsigned int max_threads=numbers::invalid_unsigned_int)
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:500
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:501
static ::ExceptionBase & ExcSLEPcError(int arg1)
#define Assert(cond, exc)
#define AssertThrowMPI(error_code)
#define AssertNothrow(cond, exc)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
InitializeLibrary
std::string get_hostname()
Definition utilities.cc:998
static const unsigned int invalid_unsigned_int
Definition types.h:220
::VectorizedArray< Number, width > max(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
****code *  *  MPI_Finalize()