partitioner.h

Go to the documentation of this file.

// ------------------------------------------------------------------------
//
// SPDX-License-Identifier: LGPL-2.1-or-later
// Copyright (C) 2012 - 2023 by the deal.II authors
//
// This file is part of the deal.II library.
//
// Part of the source code is dual licensed under Apache-2.0 WITH
// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
// governing the source code and code contributions can be found in
// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
//
// ------------------------------------------------------------------------
 
#ifndef dealii_partitioner_h
#define dealii_partitioner_h
 
#include <deal.II/base/config.h>
 
#include <deal.II/base/array_view.h>
#include <deal.II/base/communication_pattern_base.h>
#include <deal.II/base/index_set.h>
#include <deal.II/base/memory_space.h>
#include <deal.II/base/mpi_stub.h>
#include <deal.II/base/types.h>
 
#include <deal.II/lac/vector_operation.h>
 
#include <limits>
#include <memory>
 
DEAL_II_NAMESPACE_OPEN
 
namespace Utilities
{
  namespace MPI
  {
    class Partitioner : public Utilities::MPI::CommunicationPatternBase
    {
    public:
      Partitioner();
 
      Partitioner(const unsigned int size);
 
      Partitioner(const types::global_dof_index local_size,
                  const types::global_dof_index ghost_size,
                  const MPI_Comm                communicator);
 
      Partitioner(const IndexSet &locally_owned_indices,
                  const IndexSet &ghost_indices_in,
                  const MPI_Comm  communicator_in);
 
      Partitioner(const IndexSet &locally_owned_indices,
                  const MPI_Comm  communicator_in);
 
      virtual void
      reinit(const IndexSet &locally_owned_indices,
             const IndexSet &ghost_indices,
             const MPI_Comm  communicator) override;
 
      void
      set_owned_indices(const IndexSet &locally_owned_indices);
 
      void
      set_ghost_indices(const IndexSet &ghost_indices,
                        const IndexSet &larger_ghost_index_set = IndexSet());
 
      types::global_dof_index
      size() const;
 
      unsigned int
      locally_owned_size() const;
 
      const IndexSet &
      locally_owned_range() const;
 
      std::pair<types::global_dof_index, types::global_dof_index>
      local_range() const;
 
      bool
      in_local_range(const types::global_dof_index global_index) const;
 
      unsigned int
      global_to_local(const types::global_dof_index global_index) const;
 
      types::global_dof_index
      local_to_global(const unsigned int local_index) const;
 
      bool
      is_ghost_entry(const types::global_dof_index global_index) const;
 
      const IndexSet &
      ghost_indices() const;
 
      unsigned int
      n_ghost_indices() const;
 
      const std::vector<std::pair<unsigned int, unsigned int>> &
      ghost_indices_within_larger_ghost_set() const;
 
      const std::vector<std::pair<unsigned int, unsigned int>> &
      ghost_targets() const;
 
      const std::vector<std::pair<unsigned int, unsigned int>> &
      import_indices() const;
 
      unsigned int
      n_import_indices() const;
 
      const std::vector<std::pair<unsigned int, unsigned int>> &
      import_targets() const;
 
      bool
      is_compatible(const Partitioner &part) const;
 
      bool
      is_globally_compatible(const Partitioner &part) const;
 
      unsigned int
      this_mpi_process() const;
 
      unsigned int
      n_mpi_processes() const;
 
      virtual MPI_Comm
      get_mpi_communicator() const override;
 
      bool
      ghost_indices_initialized() const;
 
#ifdef DEAL_II_WITH_MPI
      template <typename Number, typename MemorySpaceType = MemorySpace::Host>
      void
      export_to_ghosted_array_start(
        const unsigned int                              communication_channel,
        const ArrayView<const Number, MemorySpaceType> &locally_owned_array,
        const ArrayView<Number, MemorySpaceType>       &temporary_storage,
        const ArrayView<Number, MemorySpaceType>       &ghost_array,
        std::vector<MPI_Request>                       &requests) const;
 
      template <typename Number, typename MemorySpaceType = MemorySpace::Host>
      void
      export_to_ghosted_array_finish(
        const ArrayView<Number, MemorySpaceType> &ghost_array,
        std::vector<MPI_Request>                 &requests) const;
 
      template <typename Number, typename MemorySpaceType = MemorySpace::Host>
      void
      import_from_ghosted_array_start(
        const VectorOperation::values             vector_operation,
        const unsigned int                        communication_channel,
        const ArrayView<Number, MemorySpaceType> &ghost_array,
        const ArrayView<Number, MemorySpaceType> &temporary_storage,
        std::vector<MPI_Request>                 &requests) const;
 
      template <typename Number, typename MemorySpaceType = MemorySpace::Host>
      void
      import_from_ghosted_array_finish(
        const VectorOperation::values                   vector_operation,
        const ArrayView<const Number, MemorySpaceType> &temporary_storage,
        const ArrayView<Number, MemorySpaceType>       &locally_owned_storage,
        const ArrayView<Number, MemorySpaceType>       &ghost_array,
        std::vector<MPI_Request>                       &requests) const;
#endif
 
      std::size_t
      memory_consumption() const;
 
      DeclException2(ExcIndexNotPresent,
                     types::global_dof_index,
                     unsigned int,
                     << "Global index " << arg1
                     << " neither owned nor ghost on proc " << arg2 << '.');
 
      DeclException3(ExcGhostIndexArrayHasWrongSize,
                     unsigned int,
                     unsigned int,
                     unsigned int,
                     << "The size of the ghost index array (" << arg1
                     << ") must either equal the number of ghost in the "
                     << "partitioner (" << arg2
                     << ") or be equal in size to a more comprehensive index"
                     << "set which contains " << arg3
                     << " elements for this partitioner.");
 
    private:
      void
      initialize_import_indices_plain_dev() const;
 
      types::global_dof_index global_size;
 
      IndexSet locally_owned_range_data;
 
      std::pair<types::global_dof_index, types::global_dof_index>
        local_range_data;
 
      IndexSet ghost_indices_data;
 
      unsigned int n_ghost_indices_data;
 
      std::vector<std::pair<unsigned int, unsigned int>> ghost_targets_data;
 
      std::vector<std::pair<unsigned int, unsigned int>> import_indices_data;
 
      // The variable is mutable to enable lazy initialization in
      // export_to_ghosted_array_start().
      mutable std::vector<
        Kokkos::View<unsigned int *, MemorySpace::Default::kokkos_space>>
        import_indices_plain_dev;
 
      unsigned int n_import_indices_data;
 
      std::vector<std::pair<unsigned int, unsigned int>> import_targets_data;
 
      std::vector<unsigned int> import_indices_chunks_by_rank_data;
 
      unsigned int n_ghost_indices_in_larger_set;
 
      std::vector<unsigned int> ghost_indices_subset_chunks_by_rank_data;
 
      std::vector<std::pair<unsigned int, unsigned int>>
        ghost_indices_subset_data;
 
      unsigned int my_pid;
 
      unsigned int n_procs;
 
      MPI_Comm communicator;
 
      bool have_ghost_indices;
    };
 
 
 
    /*--------------------- Inline functions --------------------------------*/
 
#ifndef DOXYGEN
 
    inline types::global_dof_index
    Partitioner::size() const
    {
      return global_size;
    }
 
 
 
    inline const IndexSet &
    Partitioner::locally_owned_range() const
    {
      return locally_owned_range_data;
    }
 
 
 
    inline std::pair<types::global_dof_index, types::global_dof_index>
    Partitioner::local_range() const
    {
      return local_range_data;
    }
 
 
 
    inline unsigned int
    Partitioner::locally_owned_size() const
    {
      types::global_dof_index size =
        local_range_data.second - local_range_data.first;
      Assert(size <= std::numeric_limits<unsigned int>::max(),
             ExcNotImplemented());
      return static_cast<unsigned int>(size);
    }
 
 
 
    inline bool
    Partitioner::in_local_range(
      const types::global_dof_index global_index) const
    {
      return (local_range_data.first <= global_index &&
              global_index < local_range_data.second);
    }
 
 
 
    inline bool
    Partitioner::is_ghost_entry(
      const types::global_dof_index global_index) const
    {
      // if the index is in the global range, it is trivially not a ghost
      if (in_local_range(global_index) == true)
        return false;
      else
        return ghost_indices().is_element(global_index);
    }
 
 
 
    inline unsigned int
    Partitioner::global_to_local(
      const types::global_dof_index global_index) const
    {
      Assert(in_local_range(global_index) || is_ghost_entry(global_index),
             ExcIndexNotPresent(global_index, my_pid));
      if (in_local_range(global_index))
        return static_cast<unsigned int>(global_index - local_range_data.first);
      else
        {
          // avoid checking the ghost index set via a binary search twice by
          // querying the index within set, which returns invalid_dof_index
          // for non-existent entries
          const types::global_dof_index index_within_ghosts =
            ghost_indices_data.index_within_set(global_index);
          if (index_within_ghosts == numbers::invalid_dof_index)
            return numbers::invalid_unsigned_int;
          else
            return locally_owned_size() +
                   static_cast<unsigned int>(index_within_ghosts);
        }
    }
 
 
 
    inline types::global_dof_index
    Partitioner::local_to_global(const unsigned int local_index) const
    {
      AssertIndexRange(local_index,
                       locally_owned_size() + n_ghost_indices_data);
      if (local_index < locally_owned_size())
        return local_range_data.first + types::global_dof_index(local_index);
      else
        return ghost_indices_data.nth_index_in_set(local_index -
                                                   locally_owned_size());
    }
 
 
 
    inline const IndexSet &
    Partitioner::ghost_indices() const
    {
      return ghost_indices_data;
    }
 
 
 
    inline unsigned int
    Partitioner::n_ghost_indices() const
    {
      return n_ghost_indices_data;
    }
 
 
 
    inline const std::vector<std::pair<unsigned int, unsigned int>> &
    Partitioner::ghost_indices_within_larger_ghost_set() const
    {
      return ghost_indices_subset_data;
    }
 
 
 
    inline const std::vector<std::pair<unsigned int, unsigned int>> &
    Partitioner::ghost_targets() const
    {
      return ghost_targets_data;
    }
 
 
    inline const std::vector<std::pair<unsigned int, unsigned int>> &
    Partitioner::import_indices() const
    {
      return import_indices_data;
    }
 
 
 
    inline unsigned int
    Partitioner::n_import_indices() const
    {
      return n_import_indices_data;
    }
 
 
 
    inline const std::vector<std::pair<unsigned int, unsigned int>> &
    Partitioner::import_targets() const
    {
      return import_targets_data;
    }
 
 
 
    inline unsigned int
    Partitioner::this_mpi_process() const
    {
      // return the id from the variable stored in this class instead of
      // Utilities::MPI::this_mpi_process() in order to make this query also
      // work when MPI is not initialized.
      return my_pid;
    }
 
 
 
    inline unsigned int
    Partitioner::n_mpi_processes() const
    {
      // return the number of MPI processes from the variable stored in this
      // class instead of Utilities::MPI::n_mpi_processes() in order to make
      // this query also work when MPI is not initialized.
      return n_procs;
    }
 
 
 
    inline MPI_Comm
    Partitioner::get_mpi_communicator() const
    {
      return communicator;
    }
 
 
 
    inline bool
    Partitioner::ghost_indices_initialized() const
    {
      return have_ghost_indices;
    }
 
#endif // ifndef DOXYGEN
 
  } // end of namespace MPI
 
} // end of namespace Utilities
 
 
DEAL_II_NAMESPACE_CLOSE
 
#endif