partitioner.h

// ---------------------------------------------------------------------
//
// Copyright (C) 2011 - 2018 by the deal.II authors
//
// This file is part of the deal.II library.
//
// The deal.II library is free software; you can use it, redistribute
// it, and/or modify it under the terms of the GNU Lesser General
// Public License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// The full text of the license can be found in the file LICENSE at
// the top level of the deal.II distribution.
//
// ---------------------------------------------------------------------

#ifndef dealii_partitioner_h
#define dealii_partitioner_h

#include <deal.II/base/config.h>
#include <deal.II/base/mpi.h>
#include <deal.II/base/types.h>
#include <deal.II/base/utilities.h>
#include <deal.II/base/memory_consumption.h>
#include <deal.II/base/index_set.h>
#include <deal.II/base/array_view.h>

#include <deal.II/lac/communication_pattern_base.h>
#include <deal.II/lac/vector.h>
#include <deal.II/lac/vector_operation.h>

#include <limits>


DEAL_II_NAMESPACE_OPEN

namespace Utilities
{
  namespace MPI
  {
    class Partitioner : public ::LinearAlgebra::CommunicationPatternBase
    {
    public:
      Partitioner ();

      Partitioner (const unsigned int size);

      Partitioner (const IndexSet &locally_owned_indices,
                   const IndexSet &ghost_indices_in,
                   const MPI_Comm  communicator_in);

      Partitioner (const IndexSet &locally_owned_indices,
                   const MPI_Comm  communicator_in);

      virtual void reinit(const IndexSet &vector_space_vector_index_set,
                          const IndexSet &read_write_vector_index_set,
                          const MPI_Comm &communicator);

      void set_owned_indices (const IndexSet &locally_owned_indices);

      void set_ghost_indices (const IndexSet &ghost_indices,
                              const IndexSet &larger_ghost_index_set = IndexSet());

      types::global_dof_index size() const;

      unsigned int local_size() const;

      const IndexSet &locally_owned_range() const;

      std::pair<types::global_dof_index,types::global_dof_index>
      local_range() const;

      bool in_local_range (const types::global_dof_index global_index) const;

      unsigned int
      global_to_local (const types::global_dof_index global_index) const;

      types::global_dof_index
      local_to_global (const unsigned int local_index) const;

      bool is_ghost_entry (const types::global_dof_index global_index) const;

      const IndexSet &ghost_indices() const;

      unsigned int n_ghost_indices() const;

      const std::vector<std::pair<unsigned int, unsigned int> > &
      ghost_indices_within_larger_ghost_set() const;

      const std::vector<std::pair<unsigned int, unsigned int> > &
      ghost_targets() const;

      const std::vector<std::pair<unsigned int, unsigned int> > &
      import_indices() const;

      unsigned int n_import_indices() const;

      const std::vector<std::pair<unsigned int, unsigned int> > &
      import_targets() const;

      bool is_compatible (const Partitioner &part) const;

      bool is_globally_compatible (const Partitioner &part) const;

      unsigned int this_mpi_process () const;

      unsigned int n_mpi_processes () const;

      DEAL_II_DEPRECATED
      const MPI_Comm &get_communicator() const;

      virtual const MPI_Comm &get_mpi_communicator() const;

      bool ghost_indices_initialized() const;

#ifdef DEAL_II_WITH_MPI

      template <typename Number>
      void
      export_to_ghosted_array_start(const unsigned int              communication_channel,
                                    const ArrayView<const Number>  &locally_owned_array,
                                    const ArrayView<Number>        &temporary_storage,
                                    const ArrayView<Number>        &ghost_array,
                                    std::vector<MPI_Request>       &requests) const;

      template <typename Number>
      void
      export_to_ghosted_array_finish(const ArrayView<Number>  &ghost_array,
                                     std::vector<MPI_Request> &requests) const;

      template <typename Number>
      void
      import_from_ghosted_array_start(const VectorOperation::values vector_operation,
                                      const unsigned int            communication_channel,
                                      const ArrayView<Number>      &ghost_array,
                                      const ArrayView<Number>      &temporary_storage,
                                      std::vector<MPI_Request>     &requests) const;

      template <typename Number>
      void
      import_from_ghosted_array_finish(const VectorOperation::values  vector_operation,
                                       const ArrayView<const Number> &temporary_storage,
                                       const ArrayView<Number>       &locally_owned_storage,
                                       const ArrayView<Number>       &ghost_array,
                                       std::vector<MPI_Request>      &requests) const;
#endif

      std::size_t memory_consumption() const;

      DeclException2 (ExcIndexNotPresent,
                      types::global_dof_index,
                      unsigned int,
                      << "Global index " << arg1
                      << " neither owned nor ghost on proc " << arg2 << ".");

      DeclException3 (ExcGhostIndexArrayHasWrongSize,
                      unsigned int,
                      unsigned int,
                      unsigned int,
                      << "The size of the ghost index array (" << arg1
                      << ") must either equal the number of ghost in the "
                      << "partitioner (" << arg2
                      << ") or be equal in size to a more comprehensive index"
                      << "set which contains " << arg3
                      << " elements for this partitioner.");

    private:
      types::global_dof_index global_size;

      IndexSet locally_owned_range_data;

      std::pair<types::global_dof_index,types::global_dof_index> local_range_data;

      IndexSet ghost_indices_data;

      unsigned int n_ghost_indices_data;

      std::vector<std::pair<unsigned int, unsigned int> > ghost_targets_data;

      std::vector<std::pair<unsigned int, unsigned int> > import_indices_data;

      unsigned int n_import_indices_data;

      std::vector<std::pair<unsigned int, unsigned int> > import_targets_data;

      std::vector<unsigned int> import_indices_chunks_by_rank_data;

      unsigned int n_ghost_indices_in_larger_set;

      std::vector<unsigned int> ghost_indices_subset_chunks_by_rank_data;

      std::vector<std::pair<unsigned int, unsigned int> > ghost_indices_subset_data;

      unsigned int my_pid;

      unsigned int n_procs;

      MPI_Comm communicator;

      bool have_ghost_indices;
    };



    /*----------------------- Inline functions ----------------------------------*/

#ifndef DOXYGEN

    inline
    types::global_dof_index Partitioner::size() const
    {
      return global_size;
    }



    inline
    const IndexSet &Partitioner::locally_owned_range() const
    {
      return locally_owned_range_data;
    }



    inline
    std::pair<types::global_dof_index,types::global_dof_index>
    Partitioner::local_range() const
    {
      return local_range_data;
    }



    inline
    unsigned int
    Partitioner::local_size () const
    {
      types::global_dof_index size= local_range_data.second - local_range_data.first;
      Assert(size<=std::numeric_limits<unsigned int>::max(),
             ExcNotImplemented());
      return static_cast<unsigned int>(size);
    }



    inline
    bool
    Partitioner::in_local_range (const types::global_dof_index global_index) const
    {
      return (local_range_data.first <= global_index &&
              global_index < local_range_data.second);
    }



    inline
    bool
    Partitioner::is_ghost_entry (const types::global_dof_index global_index) const
    {
      // if the index is in the global range, it is trivially not a ghost
      if (in_local_range(global_index) == true)
        return false;
      else
        return ghost_indices().is_element(global_index);
    }



    inline
    unsigned int
    Partitioner::global_to_local (const types::global_dof_index global_index) const
    {
      Assert(in_local_range(global_index) || is_ghost_entry (global_index),
             ExcIndexNotPresent(global_index, my_pid));
      if (in_local_range(global_index))
        return static_cast<unsigned int>(global_index - local_range_data.first);
      else if (is_ghost_entry (global_index))
        return (local_size() +
                static_cast<unsigned int>(ghost_indices_data.index_within_set (global_index)));
      else
        // should only end up here in optimized mode, when we use this large
        // number to trigger a segfault when using this method for array
        // access
        return numbers::invalid_unsigned_int;
    }



    inline
    types::global_dof_index
    Partitioner::local_to_global (const unsigned int local_index) const
    {
      AssertIndexRange (local_index, local_size() + n_ghost_indices_data);
      if (local_index < local_size())
        return local_range_data.first + types::global_dof_index(local_index);
      else
        return ghost_indices_data.nth_index_in_set (local_index-local_size());
    }



    inline
    const IndexSet  &Partitioner::ghost_indices() const
    {
      return ghost_indices_data;
    }



    inline
    unsigned int
    Partitioner::n_ghost_indices() const
    {
      return n_ghost_indices_data;
    }



    inline
    const std::vector<std::pair<unsigned int, unsigned int> > &
    Partitioner::ghost_indices_within_larger_ghost_set() const
    {
      return ghost_indices_subset_data;
    }



    inline
    const std::vector<std::pair<unsigned int, unsigned int> > &
    Partitioner::ghost_targets() const
    {
      return ghost_targets_data;
    }


    inline
    const std::vector<std::pair<unsigned int, unsigned int> > &
    Partitioner::import_indices() const
    {
      return import_indices_data;
    }



    inline
    unsigned int
    Partitioner::n_import_indices() const
    {
      return n_import_indices_data;
    }



    inline
    const std::vector<std::pair<unsigned int, unsigned int> > &
    Partitioner::import_targets() const
    {
      return import_targets_data;
    }



    inline
    unsigned int
    Partitioner::this_mpi_process() const
    {
      // return the id from the variable stored in this class instead of
      // Utilities::MPI::this_mpi_process() in order to make this query also
      // work when MPI is not initialized.
      return my_pid;
    }



    inline
    unsigned int
    Partitioner::n_mpi_processes() const
    {
      // return the number of MPI processes from the variable stored in this
      // class instead of Utilities::MPI::n_mpi_processes() in order to make
      // this query also work when MPI is not initialized.
      return n_procs;
    }



    inline
    const MPI_Comm &
    Partitioner::get_communicator() const
    {
      return communicator;
    }



    inline
    const MPI_Comm &
    Partitioner::get_mpi_communicator() const
    {
      return communicator;
    }



    inline
    bool
    Partitioner::ghost_indices_initialized() const
    {
      return have_ghost_indices;
    }

#endif  // ifndef DOXYGEN

  } // end of namespace MPI

} // end of namespace Utilities


DEAL_II_NAMESPACE_CLOSE

#endif