mg_transfer_matrix_free.cc

// ---------------------------------------------------------------------
//
// Copyright (C) 2016 - 2018 by the deal.II authors
//
// This file is part of the deal.II library.
//
// The deal.II library is free software; you can use it, redistribute
// it, and/or modify it under the terms of the GNU Lesser General
// Public License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// The full text of the license can be found in the file LICENSE at
// the top level of the deal.II distribution.
//
// ---------------------------------------------------------------------


#include <deal.II/base/logstream.h>
#include <deal.II/base/function.h>
#include <deal.II/base/vectorization.h>
#include <deal.II/lac/la_parallel_vector.h>
#include <deal.II/grid/tria.h>
#include <deal.II/grid/tria_iterator.h>
#include <deal.II/dofs/dof_tools.h>
#include <deal.II/fe/fe.h>
#include <deal.II/dofs/dof_accessor.h>
#include <deal.II/multigrid/mg_tools.h>
#include <deal.II/multigrid/mg_transfer_matrix_free.h>
#include <deal.II/multigrid/mg_transfer_internal.h>

#include <deal.II/matrix_free/evaluation_kernels.h>

#include <algorithm>

DEAL_II_NAMESPACE_OPEN


template <int dim, typename Number>
MGTransferMatrixFree<dim,Number>::MGTransferMatrixFree ()
  :
  fe_degree(0),
  element_is_continuous(false),
  n_components(0),
  n_child_cell_dofs(0)
{}



template <int dim, typename Number>
MGTransferMatrixFree<dim,Number>::MGTransferMatrixFree (const MGConstrainedDoFs &mg_c)
  :
  fe_degree(0),
  element_is_continuous(false),
  n_components(0),
  n_child_cell_dofs(0)
{
  this->mg_constrained_dofs = &mg_c;
}



template <int dim, typename Number>
void MGTransferMatrixFree<dim,Number>::initialize_constraints
(const MGConstrainedDoFs &mg_c)
{
  this->mg_constrained_dofs = &mg_c;
}



template <int dim, typename Number>
void MGTransferMatrixFree<dim,Number>::clear ()
{
  this->MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number> >::clear();
  fe_degree = 0;
  element_is_continuous = false;
  n_components = 0;
  n_child_cell_dofs = 0;
  level_dof_indices.clear();
  parent_child_connect.clear();
  dirichlet_indices.clear();
  n_owned_level_cells.clear();
  prolongation_matrix_1d.clear();
  evaluation_data.clear();
  weights_on_refined.clear();
}


template <int dim, typename Number>
void MGTransferMatrixFree<dim,Number>::build
(const DoFHandler<dim,dim>  &mg_dof)
{
  this->fill_and_communicate_copy_indices(mg_dof);

  std::vector<std::vector<Number> > weights_unvectorized;

  internal::MGTransfer::ElementInfo<Number> elem_info;

  internal::MGTransfer::setup_transfer<dim,Number>(mg_dof,
                                                   this->mg_constrained_dofs,
                                                   elem_info,
                                                   level_dof_indices,
                                                   parent_child_connect,
                                                   n_owned_level_cells,
                                                   dirichlet_indices,
                                                   weights_unvectorized,
                                                   this->copy_indices_global_mine,
                                                   this->ghosted_level_vector);
  // unpack element info data
  fe_degree                = elem_info.fe_degree;
  element_is_continuous    = elem_info.element_is_continuous;
  n_components             = elem_info.n_components;
  n_child_cell_dofs        = elem_info.n_child_cell_dofs;

  // duplicate and put into vectorized array
  prolongation_matrix_1d.resize(elem_info.prolongation_matrix_1d.size());
  for (unsigned int i=0; i<elem_info.prolongation_matrix_1d.size(); i++)
    prolongation_matrix_1d[i] = elem_info.prolongation_matrix_1d[i];

  // reshuffle into aligned vector of vectorized arrays
  const unsigned int vec_size = VectorizedArray<Number>::n_array_elements;
  const unsigned int n_levels = mg_dof.get_triangulation().n_global_levels();

  const unsigned int n_weights_per_cell = Utilities::fixed_power<dim>(3);
  weights_on_refined.resize(n_levels-1);
  for (unsigned int level = 1; level<n_levels; ++level)
    {
      weights_on_refined[level-1].resize(((n_owned_level_cells[level-1]+vec_size-1)/vec_size)*n_weights_per_cell);

      for (unsigned int c=0; c<n_owned_level_cells[level-1]; ++c)
        {
          const unsigned int comp = c/vec_size;
          const unsigned int v = c%vec_size;
          for (unsigned int i = 0; i<n_weights_per_cell; ++i)
            {

              weights_on_refined[level-1][comp*n_weights_per_cell+i][v] = weights_unvectorized[level-1][c*n_weights_per_cell+i];
            }
        }
    }

  evaluation_data.resize(n_child_cell_dofs);
}



template <int dim, typename Number>
void MGTransferMatrixFree<dim,Number>
::prolongate (const unsigned int                           to_level,
              LinearAlgebra::distributed::Vector<Number>       &dst,
              const LinearAlgebra::distributed::Vector<Number> &src) const
{
  Assert ((to_level >= 1) && (to_level<=level_dof_indices.size()),
          ExcIndexRange (to_level, 1, level_dof_indices.size()+1));

  AssertDimension(this->ghosted_level_vector[to_level].local_size(),
                  dst.local_size());
  AssertDimension(this->ghosted_level_vector[to_level-1].local_size(),
                  src.local_size());

  this->ghosted_level_vector[to_level-1].copy_locally_owned_data_from(src);
  this->ghosted_level_vector[to_level-1].update_ghost_values();
  this->ghosted_level_vector[to_level] = 0.;

  // the implementation in do_prolongate_add is templated in the degree of the
  // element (for efficiency reasons), so we need to find the appropriate
  // kernel here...
  if (fe_degree == 0)
    do_prolongate_add<0>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 1)
    do_prolongate_add<1>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 2)
    do_prolongate_add<2>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 3)
    do_prolongate_add<3>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 4)
    do_prolongate_add<4>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 5)
    do_prolongate_add<5>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 6)
    do_prolongate_add<6>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 7)
    do_prolongate_add<7>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 8)
    do_prolongate_add<8>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 9)
    do_prolongate_add<9>(to_level, this->ghosted_level_vector[to_level],
                         this->ghosted_level_vector[to_level-1]);
  else if (fe_degree == 10)
    do_prolongate_add<10>(to_level, this->ghosted_level_vector[to_level],
                          this->ghosted_level_vector[to_level-1]);
  else
    do_prolongate_add<-1>(to_level, this->ghosted_level_vector[to_level],
                          this->ghosted_level_vector[to_level-1]);

  this->ghosted_level_vector[to_level].compress(VectorOperation::add);
  dst.copy_locally_owned_data_from(this->ghosted_level_vector[to_level]);
}



template <int dim, typename Number>
void MGTransferMatrixFree<dim,Number>
::restrict_and_add (const unsigned int                           from_level,
                    LinearAlgebra::distributed::Vector<Number>       &dst,
                    const LinearAlgebra::distributed::Vector<Number> &src) const
{
  Assert ((from_level >= 1) && (from_level<=level_dof_indices.size()),
          ExcIndexRange (from_level, 1, level_dof_indices.size()+1));

  AssertDimension(this->ghosted_level_vector[from_level].local_size(),
                  src.local_size());
  AssertDimension(this->ghosted_level_vector[from_level-1].local_size(),
                  dst.local_size());

  this->ghosted_level_vector[from_level].copy_locally_owned_data_from(src);
  this->ghosted_level_vector[from_level].update_ghost_values();
  this->ghosted_level_vector[from_level-1] = 0.;

  if (fe_degree == 0)
    do_restrict_add<0>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 1)
    do_restrict_add<1>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 2)
    do_restrict_add<2>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 3)
    do_restrict_add<3>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 4)
    do_restrict_add<4>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 5)
    do_restrict_add<5>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 6)
    do_restrict_add<6>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 7)
    do_restrict_add<7>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 8)
    do_restrict_add<8>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 9)
    do_restrict_add<9>(from_level, this->ghosted_level_vector[from_level-1],
                       this->ghosted_level_vector[from_level]);
  else if (fe_degree == 10)
    do_restrict_add<10>(from_level, this->ghosted_level_vector[from_level-1],
                        this->ghosted_level_vector[from_level]);
  else
    // go to the non-templated version of the evaluator
    do_restrict_add<-1>(from_level, this->ghosted_level_vector[from_level-1],
                        this->ghosted_level_vector[from_level]);

  this->ghosted_level_vector[from_level-1].compress(VectorOperation::add);
  dst += this->ghosted_level_vector[from_level-1];
}



namespace
{
  template <int dim, int degree, typename Number>
  void weight_dofs_on_child (const VectorizedArray<Number> *weights,
                             const unsigned int n_components,
                             const unsigned int fe_degree,
                             VectorizedArray<Number> *data)
  {
    Assert(fe_degree > 0, ExcNotImplemented());
    Assert(fe_degree < 100, ExcNotImplemented());
    const int loop_length = degree != -1 ? 2*degree+1 : 2*fe_degree+1;
    unsigned int degree_to_3 [100];
    degree_to_3[0] = 0;
    for (int i=1; i<loop_length-1; ++i)
      degree_to_3[i] = 1;
    degree_to_3[loop_length-1] = 2;
    for (unsigned int c=0; c<n_components; ++c)
      for (int k=0; k<(dim>2 ? loop_length : 1); ++k)
        for (int j=0; j<(dim>1 ? loop_length : 1); ++j)
          {
            const unsigned int shift = 9*degree_to_3[k] + 3*degree_to_3[j];
            data[0] *= weights[shift];
            // loop bound as int avoids compiler warnings in case loop_length
            // == 1 (polynomial degree 0)
            for (int i=1; i<loop_length-1; ++i)
              data[i] *= weights[shift+1];
            data[loop_length-1] *= weights[shift+2];
            data += loop_length;
          }
  }
}



template <int dim, typename Number>
template <int degree>
void MGTransferMatrixFree<dim,Number>
::do_prolongate_add (const unsigned int                           to_level,
                     LinearAlgebra::distributed::Vector<Number>       &dst,
                     const LinearAlgebra::distributed::Vector<Number> &src) const
{
  const unsigned int vec_size = VectorizedArray<Number>::n_array_elements;
  const unsigned int degree_size = (degree > -1 ? degree : fe_degree) + 1;
  const unsigned int n_child_dofs_1d = 2*degree_size - element_is_continuous;
  const unsigned int n_scalar_cell_dofs = Utilities::fixed_power<dim>(n_child_dofs_1d);
  const unsigned int three_to_dim = Utilities::fixed_int_power<3,dim>::value;

  for (unsigned int cell=0; cell < n_owned_level_cells[to_level-1];
       cell += vec_size)
    {
      const unsigned int n_lanes = cell+vec_size > n_owned_level_cells[to_level-1] ?
                                   n_owned_level_cells[to_level-1] - cell : vec_size;

      // read from source vector
      for (unsigned int v=0; v<n_lanes; ++v)
        {
          const unsigned int shift = internal::MGTransfer::compute_shift_within_children<dim>
                                     (parent_child_connect[to_level-1][cell+v].second,
                                      fe_degree+1-element_is_continuous, fe_degree);
          const unsigned int *indices = &level_dof_indices[to_level-1][parent_child_connect[to_level-1][cell+v].first*n_child_cell_dofs+shift];
          for (unsigned int c=0, m=0; c<n_components; ++c)
            {
              for (unsigned int k=0; k<(dim>2 ? degree_size : 1); ++k)
                for (unsigned int j=0; j<(dim>1 ? degree_size : 1); ++j)
                  for (unsigned int i=0; i<degree_size; ++i, ++m)
                    evaluation_data[m][v] =
                      src.local_element(indices[c*n_scalar_cell_dofs +
                                                k*n_child_dofs_1d*n_child_dofs_1d+
                                                j*n_child_dofs_1d+i]);

              // apply Dirichlet boundary conditions on parent cell
              for (std::vector<unsigned short>::const_iterator i=dirichlet_indices[to_level-1][cell+v].begin(); i!=dirichlet_indices[to_level-1][cell+v].end(); ++i)
                evaluation_data[*i][v] = 0.;
            }
        }

      AssertDimension(prolongation_matrix_1d.size(),
                      degree_size * n_child_dofs_1d);
      // perform tensorized operation
      if (element_is_continuous)
        {
          // must go through the components backwards because we want to write
          // the output to the same array as the input
          for (int c=n_components-1; c>=0; --c)
            internal::FEEvaluationImplBasisChange<internal::evaluate_general,dim,degree+1,2*degree+1,
                     1,VectorizedArray<Number>, VectorizedArray<Number> >
                     ::do_forward(prolongation_matrix_1d,
                                  evaluation_data.begin()+ c*Utilities::fixed_power<dim>(degree_size),
                                  evaluation_data.begin()+ c*n_scalar_cell_dofs,
                                  fe_degree+1, 2*fe_degree+1);
          weight_dofs_on_child<dim,degree,Number>(&weights_on_refined[to_level-1][(cell/vec_size)*three_to_dim],
                                                  n_components, fe_degree,
                                                  evaluation_data.begin());
        }
      else
        {
          for (int c=n_components-1; c>=0; --c)
            internal::FEEvaluationImplBasisChange<internal::evaluate_general,dim,degree+1,2*degree+2,
                     1,VectorizedArray<Number>, VectorizedArray<Number> >
                     ::do_forward(prolongation_matrix_1d,
                                  evaluation_data.begin() + c*Utilities::fixed_power<dim>(degree_size),
                                  evaluation_data.begin() + c*n_scalar_cell_dofs,
                                  fe_degree+1, 2*fe_degree+2);
        }

      // write into dst vector
      const unsigned int *indices = &level_dof_indices[to_level][cell*
                                                                 n_child_cell_dofs];
      for (unsigned int v=0; v<n_lanes; ++v)
        {
          for (unsigned int i=0; i<n_child_cell_dofs; ++i)
            dst.local_element(indices[i]) += evaluation_data[i][v];
          indices += n_child_cell_dofs;
        }
    }
}



template <int dim, typename Number>
template <int degree>
void MGTransferMatrixFree<dim,Number>
::do_restrict_add (const unsigned int                           from_level,
                   LinearAlgebra::distributed::Vector<Number>       &dst,
                   const LinearAlgebra::distributed::Vector<Number> &src) const
{
  const unsigned int vec_size = VectorizedArray<Number>::n_array_elements;
  const unsigned int degree_size = (degree > -1 ? degree : fe_degree) + 1;
  const unsigned int n_child_dofs_1d = 2*degree_size - element_is_continuous;
  const unsigned int n_scalar_cell_dofs = Utilities::fixed_power<dim>(n_child_dofs_1d);
  const unsigned int three_to_dim = Utilities::fixed_int_power<3,dim>::value;

  for (unsigned int cell=0; cell < n_owned_level_cells[from_level-1];
       cell += vec_size)
    {
      const unsigned int n_lanes = cell+vec_size > n_owned_level_cells[from_level-1] ?
                                   n_owned_level_cells[from_level-1] - cell : vec_size;

      // read from source vector
      {
        const unsigned int *indices = &level_dof_indices[from_level][cell*
                                      n_child_cell_dofs];
        for (unsigned int v=0; v<n_lanes; ++v)
          {
            for (unsigned int i=0; i<n_child_cell_dofs; ++i)
              evaluation_data[i][v] = src.local_element(indices[i]);
            indices += n_child_cell_dofs;
          }
      }

      AssertDimension(prolongation_matrix_1d.size(),
                      degree_size * n_child_dofs_1d);
      // perform tensorized operation
      if (element_is_continuous)
        {
          weight_dofs_on_child<dim,degree,Number>(&weights_on_refined[from_level-1][(cell/vec_size)*three_to_dim],
                                                  n_components, fe_degree,
                                                  &evaluation_data[0]);
          for (unsigned int c=0; c<n_components; ++c)
            internal::FEEvaluationImplBasisChange<internal::evaluate_general,dim,degree+1,2*degree+1,
                     1,VectorizedArray<Number>,VectorizedArray<Number> >
                     ::do_backward(prolongation_matrix_1d, false,
                                   evaluation_data.begin() + c*n_scalar_cell_dofs,
                                   evaluation_data.begin() + c*Utilities::fixed_power<dim>(degree_size),
                                   fe_degree+1, 2*fe_degree+1);
        }
      else
        {
          for (unsigned int c=0; c<n_components; ++c)
            internal::FEEvaluationImplBasisChange<internal::evaluate_general,dim,degree+1,2*degree+2,
                     1,VectorizedArray<Number>,VectorizedArray<Number> >
                     ::do_backward(prolongation_matrix_1d, false,
                                   evaluation_data.begin() + c*n_scalar_cell_dofs,
                                   evaluation_data.begin() + c*Utilities::fixed_power<dim>(degree_size),
                                   fe_degree+1, 2*fe_degree+2);
        }

      // write into dst vector
      for (unsigned int v=0; v<n_lanes; ++v)
        {
          const unsigned int shift = internal::MGTransfer::compute_shift_within_children<dim>
                                     (parent_child_connect[from_level-1][cell+v].second,
                                      fe_degree+1-element_is_continuous, fe_degree);
          AssertIndexRange(parent_child_connect[from_level-1][cell+v].first*
                           n_child_cell_dofs+n_child_cell_dofs-1,
                           level_dof_indices[from_level-1].size());
          const unsigned int *indices = &level_dof_indices[from_level-1][parent_child_connect[from_level-1][cell+v].first*n_child_cell_dofs+shift];
          for (unsigned int c=0, m=0; c<n_components; ++c)
            {
              // apply Dirichlet boundary conditions on parent cell
              for (std::vector<unsigned short>::const_iterator i=dirichlet_indices[from_level-1][cell+v].begin(); i!=dirichlet_indices[from_level-1][cell+v].end(); ++i)
                evaluation_data[*i][v] = 0.;

              for (unsigned int k=0; k<(dim>2 ? degree_size : 1); ++k)
                for (unsigned int j=0; j<(dim>1 ? degree_size : 1); ++j)
                  for (unsigned int i=0; i<degree_size; ++i, ++m)
                    dst.local_element(indices[c*n_scalar_cell_dofs +
                                              k*n_child_dofs_1d*n_child_dofs_1d+
                                              j*n_child_dofs_1d+i])
                    += evaluation_data[m][v];
            }
        }
    }
}



template <int dim, typename Number>
std::size_t
MGTransferMatrixFree<dim,Number>::memory_consumption() const
{
  std::size_t memory = MGLevelGlobalTransfer<LinearAlgebra::distributed::Vector<Number> >::memory_consumption();
  memory += MemoryConsumption::memory_consumption(level_dof_indices);
  memory += MemoryConsumption::memory_consumption(parent_child_connect);
  memory += MemoryConsumption::memory_consumption(n_owned_level_cells);
  memory += MemoryConsumption::memory_consumption(prolongation_matrix_1d);
  memory += MemoryConsumption::memory_consumption(evaluation_data);
  memory += MemoryConsumption::memory_consumption(weights_on_refined);
  memory += MemoryConsumption::memory_consumption(dirichlet_indices);
  return memory;
}



template <int dim, typename Number>
MGTransferBlockMatrixFree<dim,Number>::MGTransferBlockMatrixFree (const MGConstrainedDoFs &mg_c)
  : same_for_all (true)
{
  matrix_free_transfer_vector.emplace_back(mg_c);
}



template <int dim, typename Number>
MGTransferBlockMatrixFree<dim,Number>::MGTransferBlockMatrixFree
(const std::vector<MGConstrainedDoFs> &mg_c)
  : same_for_all (false)
{
  for (unsigned int i=0; i<mg_c.size(); ++i)
    matrix_free_transfer_vector.emplace_back(mg_c[i]);
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>::initialize_constraints
(const MGConstrainedDoFs &mg_c)
{
  Assert(same_for_all,
         ExcMessage("This object was initialized with support for usage with "
                    "one DoFHandler for each block, but this method assumes "
                    "that the same DoFHandler is used for all the blocks!"));
  AssertDimension(matrix_free_transfer_vector.size(),1);

  matrix_free_transfer_vector[0].initialize_constraints(mg_c);
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>::initialize_constraints
(const std::vector<MGConstrainedDoFs> &mg_c)
{
  Assert(!same_for_all,
         ExcMessage("This object was initialized with support for using "
                    "the same DoFHandler for all the blocks, but this "
                    "method assumes that there is a separate DoFHandler "
                    "for each block!"));
  AssertDimension(matrix_free_transfer_vector.size(),mg_c.size());

  for (unsigned int i = 0; i<mg_c.size(); ++i)
    matrix_free_transfer_vector[i].initialize_constraints(mg_c[i]);
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>::clear ()
{
  matrix_free_transfer_vector.clear();
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>::build
(const DoFHandler<dim,dim>  &mg_dof)
{
  AssertDimension(matrix_free_transfer_vector.size(),1);
  matrix_free_transfer_vector[0].build(mg_dof);
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>::build
(const std::vector<const DoFHandler<dim,dim>*> &mg_dof)
{
  AssertDimension (matrix_free_transfer_vector.size(),mg_dof.size());
  for (unsigned int i=0; i<mg_dof.size(); ++i)
    matrix_free_transfer_vector[i].build(*mg_dof[i]);
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>
::prolongate (const unsigned int                           to_level,
              LinearAlgebra::distributed::BlockVector<Number>       &dst,
              const LinearAlgebra::distributed::BlockVector<Number> &src) const
{
  const unsigned int n_blocks = src.n_blocks();
  AssertDimension(dst.n_blocks(), n_blocks);

  if (!same_for_all)
    AssertDimension (matrix_free_transfer_vector.size(), n_blocks);

  for (unsigned int b = 0; b < n_blocks; ++b)
    {
      const unsigned int data_block = same_for_all ? 0 : b;
      matrix_free_transfer_vector[data_block].prolongate(to_level, dst.block(b), src.block(b));
    }
}



template <int dim, typename Number>
void MGTransferBlockMatrixFree<dim,Number>
::restrict_and_add (const unsigned int                           from_level,
                    LinearAlgebra::distributed::BlockVector<Number>       &dst,
                    const LinearAlgebra::distributed::BlockVector<Number> &src) const
{
  const unsigned int n_blocks = src.n_blocks();
  AssertDimension(dst.n_blocks(), n_blocks);

  if (!same_for_all)
    AssertDimension (matrix_free_transfer_vector.size(), n_blocks);

  for (unsigned int b = 0; b < n_blocks; ++b)
    {
      const unsigned int data_block = same_for_all ? 0 : b;
      matrix_free_transfer_vector[data_block].restrict_and_add(from_level, dst.block(b), src.block(b));
    }
}



template <int dim, typename Number>
std::size_t
MGTransferBlockMatrixFree<dim,Number>::memory_consumption() const
{
  std::size_t total_memory_consumption= 0;
  for (const auto &el: matrix_free_transfer_vector)
    total_memory_consumption += el.memory_consumption();
  return total_memory_consumption;
}



// explicit instantiation
#include "mg_transfer_matrix_free.inst"


DEAL_II_NAMESPACE_CLOSE