operators.h

Go to the documentation of this file.

// -----------------------------------------------------------------------------
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception OR LGPL-2.1-or-later
// Copyright (C) 2014 - 2026 by the deal.II authors
//
// This file is part of the deal.II library.
//
// Detailed license information governing the source code and contributions
// can be found in LICENSE.md and CONTRIBUTING.md at the top level directory.
//
// -----------------------------------------------------------------------------
 
 
#ifndef dealii_matrix_free_operators_h
#define dealii_matrix_free_operators_h
 
 
#include <deal.II/base/config.h>
 
#include <deal.II/base/enable_observer_pointer.h>
#include <deal.II/base/exceptions.h>
#include <deal.II/base/vectorization.h>
 
#include <deal.II/lac/diagonal_matrix.h>
#include <deal.II/lac/la_parallel_vector.h>
 
#include <deal.II/matrix_free/fe_evaluation.h>
#include <deal.II/matrix_free/matrix_free.h>
#include <deal.II/matrix_free/tools.h>
 
#include <deal.II/multigrid/mg_constrained_dofs.h>
 
#include <limits>
 
DEAL_II_NAMESPACE_OPEN
 
 
namespace MatrixFreeOperators
{
  namespace BlockHelper
  {
    // workaround for unifying non-block vector and block vector implementations
    // a non-block vector has one block and the only subblock is the vector
    // itself
    template <typename VectorType>
    std::enable_if_t<IsBlockVector<VectorType>::value, unsigned int>
    n_blocks(const VectorType &vector)
    {
      return vector.n_blocks();
    }
 
    template <typename VectorType>
    std::enable_if_t<!IsBlockVector<VectorType>::value, unsigned int>
    n_blocks(const VectorType &)
    {
      return 1;
    }
 
    template <typename VectorType>
    std::enable_if_t<IsBlockVector<VectorType>::value,
                     typename VectorType::BlockType &>
    subblock(VectorType &vector, unsigned int block_no)
    {
      AssertIndexRange(block_no, vector.n_blocks());
      return vector.block(block_no);
    }
 
    template <typename VectorType>
    std::enable_if_t<IsBlockVector<VectorType>::value,
                     const typename VectorType::BlockType &>
    subblock(const VectorType &vector, unsigned int block_no)
    {
      AssertIndexRange(block_no, vector.n_blocks());
      return vector.block(block_no);
    }
 
    template <typename VectorType>
    std::enable_if_t<!IsBlockVector<VectorType>::value, VectorType &>
    subblock(VectorType &vector, unsigned int)
    {
      return vector;
    }
 
    template <typename VectorType>
    std::enable_if_t<!IsBlockVector<VectorType>::value, const VectorType &>
    subblock(const VectorType &vector, unsigned int)
    {
      return vector;
    }
 
    template <typename VectorType>
    std::enable_if_t<IsBlockVector<VectorType>::value, void>
    collect_sizes(VectorType &vector)
    {
      vector.collect_sizes();
    }
 
    template <typename VectorType>
    std::enable_if_t<!IsBlockVector<VectorType>::value, void>
    collect_sizes(const VectorType &)
    {}
  } // namespace BlockHelper

  template <int dim,
            typename VectorType = LinearAlgebra::distributed::Vector<double>,
            typename VectorizedArrayType =
              VectorizedArray<typename VectorType::value_type>>
  class Base : public EnableObserverPointer
  {
  public:
    using value_type = typename VectorType::value_type;

    using size_type = typename VectorType::size_type;

    Base();

    virtual ~Base() override = default;

    virtual void
    clear();

    void
    initialize(std::shared_ptr<
                 const MatrixFree<dim, value_type, VectorizedArrayType>> data,
               const std::vector<unsigned int> &selected_row_blocks =
                 std::vector<unsigned int>(),
               const std::vector<unsigned int> &selected_column_blocks =
                 std::vector<unsigned int>());

    void
    initialize(std::shared_ptr<
                 const MatrixFree<dim, value_type, VectorizedArrayType>> data,
               const MGConstrainedDoFs         &mg_constrained_dofs,
               const unsigned int               level,
               const std::vector<unsigned int> &selected_row_blocks =
                 std::vector<unsigned int>());

    void
    initialize(std::shared_ptr<
                 const MatrixFree<dim, value_type, VectorizedArrayType>> data_,
               const std::vector<MGConstrainedDoFs> &mg_constrained_dofs,
               const unsigned int                    level,
               const std::vector<unsigned int>      &selected_row_blocks =
                 std::vector<unsigned int>());

    size_type
    m() const;

    size_type
    n() const;

    void
    vmult_interface_down(VectorType &dst, const VectorType &src) const;

    void
    vmult_interface_up(VectorType &dst, const VectorType &src) const;

    void
    vmult(VectorType &dst, const VectorType &src) const;

    void
    Tvmult(VectorType &dst, const VectorType &src) const;

    void
    vmult_add(VectorType &dst, const VectorType &src) const;

    void
    Tvmult_add(VectorType &dst, const VectorType &src) const;

    value_type
    el(const unsigned int row, const unsigned int col) const;

    virtual std::size_t
    memory_consumption() const;

    void
    initialize_dof_vector(VectorType &vec) const;

    virtual void
    compute_diagonal() = 0;

    std::shared_ptr<const MatrixFree<dim, value_type, VectorizedArrayType>>
    get_matrix_free() const;

    const std::shared_ptr<DiagonalMatrix<VectorType>> &
    get_matrix_diagonal_inverse() const;

    const std::shared_ptr<DiagonalMatrix<VectorType>> &
    get_matrix_diagonal() const;

    void
    precondition_Jacobi(VectorType       &dst,
                        const VectorType &src,
                        const value_type  omega) const;
 
  protected:
    void
    preprocess_constraints(VectorType &dst, const VectorType &src) const;

    void
    postprocess_constraints(VectorType &dst, const VectorType &src) const;

    void
    set_constrained_entries_to_one(VectorType &dst) const;

    virtual void
    apply_add(VectorType &dst, const VectorType &src) const = 0;

    virtual void
    Tapply_add(VectorType &dst, const VectorType &src) const;

    std::shared_ptr<const MatrixFree<dim, value_type, VectorizedArrayType>>
      data;

    std::shared_ptr<DiagonalMatrix<VectorType>> diagonal_entries;

    std::shared_ptr<DiagonalMatrix<VectorType>> inverse_diagonal_entries;

    std::vector<unsigned int> selected_rows;

    std::vector<unsigned int> selected_columns;
 
  private:
    std::vector<std::vector<unsigned int>> edge_constrained_indices;

    mutable std::vector<std::vector<std::pair<value_type, value_type>>>
      edge_constrained_values;

    bool have_interface_matrices;

    void
    mult_add(VectorType       &dst,
             const VectorType &src,
             const bool        transpose) const;

    void
    adjust_ghost_range_if_necessary(const VectorType &vec,
                                    const bool        is_row) const;
  };
 
 

  template <typename OperatorType>
  class MGInterfaceOperator : public EnableObserverPointer
  {
  public:
    using value_type = typename OperatorType::value_type;

    using size_type = typename OperatorType::size_type;

    MGInterfaceOperator();

    void
    clear();

    void
    initialize(const OperatorType &operator_in);

    template <typename VectorType>
    void
    vmult(VectorType &dst, const VectorType &src) const;

    template <typename VectorType>
    void
    Tvmult(VectorType &dst, const VectorType &src) const;

    template <typename VectorType>
    void
    initialize_dof_vector(VectorType &vec) const;
 
 
  private:
    ObserverPointer<const OperatorType> mf_base_operator;
  };
 
 

  template <int dim,
            int fe_degree,
            int n_components             = 1,
            typename Number              = double,
            typename VectorizedArrayType = VectorizedArray<Number>>
  class CellwiseInverseMassMatrix
  {
    static_assert(
      std::is_same_v<Number, typename VectorizedArrayType::value_type>,
      "Type of Number and of VectorizedArrayType do not match.");
 
  public:
    CellwiseInverseMassMatrix(
      const FEEvaluationBase<dim,
                             n_components,
                             Number,
                             false,
                             VectorizedArrayType> &fe_eval);

    void
    apply(const AlignedVector<VectorizedArrayType> &inverse_coefficient,
          const unsigned int                        n_actual_components,
          const VectorizedArrayType                *in_array,
          VectorizedArrayType                      *out_array) const;

    void
    apply(const VectorizedArrayType *in_array,
          VectorizedArrayType       *out_array) const;

    void
    apply(const AlignedVector<Tensor<2, n_components, VectorizedArrayType>>
                                    &inverse_dyadic_coefficients,
          const VectorizedArrayType *in_array,
          VectorizedArrayType       *out_array) const;

    void
    transform_from_q_points_to_basis(const unsigned int n_actual_components,
                                     const VectorizedArrayType *in_array,
                                     VectorizedArrayType *out_array) const;

    void
    fill_inverse_JxW_values(
      AlignedVector<VectorizedArrayType> &inverse_jxw) const;
 
  private:
    const FEEvaluationBase<dim,
                           n_components,
                           Number,
                           false,
                           VectorizedArrayType> &fe_eval;
  };
 
 

  template <int dim,
            int fe_degree,
            int n_q_points_1d   = fe_degree + 1,
            int n_components    = 1,
            typename VectorType = LinearAlgebra::distributed::Vector<double>,
            typename VectorizedArrayType =
              VectorizedArray<typename VectorType::value_type>>
  class MassOperator : public Base<dim, VectorType, VectorizedArrayType>
  {
  public:
    using value_type =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;

    using size_type =
      typename Base<dim, VectorType, VectorizedArrayType>::size_type;

    MassOperator();

    virtual void
    compute_diagonal() override;

    void
    compute_lumped_diagonal();

    const std::shared_ptr<DiagonalMatrix<VectorType>> &
    get_matrix_lumped_diagonal() const;

    const std::shared_ptr<DiagonalMatrix<VectorType>> &
    get_matrix_lumped_diagonal_inverse() const;
 
  private:
    virtual void
    apply_add(VectorType &dst, const VectorType &src) const override;

    void
    local_apply_cell(
      const MatrixFree<dim, value_type, VectorizedArrayType> &data,
      VectorType                                             &dst,
      const VectorType                                       &src,
      const std::pair<unsigned int, unsigned int>            &cell_range) const;

    std::shared_ptr<DiagonalMatrix<VectorType>> lumped_diagonal_entries;

    std::shared_ptr<DiagonalMatrix<VectorType>> inverse_lumped_diagonal_entries;
  };
 
 

  template <int dim,
            int fe_degree,
            int n_q_points_1d   = fe_degree + 1,
            int n_components    = 1,
            typename VectorType = LinearAlgebra::distributed::Vector<double>,
            typename VectorizedArrayType =
              VectorizedArray<typename VectorType::value_type>>
  class LaplaceOperator : public Base<dim, VectorType, VectorizedArrayType>
  {
  public:
    using value_type =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;

    using size_type =
      typename Base<dim, VectorType, VectorizedArrayType>::size_type;

    LaplaceOperator();

    virtual void
    compute_diagonal() override;

    void
    set_coefficient(
      const std::shared_ptr<Table<2, VectorizedArrayType>> &scalar_coefficient);

    virtual void
    clear() override;

    std::shared_ptr<Table<2, VectorizedArrayType>>
    get_coefficient();
 
  private:
    virtual void
    apply_add(VectorType &dst, const VectorType &src) const override;

    void
    local_apply_cell(
      const MatrixFree<dim, value_type, VectorizedArrayType> &data,
      VectorType                                             &dst,
      const VectorType                                       &src,
      const std::pair<unsigned int, unsigned int>            &cell_range) const;

    void
    local_diagonal_cell(
      const MatrixFree<dim, value_type, VectorizedArrayType> &data,
      VectorType                                             &dst,
      const VectorType &,
      const std::pair<unsigned int, unsigned int> &cell_range) const;

    template <int n_components_compute>
    void
    do_operation_on_cell(FEEvaluation<dim,
                                      fe_degree,
                                      n_q_points_1d,
                                      n_components_compute,
                                      value_type,
                                      VectorizedArrayType> &phi,
                         const unsigned int                 cell) const;

    std::shared_ptr<Table<2, VectorizedArrayType>> scalar_coefficient;
  };
 
 
 
  // ------------------------------------ inline functions ---------------------
 
  template <int dim,
            int fe_degree,
            int n_components,
            typename Number,
            typename VectorizedArrayType>
  inline CellwiseInverseMassMatrix<dim,
                                   fe_degree,
                                   n_components,
                                   Number,
                                   VectorizedArrayType>::
    CellwiseInverseMassMatrix(
      const FEEvaluationBase<dim,
                             n_components,
                             Number,
                             false,
                             VectorizedArrayType> &fe_eval)
    : fe_eval(fe_eval)
  {
    AssertDimension(fe_eval.get_shape_info().dofs_per_component_on_cell,
                    fe_eval.get_shape_info().n_q_points);
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_components,
            typename Number,
            typename VectorizedArrayType>
  inline void
  CellwiseInverseMassMatrix<dim,
                            fe_degree,
                            n_components,
                            Number,
                            VectorizedArrayType>::
    fill_inverse_JxW_values(
      AlignedVector<VectorizedArrayType> &inverse_jxw) const
  {
    const unsigned int dofs_per_component_on_cell =
      (fe_degree > -1) ?
        Utilities::pow(fe_degree + 1, dim) :
        Utilities::pow(fe_eval.get_shape_info().data.front().fe_degree + 1,
                       dim);
 
    Assert(inverse_jxw.size() > 0 &&
             inverse_jxw.size() % dofs_per_component_on_cell == 0,
           ExcMessage(
             "Expected diagonal to be a multiple of scalar dof per cells"));
 
    // compute values for the first component
    for (unsigned int q = 0; q < dofs_per_component_on_cell; ++q)
      inverse_jxw[q] = 1. / fe_eval.JxW(q);
    // copy values to rest of vector
    for (unsigned int q = dofs_per_component_on_cell; q < inverse_jxw.size();)
      for (unsigned int i = 0; i < dofs_per_component_on_cell; ++i, ++q)
        inverse_jxw[q] = inverse_jxw[i];
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_components,
            typename Number,
            typename VectorizedArrayType>
  inline void
  CellwiseInverseMassMatrix<
    dim,
    fe_degree,
    n_components,
    Number,
    VectorizedArrayType>::apply(const VectorizedArrayType *in_array,
                                VectorizedArrayType       *out_array) const
  {
    if (fe_degree > -1)
      internal::CellwiseInverseMassMatrixImplBasic<dim, VectorizedArrayType>::
        template run<fe_degree>(n_components, fe_eval, in_array, out_array);
    else
      internal::CellwiseInverseMassFactory<dim, VectorizedArrayType>::apply(
        n_components, fe_eval, in_array, out_array);
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_components,
            typename Number,
            typename VectorizedArrayType>
  inline void
  CellwiseInverseMassMatrix<dim,
                            fe_degree,
                            n_components,
                            Number,
                            VectorizedArrayType>::
    apply(const AlignedVector<VectorizedArrayType> &inverse_coefficients,
          const unsigned int                        n_actual_components,
          const VectorizedArrayType                *in_array,
          VectorizedArrayType                      *out_array) const
  {
    if (fe_degree > -1)
      internal::CellwiseInverseMassMatrixImplFlexible<
        dim,
        VectorizedArrayType>::template run<fe_degree>(n_actual_components,
                                                      fe_eval,
                                                      make_array_view(
                                                        inverse_coefficients),
                                                      false,
                                                      in_array,
                                                      out_array);
    else
      internal::CellwiseInverseMassFactory<dim, VectorizedArrayType>::apply(
        n_actual_components,
        fe_eval,
        make_array_view(inverse_coefficients),
        false,
        in_array,
        out_array);
  }
 
  template <int dim,
            int fe_degree,
            int n_components,
            typename Number,
            typename VectorizedArrayType>
  inline void
  CellwiseInverseMassMatrix<dim,
                            fe_degree,
                            n_components,
                            Number,
                            VectorizedArrayType>::
    apply(const AlignedVector<Tensor<2, n_components, VectorizedArrayType>>
                                    &inverse_dyadic_coefficients,
          const VectorizedArrayType *in_array,
          VectorizedArrayType       *out_array) const
  {
    const unsigned int unrolled_size =
      inverse_dyadic_coefficients.size() * (n_components * n_components);
 
    if (fe_degree > -1)
      internal::CellwiseInverseMassMatrixImplFlexible<dim,
                                                      VectorizedArrayType>::
        template run<fe_degree>(n_components,
                                fe_eval,
                                ArrayView<const VectorizedArrayType>(
                                  &inverse_dyadic_coefficients[0][0][0],
                                  unrolled_size),
                                true,
                                in_array,
                                out_array);
    else
      internal::CellwiseInverseMassFactory<dim, VectorizedArrayType>::apply(
        n_components,
        fe_eval,
        ArrayView<const VectorizedArrayType>(
          &inverse_dyadic_coefficients[0][0][0], unrolled_size),
        true,
        in_array,
        out_array);
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_components,
            typename Number,
            typename VectorizedArrayType>
  inline void
  CellwiseInverseMassMatrix<dim,
                            fe_degree,
                            n_components,
                            Number,
                            VectorizedArrayType>::
    transform_from_q_points_to_basis(const unsigned int n_actual_components,
                                     const VectorizedArrayType *in_array,
                                     VectorizedArrayType       *out_array) const
  {
    const auto n_q_points_1d = fe_eval.get_shape_info().data[0].n_q_points_1d;
 
    if (fe_degree > -1 && (fe_degree + 1 == n_q_points_1d))
      internal::CellwiseInverseMassMatrixImplTransformFromQPoints<
        dim,
        VectorizedArrayType>::template run<fe_degree,
                                           fe_degree + 1>(n_actual_components,
                                                          fe_eval,
                                                          in_array,
                                                          out_array);
    else
      internal::CellwiseInverseMassFactory<dim, VectorizedArrayType>::
        transform_from_q_points_to_basis(n_actual_components,
                                         fe_eval,
                                         in_array,
                                         out_array);
  }
 
 
 
  //----------------- Base operator -----------------------------
  template <int dim, typename VectorType, typename VectorizedArrayType>
  Base<dim, VectorType, VectorizedArrayType>::Base()
    : EnableObserverPointer()
    , have_interface_matrices(false)
  {}
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  typename Base<dim, VectorType, VectorizedArrayType>::size_type
  Base<dim, VectorType, VectorizedArrayType>::m() const
  {
    Assert(data.get() != nullptr, ExcNotInitialized());
    typename Base<dim, VectorType, VectorizedArrayType>::size_type total_size =
      0;
    for (const unsigned int selected_row : selected_rows)
      total_size += data->get_vector_partitioner(selected_row)->size();
    return total_size;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  typename Base<dim, VectorType, VectorizedArrayType>::size_type
  Base<dim, VectorType, VectorizedArrayType>::n() const
  {
    Assert(data.get() != nullptr, ExcNotInitialized());
    typename Base<dim, VectorType, VectorizedArrayType>::size_type total_size =
      0;
    for (const unsigned int selected_column : selected_columns)
      total_size += data->get_vector_partitioner(selected_column)->size();
    return total_size;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::clear()
  {
    data.reset();
    inverse_diagonal_entries.reset();
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  typename Base<dim, VectorType, VectorizedArrayType>::value_type
  Base<dim, VectorType, VectorizedArrayType>::el(const unsigned int row,
                                                 const unsigned int col) const
  {
    Assert(row == col, ExcNotImplemented());
    Assert(inverse_diagonal_entries.get() != nullptr &&
             inverse_diagonal_entries->m() > 0,
           ExcNotInitialized());
    return 1.0 / (*inverse_diagonal_entries)(row, row);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::initialize_dof_vector(
    VectorType &vec) const
  {
    Assert(data.get() != nullptr, ExcNotInitialized());
    AssertDimension(BlockHelper::n_blocks(vec), selected_rows.size());
    for (unsigned int i = 0; i < BlockHelper::n_blocks(vec); ++i)
      {
        const unsigned int index = selected_rows[i];
        if (!BlockHelper::subblock(vec, index)
               .partitioners_are_compatible(
                 *data->get_dof_info(index).vector_partitioner))
          data->initialize_dof_vector(BlockHelper::subblock(vec, index), index);
 
        Assert(BlockHelper::subblock(vec, index)
                 .partitioners_are_globally_compatible(
                   *data->get_dof_info(index).vector_partitioner),
               ExcInternalError());
      }
    BlockHelper::collect_sizes(vec);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::initialize(
    std::shared_ptr<const MatrixFree<dim, value_type, VectorizedArrayType>>
                                     data_,
    const std::vector<unsigned int> &given_row_selection,
    const std::vector<unsigned int> &given_column_selection)
  {
    data = data_;
 
    selected_rows.clear();
    selected_columns.clear();
    if (given_row_selection.empty())
      for (unsigned int i = 0; i < data_->n_components(); ++i)
        selected_rows.push_back(i);
    else
      {
        for (unsigned int i = 0; i < given_row_selection.size(); ++i)
          {
            AssertIndexRange(given_row_selection[i], data_->n_components());
            for (unsigned int j = 0; j < given_row_selection.size(); ++j)
              if (j != i)
                Assert(given_row_selection[j] != given_row_selection[i],
                       ExcMessage("Given row indices must be unique"));
 
            selected_rows.push_back(given_row_selection[i]);
          }
      }
    if (given_column_selection.empty())
      selected_columns = selected_rows;
    else
      {
        for (unsigned int i = 0; i < given_column_selection.size(); ++i)
          {
            AssertIndexRange(given_column_selection[i], data_->n_components());
            for (unsigned int j = 0; j < given_column_selection.size(); ++j)
              if (j != i)
                Assert(given_column_selection[j] != given_column_selection[i],
                       ExcMessage("Given column indices must be unique"));
 
            selected_columns.push_back(given_column_selection[i]);
          }
      }
 
    edge_constrained_indices.clear();
    edge_constrained_indices.resize(selected_rows.size());
    edge_constrained_values.clear();
    edge_constrained_values.resize(selected_rows.size());
    have_interface_matrices = false;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::initialize(
    std::shared_ptr<const MatrixFree<dim, value_type, VectorizedArrayType>>
                                     data_,
    const MGConstrainedDoFs         &mg_constrained_dofs,
    const unsigned int               level,
    const std::vector<unsigned int> &given_row_selection)
  {
    std::vector<MGConstrainedDoFs> mg_constrained_dofs_vector(
      1, mg_constrained_dofs);
    initialize(data_, mg_constrained_dofs_vector, level, given_row_selection);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::initialize(
    std::shared_ptr<const MatrixFree<dim, value_type, VectorizedArrayType>>
                                          data_,
    const std::vector<MGConstrainedDoFs> &mg_constrained_dofs,
    const unsigned int                    level,
    const std::vector<unsigned int>      &given_row_selection)
  {
    AssertThrow(level != numbers::invalid_unsigned_int,
                ExcMessage("level is not set"));
 
    selected_rows.clear();
    selected_columns.clear();
    if (given_row_selection.empty())
      for (unsigned int i = 0; i < data_->n_components(); ++i)
        selected_rows.push_back(i);
    else
      {
        for (unsigned int i = 0; i < given_row_selection.size(); ++i)
          {
            AssertIndexRange(given_row_selection[i], data_->n_components());
            for (unsigned int j = 0; j < given_row_selection.size(); ++j)
              if (j != i)
                Assert(given_row_selection[j] != given_row_selection[i],
                       ExcMessage("Given row indices must be unique"));
 
            selected_rows.push_back(given_row_selection[i]);
          }
      }
    selected_columns = selected_rows;
 
    AssertDimension(mg_constrained_dofs.size(), selected_rows.size());
    edge_constrained_indices.clear();
    edge_constrained_indices.resize(selected_rows.size());
    edge_constrained_values.clear();
    edge_constrained_values.resize(selected_rows.size());
 
    data = data_;
 
    for (unsigned int j = 0; j < selected_rows.size(); ++j)
      {
        if (data_->n_cell_batches() > 0)
          {
            AssertDimension(level, data_->get_cell_iterator(0, 0, j)->level());
          }
 
        // setup edge_constrained indices
        const std::vector<types::global_dof_index> interface_indices =
          mg_constrained_dofs[j]
            .get_refinement_edge_indices(level)
            .get_index_vector();
        edge_constrained_indices[j].clear();
        edge_constrained_indices[j].reserve(interface_indices.size());
        edge_constrained_values[j].resize(interface_indices.size());
        const IndexSet &locally_owned =
          data->get_dof_handler(selected_rows[j]).locally_owned_mg_dofs(level);
        for (const auto interface_index : interface_indices)
          if (locally_owned.is_element(interface_index))
            edge_constrained_indices[j].push_back(
              locally_owned.index_within_set(interface_index));
        have_interface_matrices |=
          Utilities::MPI::max(
            static_cast<unsigned int>(edge_constrained_indices[j].size()),
            data->get_vector_partitioner()->get_mpi_communicator()) > 0;
      }
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::set_constrained_entries_to_one(
    VectorType &dst) const
  {
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      {
        const std::vector<unsigned int> &constrained_dofs =
          data->get_constrained_dofs(selected_rows[j]);
        for (const auto constrained_dof : constrained_dofs)
          BlockHelper::subblock(dst, j).local_element(constrained_dof) = 1.;
        for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
          BlockHelper::subblock(dst, j).local_element(
            edge_constrained_indices[j][i]) = 1.;
      }
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::vmult(VectorType       &dst,
                                                    const VectorType &src) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    dst = Number(0.);
    vmult_add(dst, src);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::vmult_add(
    VectorType       &dst,
    const VectorType &src) const
  {
    mult_add(dst, src, false);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::Tvmult_add(
    VectorType       &dst,
    const VectorType &src) const
  {
    mult_add(dst, src, true);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::adjust_ghost_range_if_necessary(
    const VectorType &src,
    const bool        is_row) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    for (unsigned int i = 0; i < BlockHelper::n_blocks(src); ++i)
      {
        const unsigned int mf_component =
          is_row ? selected_rows[i] : selected_columns[i];
        // If both vectors use the same partitioner -> done
        if (BlockHelper::subblock(src, i).get_partitioner().get() ==
            data->get_dof_info(mf_component).vector_partitioner.get())
          continue;
 
        // If not, assert that the local ranges are the same and reset to the
        // current partitioner
        Assert(BlockHelper::subblock(src, i)
                   .get_partitioner()
                   ->locally_owned_size() ==
                 data->get_dof_info(mf_component)
                   .vector_partitioner->locally_owned_size(),
               ExcMessage(
                 "The vector passed to the vmult() function does not have "
                 "the correct size for compatibility with MatrixFree."));
 
        // copy the vector content to a temporary vector so that it does not get
        // lost
        LinearAlgebra::distributed::Vector<Number> copy_vec(
          BlockHelper::subblock(src, i));
        this->data->initialize_dof_vector(
          BlockHelper::subblock(const_cast<VectorType &>(src), i),
          mf_component);
        BlockHelper::subblock(const_cast<VectorType &>(src), i)
          .copy_locally_owned_data_from(copy_vec);
      }
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::preprocess_constraints(
    VectorType       &dst,
    const VectorType &src) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    adjust_ghost_range_if_necessary(src, false);
    adjust_ghost_range_if_necessary(dst, true);
 
    // set zero Dirichlet values on the input vector (and remember the src and
    // dst values because we need to reset them at the end)
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      {
        for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
          {
            edge_constrained_values[j][i] = std::pair<Number, Number>(
              BlockHelper::subblock(src, j).local_element(
                edge_constrained_indices[j][i]),
              BlockHelper::subblock(dst, j).local_element(
                edge_constrained_indices[j][i]));
            BlockHelper::subblock(const_cast<VectorType &>(src), j)
              .local_element(edge_constrained_indices[j][i]) = 0.;
          }
      }
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::mult_add(
    VectorType       &dst,
    const VectorType &src,
    const bool        transpose) const
  {
    AssertDimension(dst.size(), src.size());
    AssertDimension(BlockHelper::n_blocks(dst), BlockHelper::n_blocks(src));
    AssertDimension(BlockHelper::n_blocks(dst), selected_rows.size());
    preprocess_constraints(dst, src);
    if (transpose)
      Tapply_add(dst, src);
    else
      apply_add(dst, src);
    postprocess_constraints(dst, src);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::postprocess_constraints(
    VectorType       &dst,
    const VectorType &src) const
  {
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      {
        const std::vector<unsigned int> &constrained_dofs =
          data->get_constrained_dofs(selected_rows[j]);
        for (const auto constrained_dof : constrained_dofs)
          BlockHelper::subblock(dst, j).local_element(constrained_dof) +=
            BlockHelper::subblock(src, j).local_element(constrained_dof);
      }
 
    // reset edge constrained values, multiply by unit matrix and add into
    // destination
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      {
        for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
          {
            BlockHelper::subblock(const_cast<VectorType &>(src), j)
              .local_element(edge_constrained_indices[j][i]) =
              edge_constrained_values[j][i].first;
            BlockHelper::subblock(dst, j).local_element(
              edge_constrained_indices[j][i]) =
              edge_constrained_values[j][i].second +
              edge_constrained_values[j][i].first;
          }
      }
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::vmult_interface_down(
    VectorType       &dst,
    const VectorType &src) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    AssertDimension(dst.size(), src.size());
    adjust_ghost_range_if_necessary(src, false);
    adjust_ghost_range_if_necessary(dst, true);
 
    dst = Number(0.);
 
    if (!have_interface_matrices)
      return;
 
    // set zero Dirichlet values on the input vector (and remember the src and
    // dst values because we need to reset them at the end)
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
        {
          edge_constrained_values[j][i] = std::pair<Number, Number>(
            BlockHelper::subblock(src, j).local_element(
              edge_constrained_indices[j][i]),
            BlockHelper::subblock(dst, j).local_element(
              edge_constrained_indices[j][i]));
          BlockHelper::subblock(const_cast<VectorType &>(src), j)
            .local_element(edge_constrained_indices[j][i]) = 0.;
        }
 
    apply_add(dst, src);
 
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      {
        unsigned int c = 0;
        for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
          {
            for (; c < edge_constrained_indices[j][i]; ++c)
              BlockHelper::subblock(dst, j).local_element(c) = 0.;
            ++c;
 
            // reset the src values
            BlockHelper::subblock(const_cast<VectorType &>(src), j)
              .local_element(edge_constrained_indices[j][i]) =
              edge_constrained_values[j][i].first;
          }
        for (; c < BlockHelper::subblock(dst, j).locally_owned_size(); ++c)
          BlockHelper::subblock(dst, j).local_element(c) = 0.;
      }
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::vmult_interface_up(
    VectorType       &dst,
    const VectorType &src) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    AssertDimension(dst.size(), src.size());
    adjust_ghost_range_if_necessary(src, false);
    adjust_ghost_range_if_necessary(dst, true);
 
    dst = Number(0.);
 
    if (!have_interface_matrices)
      return;
 
    VectorType src_cpy(src);
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      {
        unsigned int c = 0;
        for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
          {
            for (; c < edge_constrained_indices[j][i]; ++c)
              BlockHelper::subblock(src_cpy, j).local_element(c) = 0.;
            ++c;
          }
        for (; c < BlockHelper::subblock(src_cpy, j).locally_owned_size(); ++c)
          BlockHelper::subblock(src_cpy, j).local_element(c) = 0.;
      }
 
    apply_add(dst, src_cpy);
 
    for (unsigned int j = 0; j < BlockHelper::n_blocks(dst); ++j)
      for (unsigned int i = 0; i < edge_constrained_indices[j].size(); ++i)
        BlockHelper::subblock(dst, j).local_element(
          edge_constrained_indices[j][i]) = 0.;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::Tvmult(
    VectorType       &dst,
    const VectorType &src) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    dst = Number(0.);
    Tvmult_add(dst, src);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  std::size_t
  Base<dim, VectorType, VectorizedArrayType>::memory_consumption() const
  {
    return inverse_diagonal_entries.get() != nullptr ?
             inverse_diagonal_entries->memory_consumption() :
             sizeof(*this);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  std::shared_ptr<const MatrixFree<
    dim,
    typename Base<dim, VectorType, VectorizedArrayType>::value_type,
    VectorizedArrayType>>
  Base<dim, VectorType, VectorizedArrayType>::get_matrix_free() const
  {
    return data;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  const std::shared_ptr<DiagonalMatrix<VectorType>> &
  Base<dim, VectorType, VectorizedArrayType>::get_matrix_diagonal_inverse()
    const
  {
    Assert(inverse_diagonal_entries.get() != nullptr &&
             inverse_diagonal_entries->m() > 0,
           ExcNotInitialized());
    return inverse_diagonal_entries;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  const std::shared_ptr<DiagonalMatrix<VectorType>> &
  Base<dim, VectorType, VectorizedArrayType>::get_matrix_diagonal() const
  {
    Assert(diagonal_entries.get() != nullptr && diagonal_entries->m() > 0,
           ExcNotInitialized());
    return diagonal_entries;
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::Tapply_add(
    VectorType       &dst,
    const VectorType &src) const
  {
    apply_add(dst, src);
  }
 
 
 
  template <int dim, typename VectorType, typename VectorizedArrayType>
  void
  Base<dim, VectorType, VectorizedArrayType>::precondition_Jacobi(
    VectorType                                                           &dst,
    const VectorType                                                     &src,
    const typename Base<dim, VectorType, VectorizedArrayType>::value_type omega)
    const
  {
    Assert(inverse_diagonal_entries.get() && inverse_diagonal_entries->m() > 0,
           ExcNotInitialized());
    inverse_diagonal_entries->vmult(dst, src);
    dst *= omega;
  }
 
 
 
  //------------------------- MGInterfaceOperator ------------------------------
 
  template <typename OperatorType>
  MGInterfaceOperator<OperatorType>::MGInterfaceOperator()
    : EnableObserverPointer()
    , mf_base_operator(nullptr)
  {}
 
 
 
  template <typename OperatorType>
  void
  MGInterfaceOperator<OperatorType>::clear()
  {
    mf_base_operator = nullptr;
  }
 
 
 
  template <typename OperatorType>
  void
  MGInterfaceOperator<OperatorType>::initialize(const OperatorType &operator_in)
  {
    mf_base_operator = &operator_in;
  }
 
 
 
  template <typename OperatorType>
  template <typename VectorType>
  void
  MGInterfaceOperator<OperatorType>::vmult(VectorType       &dst,
                                           const VectorType &src) const
  {
#ifndef DEAL_II_MSVC
    static_assert(
      std::is_same_v<typename VectorType::value_type, value_type>,
      "The vector type must be based on the same value type as this "
      "operator");
#endif
 
    Assert(mf_base_operator != nullptr, ExcNotInitialized());
 
    mf_base_operator->vmult_interface_down(dst, src);
  }
 
 
 
  template <typename OperatorType>
  template <typename VectorType>
  void
  MGInterfaceOperator<OperatorType>::Tvmult(VectorType       &dst,
                                            const VectorType &src) const
  {
#ifndef DEAL_II_MSVC
    static_assert(
      std::is_same_v<typename VectorType::value_type, value_type>,
      "The vector type must be based on the same value type as this "
      "operator");
#endif
 
    Assert(mf_base_operator != nullptr, ExcNotInitialized());
 
    mf_base_operator->vmult_interface_up(dst, src);
  }
 
 
 
  template <typename OperatorType>
  template <typename VectorType>
  void
  MGInterfaceOperator<OperatorType>::initialize_dof_vector(
    VectorType &vec) const
  {
    Assert(mf_base_operator != nullptr, ExcNotInitialized());
 
    mf_base_operator->initialize_dof_vector(vec);
  }
 
 
 
  //-----------------------------MassOperator----------------------------------
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::MassOperator()
    : Base<dim, VectorType, VectorizedArrayType>()
  {
    AssertThrow(
      IsBlockVector<VectorType>::value == false,
      ExcNotImplemented(
        "This class only supports the non-blocked vector variant of the Base "
        "operator because only a single FEEvaluation object is used in the "
        "apply function."));
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::compute_diagonal()
  {
    Assert((Base<dim, VectorType, VectorizedArrayType>::data.get() != nullptr),
           ExcNotInitialized());
    Assert(this->selected_rows == this->selected_columns,
           ExcMessage("This function is only implemented for square (not "
                      "rectangular) operators."));
 
    this->inverse_diagonal_entries =
      std::make_shared<DiagonalMatrix<VectorType>>();
    this->diagonal_entries = std::make_shared<DiagonalMatrix<VectorType>>();
    VectorType &inverse_diagonal_vector =
      this->inverse_diagonal_entries->get_vector();
    VectorType &diagonal_vector = this->diagonal_entries->get_vector();
    this->initialize_dof_vector(inverse_diagonal_vector);
    this->initialize_dof_vector(diagonal_vector);
 
    // Set up the action of the mass matrix in a way that's compatible with
    // MatrixFreeTools::compute_diagonal:
    auto diagonal_evaluation = [](auto &integrator) {
      integrator.evaluate(EvaluationFlags::values);
      for (unsigned int q = 0; q < integrator.n_q_points; ++q)
        integrator.submit_value(integrator.get_value(q), q);
      integrator.integrate(EvaluationFlags::values);
    };
 
    std::function<void(
      FEEvaluation<
        dim,
        fe_degree,
        n_q_points_1d,
        n_components,
        typename Base<dim, VectorType, VectorizedArrayType>::value_type,
        VectorizedArrayType> &)>
      diagonal_evaluation_f(diagonal_evaluation);
 
    Assert(this->selected_rows.size() > 0, ExcInternalError());
    for (unsigned int block_n = 0; block_n < this->selected_rows.size();
         ++block_n)
      MatrixFreeTools::compute_diagonal(*this->data,
                                        BlockHelper::subblock(diagonal_vector,
                                                              block_n),
                                        diagonal_evaluation_f,
                                        this->selected_rows[block_n]);
 
    // Constrained entries will create zeros on the main diagonal, which we
    // don't want
    this->set_constrained_entries_to_one(diagonal_vector);
 
    inverse_diagonal_vector = diagonal_vector;
 
    for (unsigned int i = 0; i < inverse_diagonal_vector.locally_owned_size();
         ++i)
      {
        if constexpr (running_in_debug_mode())
          {
            // only define the type alias in debug mode to avoid a warning
            using Number =
              typename Base<dim, VectorType, VectorizedArrayType>::value_type;
            Assert(diagonal_vector.local_element(i) > Number(0),
                   ExcInternalError());
          }
        inverse_diagonal_vector.local_element(i) =
          1. / inverse_diagonal_vector.local_element(i);
      }
 
    // We never need ghost values so don't update them
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::compute_lumped_diagonal()
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    Assert((Base<dim, VectorType, VectorizedArrayType>::data.get() != nullptr),
           ExcNotInitialized());
    Assert(this->selected_rows == this->selected_columns,
           ExcMessage("This function is only implemented for square (not "
                      "rectangular) operators."));
 
    inverse_lumped_diagonal_entries =
      std::make_shared<DiagonalMatrix<VectorType>>();
    lumped_diagonal_entries = std::make_shared<DiagonalMatrix<VectorType>>();
    VectorType &inverse_lumped_diagonal_vector =
      inverse_lumped_diagonal_entries->get_vector();
    VectorType &lumped_diagonal_vector = lumped_diagonal_entries->get_vector();
    this->initialize_dof_vector(inverse_lumped_diagonal_vector);
    this->initialize_dof_vector(lumped_diagonal_vector);
 
    // Re-use the inverse_lumped_diagonal_vector as the vector of 1s
    inverse_lumped_diagonal_vector = Number(1.);
    apply_add(lumped_diagonal_vector, inverse_lumped_diagonal_vector);
    this->set_constrained_entries_to_one(lumped_diagonal_vector);
 
    const size_type locally_owned_size =
      inverse_lumped_diagonal_vector.locally_owned_size();
    // A caller may request a lumped diagonal matrix when it doesn't make sense
    // (e.g., an element with negative-mean basis functions). Avoid division by
    // zero so we don't cause a floating point exception but permit negative
    // entries here.
    for (size_type i = 0; i < locally_owned_size; ++i)
      {
        if (lumped_diagonal_vector.local_element(i) == Number(0.))
          inverse_lumped_diagonal_vector.local_element(i) = Number(1.);
        else
          inverse_lumped_diagonal_vector.local_element(i) =
            Number(1.) / lumped_diagonal_vector.local_element(i);
      }
 
    inverse_lumped_diagonal_vector.update_ghost_values();
    lumped_diagonal_vector.update_ghost_values();
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  const std::shared_ptr<DiagonalMatrix<VectorType>> &
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::get_matrix_lumped_diagonal_inverse() const
  {
    Assert(inverse_lumped_diagonal_entries.get() != nullptr &&
             inverse_lumped_diagonal_entries->m() > 0,
           ExcNotInitialized());
    return inverse_lumped_diagonal_entries;
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  const std::shared_ptr<DiagonalMatrix<VectorType>> &
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::get_matrix_lumped_diagonal() const
  {
    Assert(lumped_diagonal_entries.get() != nullptr &&
             lumped_diagonal_entries->m() > 0,
           ExcNotInitialized());
    return lumped_diagonal_entries;
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::apply_add(VectorType       &dst,
                                               const VectorType &src) const
  {
    Base<dim, VectorType, VectorizedArrayType>::data->cell_loop(
      &MassOperator::local_apply_cell, this, dst, src);
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  MassOperator<dim,
               fe_degree,
               n_q_points_1d,
               n_components,
               VectorType,
               VectorizedArrayType>::
    local_apply_cell(
      const MatrixFree<
        dim,
        typename Base<dim, VectorType, VectorizedArrayType>::value_type,
        VectorizedArrayType>                      &data,
      VectorType                                  &dst,
      const VectorType                            &src,
      const std::pair<unsigned int, unsigned int> &cell_range) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    FEEvaluation<dim,
                 fe_degree,
                 n_q_points_1d,
                 n_components,
                 Number,
                 VectorizedArrayType>
      phi(data, this->selected_rows[0]);
    for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell)
      {
        phi.reinit(cell);
        phi.read_dof_values(src);
        phi.evaluate(EvaluationFlags::values);
        for (unsigned int q = 0; q < phi.n_q_points; ++q)
          phi.submit_value(phi.get_value(q), q);
        phi.integrate(EvaluationFlags::values);
        phi.distribute_local_to_global(dst);
      }
  }
 
 
  //-----------------------------LaplaceOperator----------------------------------
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::LaplaceOperator()
    : Base<dim, VectorType, VectorizedArrayType>()
  {}
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::clear()
  {
    Base<dim, VectorType, VectorizedArrayType>::clear();
    scalar_coefficient.reset();
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::
    set_coefficient(
      const std::shared_ptr<Table<2, VectorizedArrayType>> &scalar_coefficient_)
  {
    scalar_coefficient = scalar_coefficient_;
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  std::shared_ptr<Table<2, VectorizedArrayType>>
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::get_coefficient()
  {
    Assert(scalar_coefficient.get(), ExcNotInitialized());
    return scalar_coefficient;
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::compute_diagonal()
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    Assert((Base<dim, VectorType, VectorizedArrayType>::data.get() != nullptr),
           ExcNotInitialized());
 
    this->inverse_diagonal_entries =
      std::make_shared<DiagonalMatrix<VectorType>>();
    this->diagonal_entries = std::make_shared<DiagonalMatrix<VectorType>>();
    VectorType &inverse_diagonal_vector =
      this->inverse_diagonal_entries->get_vector();
    VectorType &diagonal_vector = this->diagonal_entries->get_vector();
    this->initialize_dof_vector(inverse_diagonal_vector);
    this->initialize_dof_vector(diagonal_vector);
 
    this->data->cell_loop(&LaplaceOperator::local_diagonal_cell,
                          this,
                          diagonal_vector,
                          /*unused*/ diagonal_vector);
    this->set_constrained_entries_to_one(diagonal_vector);
 
    inverse_diagonal_vector = diagonal_vector;
 
    for (unsigned int i = 0; i < inverse_diagonal_vector.locally_owned_size();
         ++i)
      if (std::abs(inverse_diagonal_vector.local_element(i)) >
          std::sqrt(std::numeric_limits<Number>::epsilon()))
        inverse_diagonal_vector.local_element(i) =
          1. / inverse_diagonal_vector.local_element(i);
      else
        inverse_diagonal_vector.local_element(i) = 1.;
 
    // We never need ghost values so don't update them
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::apply_add(VectorType       &dst,
                                                  const VectorType &src) const
  {
    Base<dim, VectorType, VectorizedArrayType>::data->cell_loop(
      &LaplaceOperator::local_apply_cell, this, dst, src);
  }
 
  namespace Implementation
  {
    template <typename VectorizedArrayType>
    bool
    non_negative(const VectorizedArrayType &n)
    {
      for (unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
        if (n[v] < 0.)
          return false;
 
      return true;
    }
  } // namespace Implementation
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  template <int n_components_compute>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::
    do_operation_on_cell(
      FEEvaluation<
        dim,
        fe_degree,
        n_q_points_1d,
        n_components_compute,
        typename Base<dim, VectorType, VectorizedArrayType>::value_type,
        VectorizedArrayType> &phi,
      const unsigned int      cell) const
  {
    phi.evaluate(EvaluationFlags::gradients);
    if (scalar_coefficient.get())
      {
        Assert(scalar_coefficient->size(1) == 1 ||
                 scalar_coefficient->size(1) == phi.n_q_points,
               ExcMessage("The number of columns in the coefficient table must "
                          "be either 1 or the number of quadrature points " +
                          std::to_string(phi.n_q_points) +
                          ", but the given value was " +
                          std::to_string(scalar_coefficient->size(1))));
        if (scalar_coefficient->size(1) == phi.n_q_points)
          for (unsigned int q = 0; q < phi.n_q_points; ++q)
            {
              Assert(Implementation::non_negative(
                       (*scalar_coefficient)(cell, q)),
                     ExcMessage("Coefficient must be non-negative"));
              phi.submit_gradient((*scalar_coefficient)(cell, q) *
                                    phi.get_gradient(q),
                                  q);
            }
        else
          {
            Assert(Implementation::non_negative((*scalar_coefficient)(cell, 0)),
                   ExcMessage("Coefficient must be non-negative"));
            const VectorizedArrayType coefficient =
              (*scalar_coefficient)(cell, 0);
            for (unsigned int q = 0; q < phi.n_q_points; ++q)
              phi.submit_gradient(coefficient * phi.get_gradient(q), q);
          }
      }
    else
      {
        for (unsigned int q = 0; q < phi.n_q_points; ++q)
          {
            phi.submit_gradient(phi.get_gradient(q), q);
          }
      }
    phi.integrate(EvaluationFlags::gradients);
  }
 
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::
    local_apply_cell(
      const MatrixFree<
        dim,
        typename Base<dim, VectorType, VectorizedArrayType>::value_type,
        VectorizedArrayType>                      &data,
      VectorType                                  &dst,
      const VectorType                            &src,
      const std::pair<unsigned int, unsigned int> &cell_range) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
    FEEvaluation<dim,
                 fe_degree,
                 n_q_points_1d,
                 n_components,
                 Number,
                 VectorizedArrayType>
      phi(data, this->selected_rows[0]);
    for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell)
      {
        phi.reinit(cell);
        phi.read_dof_values(src);
        do_operation_on_cell(phi, cell);
        phi.distribute_local_to_global(dst);
      }
  }
 
 
  template <int dim,
            int fe_degree,
            int n_q_points_1d,
            int n_components,
            typename VectorType,
            typename VectorizedArrayType>
  void
  LaplaceOperator<dim,
                  fe_degree,
                  n_q_points_1d,
                  n_components,
                  VectorType,
                  VectorizedArrayType>::
    local_diagonal_cell(
      const MatrixFree<
        dim,
        typename Base<dim, VectorType, VectorizedArrayType>::value_type,
        VectorizedArrayType> &data,
      VectorType             &dst,
      const VectorType &,
      const std::pair<unsigned int, unsigned int> &cell_range) const
  {
    using Number =
      typename Base<dim, VectorType, VectorizedArrayType>::value_type;
 
    FEEvaluation<dim, fe_degree, n_q_points_1d, 1, Number, VectorizedArrayType>
      eval(data, this->selected_rows[0]);
    FEEvaluation<dim,
                 fe_degree,
                 n_q_points_1d,
                 n_components,
                 Number,
                 VectorizedArrayType>
      eval_vector(data, this->selected_rows[0]);
    for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell)
      {
        eval.reinit(cell);
        eval_vector.reinit(cell);
        // This function assumes that we have the same result on all
        // components, so we only need to go through the columns of one scalar
        // component, for which we have created a separate evaluator (attached
        // to the first component, but the component does not matter because
        // we only use the underlying integrals)
        for (unsigned int i = 0; i < eval.dofs_per_cell; ++i)
          {
            for (unsigned int j = 0; j < eval.dofs_per_cell; ++j)
              eval.begin_dof_values()[j] = VectorizedArrayType();
            eval.begin_dof_values()[i] = 1.;
 
            do_operation_on_cell(eval, cell);
 
            // We now pick up the value on the diagonal (row i) and broadcast
            // it to a second evaluator for all vector components, which we
            // will distribute to the result vector afterwards
            for (unsigned int c = 0; c < n_components; ++c)
              eval_vector
                .begin_dof_values()[i + c * eval_vector.dofs_per_component] =
                eval.begin_dof_values()[i];
          }
        eval_vector.distribute_local_to_global(dst);
      }
  }
 
 
} // end of namespace MatrixFreeOperators
 
 
DEAL_II_NAMESPACE_CLOSE
 
#endif