tensor_product_matrix.h

Go to the documentation of this file.

// -----------------------------------------------------------------------------
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception OR LGPL-2.1-or-later
// Copyright (C) 2017 - 2025 by the deal.II authors
//
// This file is part of the deal.II library.
//
// Detailed license information governing the source code and contributions
// can be found in LICENSE.md and CONTRIBUTING.md at the top level directory.
//
// -----------------------------------------------------------------------------
 
#ifndef dealii_tensor_product_matrix_h
#define dealii_tensor_product_matrix_h
 
 
#include <deal.II/base/config.h>
 
#include <deal.II/base/array_view.h>
#include <deal.II/base/floating_point_comparator.h>
#include <deal.II/base/mutex.h>
#include <deal.II/base/vectorization.h>
 
#include <deal.II/lac/lapack_full_matrix.h>
 
#include <deal.II/matrix_free/tensor_product_kernels.h>
 
#include <bitset>
 
 
DEAL_II_NAMESPACE_OPEN
 
// Forward declarations
#ifndef DOXYGEN
template <typename>
class Vector;
template <typename>
class FullMatrix;
#endif
 
template <int dim, typename Number, int n_rows_1d = -1>
class TensorProductMatrixSymmetricSum
{
public:
  using value_type = Number;
 
  static constexpr int n_rows_1d_static = n_rows_1d;
 
  TensorProductMatrixSymmetricSum() = default;
 
  template <typename T>
  TensorProductMatrixSymmetricSum(const T &mass_matrix,
                                  const T &derivative_matrix);
 
  template <typename T>
  void
  reinit(const T &mass_matrix, const T &derivative_matrix);
 
  unsigned int
  m() const;
 
  unsigned int
  n() const;
 
  void
  vmult(const ArrayView<Number> &dst, const ArrayView<const Number> &src) const;
 
  void
  vmult(const ArrayView<Number>       &dst,
        const ArrayView<const Number> &src,
        AlignedVector<Number>         &tmp) const;
 
  void
  apply_inverse(const ArrayView<Number>       &dst,
                const ArrayView<const Number> &src) const;
 
  std::size_t
  memory_consumption() const;
 
protected:
  std::array<Table<2, Number>, dim> mass_matrix;
 
  std::array<Table<2, Number>, dim> derivative_matrix;
 
  std::array<AlignedVector<Number>, dim> eigenvalues;
 
  std::array<Table<2, Number>, dim> eigenvectors;
 
private:
  mutable AlignedVector<Number> tmp_array;
 
  mutable Threads::Mutex mutex;
};
 
 
 
namespace internal
{
  namespace TensorProductMatrixSymmetricSum
  {
    template <typename Number>
    struct MatrixPairComparator
    {
      using VectorizedArrayTrait =
        ::internal::VectorizedArrayTrait<Number>;
      using ScalarNumber = typename VectorizedArrayTrait::value_type;
      static constexpr std::size_t width = VectorizedArrayTrait::width();
 
      using MatrixPairType =
        std::pair<std::bitset<width>,
                  std::pair<Table<2, Number>, Table<2, Number>>>;
 
      MatrixPairComparator()
        : eps(std::sqrt(std::numeric_limits<ScalarNumber>::epsilon()))
      {}
 
      bool
      operator()(const MatrixPairType &left, const MatrixPairType &right) const
      {
        const auto &M_0 = left.second.first;
        const auto &K_0 = left.second.second;
        const auto &M_1 = right.second.first;
        const auto &K_1 = right.second.second;
 
        std::bitset<width> mask;
 
        for (unsigned int v = 0; v < width; ++v)
          mask[v] = left.first[v] && right.first[v];
 
        const FloatingPointComparator<Number> comparator(
          eps, false /*use relative tolerance*/, mask);
 
        if (comparator(M_0, M_1))
          return true;
        else if (comparator(M_1, M_0))
          return false;
        else if (comparator(K_0, K_1))
          return true;
        else
          return false;
      }
 
    private:
      const ScalarNumber eps;
    };
  } // namespace TensorProductMatrixSymmetricSum
} // namespace internal
 
 
 
template <int dim, typename Number, int n_rows_1d = -1>
class TensorProductMatrixSymmetricSumCollection
{
  using MatrixPairType = std::pair<Table<2, Number>, Table<2, Number>>;
 
  using MatrixPairTypeWithMask = std::pair<
    std::bitset<::internal::VectorizedArrayTrait<Number>::width()>,
    MatrixPairType>;
 
public:
  struct AdditionalData
  {
    AdditionalData(const bool compress_matrices           = true,
                   const bool precompute_inverse_diagonal = true);
 
    bool compress_matrices;
 
    bool precompute_inverse_diagonal;
  };
 
  TensorProductMatrixSymmetricSumCollection(
    const AdditionalData &additional_data = AdditionalData());
 
  void
  reserve(const unsigned int size);
 
  template <typename T>
  void
  insert(const unsigned int index, const T &Ms, const T &Ks);
 
  void
  finalize();
 
  void
  apply_inverse(const unsigned int             index,
                const ArrayView<Number>       &dst_in,
                const ArrayView<const Number> &src_in) const;
 
  std::size_t
  memory_consumption() const;
 
  std::size_t
  storage_size() const;
 
private:
  const bool compress_matrices;
 
  const bool precompute_inverse_diagonal;
 
  std::vector<MatrixPairType> mass_and_derivative_matrices;
 
  std::map<
    MatrixPairTypeWithMask,
    unsigned int,
    internal::TensorProductMatrixSymmetricSum::MatrixPairComparator<Number>>
    cache;
 
  std::vector<unsigned int> indices;
 
  AlignedVector<Number> mass_matrices;
 
  AlignedVector<Number> derivative_matrices;
 
  AlignedVector<Number> eigenvectors;
 
  AlignedVector<Number> eigenvalues;
 
  AlignedVector<Number> inverted_eigenvalues;
 
  std::vector<unsigned int> vector_ptr;
 
  std::vector<unsigned int> matrix_ptr;
 
  std::vector<unsigned int> vector_n_rows_1d;
};
 
 
/*----------------------- Inline functions ----------------------------------*/
 
#ifndef DOXYGEN
 
namespace internal
{
  namespace TensorProductMatrixSymmetricSum
  {
    template <typename Number>
    void
    spectral_assembly(const Number      *mass_matrix,
                      const Number      *derivative_matrix,
                      const unsigned int n_rows,
                      const unsigned int n_cols,
                      Number            *eigenvalues,
                      Number            *eigenvectors)
    {
      Assert(n_rows == n_cols, ExcNotImplemented());
 
      std::vector<bool> constrained_dofs(n_rows, false);
 
      for (unsigned int i = 0; i < n_rows; ++i)
        {
          if (mass_matrix[i + i * n_rows] == 0.0)
            {
              Assert(derivative_matrix[i + i * n_rows] == 0.0,
                     ExcInternalError());
 
              for (unsigned int j = 0; j < n_rows; ++j)
                {
                  Assert(derivative_matrix[i + j * n_rows] == 0,
                         ExcInternalError());
                  Assert(derivative_matrix[j + i * n_rows] == 0,
                         ExcInternalError());
                }
 
              constrained_dofs[i] = true;
            }
        }
 
      const auto transpose_fill_nm = [&constrained_dofs](Number            *out,
                                                         const Number      *in,
                                                         const unsigned int n,
                                                         const unsigned int m) {
        for (unsigned int mm = 0, c = 0; mm < m; ++mm)
          for (unsigned int nn = 0; nn < n; ++nn, ++c)
            out[mm + nn * m] =
              (mm == nn && constrained_dofs[mm]) ? Number(1.0) : in[c];
      };
 
      std::vector<::Vector<Number>> eigenvecs(n_rows);
      LAPACKFullMatrix<Number>            mass_copy(n_rows, n_cols);
      LAPACKFullMatrix<Number>            deriv_copy(n_rows, n_cols);
 
      transpose_fill_nm(&(mass_copy(0, 0)), mass_matrix, n_rows, n_cols);
      transpose_fill_nm(&(deriv_copy(0, 0)), derivative_matrix, n_rows, n_cols);
 
      deriv_copy.compute_generalized_eigenvalues_symmetric(mass_copy,
                                                           eigenvecs);
      AssertDimension(eigenvecs.size(), n_rows);
      for (unsigned int i = 0, c = 0; i < n_rows; ++i)
        for (unsigned int j = 0; j < n_cols; ++j, ++c)
          if (constrained_dofs[i] == false)
            eigenvectors[c] = eigenvecs[j][i];
 
      for (unsigned int i = 0; i < n_rows; ++i, ++eigenvalues)
        *eigenvalues = deriv_copy.eigenvalue(i).real();
    }
 
 
 
    template <std::size_t dim, typename Number>
    inline void
    setup(const std::array<Table<2, Number>, dim> &mass_matrix,
          const std::array<Table<2, Number>, dim> &derivative_matrix,
          std::array<Table<2, Number>, dim>       &eigenvectors,
          std::array<AlignedVector<Number>, dim>  &eigenvalues)
    {
      const unsigned int n_rows_1d = mass_matrix[0].n_cols();
 
      for (unsigned int dir = 0; dir < dim; ++dir)
        {
          AssertDimension(n_rows_1d, mass_matrix[dir].n_cols());
          AssertDimension(mass_matrix[dir].n_rows(), mass_matrix[dir].n_cols());
          AssertDimension(mass_matrix[dir].n_rows(),
                          derivative_matrix[dir].n_rows());
          AssertDimension(mass_matrix[dir].n_rows(),
                          derivative_matrix[dir].n_cols());
 
          eigenvectors[dir].reinit(mass_matrix[dir].n_cols(),
                                   mass_matrix[dir].n_rows());
          eigenvalues[dir].resize(mass_matrix[dir].n_cols());
          internal::TensorProductMatrixSymmetricSum::spectral_assembly<Number>(
            &(mass_matrix[dir](0, 0)),
            &(derivative_matrix[dir](0, 0)),
            mass_matrix[dir].n_rows(),
            mass_matrix[dir].n_cols(),
            eigenvalues[dir].begin(),
            &(eigenvectors[dir](0, 0)));
        }
    }
 
 
 
    template <std::size_t dim, typename Number, std::size_t n_lanes>
    inline void
    setup(
      const std::array<Table<2, VectorizedArray<Number, n_lanes>>, dim>
        &mass_matrix,
      const std::array<Table<2, VectorizedArray<Number, n_lanes>>, dim>
        &derivative_matrix,
      std::array<Table<2, VectorizedArray<Number, n_lanes>>, dim> &eigenvectors,
      std::array<AlignedVector<VectorizedArray<Number, n_lanes>>, dim>
        &eigenvalues)
    {
      const unsigned int     n_rows_1d = mass_matrix[0].n_cols();
      constexpr unsigned int macro_size =
        VectorizedArray<Number, n_lanes>::size();
      const std::size_t nm_flat_size_max = n_rows_1d * n_rows_1d * macro_size;
      const std::size_t n_flat_size_max  = n_rows_1d * macro_size;
 
      std::vector<Number> mass_matrix_flat;
      std::vector<Number> deriv_matrix_flat;
      std::vector<Number> eigenvalues_flat;
      std::vector<Number> eigenvectors_flat;
      mass_matrix_flat.resize(nm_flat_size_max);
      deriv_matrix_flat.resize(nm_flat_size_max);
      eigenvalues_flat.resize(n_flat_size_max);
      eigenvectors_flat.resize(nm_flat_size_max);
      std::array<unsigned int, macro_size> offsets_nm;
      std::array<unsigned int, macro_size> offsets_n;
      for (unsigned int dir = 0; dir < dim; ++dir)
        {
          AssertDimension(n_rows_1d, mass_matrix[dir].n_cols());
          AssertDimension(mass_matrix[dir].n_rows(), mass_matrix[dir].n_cols());
          AssertDimension(mass_matrix[dir].n_rows(),
                          derivative_matrix[dir].n_rows());
          AssertDimension(mass_matrix[dir].n_rows(),
                          derivative_matrix[dir].n_cols());
 
          const unsigned int n_rows = mass_matrix[dir].n_rows();
          const unsigned int n_cols = mass_matrix[dir].n_cols();
          const unsigned int nm     = n_rows * n_cols;
          for (unsigned int vv = 0; vv < macro_size; ++vv)
            offsets_nm[vv] = nm * vv;
 
          vectorized_transpose_and_store<Number, n_lanes>(
            false,
            nm,
            &(mass_matrix[dir](0, 0)),
            offsets_nm.data(),
            mass_matrix_flat.data());
          vectorized_transpose_and_store<Number, n_lanes>(
            false,
            nm,
            &(derivative_matrix[dir](0, 0)),
            offsets_nm.data(),
            deriv_matrix_flat.data());
 
          const Number *mass_cbegin    = mass_matrix_flat.data();
          const Number *deriv_cbegin   = deriv_matrix_flat.data();
          Number       *eigenvec_begin = eigenvectors_flat.data();
          Number       *eigenval_begin = eigenvalues_flat.data();
          for (unsigned int lane = 0; lane < macro_size; ++lane)
            internal::TensorProductMatrixSymmetricSum::spectral_assembly<
              Number>(mass_cbegin + nm * lane,
                      deriv_cbegin + nm * lane,
                      n_rows,
                      n_cols,
                      eigenval_begin + n_rows * lane,
                      eigenvec_begin + nm * lane);
 
          eigenvalues[dir].resize(n_rows);
          eigenvectors[dir].reinit(n_rows, n_cols);
          for (unsigned int vv = 0; vv < macro_size; ++vv)
            offsets_n[vv] = n_rows * vv;
          vectorized_load_and_transpose<Number, n_lanes>(
            n_rows,
            eigenvalues_flat.data(),
            offsets_n.data(),
            eigenvalues[dir].begin());
          vectorized_load_and_transpose<Number, n_lanes>(
            nm,
            eigenvectors_flat.data(),
            offsets_nm.data(),
            &(eigenvectors[dir](0, 0)));
        }
    }
 
 
 
    template <std::size_t dim, typename Number>
    inline std::array<Table<2, Number>, dim>
    convert(const std::array<Table<2, Number>, dim> &mass_matrix)
    {
      return mass_matrix;
    }
 
 
 
    template <std::size_t dim, typename Number>
    inline std::array<Table<2, Number>, dim>
    convert(const std::array<FullMatrix<Number>, dim> &mass_matrix)
    {
      std::array<Table<2, Number>, dim> mass_copy;
 
      std::transform(mass_matrix.cbegin(),
                     mass_matrix.cend(),
                     mass_copy.begin(),
                     [](const FullMatrix<Number> &m) -> Table<2, Number> {
                       return m;
                     });
 
      return mass_copy;
    }
 
 
 
    template <std::size_t dim, typename Number>
    inline std::array<Table<2, Number>, dim>
    convert(const Table<2, Number> &matrix)
    {
      std::array<Table<2, Number>, dim> matrices;
 
      std::fill(matrices.begin(), matrices.end(), matrix);
 
      return matrices;
    }
 
 
 
    template <int n_rows_1d_templated, std::size_t dim, typename Number>
    void
    vmult(Number                                *dst,
          const Number                          *src,
          AlignedVector<Number>                 &tmp,
          const unsigned int                     n_rows_1d_non_templated,
          const std::array<const Number *, dim> &mass_matrix,
          const std::array<const Number *, dim> &derivative_matrix)
    {
      const unsigned int n_rows_1d = n_rows_1d_templated == 0 ?
                                       n_rows_1d_non_templated :
                                       n_rows_1d_templated;
      const unsigned int n         = Utilities::fixed_power<dim>(n_rows_1d);
 
      tmp.resize_fast(n * 2);
      Number *t = tmp.begin();
 
      internal::EvaluatorTensorProduct<internal::evaluate_general,
                                       dim,
                                       n_rows_1d_templated,
                                       n_rows_1d_templated,
                                       Number>
        eval({}, {}, {}, n_rows_1d, n_rows_1d);
 
      if (dim == 1)
        {
          const Number *A = derivative_matrix[0];
          eval.template apply<0, false, false>(A, src, dst);
        }
 
      else if (dim == 2)
        {
          const Number *A0 = derivative_matrix[0];
          const Number *M0 = mass_matrix[0];
          const Number *A1 = derivative_matrix[1];
          const Number *M1 = mass_matrix[1];
          eval.template apply<0, false, false>(M0, src, t);
          eval.template apply<1, false, false>(A1, t, dst);
          eval.template apply<0, false, false>(A0, src, t);
          eval.template apply<1, false, true>(M1, t, dst);
        }
 
      else if (dim == 3)
        {
          const Number *A0 = derivative_matrix[0];
          const Number *M0 = mass_matrix[0];
          const Number *A1 = derivative_matrix[1];
          const Number *M1 = mass_matrix[1];
          const Number *A2 = derivative_matrix[2];
          const Number *M2 = mass_matrix[2];
          eval.template apply<0, false, false>(M0, src, t + n);
          eval.template apply<1, false, false>(M1, t + n, t);
          eval.template apply<2, false, false>(A2, t, dst);
          eval.template apply<1, false, false>(A1, t + n, t);
          eval.template apply<0, false, false>(A0, src, t + n);
          eval.template apply<1, false, true>(M1, t + n, t);
          eval.template apply<2, false, true>(M2, t, dst);
        }
 
      else
        AssertThrow(false, ExcNotImplemented());
    }
 
 
 
    template <int n_rows_1d_templated, std::size_t dim, typename Number>
    void
    apply_inverse(Number            *dst,
                  const Number      *src,
                  const unsigned int n_rows_1d_non_templated,
                  const std::array<const Number *, dim> &eigenvectors,
                  const std::array<const Number *, dim> &eigenvalues,
                  const Number *inverted_eigenvalues = nullptr)
    {
      const unsigned int n_rows_1d = n_rows_1d_templated == 0 ?
                                       n_rows_1d_non_templated :
                                       n_rows_1d_templated;
 
      internal::EvaluatorTensorProduct<internal::evaluate_general,
                                       dim,
                                       n_rows_1d_templated,
                                       n_rows_1d_templated,
                                       Number>
        eval({}, {}, {}, n_rows_1d, n_rows_1d);
 
      // NOTE: dof_to_quad has to be interpreted as 'dof to eigenvalue index'
      //       --> apply<.,true,.> (S,src,dst) calculates dst = S^T * src,
      //       --> apply<.,false,.> (S,src,dst) calculates dst = S * src,
      //       while the eigenvectors are stored column-wise in S, i.e.
      //       rows correspond to dofs whereas columns to eigenvalue indices!
      if (dim == 1)
        {
          const Number *S = eigenvectors[0];
          eval.template apply<0, true, false>(S, src, dst);
 
          for (unsigned int i = 0; i < n_rows_1d; ++i)
            if (inverted_eigenvalues)
              dst[i] *= inverted_eigenvalues[i];
            else
              dst[i] /= eigenvalues[0][i];
 
          eval.template apply<0, false, false>(S, dst, dst);
        }
 
      else if (dim == 2)
        {
          const Number *S0 = eigenvectors[0];
          const Number *S1 = eigenvectors[1];
          eval.template apply<0, true, false>(S0, src, dst);
          eval.template apply<1, true, false>(S1, dst, dst);
 
          for (unsigned int i1 = 0, c = 0; i1 < n_rows_1d; ++i1)
            for (unsigned int i0 = 0; i0 < n_rows_1d; ++i0, ++c)
              if (inverted_eigenvalues)
                dst[c] *= inverted_eigenvalues[c];
              else
                dst[c] /= (eigenvalues[1][i1] + eigenvalues[0][i0]);
 
          eval.template apply<1, false, false>(S1, dst, dst);
          eval.template apply<0, false, false>(S0, dst, dst);
        }
 
      else if (dim == 3)
        {
          const Number *S0 = eigenvectors[0];
          const Number *S1 = eigenvectors[1];
          const Number *S2 = eigenvectors[2];
          eval.template apply<0, true, false>(S0, src, dst);
          eval.template apply<1, true, false>(S1, dst, dst);
          eval.template apply<2, true, false>(S2, dst, dst);
 
          for (unsigned int i2 = 0, c = 0; i2 < n_rows_1d; ++i2)
            for (unsigned int i1 = 0; i1 < n_rows_1d; ++i1)
              for (unsigned int i0 = 0; i0 < n_rows_1d; ++i0, ++c)
                if (inverted_eigenvalues)
                  dst[c] *= inverted_eigenvalues[c];
                else
                  dst[c] /= (eigenvalues[2][i2] + eigenvalues[1][i1] +
                             eigenvalues[0][i0]);
 
          eval.template apply<2, false, false>(S2, dst, dst);
          eval.template apply<1, false, false>(S1, dst, dst);
          eval.template apply<0, false, false>(S0, dst, dst);
        }
 
      else
        DEAL_II_NOT_IMPLEMENTED();
    }
 
 
 
    template <int n_rows_1d_templated, std::size_t dim, typename Number>
    void
    select_vmult(Number                                *dst,
                 const Number                          *src,
                 AlignedVector<Number>                 &tmp,
                 const unsigned int                     n_rows_1d,
                 const std::array<const Number *, dim> &mass_matrix,
                 const std::array<const Number *, dim> &derivative_matrix);
 
 
 
    template <int n_rows_1d_templated, std::size_t dim, typename Number>
    void
    select_apply_inverse(Number                                *dst,
                         const Number                          *src,
                         const unsigned int                     n_rows_1d,
                         const std::array<const Number *, dim> &eigenvectors,
                         const std::array<const Number *, dim> &eigenvalues,
                         const Number *inverted_eigenvalues = nullptr);
  } // namespace TensorProductMatrixSymmetricSum
} // namespace internal
 
 
template <int dim, typename Number, int n_rows_1d>
inline unsigned int
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::m() const
{
  unsigned int m = mass_matrix[0].n_rows();
  for (unsigned int d = 1; d < dim; ++d)
    m *= mass_matrix[d].n_rows();
  return m;
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
inline unsigned int
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::n() const
{
  unsigned int n = mass_matrix[0].n_cols();
  for (unsigned int d = 1; d < dim; ++d)
    n *= mass_matrix[d].n_cols();
  return n;
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
inline void
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::vmult(
  const ArrayView<Number>       &dst_view,
  const ArrayView<const Number> &src_view) const
{
  std::scoped_lock lock(this->mutex);
  this->vmult(dst_view, src_view, this->tmp_array);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
inline void
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::vmult(
  const ArrayView<Number>       &dst_view,
  const ArrayView<const Number> &src_view,
  AlignedVector<Number>         &tmp_array) const
{
  AssertDimension(dst_view.size(), this->m());
  AssertDimension(src_view.size(), this->n());
 
  Number       *dst = dst_view.begin();
  const Number *src = src_view.begin();
 
  std::array<const Number *, dim> mass_matrix, derivative_matrix;
 
  for (unsigned int d = 0; d < dim; ++d)
    {
      mass_matrix[d]       = &this->mass_matrix[d](0, 0);
      derivative_matrix[d] = &this->derivative_matrix[d](0, 0);
    }
 
  const unsigned int n_rows_1d_non_templated = this->mass_matrix[0].n_rows();
 
  if constexpr (n_rows_1d != -1)
    internal::TensorProductMatrixSymmetricSum::vmult<n_rows_1d>(
      dst,
      src,
      tmp_array,
      n_rows_1d_non_templated,
      mass_matrix,
      derivative_matrix);
  else
    internal::TensorProductMatrixSymmetricSum::select_vmult<1>(
      dst,
      src,
      tmp_array,
      n_rows_1d_non_templated,
      mass_matrix,
      derivative_matrix);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
inline void
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::apply_inverse(
  const ArrayView<Number>       &dst_view,
  const ArrayView<const Number> &src_view) const
{
  AssertDimension(dst_view.size(), this->n());
  AssertDimension(src_view.size(), this->m());
 
  Number       *dst = dst_view.begin();
  const Number *src = src_view.begin();
 
  std::array<const Number *, dim> eigenvectors, eigenvalues;
 
  for (unsigned int d = 0; d < dim; ++d)
    {
      eigenvectors[d] = &this->eigenvectors[d](0, 0);
      eigenvalues[d]  = this->eigenvalues[d].data();
    }
 
  const unsigned int n_rows_1d_non_templated = this->mass_matrix[0].n_rows();
 
  if constexpr (n_rows_1d != -1)
    internal::TensorProductMatrixSymmetricSum::apply_inverse<n_rows_1d>(
      dst, src, n_rows_1d_non_templated, eigenvectors, eigenvalues);
  else
    internal::TensorProductMatrixSymmetricSum::select_apply_inverse<1>(
      dst, src, n_rows_1d_non_templated, eigenvectors, eigenvalues);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
std::size_t
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::memory_consumption()
  const
{
  return MemoryConsumption::memory_consumption(mass_matrix) +
         MemoryConsumption::memory_consumption(derivative_matrix) +
         MemoryConsumption::memory_consumption(eigenvalues) +
         MemoryConsumption::memory_consumption(eigenvectors) +
         MemoryConsumption::memory_consumption(tmp_array);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
template <typename T>
inline TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::
  TensorProductMatrixSymmetricSum(const T &mass_matrix,
                                  const T &derivative_matrix)
{
  reinit(mass_matrix, derivative_matrix);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
template <typename T>
inline void
TensorProductMatrixSymmetricSum<dim, Number, n_rows_1d>::reinit(
  const T &mass_matrix,
  const T &derivative_matrix)
{
  this->mass_matrix =
    internal::TensorProductMatrixSymmetricSum::convert<dim>(mass_matrix);
  this->derivative_matrix =
    internal::TensorProductMatrixSymmetricSum::convert<dim>(derivative_matrix);
 
  internal::TensorProductMatrixSymmetricSum::setup(this->mass_matrix,
                                                   this->derivative_matrix,
                                                   this->eigenvectors,
                                                   this->eigenvalues);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::
  AdditionalData::AdditionalData(const bool compress_matrices,
                                 const bool precompute_inverse_diagonal)
  : compress_matrices(compress_matrices)
  , precompute_inverse_diagonal(precompute_inverse_diagonal)
{}
 
 
 
template <int dim, typename Number, int n_rows_1d>
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::
  TensorProductMatrixSymmetricSumCollection(
    const AdditionalData &additional_data)
  : compress_matrices(additional_data.compress_matrices)
  , precompute_inverse_diagonal(additional_data.precompute_inverse_diagonal)
{}
 
 
 
template <int dim, typename Number, int n_rows_1d>
void
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::reserve(
  const unsigned int size)
{
  if (compress_matrices == false)
    mass_and_derivative_matrices.resize(size * dim);
  else
    indices.assign(size * dim, numbers::invalid_unsigned_int);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
template <typename T>
void
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::insert(
  const unsigned int index,
  const T           &Ms_in,
  const T           &Ks_in)
{
  const auto Ms =
    internal::TensorProductMatrixSymmetricSum::convert<dim>(Ms_in);
  const auto Ks =
    internal::TensorProductMatrixSymmetricSum::convert<dim>(Ks_in);
 
  for (unsigned int d = 0; d < dim; ++d)
    {
      if (compress_matrices == false)
        {
          const MatrixPairType matrix(Ms[d], Ks[d]);
          mass_and_derivative_matrices[index * dim + d] = matrix;
        }
      else
        {
          using VectorizedArrayTrait =
            ::internal::VectorizedArrayTrait<Number>;
 
          std::bitset<VectorizedArrayTrait::width()> mask;
 
          for (unsigned int v = 0; v < VectorizedArrayTrait::width(); ++v)
            {
              typename VectorizedArrayTrait::value_type a = 0.0;
 
              for (unsigned int i = 0; i < Ms[d].size(0); ++i)
                for (unsigned int j = 0; j < Ms[d].size(1); ++j)
                  {
                    a += std::abs(VectorizedArrayTrait::get(Ms[d][i][j], v));
                    a += std::abs(VectorizedArrayTrait::get(Ks[d][i][j], v));
                  }
 
              mask[v] = (a != 0.0);
            }
 
          const MatrixPairTypeWithMask matrix{mask, {Ms[d], Ks[d]}};
 
          const auto ptr = cache.find(matrix);
 
          if (ptr != cache.end())
            {
              const auto ptr_index     = ptr->second;
              indices[index * dim + d] = ptr_index;
 
              if ([&]() {
                    for (unsigned int v = 0; v < VectorizedArrayTrait::width();
                         ++v)
                      if ((mask[v] == true) && (ptr->first.first[v] == false))
                        return false;
 
                    return true;
                  }())
                {
                  // nothing to do
                }
              else
                {
                  auto mask_new = ptr->first.first;
                  auto Ms_new   = ptr->first.second.first;
                  auto Ks_new   = ptr->first.second.second;
 
                  for (unsigned int v = 0; v < VectorizedArrayTrait::width();
                       ++v)
                    if (mask_new[v] == false && mask[v] == true)
                      {
                        mask_new[v] = true;
 
                        for (unsigned int i = 0; i < Ms_new.size(0); ++i)
                          for (unsigned int j = 0; j < Ms_new.size(1); ++j)
                            {
                              VectorizedArrayTrait::get(Ms_new[i][j], v) =
                                VectorizedArrayTrait::get(Ms[d][i][j], v);
                              VectorizedArrayTrait::get(Ks_new[i][j], v) =
                                VectorizedArrayTrait::get(Ks[d][i][j], v);
                            }
                      }
 
                  cache.erase(ptr);
 
                  const MatrixPairTypeWithMask entry_new{mask_new,
                                                         {Ms_new, Ks_new}};
 
                  const auto ptr_ = cache.find(entry_new);
                  AssertThrow(ptr_ == cache.end(), ExcNotImplemented());
 
                  cache[entry_new] = ptr_index;
                }
            }
          else
            {
              const auto size          = cache.size();
              indices[index * dim + d] = size;
              cache[matrix]            = size;
            }
        }
    }
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
void
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::finalize()
{
  const auto store = [&](const unsigned int    index,
                         const MatrixPairType &M_and_K) {
    std::array<Table<2, Number>, 1> mass_matrix;
    mass_matrix[0] = M_and_K.first;
 
    std::array<Table<2, Number>, 1> derivative_matrix;
    derivative_matrix[0] = M_and_K.second;
 
    std::array<Table<2, Number>, 1>      eigenvectors;
    std::array<AlignedVector<Number>, 1> eigenvalues;
 
    internal::TensorProductMatrixSymmetricSum::setup(mass_matrix,
                                                     derivative_matrix,
                                                     eigenvectors,
                                                     eigenvalues);
 
    for (unsigned int i = 0, m = matrix_ptr[index], v = vector_ptr[index];
         i < mass_matrix[0].n_rows();
         ++i, ++v)
      {
        for (unsigned int j = 0; j < mass_matrix[0].n_cols(); ++j, ++m)
          {
            this->mass_matrices[m]       = mass_matrix[0][i][j];
            this->derivative_matrices[m] = derivative_matrix[0][i][j];
            this->eigenvectors[m]        = eigenvectors[0][i][j];
          }
 
        this->eigenvalues[v] = eigenvalues[0][i];
      }
  };
 
  if (compress_matrices == false)
    {
      // case 1) no compression requested
 
      AssertDimension(cache.size(), 0);
      AssertDimension(indices.size(), 0);
 
      this->vector_ptr.resize(mass_and_derivative_matrices.size() + 1);
      this->matrix_ptr.resize(mass_and_derivative_matrices.size() + 1);
 
      for (unsigned int i = 0; i < mass_and_derivative_matrices.size(); ++i)
        {
          const auto &M = mass_and_derivative_matrices[i].first;
 
          this->vector_ptr[i + 1] = M.n_rows();
          this->matrix_ptr[i + 1] = M.n_rows() * M.n_cols();
        }
 
      for (unsigned int i = 0; i < mass_and_derivative_matrices.size(); ++i)
        {
          this->vector_ptr[i + 1] += this->vector_ptr[i];
          this->matrix_ptr[i + 1] += this->matrix_ptr[i];
        }
 
      this->mass_matrices.resize_fast(matrix_ptr.back());
      this->derivative_matrices.resize_fast(matrix_ptr.back());
      this->eigenvectors.resize_fast(matrix_ptr.back());
      this->eigenvalues.resize_fast(vector_ptr.back());
 
      for (unsigned int i = 0; i < mass_and_derivative_matrices.size(); ++i)
        store(i, mass_and_derivative_matrices[i]);
 
      mass_and_derivative_matrices.clear();
    }
  else if (cache.size() == indices.size())
    {
      // case 2) compression requested but none possible
 
      this->vector_ptr.resize(cache.size() + 1);
      this->matrix_ptr.resize(cache.size() + 1);
 
      std::map<unsigned int, MatrixPairType> inverted_cache;
 
      for (const auto &i : cache)
        inverted_cache[i.second] = i.first.second;
 
      for (unsigned int i = 0; i < indices.size(); ++i)
        {
          const auto &M = inverted_cache[indices[i]].first;
 
          this->vector_ptr[i + 1] = M.n_rows();
          this->matrix_ptr[i + 1] = M.n_rows() * M.n_cols();
        }
 
      for (unsigned int i = 0; i < cache.size(); ++i)
        {
          this->vector_ptr[i + 1] += this->vector_ptr[i];
          this->matrix_ptr[i + 1] += this->matrix_ptr[i];
        }
 
      this->mass_matrices.resize_fast(matrix_ptr.back());
      this->derivative_matrices.resize_fast(matrix_ptr.back());
      this->eigenvectors.resize_fast(matrix_ptr.back());
      this->eigenvalues.resize_fast(vector_ptr.back());
 
      for (unsigned int i = 0; i < indices.size(); ++i)
        store(i, inverted_cache[indices[i]]);
 
      indices.clear();
      cache.clear();
    }
  else
    {
      // case 3) compress
 
      this->vector_ptr.resize(cache.size() + 1);
      this->matrix_ptr.resize(cache.size() + 1);
 
      for (const auto &i : cache)
        {
          const auto &M = i.first.second.first;
 
          this->vector_ptr[i.second + 1] = M.n_rows();
          this->matrix_ptr[i.second + 1] = M.n_rows() * M.n_cols();
        }
 
      for (unsigned int i = 0; i < cache.size(); ++i)
        {
          this->vector_ptr[i + 1] += this->vector_ptr[i];
          this->matrix_ptr[i + 1] += this->matrix_ptr[i];
        }
 
      this->mass_matrices.resize_fast(matrix_ptr.back());
      this->derivative_matrices.resize_fast(matrix_ptr.back());
      this->eigenvectors.resize_fast(matrix_ptr.back());
      this->eigenvalues.resize_fast(vector_ptr.back());
 
      for (const auto &i : cache)
        store(i.second, i.first.second);
 
      cache.clear();
    }
 
  if (precompute_inverse_diagonal)
    {
      if (dim == 1)
        {
          // 1D case: simply invert 1D eigenvalues
          for (unsigned int i = 0; i < this->eigenvalues.size(); ++i)
            this->eigenvalues[i] = Number(1.0) / this->eigenvalues[i];
          std::swap(this->inverted_eigenvalues, eigenvalues);
        }
      else
        {
          // 2D and 3D case: we have 2 or 3 1d eigenvalues so that we
          // need to combine these
 
          // step 1) if eigenvalues/eigenvectors are compressed, we
          // need to compress the diagonal (the combination of ev
          // indices) as well. This is an optional step.
          std::vector<unsigned int> indices_ev;
 
          if (indices.size() > 0)
            {
              // 1a) create cache (ev indics -> diag index)
              const unsigned int n_cells = indices.size() / dim;
              std::map<std::array<unsigned int, dim>, unsigned int> cache_ev;
              std::vector<unsigned int> cache_ev_idx(n_cells);
 
              for (unsigned int i = 0, c = 0; i < n_cells; ++i)
                {
                  std::array<unsigned int, dim> id;
 
                  for (unsigned int d = 0; d < dim; ++d, ++c)
                    id[d] = indices[c];
 
                  const auto id_ptr = cache_ev.find(id);
 
                  if (id_ptr == cache_ev.end())
                    {
                      const auto size = cache_ev.size();
                      cache_ev_idx[i] = size;
                      cache_ev[id]    = size;
                    }
                  else
                    {
                      cache_ev_idx[i] = id_ptr->second;
                    }
                }
 
              // 1b) store diagonal indices for each cell
              std::vector<unsigned int> new_indices;
              new_indices.reserve(indices.size() / dim * (dim + 1));
 
              for (unsigned int i = 0, c = 0; i < n_cells; ++i)
                {
                  for (unsigned int d = 0; d < dim; ++d, ++c)
                    new_indices.push_back(indices[c]);
                  new_indices.push_back(cache_ev_idx[i]);
                }
 
              // 1c) transpose cache (diag index -> ev indices)
              indices_ev.resize(cache_ev.size() * dim);
              for (const auto &entry : cache_ev)
                for (unsigned int d = 0; d < dim; ++d)
                  indices_ev[entry.second * dim + d] = entry.first[d];
 
              std::swap(this->indices, new_indices);
            }
 
          // step 2) allocate memory and set pointers
          const unsigned int n_diag =
            ((indices_ev.size() > 0) ? indices_ev.size() :
                                       (matrix_ptr.size() - 1)) /
            dim;
 
          std::vector<unsigned int> new_vector_ptr(n_diag + 1, 0);
          std::vector<unsigned int> new_vector_n_rows_1d(n_diag, 0);
 
          for (unsigned int i = 0; i < n_diag; ++i)
            {
              const unsigned int c = (indices_ev.size() > 0) ?
                                       indices_ev[dim * i + 0] :
                                       (dim * i + 0);
 
              const unsigned int n_rows = vector_ptr[c + 1] - vector_ptr[c];
 
              new_vector_n_rows_1d[i] = n_rows;
              new_vector_ptr[i + 1]   = Utilities::pow(n_rows, dim);
            }
 
          for (unsigned int i = 0; i < n_diag; ++i)
            new_vector_ptr[i + 1] += new_vector_ptr[i];
 
          this->inverted_eigenvalues.resize(new_vector_ptr.back());
 
          // step 3) loop over all unique diagonal entries and invert
          for (unsigned int i = 0; i < n_diag; ++i)
            {
              std::array<Number *, dim> evs;
 
              for (unsigned int d = 0; d < dim; ++d)
                evs[d] =
                  &this
                     ->eigenvalues[this->vector_ptr[(indices_ev.size() > 0) ?
                                                      indices_ev[dim * i + d] :
                                                      (dim * i + d)]];
 
              const unsigned int mm = new_vector_n_rows_1d[i];
              if (dim == 2)
                {
                  for (unsigned int i1 = 0, c = 0; i1 < mm; ++i1)
                    for (unsigned int i0 = 0; i0 < mm; ++i0, ++c)
                      this->inverted_eigenvalues[new_vector_ptr[i] + c] =
                        Number(1.0) / (evs[1][i1] + evs[0][i0]);
                }
              else
                {
                  for (unsigned int i2 = 0, c = 0; i2 < mm; ++i2)
                    for (unsigned int i1 = 0; i1 < mm; ++i1)
                      for (unsigned int i0 = 0; i0 < mm; ++i0, ++c)
                        this->inverted_eigenvalues[new_vector_ptr[i] + c] =
                          Number(1.0) / (evs[2][i2] + evs[1][i1] + evs[0][i0]);
                }
            }
 
          // step 4) clean up
          std::swap(this->vector_ptr, new_vector_ptr);
          std::swap(this->vector_n_rows_1d, new_vector_n_rows_1d);
        }
 
      this->eigenvalues.clear();
    }
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
void
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::
  apply_inverse(const unsigned int             index,
                const ArrayView<Number>       &dst_in,
                const ArrayView<const Number> &src_in) const
{
  Number       *dst = dst_in.begin();
  const Number *src = src_in.begin();
 
  if (this->eigenvalues.empty() == false)
    {
      std::array<const Number *, dim> eigenvectors;
      std::array<const Number *, dim> eigenvalues;
      unsigned int                    n_rows_1d_non_templated = 0;
 
      for (unsigned int d = 0; d < dim; ++d)
        {
          const unsigned int translated_index =
            (indices.size() > 0) ? indices[dim * index + d] : (dim * index + d);
 
          eigenvectors[d] =
            this->eigenvectors.data() + matrix_ptr[translated_index];
          eigenvalues[d] =
            this->eigenvalues.data() + vector_ptr[translated_index];
          n_rows_1d_non_templated =
            vector_ptr[translated_index + 1] - vector_ptr[translated_index];
        }
 
      if constexpr (n_rows_1d != -1)
        internal::TensorProductMatrixSymmetricSum::apply_inverse<n_rows_1d>(
          dst, src, n_rows_1d_non_templated, eigenvectors, eigenvalues);
      else
        internal::TensorProductMatrixSymmetricSum::select_apply_inverse<1>(
          dst, src, n_rows_1d_non_templated, eigenvectors, eigenvalues);
    }
  else
    {
      std::array<const Number *, dim> eigenvectors;
      const Number                   *inverted_eigenvalues    = nullptr;
      unsigned int                    n_rows_1d_non_templated = 0;
 
      for (unsigned int d = 0; d < dim; ++d)
        {
          const unsigned int translated_index =
            (indices.size() > 0) ?
              indices[((dim == 1) ? 1 : (dim + 1)) * index + d] :
              (dim * index + d);
 
          eigenvectors[d] =
            this->eigenvectors.data() + matrix_ptr[translated_index];
        }
 
      {
        const unsigned int translated_index =
          ((indices.size() > 0) && (dim != 1)) ?
            indices[(dim + 1) * index + dim] :
            index;
 
        inverted_eigenvalues =
          this->inverted_eigenvalues.data() + vector_ptr[translated_index];
        n_rows_1d_non_templated =
          (dim == 1) ?
            (vector_ptr[translated_index + 1] - vector_ptr[translated_index]) :
            vector_n_rows_1d[translated_index];
      }
 
      if constexpr (n_rows_1d != -1)
        internal::TensorProductMatrixSymmetricSum::apply_inverse<n_rows_1d>(
          dst,
          src,
          n_rows_1d_non_templated,
          eigenvectors,
          {},
          inverted_eigenvalues);
      else
        internal::TensorProductMatrixSymmetricSum::select_apply_inverse<1>(
          dst,
          src,
          n_rows_1d_non_templated,
          eigenvectors,
          {},
          inverted_eigenvalues);
    }
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
std::size_t
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::
  memory_consumption() const
{
  return MemoryConsumption::memory_consumption(indices) +
         MemoryConsumption::memory_consumption(mass_matrices) +
         MemoryConsumption::memory_consumption(derivative_matrices) +
         MemoryConsumption::memory_consumption(eigenvectors) +
         MemoryConsumption::memory_consumption(eigenvalues) +
         MemoryConsumption::memory_consumption(matrix_ptr) +
         MemoryConsumption::memory_consumption(vector_ptr);
}
 
 
 
template <int dim, typename Number, int n_rows_1d>
std::size_t
TensorProductMatrixSymmetricSumCollection<dim, Number, n_rows_1d>::
  storage_size() const
{
  if (matrix_ptr.empty())
    return 0; // if not initialized
 
  return matrix_ptr.size() - 1;
}
 
 
 
#endif
 
DEAL_II_NAMESPACE_CLOSE
 
#endif