Loading [MathJax]/extensions/TeX/newcommand.js
 deal.II version GIT relicensing-3075-gc235bd4825 2025-04-15 08:10:00+00:00
\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}} \newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=} \newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]} \newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages Concepts
vector_operations_internal.h
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2016 - 2023 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15
16#ifndef dealii_vector_operations_internal_h
17#define dealii_vector_operations_internal_h
18
19#include <deal.II/base/config.h>
20
25#include <deal.II/base/types.h>
27
29
30#include <Kokkos_Core.hpp>
31
32#include <cstdio>
33#include <cstring>
34
35#ifdef DEAL_II_WITH_TBB
36# include <tbb/blocked_range.h>
37# include <tbb/partitioner.h>
38#endif
39
40
42
43namespace internal
44{
45 namespace VectorOperations
46 {
48
49 template <typename T>
50 bool
51 is_non_negative(const T &t)
52 {
53 return t >= 0;
54 }
55
56
57 template <typename T>
58 bool
59 is_non_negative(const std::complex<T> &)
60 {
61 Assert(false, ExcMessage("Complex numbers do not have an ordering."));
62
63 return false;
64 }
65
66
67 // call std::copy, except for in
68 // the case where we want to copy
69 // from std::complex to a
70 // non-complex type
71 template <typename T, typename U>
72 void
73 copy(const T *begin, const T *end, U *dest)
74 {
75 std::copy(begin, end, dest);
76 }
77
78 template <typename T, typename U>
79 void
80 copy(const std::complex<T> *begin,
81 const std::complex<T> *end,
82 std::complex<U> *dest)
83 {
84 std::copy(begin, end, dest);
85 }
86
87 template <typename T, typename U>
88 void
89 copy(const std::complex<T> *, const std::complex<T> *, U *)
90 {
91 Assert(false,
92 ExcMessage("Can't convert a vector of complex numbers "
93 "into a vector of reals/doubles"));
94 }
95
96
97
98#ifdef DEAL_II_WITH_TBB
107 template <typename Functor>
109 {
111 const size_type start,
112 const size_type end)
114 , start(start)
115 , end(end)
116 {
117 const size_type vec_size = end - start;
118 // set chunk size for sub-tasks
119 const unsigned int gs =
121 n_chunks =
123 vec_size / gs);
124 chunk_size = vec_size / n_chunks;
125
126 // round to next multiple of 512 (or minimum grain size if that happens
127 // to be smaller). this is advantageous because our accumulation
128 // algorithms favor lengths of a power of 2 due to pairwise summation ->
129 // at most one 'oddly' sized chunk
130 if (chunk_size > 512)
131 chunk_size = ((chunk_size + 511) / 512) * 512;
132 n_chunks = (vec_size + chunk_size - 1) / chunk_size;
133 AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
134 AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
135 }
136
137 void
138 operator()(const tbb::blocked_range<size_type> &range) const
139 {
140 const size_type r_begin = start + range.begin() * chunk_size;
141 const size_type r_end = std::min(start + range.end() * chunk_size, end);
142 functor(r_begin, r_end);
143 }
144
145 Functor &functor;
148 unsigned int n_chunks;
150 };
151#endif
152
153 template <typename Functor>
154 void
156 Functor &functor,
157 const size_type start,
158 const size_type end,
159 const std::shared_ptr<::parallel::internal::TBBPartitioner>
160 &partitioner)
161 {
162#ifdef DEAL_II_WITH_TBB
163 const size_type vec_size = end - start;
164 // only go to the parallel function in case there are at least 4 parallel
165 // items, otherwise the overhead is too large
166 if (vec_size >=
169 {
170 Assert(partitioner.get() != nullptr,
172 "Unexpected initialization of Vector that does "
173 "not set the TBB partitioner to a usable state."));
174 std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
175 partitioner->acquire_one_partitioner();
176
177 TBBForFunctor<Functor> generic_functor(functor, start, end);
178 // We use a minimum grain size of 1 here since the grains at this
179 // stage of dividing the work refer to the number of vector chunks
180 // that are processed by (possibly different) threads in the
181 // parallelized for loop (i.e., they do not refer to individual
182 // vector entries). The number of chunks here is calculated inside
183 // TBBForFunctor. See also GitHub issue #2496 for further discussion
184 // of this strategy.
186 static_cast<size_type>(0),
187 static_cast<size_type>(generic_functor.n_chunks),
188 generic_functor,
189 1,
190 tbb_partitioner);
191 partitioner->release_one_partitioner(tbb_partitioner);
192 }
193 else if (vec_size > 0)
194 functor(start, end);
195#else
196 functor(start, end);
197 (void)partitioner;
198#endif
199 }
200
201
202 // Define the functors necessary to use SIMD with TBB. we also include the
203 // simple copy and set operations
204
205 template <typename Number>
207 {
208 Vector_set(const Number value, Number *const dst)
209 : value(value)
210 , dst(dst)
211 {
212 Assert(dst != nullptr, ExcInternalError());
213 }
214
215 void
216 operator()(const size_type begin, const size_type end) const
217 {
218 Assert(end >= begin, ExcInternalError());
219
220 if (value == Number())
221 std::fill(dst + begin, dst + end, Number());
222 else
223 std::fill(dst + begin, dst + end, value);
224 }
225
226 const Number value;
227 Number *const dst;
228 };
229
230 template <typename Number, typename OtherNumber>
232 {
233 Vector_copy(const OtherNumber *const src, Number *const dst)
234 : src(src)
235 , dst(dst)
236 {
237 Assert(src != nullptr, ExcInternalError());
238 Assert(dst != nullptr, ExcInternalError());
239 }
240
241 void
242 operator()(const size_type begin, const size_type end) const
243 {
244 Assert(end >= begin, ExcInternalError());
245
246 if constexpr (std::is_trivially_copyable<Number>() &&
247 std::is_same_v<Number, OtherNumber>)
248 std::memcpy(dst + begin, src + begin, (end - begin) * sizeof(Number));
249 else
250 {
252 for (size_type i = begin; i < end; ++i)
253 dst[i] = src[i];
254 }
255 }
256
257 const OtherNumber *const src;
258 Number *const dst;
259 };
260
261 template <typename Number>
263 {
264 Vectorization_multiply_factor(Number *const val, const Number factor)
265 : val(val)
266 , stored_factor(factor)
267 {}
268
269 void
270 operator()(const size_type begin, const size_type end) const
271 {
272 // create a local copy of the variable to help the compiler with the
273 // aliasing analysis
274 const Number factor = stored_factor;
275
277 {
279 for (size_type i = begin; i < end; ++i)
280 val[i] *= factor;
281 }
282 else
283 {
284 for (size_type i = begin; i < end; ++i)
285 val[i] *= factor;
286 }
287 }
288
289 Number *const val;
290 const Number stored_factor;
291 };
292
293 template <typename Number>
295 {
297 const Number *const v_val,
298 const Number factor)
299 : val(val)
300 , v_val(v_val)
301 , stored_factor(factor)
302 {}
303
304 void
305 operator()(const size_type begin, const size_type end) const
306 {
307 // create a local copy of the variable to help the compiler with the
308 // aliasing analysis
309 const Number factor = stored_factor;
311 {
313 for (size_type i = begin; i < end; ++i)
314 val[i] += factor * v_val[i];
315 }
316 else
317 {
318 for (size_type i = begin; i < end; ++i)
319 val[i] += factor * v_val[i];
320 }
321 }
322
323 Number *const val;
324 const Number *const v_val;
325 const Number stored_factor;
326 };
327
328 template <typename Number>
330 {
332 const Number *const v_val,
333 const Number a,
334 const Number x)
335 : val(val)
336 , v_val(v_val)
337 , stored_a(a)
338 , stored_x(x)
339 {}
340
341 void
342 operator()(const size_type begin, const size_type end) const
343 {
344 // create a local copy of the variable to help the compiler with the
345 // aliasing analysis
346 const Number x = stored_x, a = stored_a;
347
349 {
351 for (size_type i = begin; i < end; ++i)
352 val[i] = x * val[i] + a * v_val[i];
353 }
354 else
355 {
356 for (size_type i = begin; i < end; ++i)
357 val[i] = x * val[i] + a * v_val[i];
358 }
359 }
360
361 Number *const val;
362 const Number *const v_val;
363 const Number stored_a;
364 const Number stored_x;
365 };
366
367 template <typename Number>
369 {
370 Vectorization_subtract_v(Number *val, const Number *const v_val)
371 : val(val)
372 , v_val(v_val)
373 {}
374
375 void
376 operator()(const size_type begin, const size_type end) const
377 {
379 {
381 for (size_type i = begin; i < end; ++i)
382 val[i] -= v_val[i];
383 }
384 else
385 {
386 for (size_type i = begin; i < end; ++i)
387 val[i] -= v_val[i];
388 }
389 }
390
391 Number *const val;
392 const Number *const v_val;
393 };
394
395 template <typename Number>
397 {
398 Vectorization_add_factor(Number *const val, const Number factor)
399 : val(val)
400 , stored_factor(factor)
401 {}
402
403 void
404 operator()(const size_type begin, const size_type end) const
405 {
406 const Number factor = stored_factor;
407
409 {
411 for (size_type i = begin; i < end; ++i)
412 val[i] += factor;
413 }
414 else
415 {
416 for (size_type i = begin; i < end; ++i)
417 val[i] += factor;
418 }
419 }
420
421 Number *const val;
422 const Number stored_factor;
423 };
424
425 template <typename Number>
427 {
428 Vectorization_add_v(Number *const val, const Number *const v_val)
429 : val(val)
430 , v_val(v_val)
431 {}
432
433 void
434 operator()(const size_type begin, const size_type end) const
435 {
437 {
439 for (size_type i = begin; i < end; ++i)
440 val[i] += v_val[i];
441 }
442 else
443 {
444 for (size_type i = begin; i < end; ++i)
445 val[i] += v_val[i];
446 }
447 }
448
449 Number *const val;
450 const Number *const v_val;
451 };
452
453 template <typename Number>
455 {
457 const Number *const v_val,
458 const Number *const w_val,
459 const Number a,
460 const Number b)
461 : val(val)
462 , v_val(v_val)
463 , w_val(w_val)
464 , stored_a(a)
465 , stored_b(b)
466 {}
467
468 void
469 operator()(const size_type begin, const size_type end) const
470 {
471 const Number a = stored_a, b = stored_b;
472
474 {
476 for (size_type i = begin; i < end; ++i)
477 val[i] = val[i] + a * v_val[i] + b * w_val[i];
478 }
479 else
480 {
481 for (size_type i = begin; i < end; ++i)
482 val[i] = val[i] + a * v_val[i] + b * w_val[i];
483 }
484 }
485
486 Number *const val;
487 const Number *const v_val;
488 const Number *const w_val;
489 const Number stored_a;
490 const Number stored_b;
491 };
492
493 template <typename Number>
495 {
497 const Number *const v_val,
498 const Number x)
499 : val(val)
500 , v_val(v_val)
501 , stored_x(x)
502 {}
503
504 void
505 operator()(const size_type begin, const size_type end) const
506 {
507 const Number x = stored_x;
508
510 {
512 for (size_type i = begin; i < end; ++i)
513 val[i] = x * val[i] + v_val[i];
514 }
515 else
516 {
517 for (size_type i = begin; i < end; ++i)
518 val[i] = x * val[i] + v_val[i];
519 }
520 }
521
522 Number *const val;
523 const Number *const v_val;
524 const Number stored_x;
525 };
526
527 template <typename Number>
529 {
531 const Number *v_val,
532 const Number *w_val,
533 Number x,
534 Number a,
535 Number b)
536 : val(val)
537 , v_val(v_val)
538 , w_val(w_val)
539 , stored_x(x)
540 , stored_a(a)
541 , stored_b(b)
542 {}
543
544 void
545 operator()(const size_type begin, const size_type end) const
546 {
547 const Number x = stored_x, a = stored_a, b = stored_b;
548
550 {
552 for (size_type i = begin; i < end; ++i)
553 val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
554 }
555 else
556 {
557 for (size_type i = begin; i < end; ++i)
558 val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
559 }
560 }
561
562 Number *const val;
563 const Number *const v_val;
564 const Number *const w_val;
565 const Number stored_x;
566 const Number stored_a;
567 const Number stored_b;
568 };
569
570 template <typename Number>
572 {
573 Vectorization_scale(Number *const val, const Number *const v_val)
574 : val(val)
575 , v_val(v_val)
576 {}
577
578 void
579 operator()(const size_type begin, const size_type end) const
580 {
582 {
584 for (size_type i = begin; i < end; ++i)
585 val[i] *= v_val[i];
586 }
587 else
588 {
589 for (size_type i = begin; i < end; ++i)
590 val[i] *= v_val[i];
591 }
592 }
593
594 Number *const val;
595 const Number *const v_val;
596 };
597
598 template <typename Number>
600 {
602 const Number *const u_val,
603 const Number a)
604 : val(val)
605 , u_val(u_val)
606 , stored_a(a)
607 {}
608
609 void
610 operator()(const size_type begin, const size_type end) const
611 {
612 const Number a = stored_a;
613
615 {
617 for (size_type i = begin; i < end; ++i)
618 val[i] = a * u_val[i];
619 }
620 else
621 {
622 for (size_type i = begin; i < end; ++i)
623 val[i] = a * u_val[i];
624 }
625 }
626
627 Number *const val;
628 const Number *const u_val;
629 const Number stored_a;
630 };
631
632 template <typename Number>
634 {
636 const Number *const u_val,
637 const Number *const v_val,
638 const Number a,
639 const Number b)
640 : val(val)
641 , u_val(u_val)
642 , v_val(v_val)
643 , stored_a(a)
644 , stored_b(b)
645 {}
646
647 void
648 operator()(const size_type begin, const size_type end) const
649 {
650 const Number a = stored_a, b = stored_b;
651
653 {
655 for (size_type i = begin; i < end; ++i)
656 val[i] = a * u_val[i] + b * v_val[i];
657 }
658 else
659 {
660 for (size_type i = begin; i < end; ++i)
661 val[i] = a * u_val[i] + b * v_val[i];
662 }
663 }
664
665 Number *const val;
666 const Number *const u_val;
667 const Number *const v_val;
668 const Number stored_a;
669 const Number stored_b;
670 };
671
672 template <typename Number>
674 {
676 const Number *u_val,
677 const Number *v_val,
678 const Number *w_val,
679 const Number a,
680 const Number b,
681 const Number c)
682 : val(val)
683 , u_val(u_val)
684 , v_val(v_val)
685 , w_val(w_val)
686 , stored_a(a)
687 , stored_b(b)
688 , stored_c(c)
689 {}
690
691 void
692 operator()(const size_type begin, const size_type end) const
693 {
694 const Number a = stored_a, b = stored_b, c = stored_c;
695
697 {
699 for (size_type i = begin; i < end; ++i)
700 val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
701 }
702 else
703 {
704 for (size_type i = begin; i < end; ++i)
705 val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
706 }
707 }
708
709 Number *const val;
710 const Number *const u_val;
711 const Number *const v_val;
712 const Number *const w_val;
713 const Number stored_a;
714 const Number stored_b;
715 const Number stored_c;
716 };
717
718 template <typename Number>
720 {
721 Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
722 : val(val)
723 , a_val(a_val)
724 , b_val(b_val)
725 {}
726
727 void
728 operator()(const size_type begin, const size_type end) const
729 {
731 {
733 for (size_type i = begin; i < end; ++i)
734 val[i] = a_val[i] / b_val[i];
735 }
736 else
737 {
738 for (size_type i = begin; i < end; ++i)
739 val[i] = a_val[i] / b_val[i];
740 }
741 }
742
743 Number *const val;
744 const Number *const a_val;
745 const Number *const b_val;
746 };
747
748
749
750 // All sums over all the vector entries (l2-norm, inner product, etc.) are
751 // performed with the same code, using a templated operation defined
752 // here. There are always two versions defined, a standard one that covers
753 // most cases and a vectorized one which is only for equal types and float
754 // and double.
755 template <typename Number, typename Number2>
756 struct Dot
757 {
758 static constexpr bool vectorizes = std::is_same_v<Number, Number2> &&
760
761 Dot(const Number *const X, const Number2 *const Y)
762 : X(X)
763 , Y(Y)
764 {}
765
766 Number
767 operator()(const size_type i) const
768 {
769 return X[i] * Number(numbers::NumberTraits<Number2>::conjugate(Y[i]));
770 }
771
774 {
776 x.load(X + i);
777 y.load(Y + i);
778
779 // the following operation in VectorizedArray does an element-wise
780 // scalar product without taking into account complex values and
781 // the need to take the complex-conjugate of one argument. this
782 // may be a bug, but because all VectorizedArray classes only
783 // work on real scalars, it doesn't really matter very much.
784 // in any case, assert that we really don't get here for
785 // complex-valued objects
786 static_assert(numbers::NumberTraits<Number>::is_complex == false,
787 "This operation is not correctly implemented for "
788 "complex-valued objects.");
789 return x * y;
790 }
791
792 const Number *const X;
793 const Number2 *const Y;
794 };
795
796 template <typename Number, typename RealType>
797 struct Norm2
798 {
799 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
800
801 Norm2(const Number *const X)
802 : X(X)
803 {}
804
805 RealType
806 operator()(const size_type i) const
807 {
809 }
810
813 {
815 x.load(X + i);
816 return x * x;
817 }
818
819 const Number *const X;
820 };
821
822 template <typename Number, typename RealType>
823 struct Norm1
824 {
825 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
826
827 Norm1(const Number *X)
828 : X(X)
829 {}
830
831 RealType
832 operator()(const size_type i) const
833 {
835 }
836
839 {
841 x.load(X + i);
842 return std::abs(x);
843 }
844
845 const Number *X;
846 };
847
848 template <typename Number, typename RealType>
849 struct NormP
850 {
851 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
852
853 NormP(const Number *X, RealType p)
854 : X(X)
855 , p(p)
856 {}
857
858 RealType
859 operator()(const size_type i) const
860 {
862 }
863
866 {
868 x.load(X + i);
869 return std::pow(std::abs(x), p);
870 }
871
872 const Number *X;
873 const RealType p;
874 };
875
876 template <typename Number>
878 {
879 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
880
881 MeanValue(const Number *X)
882 : X(X)
883 {}
884
885 Number
886 operator()(const size_type i) const
887 {
888 return X[i];
889 }
890
893 {
895 x.load(X + i);
896 return x;
897 }
898
899 const Number *X;
900 };
901
902 template <typename Number>
904 {
905 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
906
907 AddAndDot(Number *const X,
908 const Number *const V,
909 const Number *const W,
910 const Number a)
911 : X(X)
912 , V(V)
913 , W(W)
914 , a(a)
915 {}
916
917 Number
918 operator()(const size_type i) const
919 {
920 X[i] += a * V[i];
921 return X[i] * Number(numbers::NumberTraits<Number>::conjugate(W[i]));
922 }
923
926 {
928 x.load(X + i);
929 v.load(V + i);
930 x += a * v;
931 x.store(X + i);
932 // may only load from W after storing in X because the pointers might
933 // point to the same memory
934 w.load(W + i);
935
936 // the following operation in VectorizedArray does an element-wise
937 // scalar product without taking into account complex values and
938 // the need to take the complex-conjugate of one argument. this
939 // may be a bug, but because all VectorizedArray classes only
940 // work on real scalars, it doesn't really matter very much.
941 // in any case, assert that we really don't get here for
942 // complex-valued objects
943 static_assert(numbers::NumberTraits<Number>::is_complex == false,
944 "This operation is not correctly implemented for "
945 "complex-valued objects.");
946 return x * w;
947 }
948
949 Number *const X;
950 const Number *const V;
951 const Number *const W;
952 const Number a;
953 };
954
955
956
957 // this is the main working loop for all vector sums using the templated
958 // operation above. it accumulates the sums using a block-wise summation
959 // algorithm with post-update. this blocked algorithm has been proposed in
960 // a similar form by Castaldo, Whaley and Chronopoulos (SIAM
961 // J. Sci. Comput. 31, 1156-1174, 2008) and we use the smallest possible
962 // block size, 2. Sometimes it is referred to as pairwise summation. The
963 // worst case error made by this algorithm is on the order O(eps *
964 // log2(vec_size)), whereas a naive summation is O(eps * vec_size). Even
965 // though the Kahan summation is even more accurate with an error O(eps)
966 // by carrying along remainders not captured by the main sum, that involves
967 // additional costs which are not worthwhile. See the Wikipedia article on
968 // the Kahan summation algorithm.
969
970 // The algorithm implemented here has the additional benefit that it is
971 // easily parallelized without changing the order of how the elements are
972 // added (floating point addition is not associative). For the same vector
973 // size and minimum_parallel_grainsize, the blocks are always the
974 // same and added pairwise.
975
976 // The depth of recursion is controlled by the 'magic' parameter
977 // vector_accumulation_recursion_threshold: If the length is below
978 // vector_accumulation_recursion_threshold * 32 (32 is the part of code we
979 // unroll), a straight loop instead of recursion will be used. At the
980 // innermost level, eight values are added consecutively in order to better
981 // balance multiplications and additions.
982
983 // Loops are unrolled as follows: the range [first,last) is broken into
984 // @p n_chunks each of size 32 plus the @p remainder.
985 // accumulate_regular() does the work on 32*n_chunks elements employing SIMD
986 // if possible and stores the result of the operation for each chunk in @p outer_results.
987
988 // The code returns the result as the last argument in order to make
989 // spawning tasks simpler and use automatic template deduction.
990
991
998
999 template <typename Operation, typename ResultType>
1000 void
1001 accumulate_recursive(const Operation &op,
1002 const size_type first,
1003 const size_type last,
1004 ResultType &result)
1005 {
1006 if (first == last)
1007 {
1008 result = ResultType();
1009 return;
1010 }
1011
1012 const size_type vec_size = last - first;
1013 if (vec_size <= vector_accumulation_recursion_threshold * 32)
1014 {
1015 // The vector is short enough so we perform the summation. We store
1016 // the number of chunks (each 32 indices) for the given vector
1017 // length; all results are stored in outer_results[0,n_chunks). We
1018 // keep twice the number around to be able to do the pairwise
1019 // summation with a single for loop (see the loop over j below)
1020 ResultType outer_results[vector_accumulation_recursion_threshold * 2];
1021
1022 // Select between the regular version and vectorized version based
1023 // on the number types we are given. To choose the vectorized
1024 // version often enough, we need to have all tasks but the last one
1025 // to be divisible by the vectorization length
1026 size_type n_chunks =
1027 do_accumulate(op,
1028 vec_size,
1029 first,
1030 outer_results,
1031 std::bool_constant<Operation::vectorizes>());
1032
1033 AssertIndexRange(n_chunks,
1035
1036 // now sum the results from the chunks stored in
1037 // outer_results[0,n_chunks) recursively
1038 unsigned int j = 0;
1039 constexpr unsigned int n_lanes = VectorizedArray<ResultType>::size();
1040 for (; j + 2 * n_lanes - 1 < n_chunks;
1041 j += 2 * n_lanes, n_chunks += n_lanes)
1042 {
1044 a.load(outer_results + j);
1045 b.load(outer_results + j + n_lanes);
1046 a += b;
1047 a.store(outer_results + n_chunks);
1048 }
1049
1050 // In the vectorized case, we know the loop bounds and can do things
1051 // more efficiently
1052 if (Operation::vectorizes)
1053 {
1054 AssertDimension(j + n_lanes, n_chunks);
1055 AssertIndexRange(n_chunks,
1057 ResultType *result_ptr = outer_results + j;
1058 if (n_lanes >= 16)
1059 for (unsigned int i = 0; i < 8; ++i)
1060 result_ptr[i] = result_ptr[i] + result_ptr[i + 8];
1061 if (n_lanes >= 8)
1062 for (unsigned int i = 0; i < 4; ++i)
1063 result_ptr[i] = result_ptr[i] + result_ptr[i + 4];
1064 if (n_lanes >= 4)
1065 for (unsigned int i = 0; i < 2; ++i)
1066 result_ptr[i] = result_ptr[i] + result_ptr[i + 2];
1067 result = result_ptr[0] + result_ptr[1];
1068 }
1069 else
1070 {
1071 // Without vectorization, we do not know the exact bounds, so we
1072 // need to continue the variable-length pairwise summation loop
1073 // from above
1074 for (; j + 1 < n_chunks; j += 2, ++n_chunks)
1075 outer_results[n_chunks] =
1076 outer_results[j] + outer_results[j + 1];
1077
1078 AssertIndexRange(n_chunks,
1080 Assert(n_chunks > 0, ExcInternalError());
1081 result = outer_results[n_chunks - 1];
1082 }
1083 }
1084 else
1085 {
1086 // split vector into four pieces and work on the pieces
1087 // recursively. Make pieces (except last) divisible by one fourth the
1088 // recursion threshold.
1089 const size_type new_size =
1090 (vec_size / (vector_accumulation_recursion_threshold * 32)) *
1092 Assert(first + 3 * new_size < last, ExcInternalError());
1093 ResultType r0, r1, r2, r3;
1094 accumulate_recursive(op, first, first + new_size, r0);
1095 accumulate_recursive(op, first + new_size, first + 2 * new_size, r1);
1097 first + 2 * new_size,
1098 first + 3 * new_size,
1099 r2);
1100 accumulate_recursive(op, first + 3 * new_size, last, r3);
1101 result = (r0 + r1) + (r2 + r3);
1102 }
1103 }
1104
1105
1106 // this is the inner working routine for the accumulation loops below. We
1107 // pulled this part out of the regular accumulate routine because we might
1108 // do this thing vectorized (see specialized function below; this is the
1109 // un-vectorized version). As opposed to the vector add functions above,
1110 // we here pass the functor 'op' by value, because we cannot create a copy
1111 // of the scalar inline, and instead make sure that the numbers get local
1112 // (and thus definitely not aliased) for the compiler
1113 template <typename Operation, typename ResultType>
1114 size_type
1115 do_accumulate(const Operation op,
1116 const size_type vec_size,
1117 const size_type start_index,
1118 ResultType *outer_results,
1119 std::bool_constant<false>)
1120 {
1121 // Create local copy to indicate no aliasing to the compiler
1122 size_type index = start_index;
1123
1124 // choose each chunk to have a width of 32, thereby the index
1125 // is incremented by 4*8 for each @p i.
1126 size_type n_chunks = vec_size / 32;
1127 for (size_type i = 0; i < n_chunks; ++i)
1128 {
1129 ResultType r = {};
1130 for (unsigned int k = 0; k < 2; ++k)
1131 {
1132 ResultType r0 = op(index);
1133 ResultType r1 = op(index + 1);
1134 ResultType r2 = op(index + 2);
1135 ResultType r3 = op(index + 3);
1136 index += 4;
1137 for (size_type j = 1; j < 4; ++j, index += 4)
1138 {
1139 r0 += op(index);
1140 r1 += op(index + 1);
1141 r2 += op(index + 2);
1142 r3 += op(index + 3);
1143 }
1144 r += (r0 + r1) + (r2 + r3);
1145 }
1146 outer_results[i] = r;
1147 }
1148
1149 if (n_chunks * 32 < vec_size)
1150 {
1151 const size_type remainder = vec_size - n_chunks * 32;
1152 const size_type inner_chunks = remainder / 8;
1153 const size_type remainder_inner = remainder % 8;
1154 ResultType r0 = ResultType(), r1 = ResultType(), r2 = ResultType();
1155 switch (inner_chunks)
1156 {
1157 case 3:
1158 r2 = op(index++);
1159 for (size_type j = 1; j < 8; ++j)
1160 r2 += op(index++);
1162 case 2:
1163 r1 = op(index++);
1164 for (size_type j = 1; j < 8; ++j)
1165 r1 += op(index++);
1166 r1 += r2;
1168 case 1:
1169 r2 = op(index++);
1170 for (size_type j = 1; j < 8; ++j)
1171 r2 += op(index++);
1173 default:
1174 for (size_type j = 0; j < remainder_inner; ++j)
1175 r0 += op(index++);
1176 outer_results[n_chunks++] = (r0 + r2) + r1;
1177 break;
1178 }
1179 }
1180
1181 // make sure we worked through all indices
1182 AssertDimension(index, start_index + vec_size);
1183
1184 return n_chunks;
1185 }
1186
1187
1188
1189 // this is the inner working routine for the accumulation loops
1190 // below. This is the specialized case where we can vectorize. We request
1191 // the 'do_vectorized' routine of the operation instead of the regular one
1192 // which does several operations at once. As above, pass in the functor by
1193 // value to create a local copy of the scalar factors in the function (if
1194 // there are any).
1195 template <typename Operation, typename Number>
1196 size_type
1197 do_accumulate(const Operation op,
1198 const size_type vec_size,
1199 const size_type start_index,
1200 Number *outer_results,
1201 std::bool_constant<true>)
1202 {
1203 // Create local copy to indicate no aliasing to the compiler
1204 size_type index = start_index;
1205
1206 // we start from @p index and workout @p n_chunks each of size 32.
1207 // in order employ SIMD and work on @p nvecs at a time, we split this
1208 // loop yet again:
1209 // First we work on (n_chunks/nvecs) chunks, where each chunk processes
1210 // nvecs*(4*8) elements.
1211
1212 constexpr size_type n_lanes = VectorizedArray<Number>::size();
1213 const size_type regular_chunks = vec_size / (32 * n_lanes);
1214 for (size_type i = 0; i < regular_chunks; ++i)
1215 {
1217 for (unsigned int k = 0; k < 2; ++k)
1218 {
1219 VectorizedArray<Number> r0 = op.do_vectorized(index);
1220 VectorizedArray<Number> r1 = op.do_vectorized(index + n_lanes);
1222 op.do_vectorized(index + 2 * n_lanes);
1224 op.do_vectorized(index + 3 * n_lanes);
1225 index += n_lanes * 4;
1226 for (size_type j = 1; j < 4; ++j, index += n_lanes * 4)
1227 {
1228 r0 += op.do_vectorized(index);
1229 r1 += op.do_vectorized(index + n_lanes);
1230 r2 += op.do_vectorized(index + 2 * n_lanes);
1231 r3 += op.do_vectorized(index + 3 * n_lanes);
1232 }
1233 r += (r0 + r1) + (r2 + r3);
1234 }
1235 r.store(&outer_results[i * n_lanes]);
1236 }
1237
1238 // If we are treating a case where the vector length is not divisible by
1239 // the vectorization length, need a cleanup loop
1240 // The remaining chunks are processed one by one starting from
1241 // regular_chunks * n_lanes; We do as much as possible with 2 SIMD
1242 // operations within each chunk. Here we assume that n_lanes < 32/2 = 16
1243 // as well as 16 % n_lanes == 0.
1244 static_assert(n_lanes <= 16 && 16 % n_lanes == 0,
1245 "VectorizedArray::size() must be 1, 2, 4, 8, or 16");
1246 size_type n_chunks = regular_chunks * n_lanes;
1247 const size_type start_irregular = regular_chunks * n_lanes * 32;
1248 if (start_irregular < vec_size)
1249 {
1252 const size_type remainder = vec_size - start_irregular;
1253 const size_type loop_length = remainder / (2 * n_lanes);
1254 for (size_type j = 0; j < loop_length; ++j, index += 2 * n_lanes)
1255 {
1256 r0 += op.do_vectorized(index);
1257 r1 += op.do_vectorized(index + n_lanes);
1258 }
1259 Number scalar_part = Number();
1260 size_type last = remainder % (2 * n_lanes);
1261 if (last > 0)
1262 {
1263 if (last >= n_lanes)
1264 {
1265 r0 += op.do_vectorized(index);
1266 index += n_lanes;
1267 last -= n_lanes;
1268 }
1269 for (unsigned int i = 0; i < last; ++i)
1270 scalar_part += op(index++);
1271 }
1272
1273 r0 += r1;
1274 r0.store(&outer_results[n_chunks]);
1275 outer_results[n_chunks] += scalar_part;
1276
1277 // update n_chunks to denote range of entries to sum up in
1278 // outer_results[].
1279 n_chunks += n_lanes;
1280 }
1281
1282 // make sure we worked through all indices
1283 AssertDimension(index, start_index + vec_size);
1284
1285 return n_chunks;
1286 }
1287
1288
1289
1290#ifdef DEAL_II_WITH_TBB
1319 template <typename Operation, typename ResultType>
1321 {
1322 static const unsigned int threshold_array_allocate = 512;
1323
1324 TBBReduceFunctor(const Operation &op,
1325 const size_type start,
1326 const size_type end)
1327 : op(op)
1328 , start(start)
1329 , end(end)
1330 {
1331 const size_type vec_size = end - start;
1332 // set chunk size for sub-tasks
1333 const unsigned int gs =
1335 n_chunks =
1336 std::min(static_cast<size_type>(4 * MultithreadInfo::n_threads()),
1337 vec_size / gs);
1338 chunk_size = vec_size / n_chunks;
1339
1340 // round to next multiple of 512 (or leave it at the minimum grain size
1341 // if that happens to be smaller). this is advantageous because our
1342 // algorithm favors lengths of a power of 2 due to pairwise summation ->
1343 // at most one 'oddly' sized chunk
1344 if (chunk_size > 512)
1345 chunk_size = ((chunk_size + 511) / 512) * 512;
1346 n_chunks = (vec_size + chunk_size - 1) / chunk_size;
1347 AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
1348 AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
1349
1351 {
1352 // make sure we allocate an even number of elements,
1353 // access to the new last element is needed in do_sum()
1354 large_array.resize(2 * ((n_chunks + 1) / 2));
1355 array_ptr = large_array.data();
1356 }
1357 else
1358 array_ptr = &small_array[0];
1359 }
1360
1365 void
1366 operator()(const tbb::blocked_range<size_type> &range) const
1367 {
1368 for (size_type i = range.begin(); i < range.end(); ++i)
1370 start + i * chunk_size,
1371 std::min(start + (i + 1) * chunk_size, end),
1372 array_ptr[i]);
1373 }
1374
1375 ResultType
1376 do_sum() const
1377 {
1378 while (n_chunks > 1)
1379 {
1380 if (n_chunks % 2 == 1)
1381 array_ptr[n_chunks++] = ResultType();
1382 for (size_type i = 0; i < n_chunks; i += 2)
1383 array_ptr[i / 2] = array_ptr[i] + array_ptr[i + 1];
1384 n_chunks /= 2;
1385 }
1386 return array_ptr[0];
1387 }
1388
1389 const Operation &op;
1392
1393 mutable unsigned int n_chunks;
1394 unsigned int chunk_size;
1396 std::vector<ResultType> large_array;
1397 // this variable either points to small_array or large_array depending on
1398 // the number of threads we want to feed
1399 mutable ResultType *array_ptr;
1400 };
1401#endif
1402
1403
1404
1409 template <typename Operation, typename ResultType>
1410#ifndef DEBUG
1412#endif
1413 inline void
1415 const Operation &op,
1416 const size_type start,
1417 const size_type end,
1418 ResultType &result,
1419 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1420 &partitioner)
1421 {
1422#ifdef DEAL_II_WITH_TBB
1423 const size_type vec_size = end - start;
1424 // only go to the parallel function in case there are at least 4 parallel
1425 // items, otherwise the overhead is too large
1426 if (vec_size >=
1429 {
1430 Assert(partitioner.get() != nullptr,
1432 "Unexpected initialization of Vector that does "
1433 "not set the TBB partitioner to a usable state."));
1434 std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
1435 partitioner->acquire_one_partitioner();
1436
1437 TBBReduceFunctor<Operation, ResultType> generic_functor(op,
1438 start,
1439 end);
1440 // We use a minimum grain size of 1 here since the grains at this
1441 // stage of dividing the work refer to the number of vector chunks
1442 // that are processed by (possibly different) threads in the
1443 // parallelized for loop (i.e., they do not refer to individual
1444 // vector entries). The number of chunks here is calculated inside
1445 // TBBForFunctor. See also GitHub issue #2496 for further discussion
1446 // of this strategy.
1448 static_cast<size_type>(0),
1449 static_cast<size_type>(generic_functor.n_chunks),
1450 generic_functor,
1451 1,
1452 tbb_partitioner);
1453 partitioner->release_one_partitioner(tbb_partitioner);
1454 result = generic_functor.do_sum();
1455 }
1456 else
1457 accumulate_recursive(op, start, end, result);
1458#else
1459 accumulate_recursive(op, start, end, result);
1460 (void)partitioner;
1461#endif
1462 }
1463
1464
1465 template <typename Number, typename Number2, typename MemorySpace>
1467 {
1468 static void
1470 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1471 /*thread_loop_partitioner*/,
1472 const size_type /*size*/,
1473 const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1474 & /*v_data*/,
1476 {
1477 static_assert(
1478 std::is_same_v<MemorySpace, ::MemorySpace::Default> &&
1479 std::is_same_v<Number, Number2>,
1480 "For the Default MemorySpace Number and Number2 should be the same type");
1481 }
1482
1483 static void
1485 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1486 /*thread_loop_partitioner*/,
1487 const size_type /*size*/,
1488 const Number /*s*/,
1490 {}
1491
1492 static void
1494 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1495 /*thread_loop_partitioner*/,
1496 const size_type /*size*/,
1497 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1498 & /*v_data*/,
1500 {}
1501
1502 static void
1504 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1505 /*thread_loop_partitioner*/,
1506 const size_type /*size*/,
1507 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1508 & /*v_data*/,
1510 {}
1511
1512 static void
1514 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1515 /*thread_loop_partitioner*/,
1516 const size_type /*size*/,
1517 Number /*a*/,
1519 {}
1520
1521 static void
1523 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1524 /*thread_loop_partitioner*/,
1525 const size_type /*size*/,
1526 const Number /*a*/,
1527 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1528 & /*v_data*/,
1530 {}
1531
1532 static void
1534 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1535 /*thread_loop_partitioner*/,
1536 const size_type /*size*/,
1537 const Number /*a*/,
1538 const Number /*b*/,
1539 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1540 & /*v_data*/,
1541 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1542 & /*w_data*/,
1544 {}
1545
1546 static void
1548 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1549 /*thread_loop_partitioner*/,
1550 const size_type /*size*/,
1551 const Number /*x*/,
1552 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1553 & /*v_data*/,
1555 {}
1556
1557 static void
1559 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1560 /*thread_loop_partitioner*/,
1561 const size_type /*size*/,
1562 const Number /*x*/,
1563 const Number /*a*/,
1564 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1565 & /*v_data*/,
1567 {}
1568
1569 static void
1571 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1572 /*thread_loop_partitioner*/,
1573 const size_type /*size*/,
1574 const Number /*x*/,
1575 const Number /*a*/,
1576 const Number /*b*/,
1577 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1578 & /*v_data*/,
1579 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1580 & /*w_data*/,
1582 {}
1583
1584 static void
1586 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1587 /*thread_loop_partitioner*/,
1588 const size_type /*size*/,
1589 const Number /*factor*/,
1591 {}
1592
1593 static void
1595 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1596 /*thread_loop_partitioner*/,
1597 const size_type /*size*/,
1598 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1599 & /*v_data*/,
1601 {}
1602
1603 static void
1605 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1606 /*thread_loop_partitioner*/,
1607 const size_type /*size*/,
1608 const Number /*a*/,
1609 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1610 & /*v_data*/,
1612 {}
1613
1614 static void
1616 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1617 /*thread_loop_partitioner*/,
1618 const size_type /*size*/,
1619 const Number /*a*/,
1620 const Number /*b*/,
1621 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1622 & /*v_data*/,
1623 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1624 & /*w_data*/,
1626 {}
1627
1628 static Number
1630 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1631 /*thread_loop_partitioner*/,
1632 const size_type /*size*/,
1633 const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1634 & /*v_data*/,
1636 {
1637 return Number();
1638 }
1639
1640 template <typename real_type>
1641 static void
1643 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1644 /*thread_loop_partitioner*/,
1645 const size_type /*size*/,
1646 real_type & /*sum*/,
1647 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1648 & /*v_data*/,
1650 {}
1651
1652 static Number
1654 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1655 /*thread_loop_partitioner*/,
1656 const size_type /*size*/,
1657 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1658 & /*data*/)
1659 {
1660 return Number();
1661 }
1662
1663 template <typename real_type>
1664 static void
1666 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1667 /*thread_loop_partitioner*/,
1668 const size_type /*size*/,
1669 real_type & /*sum*/,
1670 Number * /*values*/,
1671 Number * /*values*/)
1672 {}
1673
1674 template <typename real_type>
1675 static void
1677 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1678 /*thread_loop_partitioner*/,
1679 const size_type /*size*/,
1680 real_type & /*sum*/,
1681 real_type /*p*/,
1683 {}
1684
1685 static Number
1687 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1688 /*thread_loop_partitioner*/,
1689 const size_type /*size*/,
1690 const Number /*a*/,
1691 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1692 & /*v_data*/,
1693 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1694 & /*w_data*/,
1696 {
1697 return Number();
1698 }
1699
1700 template <typename MemorySpace2>
1701 static void
1703 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1704 /*thread_loop_partitioner*/,
1705 const size_type /*size*/,
1706 VectorOperation::values /*operation*/,
1707 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
1708 & /*v_data*/,
1710 {}
1711 };
1712
1713
1714
1715 template <typename Number, typename Number2>
1716 struct functions<Number, Number2, ::MemorySpace::Host>
1717 {
1718 static void
1719 copy(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1720 &thread_loop_partitioner,
1721 const size_type size,
1722 const ::MemorySpace::
1723 MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1724 ::MemorySpace::MemorySpaceData<Number,
1725 ::MemorySpace::Host>
1726 &data)
1727 {
1728 Vector_copy<Number, Number2> copier(v_data.values.data(),
1729 data.values.data());
1730 parallel_for(copier, 0, size, thread_loop_partitioner);
1731 }
1732
1733 static void
1734 set(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1735 &thread_loop_partitioner,
1736 const size_type size,
1737 const Number s,
1740 &data)
1741 {
1742 Vector_set<Number> setter(s, data.values.data());
1743 parallel_for(setter, 0, size, thread_loop_partitioner);
1744 }
1745
1746 static void
1748 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1749 &thread_loop_partitioner,
1750 const size_type size,
1751 const ::MemorySpace::
1752 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1753 ::MemorySpace::MemorySpaceData<Number,
1754 ::MemorySpace::Host>
1755 &data)
1756 {
1757 Vectorization_add_v<Number> vector_add(data.values.data(),
1758 v_data.values.data());
1759 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1760 }
1761
1762 static void
1764 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1765 &thread_loop_partitioner,
1766 const size_type size,
1767 const ::MemorySpace::
1768 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1769 ::MemorySpace::MemorySpaceData<Number,
1770 ::MemorySpace::Host>
1771 &data)
1772 {
1773 Vectorization_subtract_v<Number> vector_subtract(data.values.data(),
1774 v_data.values.data());
1775 parallel_for(vector_subtract, 0, size, thread_loop_partitioner);
1776 }
1777
1778 static void
1780 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1781 &thread_loop_partitioner,
1782 const size_type size,
1783 Number a,
1786 &data)
1787 {
1788 Vectorization_add_factor<Number> vector_add(data.values.data(), a);
1789 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1790 }
1791
1792 static void
1793 add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1794 &thread_loop_partitioner,
1795 const size_type size,
1796 const Number a,
1797 const ::MemorySpace::
1798 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1799 ::MemorySpace::MemorySpaceData<Number,
1800 ::MemorySpace::Host>
1801 &data)
1802 {
1803 Vectorization_add_av<Number> vector_add(data.values.data(),
1804 v_data.values.data(),
1805 a);
1806 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1807 }
1808
1809 static void
1811 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1812 &thread_loop_partitioner,
1813 const size_type size,
1814 const Number a,
1815 const Number b,
1816 const ::MemorySpace::
1817 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1818 const ::MemorySpace::
1819 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1820 ::MemorySpace::MemorySpaceData<Number,
1821 ::MemorySpace::Host>
1822 &data)
1823 {
1825 data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1826 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1827 }
1828
1829 static void
1831 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1832 &thread_loop_partitioner,
1833 const size_type size,
1834 const Number x,
1835 const ::MemorySpace::
1836 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1837 ::MemorySpace::MemorySpaceData<Number,
1838 ::MemorySpace::Host>
1839 &data)
1840 {
1841 Vectorization_sadd_xv<Number> vector_sadd(data.values.data(),
1842 v_data.values.data(),
1843 x);
1844 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1845 }
1846
1847 static void
1849 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1850 &thread_loop_partitioner,
1851 const size_type size,
1852 const Number x,
1853 const Number a,
1854 const ::MemorySpace::
1855 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1856 ::MemorySpace::MemorySpaceData<Number,
1857 ::MemorySpace::Host>
1858 &data)
1859 {
1860 Vectorization_sadd_xav<Number> vector_sadd(data.values.data(),
1861 v_data.values.data(),
1862 a,
1863 x);
1864 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1865 }
1866
1867 static void
1869 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1870 &thread_loop_partitioner,
1871 const size_type size,
1872 const Number x,
1873 const Number a,
1874 const Number b,
1875 const ::MemorySpace::
1876 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1877 const ::MemorySpace::
1878 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1879 ::MemorySpace::MemorySpaceData<Number,
1880 ::MemorySpace::Host>
1881 &data)
1882 {
1883 Vectorization_sadd_xavbw<Number> vector_sadd(data.values.data(),
1884 v_data.values.data(),
1885 w_data.values.data(),
1886 x,
1887 a,
1888 b);
1889 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1890 }
1891
1892 static void
1894 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1895 &thread_loop_partitioner,
1896 const size_type size,
1897 const Number factor,
1900 &data)
1901 {
1903 data.values.data(), factor);
1904 parallel_for(vector_multiply, 0, size, thread_loop_partitioner);
1905 }
1906
1907 static void
1908 scale(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1909 &thread_loop_partitioner,
1910 const size_type size,
1911 const ::MemorySpace::
1912 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1913 ::MemorySpace::MemorySpaceData<Number,
1914 ::MemorySpace::Host>
1915 &data)
1916 {
1917 Vectorization_scale<Number> vector_scale(data.values.data(),
1918 v_data.values.data());
1919 parallel_for(vector_scale, 0, size, thread_loop_partitioner);
1920 }
1921
1922 static void
1923 equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1924 &thread_loop_partitioner,
1925 const size_type size,
1926 const Number a,
1927 const ::MemorySpace::
1928 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1929 ::MemorySpace::MemorySpaceData<Number,
1930 ::MemorySpace::Host>
1931 &data)
1932 {
1933 Vectorization_equ_au<Number> vector_equ(data.values.data(),
1934 v_data.values.data(),
1935 a);
1936 parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1937 }
1938
1939 static void
1941 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1942 &thread_loop_partitioner,
1943 const size_type size,
1944 const Number a,
1945 const Number b,
1946 const ::MemorySpace::
1947 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1948 const ::MemorySpace::
1949 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1950 ::MemorySpace::MemorySpaceData<Number,
1951 ::MemorySpace::Host>
1952 &data)
1953 {
1955 data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1956 parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1957 }
1958
1959 static Number
1960 dot(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1961 &thread_loop_partitioner,
1962 const size_type size,
1963 const ::MemorySpace::
1964 MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1965 ::MemorySpace::MemorySpaceData<Number,
1966 ::MemorySpace::Host>
1967 &data)
1968 {
1969 Number sum;
1971 data.values.data(), v_data.values.data());
1973 dot, 0, size, sum, thread_loop_partitioner);
1974 AssertIsFinite(sum);
1975
1976 return sum;
1977 }
1978
1979 template <typename real_type>
1980 static void
1981 norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1982 &thread_loop_partitioner,
1983 const size_type size,
1984 real_type &sum,
1987 &data)
1988 {
1989 Norm2<Number, real_type> norm2(data.values.data());
1990 parallel_reduce(norm2, 0, size, sum, thread_loop_partitioner);
1991 }
1992
1993 static Number
1995 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1996 &thread_loop_partitioner,
1997 const size_type size,
1998 const ::MemorySpace::
1999 MemorySpaceData<Number, ::MemorySpace::Host> &data)
2000 {
2001 Number sum;
2002 MeanValue<Number> mean(data.values.data());
2003 parallel_reduce(mean, 0, size, sum, thread_loop_partitioner);
2004
2005 return sum;
2006 }
2007
2008 template <typename real_type>
2009 static void
2010 norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2011 &thread_loop_partitioner,
2012 const size_type size,
2013 real_type &sum,
2016 &data,
2017 const size_type optional_offset = 0)
2018 {
2019 Norm1<Number, real_type> norm1(data.values.data());
2020 parallel_reduce(norm1,
2021 optional_offset,
2022 optional_offset + size,
2023 sum,
2024 thread_loop_partitioner);
2025 }
2026
2027 template <typename real_type>
2028 static void
2029 norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2030 &thread_loop_partitioner,
2031 const size_type size,
2032 real_type &sum,
2033 const real_type p,
2036 &data)
2037 {
2038 NormP<Number, real_type> normp(data.values.data(), p);
2039 parallel_reduce(normp, 0, size, sum, thread_loop_partitioner);
2040 }
2041
2042 static Number
2044 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2045 &thread_loop_partitioner,
2046 const size_type size,
2047 const Number a,
2048 const ::MemorySpace::
2049 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
2050 const ::MemorySpace::
2051 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
2052 ::MemorySpace::MemorySpaceData<Number,
2053 ::MemorySpace::Host>
2054 &data)
2055 {
2056 Number sum;
2057 AddAndDot<Number> adder(data.values.data(),
2058 v_data.values.data(),
2059 w_data.values.data(),
2060 a);
2061 parallel_reduce(adder, 0, size, sum, thread_loop_partitioner);
2062
2063 return sum;
2064 }
2065
2066 template <typename MemorySpace2>
2067 static void
2069 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2070 &thread_loop_partitioner,
2071 const size_type size,
2072 VectorOperation::values operation,
2073 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2074 &v_data,
2077 &data,
2078 std::enable_if_t<
2079 std::is_same_v<MemorySpace2, ::MemorySpace::Host>,
2080 int> = 0)
2081 {
2082 if (operation == VectorOperation::insert)
2083 {
2084 copy(thread_loop_partitioner, size, v_data, data);
2085 }
2086 else if (operation == VectorOperation::add)
2087 {
2088 add_vector(thread_loop_partitioner, size, v_data, data);
2089 }
2090 else
2091 {
2093 }
2094 }
2095
2096 template <typename MemorySpace2>
2097 static void
2099 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2100 & /*thread_loop_partitioner*/,
2101 const size_type size,
2102 VectorOperation::values operation,
2103 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2104 &v_data,
2107 &data,
2108 std::enable_if_t<
2109 std::is_same_v<MemorySpace2, ::MemorySpace::Default>,
2110 int> = 0)
2111 {
2112 if (operation == VectorOperation::insert)
2113 {
2114 Kokkos::deep_copy(
2115 Kokkos::subview(data.values,
2116 Kokkos::pair<size_type, size_type>(0, size)),
2117 Kokkos::subview(v_data.values,
2118 Kokkos::pair<size_type, size_type>(0, size)));
2119 }
2120 else
2121 {
2123 }
2124 }
2125 };
2126
2127
2128
2129 template <typename Number>
2130 struct functions<Number, Number, ::MemorySpace::Default>
2131 {
2132 static void
2134 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2135 const size_type size,
2136 const ::MemorySpace::
2137 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2138 ::MemorySpace::MemorySpaceData<Number,
2139 ::MemorySpace::Default>
2140 &data)
2141 {
2142 Kokkos::deep_copy(
2143 Kokkos::subview(data.values,
2144 Kokkos::pair<size_type, size_type>(0, size)),
2145 Kokkos::subview(v_data.values,
2146 Kokkos::pair<size_type, size_type>(0, size)));
2147 }
2148
2149 static void
2150 set(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2151 const size_type size,
2152 const Number s,
2155 &data)
2156 {
2157 Kokkos::deep_copy(
2158 Kokkos::subview(data.values,
2159 Kokkos::pair<size_type, size_type>(0, size)),
2160 s);
2161 }
2162
2163 static void
2165 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2166 const size_type size,
2167 const ::MemorySpace::
2168 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2169 ::MemorySpace::MemorySpaceData<Number,
2170 ::MemorySpace::Default>
2171 &data)
2172 {
2173 auto exec = typename ::MemorySpace::Default::kokkos_space::
2174 execution_space{};
2175 Kokkos::parallel_for(
2176 "::add_vector",
2177 Kokkos::RangePolicy<
2178 ::MemorySpace::Default::kokkos_space::execution_space>(
2179 exec, 0, size),
2180 KOKKOS_LAMBDA(int i) { data.values(i) += v_data.values(i); });
2181 exec.fence();
2182 }
2183
2184 static void
2186 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2187 const size_type size,
2188 const ::MemorySpace::
2189 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2190 ::MemorySpace::MemorySpaceData<Number,
2191 ::MemorySpace::Default>
2192 &data)
2193 {
2194 auto exec = typename ::MemorySpace::Default::kokkos_space::
2195 execution_space{};
2196 Kokkos::parallel_for(
2197 "::subtract_vector",
2198 Kokkos::RangePolicy<
2199 ::MemorySpace::Default::kokkos_space::execution_space>(
2200 exec, 0, size),
2201 KOKKOS_LAMBDA(size_type i) { data.values(i) -= v_data.values(i); });
2202 exec.fence();
2203 }
2204
2205 static void
2207 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2208 const size_type size,
2209 Number a,
2212 &data)
2213 {
2214 auto exec = typename ::MemorySpace::Default::kokkos_space::
2215 execution_space{};
2216 Kokkos::parallel_for(
2217 "::add_factor",
2218 Kokkos::RangePolicy<
2219 ::MemorySpace::Default::kokkos_space::execution_space>(
2220 exec, 0, size),
2221 KOKKOS_LAMBDA(size_type i) { data.values(i) += a; });
2222 exec.fence();
2223 }
2224
2225 static void
2227 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2228 const size_type size,
2229 const Number a,
2230 const ::MemorySpace::
2231 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2232 ::MemorySpace::MemorySpaceData<Number,
2233 ::MemorySpace::Default>
2234 &data)
2235 {
2236 auto exec = typename ::MemorySpace::Default::kokkos_space::
2237 execution_space{};
2238 Kokkos::parallel_for(
2239 "::add_av",
2240 Kokkos::RangePolicy<
2241 ::MemorySpace::Default::kokkos_space::execution_space>(
2242 exec, 0, size),
2243 KOKKOS_LAMBDA(size_type i) {
2244 data.values(i) += a * v_data.values(i);
2245 });
2246 exec.fence();
2247 }
2248
2249 static void
2251 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2252 const size_type size,
2253 const Number a,
2254 const Number b,
2255 const ::MemorySpace::
2256 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2257 const ::MemorySpace::
2258 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2259 ::MemorySpace::MemorySpaceData<Number,
2260 ::MemorySpace::Default>
2261 &data)
2262 {
2263 auto exec = typename ::MemorySpace::Default::kokkos_space::
2264 execution_space{};
2265 Kokkos::parallel_for(
2266 "::add_avpbw",
2267 Kokkos::RangePolicy<
2268 ::MemorySpace::Default::kokkos_space::execution_space>(
2269 exec, 0, size),
2270 KOKKOS_LAMBDA(size_type i) {
2271 data.values(i) += a * v_data.values(i) + b * w_data.values(i);
2272 });
2273 exec.fence();
2274 }
2275
2276 static void
2278 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2279 const size_type size,
2280 const Number x,
2281 const ::MemorySpace::
2282 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2283 ::MemorySpace::MemorySpaceData<Number,
2284 ::MemorySpace::Default>
2285 &data)
2286 {
2287 auto exec = typename ::MemorySpace::Default::kokkos_space::
2288 execution_space{};
2289 Kokkos::parallel_for(
2290 "::sadd_xv",
2291 Kokkos::RangePolicy<
2292 ::MemorySpace::Default::kokkos_space::execution_space>(
2293 exec, 0, size),
2294 KOKKOS_LAMBDA(size_type i) {
2295 data.values(i) = x * data.values(i) + v_data.values(i);
2296 });
2297 exec.fence();
2298 }
2299
2300 static void
2302 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2303 const size_type size,
2304 const Number x,
2305 const Number a,
2306 const ::MemorySpace::
2307 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2308 ::MemorySpace::MemorySpaceData<Number,
2309 ::MemorySpace::Default>
2310 &data)
2311 {
2312 auto exec = typename ::MemorySpace::Default::kokkos_space::
2313 execution_space{};
2314 Kokkos::parallel_for(
2315 "::sadd_xav",
2316 Kokkos::RangePolicy<
2317 ::MemorySpace::Default::kokkos_space::execution_space>(
2318 exec, 0, size),
2319 KOKKOS_LAMBDA(size_type i) {
2320 data.values(i) = x * data.values(i) + a * v_data.values(i);
2321 });
2322 exec.fence();
2323 }
2324
2325 static void
2327 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2328 const size_type size,
2329 const Number x,
2330 const Number a,
2331 const Number b,
2332 const ::MemorySpace::
2333 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2334 const ::MemorySpace::
2335 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2336 ::MemorySpace::MemorySpaceData<Number,
2337 ::MemorySpace::Default>
2338 &data)
2339 {
2340 auto exec = typename ::MemorySpace::Default::kokkos_space::
2341 execution_space{};
2342 Kokkos::parallel_for(
2343 "::sadd_xavbw",
2344 Kokkos::RangePolicy<
2345 ::MemorySpace::Default::kokkos_space::execution_space>(
2346 exec, 0, size),
2347 KOKKOS_LAMBDA(size_type i) {
2348 data.values(i) =
2349 x * data.values(i) + a * v_data.values(i) + b * w_data.values(i);
2350 });
2351 exec.fence();
2352 }
2353
2354 static void
2356 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2357 const size_type size,
2358 const Number factor,
2361 &data)
2362 {
2363 auto exec = typename ::MemorySpace::Default::kokkos_space::
2364 execution_space{};
2365 Kokkos::parallel_for(
2366 "::multiply_factor",
2367 Kokkos::RangePolicy<
2368 ::MemorySpace::Default::kokkos_space::execution_space>(
2369 exec, 0, size),
2370 KOKKOS_LAMBDA(size_type i) { data.values(i) *= factor; });
2371 exec.fence();
2372 }
2373
2374 static void
2376 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2377 const size_type size,
2378 const ::MemorySpace::
2379 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2380 ::MemorySpace::MemorySpaceData<Number,
2381 ::MemorySpace::Default>
2382 &data)
2383 {
2384 auto exec = typename ::MemorySpace::Default::kokkos_space::
2385 execution_space{};
2386 Kokkos::parallel_for(
2387 "::scale",
2388 Kokkos::RangePolicy<
2389 ::MemorySpace::Default::kokkos_space::execution_space>(
2390 exec, 0, size),
2391 KOKKOS_LAMBDA(size_type i) { data.values(i) *= v_data.values(i); });
2392 exec.fence();
2393 }
2394
2395 static void
2397 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2398 const size_type size,
2399 const Number a,
2400 const ::MemorySpace::
2401 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2402 ::MemorySpace::MemorySpaceData<Number,
2403 ::MemorySpace::Default>
2404 &data)
2405 {
2406 auto exec = typename ::MemorySpace::Default::kokkos_space::
2407 execution_space{};
2408 Kokkos::parallel_for(
2409 "::equ_au",
2410 Kokkos::RangePolicy<
2411 ::MemorySpace::Default::kokkos_space::execution_space>(
2412 exec, 0, size),
2413 KOKKOS_LAMBDA(size_type i) {
2414 data.values(i) = a * v_data.values(i);
2415 });
2416 exec.fence();
2417 }
2418
2419 static void
2421 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2422 const size_type size,
2423 const Number a,
2424 const Number b,
2425 const ::MemorySpace::
2426 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2427 const ::MemorySpace::
2428 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2429 ::MemorySpace::MemorySpaceData<Number,
2430 ::MemorySpace::Default>
2431 &data)
2432 {
2433 auto exec = typename ::MemorySpace::Default::kokkos_space::
2434 execution_space{};
2435 Kokkos::parallel_for(
2436 "::equ_aubv",
2437 Kokkos::RangePolicy<
2438 ::MemorySpace::Default::kokkos_space::execution_space>(
2439 exec, 0, size),
2440 KOKKOS_LAMBDA(size_type i) {
2441 data.values(i) = a * v_data.values(i) + b * w_data.values(i);
2442 });
2443 exec.fence();
2444 }
2445
2446 static Number
2447 dot(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2448 const size_type size,
2449 const ::MemorySpace::
2450 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2451 ::MemorySpace::MemorySpaceData<Number,
2452 ::MemorySpace::Default>
2453 &data)
2454 {
2455 Number result;
2456
2457 auto exec = typename ::MemorySpace::Default::kokkos_space::
2458 execution_space{};
2459 Kokkos::parallel_reduce(
2460 "::dot",
2461 Kokkos::RangePolicy<
2462 ::MemorySpace::Default::kokkos_space::execution_space>(
2463 exec, 0, size),
2464 KOKKOS_LAMBDA(size_type i, Number & update) {
2465 update += data.values(i) * v_data.values(i);
2466 },
2467 result);
2468
2469 AssertIsFinite(result);
2470 return result;
2471 }
2472
2473 template <typename real_type>
2474 static void
2475 norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2476 &thread_loop_partitioner,
2477 const size_type size,
2478 real_type &sum,
2479 ::MemorySpace::
2480 MemorySpaceData<Number, ::MemorySpace::Default> &data)
2481 {
2482 sum = dot(thread_loop_partitioner, size, data, data);
2483 }
2484
2485 static Number
2487 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2488 const size_type size,
2489 const ::MemorySpace::
2490 MemorySpaceData<Number, ::MemorySpace::Default> &data)
2491 {
2492 Number result;
2493
2494 auto exec = typename ::MemorySpace::Default::kokkos_space::
2495 execution_space{};
2496 Kokkos::parallel_reduce(
2497 "::mean_value",
2498 Kokkos::RangePolicy<
2499 ::MemorySpace::Default::kokkos_space::execution_space>(
2500 exec, 0, size),
2501 KOKKOS_LAMBDA(size_type i, Number & update) {
2502 update += data.values(i);
2503 },
2504 result);
2505
2506 AssertIsFinite(result);
2507 return result;
2508 }
2509
2510 template <typename real_type>
2511 static void
2513 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2514 const size_type size,
2515 real_type &sum,
2518 &data,
2519 const size_type optional_offset = 0)
2520 {
2521 auto exec = typename ::MemorySpace::Default::kokkos_space::
2522 execution_space{};
2523 Kokkos::parallel_reduce(
2524 "::norm_1",
2525 Kokkos::RangePolicy<
2526 ::MemorySpace::Default::kokkos_space::execution_space>(
2527 exec, optional_offset, optional_offset + size),
2528 KOKKOS_LAMBDA(size_type i, Number & update) {
2529#if DEAL_II_KOKKOS_VERSION_GTE(3, 7, 0)
2530 update += Kokkos::abs(data.values(i));
2531#else
2532 update += Kokkos::Experimental::fabs(data.values(i));
2533#endif
2534 },
2535 sum);
2536 }
2537
2538 template <typename real_type>
2539 static void
2541 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2542 const size_type size,
2543 real_type &sum,
2544 real_type exp,
2547 &data)
2548 {
2549 auto exec = typename ::MemorySpace::Default::kokkos_space::
2550 execution_space{};
2551 Kokkos::parallel_reduce(
2552 "::norm_p",
2553 Kokkos::RangePolicy<
2554 ::MemorySpace::Default::kokkos_space::execution_space>(
2555 exec, 0, size),
2556 KOKKOS_LAMBDA(size_type i, Number & update) {
2557#if DEAL_II_KOKKOS_VERSION_GTE(3, 7, 0)
2558 update += Kokkos::pow(Kokkos::abs(data.values(i)), exp);
2559#else
2560 update += Kokkos::Experimental::pow(
2561 Kokkos::Experimental::fabs(data.values(i)), exp);
2562#endif
2563 },
2564 sum);
2565 }
2566
2567 static Number
2569 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2570 const size_type size,
2571 const Number a,
2572 const ::MemorySpace::
2573 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2574 const ::MemorySpace::
2575 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2576 ::MemorySpace::MemorySpaceData<Number,
2577 ::MemorySpace::Default>
2578 &data)
2579 {
2580 Number res;
2581
2582 auto exec = typename ::MemorySpace::Default::kokkos_space::
2583 execution_space{};
2584 Kokkos::parallel_reduce(
2585 "::add_and_dot",
2586 Kokkos::RangePolicy<
2587 ::MemorySpace::Default::kokkos_space::execution_space>(
2588 exec, 0, size),
2589 KOKKOS_LAMBDA(size_type i, Number & update) {
2590 data.values(i) += a * v_data.values(i);
2591 update +=
2593 w_data.values(i)));
2594 },
2595 res);
2596
2597 return res;
2598 }
2599
2600 template <typename MemorySpace2>
2601 static void
2603 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2604 &thread_loop_partitioner,
2605 const size_type size,
2606 VectorOperation::values operation,
2607 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2608 &v_data,
2611 &data,
2612 std::enable_if_t<
2613 std::is_same_v<MemorySpace2, ::MemorySpace::Default>,
2614 int> = 0)
2615 {
2616 if (operation == VectorOperation::insert)
2617 {
2618 copy(thread_loop_partitioner, size, v_data, data);
2619 }
2620 else if (operation == VectorOperation::add)
2621 {
2622 add_vector(thread_loop_partitioner, size, v_data, data);
2623 }
2624 else
2625 {
2627 }
2628 }
2629
2630 template <typename MemorySpace2>
2631 static void
2633 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2634 & /*thread_loop_partitioner*/,
2635 const size_type size,
2636 VectorOperation::values operation,
2637 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2638 &v_data,
2641 &data,
2642 std::enable_if_t<
2643 std::is_same_v<MemorySpace2, ::MemorySpace::Host>,
2644 int> = 0)
2645 {
2646 if (operation == VectorOperation::insert)
2647 {
2648 Kokkos::deep_copy(
2649 Kokkos::subview(data.values,
2650 Kokkos::pair<size_type, size_type>(0, size)),
2651 Kokkos::subview(v_data.values,
2652 Kokkos::pair<size_type, size_type>(0, size)));
2653 }
2654 else
2655 {
2657 }
2658 }
2659 };
2660 } // namespace VectorOperations
2661} // namespace internal
2662
2664
2665#endif
static unsigned int n_threads()
void store(OtherNumber *ptr) const
void load(const OtherNumber *ptr)
#define DEAL_II_ALWAYS_INLINE
Definition config.h:161
#define DEAL_II_OPENMP_SIMD_PRAGMA
Definition config.h:208
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:35
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:36
#define DEAL_II_FALLTHROUGH
Definition config.h:267
Point< 2 > first
Definition grid_out.cc:4632
static ::ExceptionBase & ExcNotImplemented()
#define Assert(cond, exc)
#define AssertIsFinite(number)
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
std::vector< index_type > data
Definition mpi.cc:746
std::size_t size
Definition mpi.cc:745
unsigned int minimum_parallel_grain_size
Definition parallel.cc:50
void accumulate_recursive(const Operation &op, const size_type first, const size_type last, ResultType &result)
size_type do_accumulate(const Operation op, const size_type vec_size, const size_type start_index, ResultType *outer_results, std::bool_constant< false >)
void parallel_reduce(const Operation &op, const size_type start, const size_type end, ResultType &result, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
void copy(const T *begin, const T *end, U *dest)
void parallel_for(Functor &functor, const size_type start, const size_type end, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
const unsigned int vector_accumulation_recursion_threshold
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
Definition parallel.h:122
::VectorizedArray< Number, width > min(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
::VectorizedArray< Number, width > pow(const ::VectorizedArray< Number, width > &, const Number p)
::VectorizedArray< Number, width > abs(const ::VectorizedArray< Number, width > &)
unsigned int global_dof_index
Definition types.h:94
AddAndDot(Number *const X, const Number *const V, const Number *const W, const Number a)
VectorizedArray< Number > do_vectorized(const size_type i) const
Dot(const Number *const X, const Number2 *const Y)
Number operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
void operator()(const tbb::blocked_range< size_type > &range) const
TBBForFunctor(Functor &functor, const size_type start, const size_type end)
TBBReduceFunctor(const Operation &op, const size_type start, const size_type end)
void operator()(const tbb::blocked_range< size_type > &range) const
Vector_copy(const OtherNumber *const src, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vector_set(const Number value, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_av(Number *const val, const Number *const v_val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_avpbw(Number *const val, const Number *const v_val, const Number *const w_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_add_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_v(Number *const val, const Number *const v_val)
Vectorization_equ_au(Number *const val, const Number *const u_val, const Number a)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubv(Number *const val, const Number *const u_val, const Number *const v_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubvcw(Number *val, const Number *u_val, const Number *v_val, const Number *w_val, const Number a, const Number b, const Number c)
void operator()(const size_type begin, const size_type end) const
Vectorization_multiply_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
Vectorization_sadd_xav(Number *val, const Number *const v_val, const Number a, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_sadd_xavbw(Number *val, const Number *v_val, const Number *w_val, Number x, Number a, Number b)
Vectorization_sadd_xv(Number *const val, const Number *const v_val, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_scale(Number *const val, const Number *const v_val)
Vectorization_subtract_v(Number *val, const Number *const v_val)
void operator()(const size_type begin, const size_type end) const
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, const real_type p, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Default >, int >=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Host >, int >=0)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, const size_type optional_offset=0)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, real_type exp, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, const size_type optional_offset=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Host >, int >=0)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same_v< MemorySpace2, ::MemorySpace::Default >, int >=0)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, VectorOperation::values, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, Number *, Number *)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, real_type, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static real_type abs(const number &x)
Definition numbers.h:574
static constexpr real_type abs_square(const number &x)
Definition numbers.h:565