Reference documentation for deal.II version 9.5.0
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
vector_operations_internal.h
Go to the documentation of this file.
1// ---------------------------------------------------------------------
2//
3// Copyright (C) 2016 - 2023 by the deal.II authors
4//
5// This file is part of the deal.II library.
6//
7// The deal.II library is free software; you can use it, redistribute
8// it, and/or modify it under the terms of the GNU Lesser General
9// Public License as published by the Free Software Foundation; either
10// version 2.1 of the License, or (at your option) any later version.
11// The full text of the license can be found in the file LICENSE.md at
12// the top level directory of deal.II.
13//
14// ---------------------------------------------------------------------
15
16
17#ifndef dealii_vector_operations_internal_h
18#define dealii_vector_operations_internal_h
19
20#include <deal.II/base/config.h>
21
26#include <deal.II/base/types.h>
28
30
31#include <cstdio>
32#include <cstring>
33
35
36namespace internal
37{
38 namespace VectorOperations
39 {
41
42 template <typename T>
43 bool
44 is_non_negative(const T &t)
45 {
46 return t >= 0;
47 }
48
49
50 template <typename T>
51 bool
52 is_non_negative(const std::complex<T> &)
53 {
54 Assert(false, ExcMessage("Complex numbers do not have an ordering."));
55
56 return false;
57 }
58
59
60 // call std::copy, except for in
61 // the case where we want to copy
62 // from std::complex to a
63 // non-complex type
64 template <typename T, typename U>
65 void
66 copy(const T *begin, const T *end, U *dest)
67 {
68 std::copy(begin, end, dest);
69 }
70
71 template <typename T, typename U>
72 void
73 copy(const std::complex<T> *begin,
74 const std::complex<T> *end,
75 std::complex<U> * dest)
76 {
77 std::copy(begin, end, dest);
78 }
79
80 template <typename T, typename U>
81 void
82 copy(const std::complex<T> *, const std::complex<T> *, U *)
83 {
84 Assert(false,
85 ExcMessage("Can't convert a vector of complex numbers "
86 "into a vector of reals/doubles"));
87 }
88
89
90
91#ifdef DEAL_II_WITH_TBB
100 template <typename Functor>
102 {
104 const size_type start,
105 const size_type end)
107 , start(start)
108 , end(end)
109 {
110 const size_type vec_size = end - start;
111 // set chunk size for sub-tasks
112 const unsigned int gs =
114 n_chunks =
116 vec_size / gs);
117 chunk_size = vec_size / n_chunks;
118
119 // round to next multiple of 512 (or minimum grain size if that happens
120 // to be smaller). this is advantageous because our accumulation
121 // algorithms favor lengths of a power of 2 due to pairwise summation ->
122 // at most one 'oddly' sized chunk
123 if (chunk_size > 512)
124 chunk_size = ((chunk_size + 511) / 512) * 512;
125 n_chunks = (vec_size + chunk_size - 1) / chunk_size;
126 AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
127 AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
128 }
129
130 void
131 operator()(const tbb::blocked_range<size_type> &range) const
132 {
133 const size_type r_begin = start + range.begin() * chunk_size;
134 const size_type r_end = std::min(start + range.end() * chunk_size, end);
135 functor(r_begin, r_end);
136 }
137
138 Functor & functor;
141 unsigned int n_chunks;
143 };
144#endif
145
146 template <typename Functor>
147 void
149 Functor & functor,
150 const size_type start,
151 const size_type end,
152 const std::shared_ptr<::parallel::internal::TBBPartitioner>
153 &partitioner)
154 {
155#ifdef DEAL_II_WITH_TBB
156 const size_type vec_size = end - start;
157 // only go to the parallel function in case there are at least 4 parallel
158 // items, otherwise the overhead is too large
159 if (vec_size >=
162 {
163 Assert(partitioner.get() != nullptr,
165 "Unexpected initialization of Vector that does "
166 "not set the TBB partitioner to a usable state."));
167 std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
168 partitioner->acquire_one_partitioner();
169
170 TBBForFunctor<Functor> generic_functor(functor, start, end);
171 // We use a minimum grain size of 1 here since the grains at this
172 // stage of dividing the work refer to the number of vector chunks
173 // that are processed by (possibly different) threads in the
174 // parallelized for loop (i.e., they do not refer to individual
175 // vector entries). The number of chunks here is calculated inside
176 // TBBForFunctor. See also GitHub issue #2496 for further discussion
177 // of this strategy.
179 static_cast<size_type>(0),
180 static_cast<size_type>(generic_functor.n_chunks),
181 generic_functor,
182 1,
183 tbb_partitioner);
184 partitioner->release_one_partitioner(tbb_partitioner);
185 }
186 else if (vec_size > 0)
187 functor(start, end);
188#else
189 functor(start, end);
190 (void)partitioner;
191#endif
192 }
193
194
195 // Define the functors necessary to use SIMD with TBB. we also include the
196 // simple copy and set operations
197
198 template <typename Number>
200 {
201 Vector_set(const Number value, Number *const dst)
202 : value(value)
203 , dst(dst)
204 {
205 Assert(dst != nullptr, ExcInternalError());
206 }
207
208 void
209 operator()(const size_type begin, const size_type end) const
210 {
211 Assert(end >= begin, ExcInternalError());
212
213 if (value == Number())
214 {
215 if DEAL_II_CONSTEXPR_IN_CONDITIONAL (std::is_trivial<Number>::value)
216 {
217 std::memset(dst + begin, 0, sizeof(Number) * (end - begin));
218 return;
219 }
220 }
221 std::fill(dst + begin, dst + end, value);
222 }
223
224 const Number value;
225 Number *const dst;
226 };
227
228 template <typename Number, typename OtherNumber>
230 {
231 Vector_copy(const OtherNumber *const src, Number *const dst)
232 : src(src)
233 , dst(dst)
234 {
235 Assert(src != nullptr, ExcInternalError());
236 Assert(dst != nullptr, ExcInternalError());
237 }
238
239 void
240 operator()(const size_type begin, const size_type end) const
241 {
242 Assert(end >= begin, ExcInternalError());
243
244 if DEAL_II_CONSTEXPR_IN_CONDITIONAL (std::is_trivially_copyable<
245 Number>() &&
246 std::is_same<Number,
247 OtherNumber>::value)
248 std::memcpy(dst + begin, src + begin, (end - begin) * sizeof(Number));
249 else
250 {
252 for (size_type i = begin; i < end; ++i)
253 dst[i] = src[i];
254 }
255 }
256
257 const OtherNumber *const src;
258 Number *const dst;
259 };
260
261 template <typename Number>
263 {
264 Vectorization_multiply_factor(Number *const val, const Number factor)
265 : val(val)
266 , stored_factor(factor)
267 {}
268
269 void
270 operator()(const size_type begin, const size_type end) const
271 {
272 // create a local copy of the variable to help the compiler with the
273 // aliasing analysis
274 const Number factor = stored_factor;
275
277 {
279 for (size_type i = begin; i < end; ++i)
280 val[i] *= factor;
281 }
282 else
283 {
284 for (size_type i = begin; i < end; ++i)
285 val[i] *= factor;
286 }
287 }
288
289 Number *const val;
290 const Number stored_factor;
291 };
292
293 template <typename Number>
295 {
297 const Number *const v_val,
298 const Number factor)
299 : val(val)
300 , v_val(v_val)
301 , stored_factor(factor)
302 {}
303
304 void
305 operator()(const size_type begin, const size_type end) const
306 {
307 // create a local copy of the variable to help the compiler with the
308 // aliasing analysis
309 const Number factor = stored_factor;
311 {
313 for (size_type i = begin; i < end; ++i)
314 val[i] += factor * v_val[i];
315 }
316 else
317 {
318 for (size_type i = begin; i < end; ++i)
319 val[i] += factor * v_val[i];
320 }
321 }
322
323 Number *const val;
324 const Number *const v_val;
325 const Number stored_factor;
326 };
327
328 template <typename Number>
330 {
332 const Number *const v_val,
333 const Number a,
334 const Number x)
335 : val(val)
336 , v_val(v_val)
337 , stored_a(a)
338 , stored_x(x)
339 {}
340
341 void
342 operator()(const size_type begin, const size_type end) const
343 {
344 // create a local copy of the variable to help the compiler with the
345 // aliasing analysis
346 const Number x = stored_x, a = stored_a;
347
349 {
351 for (size_type i = begin; i < end; ++i)
352 val[i] = x * val[i] + a * v_val[i];
353 }
354 else
355 {
356 for (size_type i = begin; i < end; ++i)
357 val[i] = x * val[i] + a * v_val[i];
358 }
359 }
360
361 Number *const val;
362 const Number *const v_val;
363 const Number stored_a;
364 const Number stored_x;
365 };
366
367 template <typename Number>
369 {
370 Vectorization_subtract_v(Number *val, const Number *const v_val)
371 : val(val)
372 , v_val(v_val)
373 {}
374
375 void
376 operator()(const size_type begin, const size_type end) const
377 {
379 {
381 for (size_type i = begin; i < end; ++i)
382 val[i] -= v_val[i];
383 }
384 else
385 {
386 for (size_type i = begin; i < end; ++i)
387 val[i] -= v_val[i];
388 }
389 }
390
391 Number *const val;
392 const Number *const v_val;
393 };
394
395 template <typename Number>
397 {
398 Vectorization_add_factor(Number *const val, const Number factor)
399 : val(val)
400 , stored_factor(factor)
401 {}
402
403 void
404 operator()(const size_type begin, const size_type end) const
405 {
406 const Number factor = stored_factor;
407
409 {
411 for (size_type i = begin; i < end; ++i)
412 val[i] += factor;
413 }
414 else
415 {
416 for (size_type i = begin; i < end; ++i)
417 val[i] += factor;
418 }
419 }
420
421 Number *const val;
422 const Number stored_factor;
423 };
424
425 template <typename Number>
427 {
428 Vectorization_add_v(Number *const val, const Number *const v_val)
429 : val(val)
430 , v_val(v_val)
431 {}
432
433 void
434 operator()(const size_type begin, const size_type end) const
435 {
437 {
439 for (size_type i = begin; i < end; ++i)
440 val[i] += v_val[i];
441 }
442 else
443 {
444 for (size_type i = begin; i < end; ++i)
445 val[i] += v_val[i];
446 }
447 }
448
449 Number *const val;
450 const Number *const v_val;
451 };
452
453 template <typename Number>
455 {
457 const Number *const v_val,
458 const Number *const w_val,
459 const Number a,
460 const Number b)
461 : val(val)
462 , v_val(v_val)
463 , w_val(w_val)
464 , stored_a(a)
465 , stored_b(b)
466 {}
467
468 void
469 operator()(const size_type begin, const size_type end) const
470 {
471 const Number a = stored_a, b = stored_b;
472
474 {
476 for (size_type i = begin; i < end; ++i)
477 val[i] = val[i] + a * v_val[i] + b * w_val[i];
478 }
479 else
480 {
481 for (size_type i = begin; i < end; ++i)
482 val[i] = val[i] + a * v_val[i] + b * w_val[i];
483 }
484 }
485
486 Number *const val;
487 const Number *const v_val;
488 const Number *const w_val;
489 const Number stored_a;
490 const Number stored_b;
491 };
492
493 template <typename Number>
495 {
497 const Number *const v_val,
498 const Number x)
499 : val(val)
500 , v_val(v_val)
501 , stored_x(x)
502 {}
503
504 void
505 operator()(const size_type begin, const size_type end) const
506 {
507 const Number x = stored_x;
508
510 {
512 for (size_type i = begin; i < end; ++i)
513 val[i] = x * val[i] + v_val[i];
514 }
515 else
516 {
517 for (size_type i = begin; i < end; ++i)
518 val[i] = x * val[i] + v_val[i];
519 }
520 }
521
522 Number *const val;
523 const Number *const v_val;
524 const Number stored_x;
525 };
526
527 template <typename Number>
529 {
531 const Number *v_val,
532 const Number *w_val,
533 Number x,
534 Number a,
535 Number b)
536 : val(val)
537 , v_val(v_val)
538 , w_val(w_val)
539 , stored_x(x)
540 , stored_a(a)
541 , stored_b(b)
542 {}
543
544 void
545 operator()(const size_type begin, const size_type end) const
546 {
547 const Number x = stored_x, a = stored_a, b = stored_b;
548
550 {
552 for (size_type i = begin; i < end; ++i)
553 val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
554 }
555 else
556 {
557 for (size_type i = begin; i < end; ++i)
558 val[i] = x * val[i] + a * v_val[i] + b * w_val[i];
559 }
560 }
561
562 Number *const val;
563 const Number *const v_val;
564 const Number *const w_val;
565 const Number stored_x;
566 const Number stored_a;
567 const Number stored_b;
568 };
569
570 template <typename Number>
572 {
573 Vectorization_scale(Number *const val, const Number *const v_val)
574 : val(val)
575 , v_val(v_val)
576 {}
577
578 void
579 operator()(const size_type begin, const size_type end) const
580 {
582 {
584 for (size_type i = begin; i < end; ++i)
585 val[i] *= v_val[i];
586 }
587 else
588 {
589 for (size_type i = begin; i < end; ++i)
590 val[i] *= v_val[i];
591 }
592 }
593
594 Number *const val;
595 const Number *const v_val;
596 };
597
598 template <typename Number>
600 {
602 const Number *const u_val,
603 const Number a)
604 : val(val)
605 , u_val(u_val)
606 , stored_a(a)
607 {}
608
609 void
610 operator()(const size_type begin, const size_type end) const
611 {
612 const Number a = stored_a;
613
615 {
617 for (size_type i = begin; i < end; ++i)
618 val[i] = a * u_val[i];
619 }
620 else
621 {
622 for (size_type i = begin; i < end; ++i)
623 val[i] = a * u_val[i];
624 }
625 }
626
627 Number *const val;
628 const Number *const u_val;
629 const Number stored_a;
630 };
631
632 template <typename Number>
634 {
636 const Number *const u_val,
637 const Number *const v_val,
638 const Number a,
639 const Number b)
640 : val(val)
641 , u_val(u_val)
642 , v_val(v_val)
643 , stored_a(a)
644 , stored_b(b)
645 {}
646
647 void
648 operator()(const size_type begin, const size_type end) const
649 {
650 const Number a = stored_a, b = stored_b;
651
653 {
655 for (size_type i = begin; i < end; ++i)
656 val[i] = a * u_val[i] + b * v_val[i];
657 }
658 else
659 {
660 for (size_type i = begin; i < end; ++i)
661 val[i] = a * u_val[i] + b * v_val[i];
662 }
663 }
664
665 Number *const val;
666 const Number *const u_val;
667 const Number *const v_val;
668 const Number stored_a;
669 const Number stored_b;
670 };
671
672 template <typename Number>
674 {
676 const Number *u_val,
677 const Number *v_val,
678 const Number *w_val,
679 const Number a,
680 const Number b,
681 const Number c)
682 : val(val)
683 , u_val(u_val)
684 , v_val(v_val)
685 , w_val(w_val)
686 , stored_a(a)
687 , stored_b(b)
688 , stored_c(c)
689 {}
690
691 void
692 operator()(const size_type begin, const size_type end) const
693 {
694 const Number a = stored_a, b = stored_b, c = stored_c;
695
697 {
699 for (size_type i = begin; i < end; ++i)
700 val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
701 }
702 else
703 {
704 for (size_type i = begin; i < end; ++i)
705 val[i] = a * u_val[i] + b * v_val[i] + c * w_val[i];
706 }
707 }
708
709 Number *const val;
710 const Number *const u_val;
711 const Number *const v_val;
712 const Number *const w_val;
713 const Number stored_a;
714 const Number stored_b;
715 const Number stored_c;
716 };
717
718 template <typename Number>
720 {
721 Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
722 : val(val)
723 , a_val(a_val)
724 , b_val(b_val)
725 {}
726
727 void
728 operator()(const size_type begin, const size_type end) const
729 {
731 {
733 for (size_type i = begin; i < end; ++i)
734 val[i] = a_val[i] / b_val[i];
735 }
736 else
737 {
738 for (size_type i = begin; i < end; ++i)
739 val[i] = a_val[i] / b_val[i];
740 }
741 }
742
743 Number *const val;
744 const Number *const a_val;
745 const Number *const b_val;
746 };
747
748
749
750 // All sums over all the vector entries (l2-norm, inner product, etc.) are
751 // performed with the same code, using a templated operation defined
752 // here. There are always two versions defined, a standard one that covers
753 // most cases and a vectorized one which is only for equal types and float
754 // and double.
755 template <typename Number, typename Number2>
756 struct Dot
757 {
758 static constexpr bool vectorizes = std::is_same<Number, Number2>::value &&
760
761 Dot(const Number *const X, const Number2 *const Y)
762 : X(X)
763 , Y(Y)
764 {}
765
766 Number
767 operator()(const size_type i) const
768 {
769 return X[i] * Number(numbers::NumberTraits<Number2>::conjugate(Y[i]));
770 }
771
774 {
776 x.load(X + i);
777 y.load(Y + i);
778
779 // the following operation in VectorizedArray does an element-wise
780 // scalar product without taking into account complex values and
781 // the need to take the complex-conjugate of one argument. this
782 // may be a bug, but because all VectorizedArray classes only
783 // work on real scalars, it doesn't really matter very much.
784 // in any case, assert that we really don't get here for
785 // complex-valued objects
786 static_assert(numbers::NumberTraits<Number>::is_complex == false,
787 "This operation is not correctly implemented for "
788 "complex-valued objects.");
789 return x * y;
790 }
791
792 const Number *const X;
793 const Number2 *const Y;
794 };
795
796 template <typename Number, typename RealType>
797 struct Norm2
798 {
799 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
800
801 Norm2(const Number *const X)
802 : X(X)
803 {}
804
805 RealType
806 operator()(const size_type i) const
807 {
809 }
810
813 {
815 x.load(X + i);
816 return x * x;
817 }
818
819 const Number *const X;
820 };
821
822 template <typename Number, typename RealType>
823 struct Norm1
824 {
825 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
826
827 Norm1(const Number *X)
828 : X(X)
829 {}
830
831 RealType
832 operator()(const size_type i) const
833 {
835 }
836
839 {
841 x.load(X + i);
842 return std::abs(x);
843 }
844
845 const Number *X;
846 };
847
848 template <typename Number, typename RealType>
849 struct NormP
850 {
851 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
852
853 NormP(const Number *X, RealType p)
854 : X(X)
855 , p(p)
856 {}
857
858 RealType
859 operator()(const size_type i) const
860 {
862 }
863
866 {
868 x.load(X + i);
869 return std::pow(std::abs(x), p);
870 }
871
872 const Number * X;
873 const RealType p;
874 };
875
876 template <typename Number>
878 {
879 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
880
881 MeanValue(const Number *X)
882 : X(X)
883 {}
884
885 Number
886 operator()(const size_type i) const
887 {
888 return X[i];
889 }
890
893 {
895 x.load(X + i);
896 return x;
897 }
898
899 const Number *X;
900 };
901
902 template <typename Number>
904 {
905 static const bool vectorizes = VectorizedArray<Number>::size() > 1;
906
907 AddAndDot(Number *const X,
908 const Number *const V,
909 const Number *const W,
910 const Number a)
911 : X(X)
912 , V(V)
913 , W(W)
914 , a(a)
915 {}
916
917 Number
918 operator()(const size_type i) const
919 {
920 X[i] += a * V[i];
921 return X[i] * Number(numbers::NumberTraits<Number>::conjugate(W[i]));
922 }
923
926 {
928 x.load(X + i);
929 v.load(V + i);
930 x += a * v;
931 x.store(X + i);
932 // may only load from W after storing in X because the pointers might
933 // point to the same memory
934 w.load(W + i);
935
936 // the following operation in VectorizedArray does an element-wise
937 // scalar product without taking into account complex values and
938 // the need to take the complex-conjugate of one argument. this
939 // may be a bug, but because all VectorizedArray classes only
940 // work on real scalars, it doesn't really matter very much.
941 // in any case, assert that we really don't get here for
942 // complex-valued objects
943 static_assert(numbers::NumberTraits<Number>::is_complex == false,
944 "This operation is not correctly implemented for "
945 "complex-valued objects.");
946 return x * w;
947 }
948
949 Number *const X;
950 const Number *const V;
951 const Number *const W;
952 const Number a;
953 };
954
955
956
957 // this is the main working loop for all vector sums using the templated
958 // operation above. it accumulates the sums using a block-wise summation
959 // algorithm with post-update. this blocked algorithm has been proposed in
960 // a similar form by Castaldo, Whaley and Chronopoulos (SIAM
961 // J. Sci. Comput. 31, 1156-1174, 2008) and we use the smallest possible
962 // block size, 2. Sometimes it is referred to as pairwise summation. The
963 // worst case error made by this algorithm is on the order O(eps *
964 // log2(vec_size)), whereas a naive summation is O(eps * vec_size). Even
965 // though the Kahan summation is even more accurate with an error O(eps)
966 // by carrying along remainders not captured by the main sum, that involves
967 // additional costs which are not worthwhile. See the Wikipedia article on
968 // the Kahan summation algorithm.
969
970 // The algorithm implemented here has the additional benefit that it is
971 // easily parallelized without changing the order of how the elements are
972 // added (floating point addition is not associative). For the same vector
973 // size and minimum_parallel_grainsize, the blocks are always the
974 // same and added pairwise.
975
976 // The depth of recursion is controlled by the 'magic' parameter
977 // vector_accumulation_recursion_threshold: If the length is below
978 // vector_accumulation_recursion_threshold * 32 (32 is the part of code we
979 // unroll), a straight loop instead of recursion will be used. At the
980 // innermost level, eight values are added consecutively in order to better
981 // balance multiplications and additions.
982
983 // Loops are unrolled as follows: the range [first,last) is broken into
984 // @p n_chunks each of size 32 plus the @p remainder.
985 // accumulate_regular() does the work on 32*n_chunks elements employing SIMD
986 // if possible and stores the result of the operation for each chunk in @p outer_results.
987
988 // The code returns the result as the last argument in order to make
989 // spawning tasks simpler and use automatic template deduction.
990
991
998
999 template <typename Operation, typename ResultType>
1000 void
1001 accumulate_recursive(const Operation &op,
1002 const size_type first,
1003 const size_type last,
1004 ResultType & result)
1005 {
1006 if (first == last)
1007 {
1008 result = ResultType();
1009 return;
1010 }
1011
1012 const size_type vec_size = last - first;
1013 if (vec_size <= vector_accumulation_recursion_threshold * 32)
1014 {
1015 // The vector is short enough so we perform the summation. We store
1016 // the number of chunks (each 32 indices) for the given vector
1017 // length; all results are stored in outer_results[0,n_chunks). We
1018 // keep twice the number around to be able to do the pairwise
1019 // summation with a single for loop (see the loop over j below)
1020 ResultType outer_results[vector_accumulation_recursion_threshold * 2];
1021
1022 // Select between the regular version and vectorized version based
1023 // on the number types we are given. To choose the vectorized
1024 // version often enough, we need to have all tasks but the last one
1025 // to be divisible by the vectorization length
1026 size_type n_chunks = do_accumulate(
1027 op,
1028 vec_size,
1029 first,
1030 outer_results,
1031 std::integral_constant<bool, Operation::vectorizes>());
1032
1033 AssertIndexRange(n_chunks,
1035
1036 // now sum the results from the chunks stored in
1037 // outer_results[0,n_chunks) recursively
1038 unsigned int j = 0;
1039 constexpr unsigned int n_lanes = VectorizedArray<ResultType>::size();
1040 for (; j + 2 * n_lanes - 1 < n_chunks;
1041 j += 2 * n_lanes, n_chunks += n_lanes)
1042 {
1044 a.load(outer_results + j);
1045 b.load(outer_results + j + n_lanes);
1046 a += b;
1047 a.store(outer_results + n_chunks);
1048 }
1049
1050 // In the vectorized case, we know the loop bounds and can do things
1051 // more efficiently
1052 if (Operation::vectorizes)
1053 {
1054 AssertDimension(j + n_lanes, n_chunks);
1055 AssertIndexRange(n_chunks,
1057 ResultType *result_ptr = outer_results + j;
1058 if (n_lanes >= 16)
1059 for (unsigned int i = 0; i < 8; ++i)
1060 result_ptr[i] = result_ptr[i] + result_ptr[i + 8];
1061 if (n_lanes >= 8)
1062 for (unsigned int i = 0; i < 4; ++i)
1063 result_ptr[i] = result_ptr[i] + result_ptr[i + 4];
1064 if (n_lanes >= 4)
1065 for (unsigned int i = 0; i < 2; ++i)
1066 result_ptr[i] = result_ptr[i] + result_ptr[i + 2];
1067 result = result_ptr[0] + result_ptr[1];
1068 }
1069 else
1070 {
1071 // Without vectorization, we do not know the exact bounds, so we
1072 // need to continue the variable-length pairwise summation loop
1073 // from above
1074 for (; j + 1 < n_chunks; j += 2, ++n_chunks)
1075 outer_results[n_chunks] =
1076 outer_results[j] + outer_results[j + 1];
1077
1078 AssertIndexRange(n_chunks,
1080 Assert(n_chunks > 0, ExcInternalError());
1081 result = outer_results[n_chunks - 1];
1082 }
1083 }
1084 else
1085 {
1086 // split vector into four pieces and work on the pieces
1087 // recursively. Make pieces (except last) divisible by one fourth the
1088 // recursion threshold.
1089 const size_type new_size =
1090 (vec_size / (vector_accumulation_recursion_threshold * 32)) *
1092 Assert(first + 3 * new_size < last, ExcInternalError());
1093 ResultType r0, r1, r2, r3;
1094 accumulate_recursive(op, first, first + new_size, r0);
1095 accumulate_recursive(op, first + new_size, first + 2 * new_size, r1);
1097 first + 2 * new_size,
1098 first + 3 * new_size,
1099 r2);
1100 accumulate_recursive(op, first + 3 * new_size, last, r3);
1101 result = (r0 + r1) + (r2 + r3);
1102 }
1103 }
1104
1105
1106 // this is the inner working routine for the accumulation loops below. We
1107 // pulled this part out of the regular accumulate routine because we might
1108 // do this thing vectorized (see specialized function below; this is the
1109 // un-vectorized version). As opposed to the vector add functions above,
1110 // we here pass the functor 'op' by value, because we cannot create a copy
1111 // of the scalar inline, and instead make sure that the numbers get local
1112 // (and thus definitely not aliased) for the compiler
1113 template <typename Operation, typename ResultType>
1114 size_type
1115 do_accumulate(const Operation op,
1116 const size_type vec_size,
1117 const size_type start_index,
1118 ResultType * outer_results,
1119 std::integral_constant<bool, false>)
1120 {
1121 // Create local copy to indicate no aliasing to the compiler
1122 size_type index = start_index;
1123
1124 // choose each chunk to have a width of 32, thereby the index
1125 // is incremented by 4*8 for each @p i.
1126 size_type n_chunks = vec_size / 32;
1127 for (size_type i = 0; i < n_chunks; ++i)
1128 {
1129 ResultType r = {};
1130 for (unsigned int k = 0; k < 2; ++k)
1131 {
1132 ResultType r0 = op(index);
1133 ResultType r1 = op(index + 1);
1134 ResultType r2 = op(index + 2);
1135 ResultType r3 = op(index + 3);
1136 index += 4;
1137 for (size_type j = 1; j < 4; ++j, index += 4)
1138 {
1139 r0 += op(index);
1140 r1 += op(index + 1);
1141 r2 += op(index + 2);
1142 r3 += op(index + 3);
1143 }
1144 r += (r0 + r1) + (r2 + r3);
1145 }
1146 outer_results[i] = r;
1147 }
1148
1149 if (n_chunks * 32 < vec_size)
1150 {
1151 const size_type remainder = vec_size - n_chunks * 32;
1152 const size_type inner_chunks = remainder / 8;
1153 const size_type remainder_inner = remainder % 8;
1154 ResultType r0 = ResultType(), r1 = ResultType(), r2 = ResultType();
1155 switch (inner_chunks)
1156 {
1157 case 3:
1158 r2 = op(index++);
1159 for (size_type j = 1; j < 8; ++j)
1160 r2 += op(index++);
1162 case 2:
1163 r1 = op(index++);
1164 for (size_type j = 1; j < 8; ++j)
1165 r1 += op(index++);
1166 r1 += r2;
1168 case 1:
1169 r2 = op(index++);
1170 for (size_type j = 1; j < 8; ++j)
1171 r2 += op(index++);
1173 default:
1174 for (size_type j = 0; j < remainder_inner; ++j)
1175 r0 += op(index++);
1176 outer_results[n_chunks++] = (r0 + r2) + r1;
1177 break;
1178 }
1179 }
1180
1181 // make sure we worked through all indices
1182 AssertDimension(index, start_index + vec_size);
1183
1184 return n_chunks;
1185 }
1186
1187
1188
1189 // this is the inner working routine for the accumulation loops
1190 // below. This is the specialized case where we can vectorize. We request
1191 // the 'do_vectorized' routine of the operation instead of the regular one
1192 // which does several operations at once. As above, pass in the functor by
1193 // value to create a local copy of the scalar factors in the function (if
1194 // there are any).
1195 template <typename Operation, typename Number>
1196 size_type
1197 do_accumulate(const Operation op,
1198 const size_type vec_size,
1199 const size_type start_index,
1200 Number * outer_results,
1201 std::integral_constant<bool, true>)
1202 {
1203 // Create local copy to indicate no aliasing to the compiler
1204 size_type index = start_index;
1205
1206 // we start from @p index and workout @p n_chunks each of size 32.
1207 // in order employ SIMD and work on @p nvecs at a time, we split this
1208 // loop yet again:
1209 // First we work on (n_chunks/nvecs) chunks, where each chunk processes
1210 // nvecs*(4*8) elements.
1211
1212 constexpr size_type n_lanes = VectorizedArray<Number>::size();
1213 const size_type regular_chunks = vec_size / (32 * n_lanes);
1214 for (size_type i = 0; i < regular_chunks; ++i)
1215 {
1217 for (unsigned int k = 0; k < 2; ++k)
1218 {
1219 VectorizedArray<Number> r0 = op.do_vectorized(index);
1220 VectorizedArray<Number> r1 = op.do_vectorized(index + n_lanes);
1222 op.do_vectorized(index + 2 * n_lanes);
1224 op.do_vectorized(index + 3 * n_lanes);
1225 index += n_lanes * 4;
1226 for (size_type j = 1; j < 4; ++j, index += n_lanes * 4)
1227 {
1228 r0 += op.do_vectorized(index);
1229 r1 += op.do_vectorized(index + n_lanes);
1230 r2 += op.do_vectorized(index + 2 * n_lanes);
1231 r3 += op.do_vectorized(index + 3 * n_lanes);
1232 }
1233 r += (r0 + r1) + (r2 + r3);
1234 }
1235 r.store(&outer_results[i * n_lanes]);
1236 }
1237
1238 // If we are treating a case where the vector length is not divisible by
1239 // the vectorization length, need a cleanup loop
1240 // The remaining chunks are processed one by one starting from
1241 // regular_chunks * n_lanes; We do as much as possible with 2 SIMD
1242 // operations within each chunk. Here we assume that n_lanes < 32/2 = 16
1243 // as well as 16 % n_lanes == 0.
1244 static_assert(n_lanes <= 16 && 16 % n_lanes == 0,
1245 "VectorizedArray::size() must be 1, 2, 4, 8, or 16");
1246 size_type n_chunks = regular_chunks * n_lanes;
1247 const size_type start_irregular = regular_chunks * n_lanes * 32;
1248 if (start_irregular < vec_size)
1249 {
1252 const size_type remainder = vec_size - start_irregular;
1253 const size_type loop_length = remainder / (2 * n_lanes);
1254 for (size_type j = 0; j < loop_length; ++j, index += 2 * n_lanes)
1255 {
1256 r0 += op.do_vectorized(index);
1257 r1 += op.do_vectorized(index + n_lanes);
1258 }
1259 Number scalar_part = Number();
1260 size_type last = remainder % (2 * n_lanes);
1261 if (last > 0)
1262 {
1263 if (last >= n_lanes)
1264 {
1265 r0 += op.do_vectorized(index);
1266 index += n_lanes;
1267 last -= n_lanes;
1268 }
1269 for (unsigned int i = 0; i < last; ++i)
1270 scalar_part += op(index++);
1271 }
1272
1273 r0 += r1;
1274 r0.store(&outer_results[n_chunks]);
1275 outer_results[n_chunks] += scalar_part;
1276
1277 // update n_chunks to denote range of entries to sum up in
1278 // outer_results[].
1279 n_chunks += n_lanes;
1280 }
1281
1282 // make sure we worked through all indices
1283 AssertDimension(index, start_index + vec_size);
1284
1285 return n_chunks;
1286 }
1287
1288
1289
1290#ifdef DEAL_II_WITH_TBB
1319 template <typename Operation, typename ResultType>
1321 {
1322 static const unsigned int threshold_array_allocate = 512;
1323
1324 TBBReduceFunctor(const Operation &op,
1325 const size_type start,
1326 const size_type end)
1327 : op(op)
1328 , start(start)
1329 , end(end)
1330 {
1331 const size_type vec_size = end - start;
1332 // set chunk size for sub-tasks
1333 const unsigned int gs =
1335 n_chunks =
1336 std::min(static_cast<size_type>(4 * MultithreadInfo::n_threads()),
1337 vec_size / gs);
1338 chunk_size = vec_size / n_chunks;
1339
1340 // round to next multiple of 512 (or leave it at the minimum grain size
1341 // if that happens to be smaller). this is advantageous because our
1342 // algorithm favors lengths of a power of 2 due to pairwise summation ->
1343 // at most one 'oddly' sized chunk
1344 if (chunk_size > 512)
1345 chunk_size = ((chunk_size + 511) / 512) * 512;
1346 n_chunks = (vec_size + chunk_size - 1) / chunk_size;
1347 AssertIndexRange((n_chunks - 1) * chunk_size, vec_size);
1348 AssertIndexRange(vec_size, n_chunks * chunk_size + 1);
1349
1351 {
1352 // make sure we allocate an even number of elements,
1353 // access to the new last element is needed in do_sum()
1354 large_array.resize(2 * ((n_chunks + 1) / 2));
1355 array_ptr = large_array.data();
1356 }
1357 else
1358 array_ptr = &small_array[0];
1359 }
1360
1365 void
1366 operator()(const tbb::blocked_range<size_type> &range) const
1367 {
1368 for (size_type i = range.begin(); i < range.end(); ++i)
1370 start + i * chunk_size,
1371 std::min(start + (i + 1) * chunk_size, end),
1372 array_ptr[i]);
1373 }
1374
1375 ResultType
1376 do_sum() const
1377 {
1378 while (n_chunks > 1)
1379 {
1380 if (n_chunks % 2 == 1)
1381 array_ptr[n_chunks++] = ResultType();
1382 for (size_type i = 0; i < n_chunks; i += 2)
1383 array_ptr[i / 2] = array_ptr[i] + array_ptr[i + 1];
1384 n_chunks /= 2;
1385 }
1386 return array_ptr[0];
1387 }
1388
1389 const Operation &op;
1392
1393 mutable unsigned int n_chunks;
1394 unsigned int chunk_size;
1396 std::vector<ResultType> large_array;
1397 // this variable either points to small_array or large_array depending on
1398 // the number of threads we want to feed
1399 mutable ResultType *array_ptr;
1400 };
1401#endif
1402
1403
1404
1409 template <typename Operation, typename ResultType>
1410#ifndef DEBUG
1412#endif
1413 inline void
1415 const Operation &op,
1416 const size_type start,
1417 const size_type end,
1418 ResultType & result,
1419 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1420 &partitioner)
1421 {
1422#ifdef DEAL_II_WITH_TBB
1423 const size_type vec_size = end - start;
1424 // only go to the parallel function in case there are at least 4 parallel
1425 // items, otherwise the overhead is too large
1426 if (vec_size >=
1429 {
1430 Assert(partitioner.get() != nullptr,
1432 "Unexpected initialization of Vector that does "
1433 "not set the TBB partitioner to a usable state."));
1434 std::shared_ptr<tbb::affinity_partitioner> tbb_partitioner =
1435 partitioner->acquire_one_partitioner();
1436
1437 TBBReduceFunctor<Operation, ResultType> generic_functor(op,
1438 start,
1439 end);
1440 // We use a minimum grain size of 1 here since the grains at this
1441 // stage of dividing the work refer to the number of vector chunks
1442 // that are processed by (possibly different) threads in the
1443 // parallelized for loop (i.e., they do not refer to individual
1444 // vector entries). The number of chunks here is calculated inside
1445 // TBBForFunctor. See also GitHub issue #2496 for further discussion
1446 // of this strategy.
1448 static_cast<size_type>(0),
1449 static_cast<size_type>(generic_functor.n_chunks),
1450 generic_functor,
1451 1,
1452 tbb_partitioner);
1453 partitioner->release_one_partitioner(tbb_partitioner);
1454 result = generic_functor.do_sum();
1455 }
1456 else
1457 accumulate_recursive(op, start, end, result);
1458#else
1459 accumulate_recursive(op, start, end, result);
1460 (void)partitioner;
1461#endif
1462 }
1463
1464
1465 template <typename Number, typename Number2, typename MemorySpace>
1467 {
1468 static void
1470 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1471 /*thread_loop_partitioner*/,
1472 const size_type /*size*/,
1473 const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1474 & /*v_data*/,
1476 {
1477 static_assert(
1478 std::is_same<MemorySpace, ::MemorySpace::Default>::value &&
1479 std::is_same<Number, Number2>::value,
1480 "For the Default MemorySpace Number and Number2 should be the same type");
1481 }
1482
1483 static void
1485 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1486 /*thread_loop_partitioner*/,
1487 const size_type /*size*/,
1488 const Number /*s*/,
1490 {}
1491
1492 static void
1494 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1495 /*thread_loop_partitioner*/,
1496 const size_type /*size*/,
1497 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1498 & /*v_data*/,
1500 {}
1501
1502 static void
1504 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1505 /*thread_loop_partitioner*/,
1506 const size_type /*size*/,
1507 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1508 & /*v_data*/,
1510 {}
1511
1512 static void
1514 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1515 /*thread_loop_partitioner*/,
1516 const size_type /*size*/,
1517 Number /*a*/,
1519 {}
1520
1521 static void
1523 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1524 /*thread_loop_partitioner*/,
1525 const size_type /*size*/,
1526 const Number /*a*/,
1527 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1528 & /*v_data*/,
1530 {}
1531
1532 static void
1534 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1535 /*thread_loop_partitioner*/,
1536 const size_type /*size*/,
1537 const Number /*a*/,
1538 const Number /*b*/,
1539 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1540 & /*v_data*/,
1541 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1542 & /*w_data*/,
1544 {}
1545
1546 static void
1548 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1549 /*thread_loop_partitioner*/,
1550 const size_type /*size*/,
1551 const Number /*x*/,
1552 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1553 & /*v_data*/,
1555 {}
1556
1557 static void
1559 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1560 /*thread_loop_partitioner*/,
1561 const size_type /*size*/,
1562 const Number /*x*/,
1563 const Number /*a*/,
1564 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1565 & /*v_data*/,
1567 {}
1568
1569 static void
1571 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1572 /*thread_loop_partitioner*/,
1573 const size_type /*size*/,
1574 const Number /*x*/,
1575 const Number /*a*/,
1576 const Number /*b*/,
1577 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1578 & /*v_data*/,
1579 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1580 & /*w_data*/,
1582 {}
1583
1584 static void
1586 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1587 /*thread_loop_partitioner*/,
1588 const size_type /*size*/,
1589 const Number /*factor*/,
1591 {}
1592
1593 static void
1595 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1596 /*thread_loop_partitioner*/,
1597 const size_type /*size*/,
1598 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1599 & /*v_data*/,
1601 {}
1602
1603 static void
1605 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1606 /*thread_loop_partitioner*/,
1607 const size_type /*size*/,
1608 const Number /*a*/,
1609 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1610 & /*v_data*/,
1612 {}
1613
1614 static void
1616 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1617 /*thread_loop_partitioner*/,
1618 const size_type /*size*/,
1619 const Number /*a*/,
1620 const Number /*b*/,
1621 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1622 & /*v_data*/,
1623 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1624 & /*w_data*/,
1626 {}
1627
1628 static Number
1630 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1631 /*thread_loop_partitioner*/,
1632 const size_type /*size*/,
1633 const ::MemorySpace::MemorySpaceData<Number2, MemorySpace>
1634 & /*v_data*/,
1636 {
1637 return Number();
1638 }
1639
1640 template <typename real_type>
1641 static void
1643 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1644 /*thread_loop_partitioner*/,
1645 const size_type /*size*/,
1646 real_type & /*sum*/,
1647 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1648 & /*v_data*/,
1650 {}
1651
1652 static Number
1654 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1655 /*thread_loop_partitioner*/,
1656 const size_type /*size*/,
1657 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1658 & /*data*/)
1659 {
1660 return Number();
1661 }
1662
1663 template <typename real_type>
1664 static void
1666 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1667 /*thread_loop_partitioner*/,
1668 const size_type /*size*/,
1669 real_type & /*sum*/,
1670 Number * /*values*/,
1671 Number * /*values*/)
1672 {}
1673
1674 template <typename real_type>
1675 static void
1677 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1678 /*thread_loop_partitioner*/,
1679 const size_type /*size*/,
1680 real_type & /*sum*/,
1681 real_type /*p*/,
1683 {}
1684
1685 static Number
1687 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1688 /*thread_loop_partitioner*/,
1689 const size_type /*size*/,
1690 const Number /*a*/,
1691 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1692 & /*v_data*/,
1693 const ::MemorySpace::MemorySpaceData<Number, MemorySpace>
1694 & /*w_data*/,
1696 {
1697 return Number();
1698 }
1699
1700 template <typename MemorySpace2>
1701 static void
1703 const std::shared_ptr<::parallel::internal::TBBPartitioner> &
1704 /*thread_loop_partitioner*/,
1705 const size_type /*size*/,
1706 VectorOperation::values /*operation*/,
1707 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
1708 & /*v_data*/,
1710 {}
1711 };
1712
1713
1714
1715 template <typename Number, typename Number2>
1716 struct functions<Number, Number2, ::MemorySpace::Host>
1717 {
1718 static void
1719 copy(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1720 & thread_loop_partitioner,
1721 const size_type size,
1722 const ::MemorySpace::
1723 MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1724 ::MemorySpace::MemorySpaceData<Number,
1725 ::MemorySpace::Host>
1726 &data)
1727 {
1728 Vector_copy<Number, Number2> copier(v_data.values.data(),
1729 data.values.data());
1730 parallel_for(copier, 0, size, thread_loop_partitioner);
1731 }
1732
1733 static void
1734 set(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1735 & thread_loop_partitioner,
1736 const size_type size,
1737 const Number s,
1740 &data)
1741 {
1742 Vector_set<Number> setter(s, data.values.data());
1743 parallel_for(setter, 0, size, thread_loop_partitioner);
1744 }
1745
1746 static void
1748 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1749 & thread_loop_partitioner,
1750 const size_type size,
1751 const ::MemorySpace::
1752 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1753 ::MemorySpace::MemorySpaceData<Number,
1754 ::MemorySpace::Host>
1755 &data)
1756 {
1757 Vectorization_add_v<Number> vector_add(data.values.data(),
1758 v_data.values.data());
1759 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1760 }
1761
1762 static void
1764 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1765 & thread_loop_partitioner,
1766 const size_type size,
1767 const ::MemorySpace::
1768 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1769 ::MemorySpace::MemorySpaceData<Number,
1770 ::MemorySpace::Host>
1771 &data)
1772 {
1773 Vectorization_subtract_v<Number> vector_subtract(data.values.data(),
1774 v_data.values.data());
1775 parallel_for(vector_subtract, 0, size, thread_loop_partitioner);
1776 }
1777
1778 static void
1780 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1781 & thread_loop_partitioner,
1782 const size_type size,
1783 Number a,
1786 &data)
1787 {
1788 Vectorization_add_factor<Number> vector_add(data.values.data(), a);
1789 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1790 }
1791
1792 static void
1793 add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1794 & thread_loop_partitioner,
1795 const size_type size,
1796 const Number a,
1797 const ::MemorySpace::
1798 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1799 ::MemorySpace::MemorySpaceData<Number,
1800 ::MemorySpace::Host>
1801 &data)
1802 {
1803 Vectorization_add_av<Number> vector_add(data.values.data(),
1804 v_data.values.data(),
1805 a);
1806 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1807 }
1808
1809 static void
1811 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1812 & thread_loop_partitioner,
1813 const size_type size,
1814 const Number a,
1815 const Number b,
1816 const ::MemorySpace::
1817 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1818 const ::MemorySpace::
1819 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1820 ::MemorySpace::MemorySpaceData<Number,
1821 ::MemorySpace::Host>
1822 &data)
1823 {
1825 data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1826 parallel_for(vector_add, 0, size, thread_loop_partitioner);
1827 }
1828
1829 static void
1831 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1832 & thread_loop_partitioner,
1833 const size_type size,
1834 const Number x,
1835 const ::MemorySpace::
1836 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1837 ::MemorySpace::MemorySpaceData<Number,
1838 ::MemorySpace::Host>
1839 &data)
1840 {
1841 Vectorization_sadd_xv<Number> vector_sadd(data.values.data(),
1842 v_data.values.data(),
1843 x);
1844 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1845 }
1846
1847 static void
1849 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1850 & thread_loop_partitioner,
1851 const size_type size,
1852 const Number x,
1853 const Number a,
1854 const ::MemorySpace::
1855 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1856 ::MemorySpace::MemorySpaceData<Number,
1857 ::MemorySpace::Host>
1858 &data)
1859 {
1860 Vectorization_sadd_xav<Number> vector_sadd(data.values.data(),
1861 v_data.values.data(),
1862 a,
1863 x);
1864 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1865 }
1866
1867 static void
1869 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1870 & thread_loop_partitioner,
1871 const size_type size,
1872 const Number x,
1873 const Number a,
1874 const Number b,
1875 const ::MemorySpace::
1876 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1877 const ::MemorySpace::
1878 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1879 ::MemorySpace::MemorySpaceData<Number,
1880 ::MemorySpace::Host>
1881 &data)
1882 {
1883 Vectorization_sadd_xavbw<Number> vector_sadd(data.values.data(),
1884 v_data.values.data(),
1885 w_data.values.data(),
1886 x,
1887 a,
1888 b);
1889 parallel_for(vector_sadd, 0, size, thread_loop_partitioner);
1890 }
1891
1892 static void
1894 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1895 & thread_loop_partitioner,
1896 const size_type size,
1897 const Number factor,
1900 &data)
1901 {
1903 data.values.data(), factor);
1904 parallel_for(vector_multiply, 0, size, thread_loop_partitioner);
1905 }
1906
1907 static void
1908 scale(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1909 & thread_loop_partitioner,
1910 const size_type size,
1911 const ::MemorySpace::
1912 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1913 ::MemorySpace::MemorySpaceData<Number,
1914 ::MemorySpace::Host>
1915 &data)
1916 {
1917 Vectorization_scale<Number> vector_scale(data.values.data(),
1918 v_data.values.data());
1919 parallel_for(vector_scale, 0, size, thread_loop_partitioner);
1920 }
1921
1922 static void
1923 equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1924 & thread_loop_partitioner,
1925 const size_type size,
1926 const Number a,
1927 const ::MemorySpace::
1928 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1929 ::MemorySpace::MemorySpaceData<Number,
1930 ::MemorySpace::Host>
1931 &data)
1932 {
1933 Vectorization_equ_au<Number> vector_equ(data.values.data(),
1934 v_data.values.data(),
1935 a);
1936 parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1937 }
1938
1939 static void
1941 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1942 & thread_loop_partitioner,
1943 const size_type size,
1944 const Number a,
1945 const Number b,
1946 const ::MemorySpace::
1947 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
1948 const ::MemorySpace::
1949 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
1950 ::MemorySpace::MemorySpaceData<Number,
1951 ::MemorySpace::Host>
1952 &data)
1953 {
1955 data.values.data(), v_data.values.data(), w_data.values.data(), a, b);
1956 parallel_for(vector_equ, 0, size, thread_loop_partitioner);
1957 }
1958
1959 static Number
1960 dot(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1961 & thread_loop_partitioner,
1962 const size_type size,
1963 const ::MemorySpace::
1964 MemorySpaceData<Number2, ::MemorySpace::Host> &v_data,
1965 ::MemorySpace::MemorySpaceData<Number,
1966 ::MemorySpace::Host>
1967 &data)
1968 {
1969 Number sum;
1971 data.values.data(), v_data.values.data());
1973 dot, 0, size, sum, thread_loop_partitioner);
1974 AssertIsFinite(sum);
1975
1976 return sum;
1977 }
1978
1979 template <typename real_type>
1980 static void
1981 norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
1982 & thread_loop_partitioner,
1983 const size_type size,
1984 real_type & sum,
1987 &data)
1988 {
1989 Norm2<Number, real_type> norm2(data.values.data());
1990 parallel_reduce(norm2, 0, size, sum, thread_loop_partitioner);
1991 }
1992
1993 static Number
1995 const std::shared_ptr<::parallel::internal::TBBPartitioner>
1996 & thread_loop_partitioner,
1997 const size_type size,
1998 const ::MemorySpace::
1999 MemorySpaceData<Number, ::MemorySpace::Host> &data)
2000 {
2001 Number sum;
2002 MeanValue<Number> mean(data.values.data());
2003 parallel_reduce(mean, 0, size, sum, thread_loop_partitioner);
2004
2005 return sum;
2006 }
2007
2008 template <typename real_type>
2009 static void
2010 norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2011 & thread_loop_partitioner,
2012 const size_type size,
2013 real_type & sum,
2016 &data)
2017 {
2018 Norm1<Number, real_type> norm1(data.values.data());
2019 parallel_reduce(norm1, 0, size, sum, thread_loop_partitioner);
2020 }
2021
2022 template <typename real_type>
2023 static void
2024 norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2025 & thread_loop_partitioner,
2026 const size_type size,
2027 real_type & sum,
2028 const real_type p,
2031 &data)
2032 {
2033 NormP<Number, real_type> normp(data.values.data(), p);
2034 parallel_reduce(normp, 0, size, sum, thread_loop_partitioner);
2035 }
2036
2037 static Number
2039 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2040 & thread_loop_partitioner,
2041 const size_type size,
2042 const Number a,
2043 const ::MemorySpace::
2044 MemorySpaceData<Number, ::MemorySpace::Host> &v_data,
2045 const ::MemorySpace::
2046 MemorySpaceData<Number, ::MemorySpace::Host> &w_data,
2047 ::MemorySpace::MemorySpaceData<Number,
2048 ::MemorySpace::Host>
2049 &data)
2050 {
2051 Number sum;
2052 AddAndDot<Number> adder(data.values.data(),
2053 v_data.values.data(),
2054 w_data.values.data(),
2055 a);
2056 parallel_reduce(adder, 0, size, sum, thread_loop_partitioner);
2057
2058 return sum;
2059 }
2060
2061 template <typename MemorySpace2>
2062 static void
2064 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2065 & thread_loop_partitioner,
2066 const size_type size,
2067 VectorOperation::values operation,
2068 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2069 &v_data,
2072 &data,
2073 std::enable_if_t<
2074 std::is_same<MemorySpace2, ::MemorySpace::Host>::value,
2075 int> = 0)
2076 {
2077 if (operation == VectorOperation::insert)
2078 {
2079 copy(thread_loop_partitioner, size, v_data, data);
2080 }
2081 else if (operation == VectorOperation::add)
2082 {
2083 add_vector(thread_loop_partitioner, size, v_data, data);
2084 }
2085 else
2086 {
2088 }
2089 }
2090
2091 template <typename MemorySpace2>
2092 static void
2094 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2095 & /*thread_loop_partitioner*/,
2096 const size_type size,
2097 VectorOperation::values operation,
2098 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2099 &v_data,
2102 &data,
2103 std::enable_if_t<
2104 std::is_same<MemorySpace2, ::MemorySpace::Default>::value,
2105 int> = 0)
2106 {
2107 if (operation == VectorOperation::insert)
2108 {
2109 Kokkos::deep_copy(
2110 Kokkos::subview(data.values,
2111 Kokkos::pair<size_type, size_type>(0, size)),
2112 Kokkos::subview(v_data.values,
2113 Kokkos::pair<size_type, size_type>(0, size)));
2114 }
2115 else
2116 {
2118 }
2119 }
2120 };
2121
2122
2123
2124 template <typename Number>
2125 struct functions<Number, Number, ::MemorySpace::Default>
2126 {
2127 static void
2129 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2130 const size_type size,
2131 const ::MemorySpace::
2132 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2133 ::MemorySpace::MemorySpaceData<Number,
2134 ::MemorySpace::Default>
2135 &data)
2136 {
2137 Kokkos::deep_copy(
2138 Kokkos::subview(data.values,
2139 Kokkos::pair<size_type, size_type>(0, size)),
2140 Kokkos::subview(v_data.values,
2141 Kokkos::pair<size_type, size_type>(0, size)));
2142 }
2143
2144 static void
2145 set(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2146 const size_type size,
2147 const Number s,
2150 &data)
2151 {
2152 Kokkos::deep_copy(
2153 Kokkos::subview(data.values,
2154 Kokkos::pair<size_type, size_type>(0, size)),
2155 s);
2156 }
2157
2158 static void
2160 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2161 const size_type size,
2162 const ::MemorySpace::
2163 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2164 ::MemorySpace::MemorySpaceData<Number,
2165 ::MemorySpace::Default>
2166 &data)
2167 {
2168 auto exec = typename ::MemorySpace::Default::kokkos_space::
2169 execution_space{};
2170 Kokkos::parallel_for(
2171 "::add_vector",
2172 Kokkos::RangePolicy<
2173 ::MemorySpace::Default::kokkos_space::execution_space>(
2174 exec, 0, size),
2175 KOKKOS_LAMBDA(int i) { data.values(i) += v_data.values(i); });
2176 exec.fence();
2177 }
2178
2179 static void
2181 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2182 const size_type size,
2183 const ::MemorySpace::
2184 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2185 ::MemorySpace::MemorySpaceData<Number,
2186 ::MemorySpace::Default>
2187 &data)
2188 {
2189 auto exec = typename ::MemorySpace::Default::kokkos_space::
2190 execution_space{};
2191 Kokkos::parallel_for(
2192 "::subtract_vector",
2193 Kokkos::RangePolicy<
2194 ::MemorySpace::Default::kokkos_space::execution_space>(
2195 exec, 0, size),
2196 KOKKOS_LAMBDA(size_type i) { data.values(i) -= v_data.values(i); });
2197 exec.fence();
2198 }
2199
2200 static void
2202 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2203 const size_type size,
2204 Number a,
2207 &data)
2208 {
2209 auto exec = typename ::MemorySpace::Default::kokkos_space::
2210 execution_space{};
2211 Kokkos::parallel_for(
2212 "::add_factor",
2213 Kokkos::RangePolicy<
2214 ::MemorySpace::Default::kokkos_space::execution_space>(
2215 exec, 0, size),
2216 KOKKOS_LAMBDA(size_type i) { data.values(i) += a; });
2217 exec.fence();
2218 }
2219
2220 static void
2222 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2223 const size_type size,
2224 const Number a,
2225 const ::MemorySpace::
2226 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2227 ::MemorySpace::MemorySpaceData<Number,
2228 ::MemorySpace::Default>
2229 &data)
2230 {
2231 auto exec = typename ::MemorySpace::Default::kokkos_space::
2232 execution_space{};
2233 Kokkos::parallel_for(
2234 "::add_av",
2235 Kokkos::RangePolicy<
2236 ::MemorySpace::Default::kokkos_space::execution_space>(
2237 exec, 0, size),
2238 KOKKOS_LAMBDA(size_type i) {
2239 data.values(i) += a * v_data.values(i);
2240 });
2241 exec.fence();
2242 }
2243
2244 static void
2246 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2247 const size_type size,
2248 const Number a,
2249 const Number b,
2250 const ::MemorySpace::
2251 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2252 const ::MemorySpace::
2253 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2254 ::MemorySpace::MemorySpaceData<Number,
2255 ::MemorySpace::Default>
2256 &data)
2257 {
2258 auto exec = typename ::MemorySpace::Default::kokkos_space::
2259 execution_space{};
2260 Kokkos::parallel_for(
2261 "::add_avpbw",
2262 Kokkos::RangePolicy<
2263 ::MemorySpace::Default::kokkos_space::execution_space>(
2264 exec, 0, size),
2265 KOKKOS_LAMBDA(size_type i) {
2266 data.values(i) += a * v_data.values(i) + b * w_data.values(i);
2267 });
2268 exec.fence();
2269 }
2270
2271 static void
2273 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2274 const size_type size,
2275 const Number x,
2276 const ::MemorySpace::
2277 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2278 ::MemorySpace::MemorySpaceData<Number,
2279 ::MemorySpace::Default>
2280 &data)
2281 {
2282 auto exec = typename ::MemorySpace::Default::kokkos_space::
2283 execution_space{};
2284 Kokkos::parallel_for(
2285 "::sadd_xv",
2286 Kokkos::RangePolicy<
2287 ::MemorySpace::Default::kokkos_space::execution_space>(
2288 exec, 0, size),
2289 KOKKOS_LAMBDA(size_type i) {
2290 data.values(i) = x * data.values(i) + v_data.values(i);
2291 });
2292 exec.fence();
2293 }
2294
2295 static void
2297 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2298 const size_type size,
2299 const Number x,
2300 const Number a,
2301 const ::MemorySpace::
2302 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2303 ::MemorySpace::MemorySpaceData<Number,
2304 ::MemorySpace::Default>
2305 &data)
2306 {
2307 auto exec = typename ::MemorySpace::Default::kokkos_space::
2308 execution_space{};
2309 Kokkos::parallel_for(
2310 "::sadd_xav",
2311 Kokkos::RangePolicy<
2312 ::MemorySpace::Default::kokkos_space::execution_space>(
2313 exec, 0, size),
2314 KOKKOS_LAMBDA(size_type i) {
2315 data.values(i) = x * data.values(i) + a * v_data.values(i);
2316 });
2317 exec.fence();
2318 }
2319
2320 static void
2322 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2323 const size_type size,
2324 const Number x,
2325 const Number a,
2326 const Number b,
2327 const ::MemorySpace::
2328 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2329 const ::MemorySpace::
2330 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2331 ::MemorySpace::MemorySpaceData<Number,
2332 ::MemorySpace::Default>
2333 &data)
2334 {
2335 auto exec = typename ::MemorySpace::Default::kokkos_space::
2336 execution_space{};
2337 Kokkos::parallel_for(
2338 "::sadd_xavbw",
2339 Kokkos::RangePolicy<
2340 ::MemorySpace::Default::kokkos_space::execution_space>(
2341 exec, 0, size),
2342 KOKKOS_LAMBDA(size_type i) {
2343 data.values(i) =
2344 x * data.values(i) + a * v_data.values(i) + b * w_data.values(i);
2345 });
2346 exec.fence();
2347 }
2348
2349 static void
2351 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2352 const size_type size,
2353 const Number factor,
2356 &data)
2357 {
2358 auto exec = typename ::MemorySpace::Default::kokkos_space::
2359 execution_space{};
2360 Kokkos::parallel_for(
2361 "::multiply_factor",
2362 Kokkos::RangePolicy<
2363 ::MemorySpace::Default::kokkos_space::execution_space>(
2364 exec, 0, size),
2365 KOKKOS_LAMBDA(size_type i) { data.values(i) *= factor; });
2366 exec.fence();
2367 }
2368
2369 static void
2371 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2372 const size_type size,
2373 const ::MemorySpace::
2374 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2375 ::MemorySpace::MemorySpaceData<Number,
2376 ::MemorySpace::Default>
2377 &data)
2378 {
2379 auto exec = typename ::MemorySpace::Default::kokkos_space::
2380 execution_space{};
2381 Kokkos::parallel_for(
2382 "::scale",
2383 Kokkos::RangePolicy<
2384 ::MemorySpace::Default::kokkos_space::execution_space>(
2385 exec, 0, size),
2386 KOKKOS_LAMBDA(size_type i) { data.values(i) *= v_data.values(i); });
2387 exec.fence();
2388 }
2389
2390 static void
2392 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2393 const size_type size,
2394 const Number a,
2395 const ::MemorySpace::
2396 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2397 ::MemorySpace::MemorySpaceData<Number,
2398 ::MemorySpace::Default>
2399 &data)
2400 {
2401 auto exec = typename ::MemorySpace::Default::kokkos_space::
2402 execution_space{};
2403 Kokkos::parallel_for(
2404 "::equ_au",
2405 Kokkos::RangePolicy<
2406 ::MemorySpace::Default::kokkos_space::execution_space>(
2407 exec, 0, size),
2408 KOKKOS_LAMBDA(size_type i) {
2409 data.values(i) = a * v_data.values(i);
2410 });
2411 exec.fence();
2412 }
2413
2414 static void
2416 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2417 const size_type size,
2418 const Number a,
2419 const Number b,
2420 const ::MemorySpace::
2421 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2422 const ::MemorySpace::
2423 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2424 ::MemorySpace::MemorySpaceData<Number,
2425 ::MemorySpace::Default>
2426 &data)
2427 {
2428 auto exec = typename ::MemorySpace::Default::kokkos_space::
2429 execution_space{};
2430 Kokkos::parallel_for(
2431 "::equ_aubv",
2432 Kokkos::RangePolicy<
2433 ::MemorySpace::Default::kokkos_space::execution_space>(
2434 exec, 0, size),
2435 KOKKOS_LAMBDA(size_type i) {
2436 data.values(i) = a * v_data.values(i) + b * w_data.values(i);
2437 });
2438 exec.fence();
2439 }
2440
2441 static Number
2442 dot(const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2443 const size_type size,
2444 const ::MemorySpace::
2445 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2446 ::MemorySpace::MemorySpaceData<Number,
2447 ::MemorySpace::Default>
2448 &data)
2449 {
2450 Number result;
2451
2452 auto exec = typename ::MemorySpace::Default::kokkos_space::
2453 execution_space{};
2454 Kokkos::parallel_reduce(
2455 "::dot",
2456 Kokkos::RangePolicy<
2457 ::MemorySpace::Default::kokkos_space::execution_space>(
2458 exec, 0, size),
2459 KOKKOS_LAMBDA(size_type i, Number & update) {
2460 update += data.values(i) * v_data.values(i);
2461 },
2462 result);
2463
2464 AssertIsFinite(result);
2465 return result;
2466 }
2467
2468 template <typename real_type>
2469 static void
2470 norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner>
2471 & thread_loop_partitioner,
2472 const size_type size,
2473 real_type & sum,
2474 ::MemorySpace::
2475 MemorySpaceData<Number, ::MemorySpace::Default> &data)
2476 {
2477 sum = dot(thread_loop_partitioner, size, data, data);
2478 }
2479
2480 static Number
2482 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2483 const size_type size,
2484 const ::MemorySpace::
2485 MemorySpaceData<Number, ::MemorySpace::Default> &data)
2486 {
2487 Number result;
2488
2489 auto exec = typename ::MemorySpace::Default::kokkos_space::
2490 execution_space{};
2491 Kokkos::parallel_reduce(
2492 "::mean_value",
2493 Kokkos::RangePolicy<
2494 ::MemorySpace::Default::kokkos_space::execution_space>(
2495 exec, 0, size),
2496 KOKKOS_LAMBDA(size_type i, Number & update) {
2497 update += data.values(i);
2498 },
2499 result);
2500
2501 AssertIsFinite(result);
2502 return result;
2503 }
2504
2505 template <typename real_type>
2506 static void
2508 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2509 const size_type size,
2510 real_type & sum,
2513 &data)
2514 {
2515 auto exec = typename ::MemorySpace::Default::kokkos_space::
2516 execution_space{};
2517 Kokkos::parallel_reduce(
2518 "::norm_1",
2519 Kokkos::RangePolicy<
2520 ::MemorySpace::Default::kokkos_space::execution_space>(
2521 exec, 0, size),
2522 KOKKOS_LAMBDA(size_type i, Number & update) {
2523#if KOKKOS_VERSION < 30400
2524 update += std::abs(data.values(i));
2525#elif KOKKOS_VERSION < 30700
2526 update += Kokkos::Experimental::fabs(data.values(i));
2527#else
2528 update += Kokkos::abs(data.values(i));
2529#endif
2530 },
2531 sum);
2532 }
2533
2534 template <typename real_type>
2535 static void
2537 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2538 const size_type size,
2539 real_type & sum,
2540 real_type exp,
2543 &data)
2544 {
2545 auto exec = typename ::MemorySpace::Default::kokkos_space::
2546 execution_space{};
2547 Kokkos::parallel_reduce(
2548 "::norm_p",
2549 Kokkos::RangePolicy<
2550 ::MemorySpace::Default::kokkos_space::execution_space>(
2551 exec, 0, size),
2552 KOKKOS_LAMBDA(size_type i, Number & update) {
2553#if KOKKOS_VERSION < 30400
2554 update += std::pow(fabs(data.values(i)), exp);
2555#elif KOKKOS_VERSION < 30700
2556 update += Kokkos::Experimental::pow(
2557 Kokkos::Experimental::fabs(data.values(i)), exp);
2558#else
2559 update += Kokkos::pow(Kokkos::abs(data.values(i)), exp);
2560#endif
2561 },
2562 sum);
2563 }
2564
2565 static Number
2567 const std::shared_ptr<::parallel::internal::TBBPartitioner> &,
2568 const size_type size,
2569 const Number a,
2570 const ::MemorySpace::
2571 MemorySpaceData<Number, ::MemorySpace::Default> &v_data,
2572 const ::MemorySpace::
2573 MemorySpaceData<Number, ::MemorySpace::Default> &w_data,
2574 ::MemorySpace::MemorySpaceData<Number,
2575 ::MemorySpace::Default>
2576 &data)
2577 {
2578 Number res;
2579
2580 auto exec = typename ::MemorySpace::Default::kokkos_space::
2581 execution_space{};
2582 Kokkos::parallel_reduce(
2583 "::add_and_dot",
2584 Kokkos::RangePolicy<
2585 ::MemorySpace::Default::kokkos_space::execution_space>(
2586 exec, 0, size),
2587 KOKKOS_LAMBDA(size_type i, Number & update) {
2588 data.values(i) += a * v_data.values(i);
2589 update +=
2590 data.values(i) * Number(numbers::NumberTraits<Number>::conjugate(
2591 w_data.values(i)));
2592 },
2593 res);
2594
2595 return res;
2596 }
2597
2598 template <typename MemorySpace2>
2599 static void
2601 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2602 & thread_loop_partitioner,
2603 const size_type size,
2604 VectorOperation::values operation,
2605 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2606 &v_data,
2609 &data,
2610 std::enable_if_t<
2611 std::is_same<MemorySpace2, ::MemorySpace::Default>::value,
2612 int> = 0)
2613 {
2614 if (operation == VectorOperation::insert)
2615 {
2616 copy(thread_loop_partitioner, size, v_data, data);
2617 }
2618 else if (operation == VectorOperation::add)
2619 {
2620 add_vector(thread_loop_partitioner, size, v_data, data);
2621 }
2622 else
2623 {
2625 }
2626 }
2627
2628 template <typename MemorySpace2>
2629 static void
2631 const std::shared_ptr<::parallel::internal::TBBPartitioner>
2632 & /*thread_loop_partitioner*/,
2633 const size_type size,
2634 VectorOperation::values operation,
2635 const ::MemorySpace::MemorySpaceData<Number, MemorySpace2>
2636 &v_data,
2639 &data,
2640 std::enable_if_t<
2641 std::is_same<MemorySpace2, ::MemorySpace::Host>::value,
2642 int> = 0)
2643 {
2644 if (operation == VectorOperation::insert)
2645 {
2646 Kokkos::deep_copy(
2647 Kokkos::subview(data.values,
2648 Kokkos::pair<size_type, size_type>(0, size)),
2649 Kokkos::subview(v_data.values,
2650 Kokkos::pair<size_type, size_type>(0, size)));
2651 }
2652 else
2653 {
2655 }
2656 }
2657 };
2658 } // namespace VectorOperations
2659} // namespace internal
2660
2662
2663#endif
static unsigned int n_threads()
void store(OtherNumber *ptr) const
void load(const OtherNumber *ptr)
#define DEAL_II_CONSTEXPR_IN_CONDITIONAL
Definition config.h:572
#define DEAL_II_ALWAYS_INLINE
Definition config.h:106
#define DEAL_II_OPENMP_SIMD_PRAGMA
Definition config.h:138
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:472
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:473
#define DEAL_II_FALLTHROUGH
Definition config.h:184
Point< 2 > first
Definition grid_out.cc:4615
static ::ExceptionBase & ExcNotImplemented()
#define Assert(cond, exc)
#define AssertIsFinite(number)
#define AssertDimension(dim1, dim2)
#define AssertIndexRange(index, range)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
#define AssertThrow(cond, exc)
unsigned int minimum_parallel_grain_size
Definition parallel.cc:34
void accumulate_recursive(const Operation &op, const size_type first, const size_type last, ResultType &result)
void parallel_reduce(const Operation &op, const size_type start, const size_type end, ResultType &result, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
void copy(const T *begin, const T *end, U *dest)
void parallel_for(Functor &functor, const size_type start, const size_type end, const std::shared_ptr<::parallel::internal::TBBPartitioner > &partitioner)
const unsigned int vector_accumulation_recursion_threshold
size_type do_accumulate(const Operation op, const size_type vec_size, const size_type start_index, ResultType *outer_results, std::integral_constant< bool, false >)
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
Definition parallel.h:82
::VectorizedArray< Number, width > min(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
::VectorizedArray< Number, width > pow(const ::VectorizedArray< Number, width > &, const Number p)
::VectorizedArray< Number, width > abs(const ::VectorizedArray< Number, width > &)
unsigned int global_dof_index
Definition types.h:82
AddAndDot(Number *const X, const Number *const V, const Number *const W, const Number a)
VectorizedArray< Number > do_vectorized(const size_type i) const
Dot(const Number *const X, const Number2 *const Y)
Number operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
RealType operator()(const size_type i) const
VectorizedArray< Number > do_vectorized(const size_type i) const
void operator()(const tbb::blocked_range< size_type > &range) const
TBBForFunctor(Functor &functor, const size_type start, const size_type end)
TBBReduceFunctor(const Operation &op, const size_type start, const size_type end)
void operator()(const tbb::blocked_range< size_type > &range) const
Vector_copy(const OtherNumber *const src, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vector_set(const Number value, Number *const dst)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_av(Number *const val, const Number *const v_val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_avpbw(Number *const val, const Number *const v_val, const Number *const w_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_add_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
Vectorization_add_v(Number *const val, const Number *const v_val)
Vectorization_equ_au(Number *const val, const Number *const u_val, const Number a)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubv(Number *const val, const Number *const u_val, const Number *const v_val, const Number a, const Number b)
void operator()(const size_type begin, const size_type end) const
Vectorization_equ_aubvcw(Number *val, const Number *u_val, const Number *v_val, const Number *w_val, const Number a, const Number b, const Number c)
void operator()(const size_type begin, const size_type end) const
Vectorization_multiply_factor(Number *const val, const Number factor)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_ratio(Number *val, const Number *a_val, const Number *b_val)
Vectorization_sadd_xav(Number *val, const Number *const v_val, const Number a, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_sadd_xavbw(Number *val, const Number *v_val, const Number *w_val, Number x, Number a, Number b)
Vectorization_sadd_xv(Number *const val, const Number *const v_val, const Number x)
void operator()(const size_type begin, const size_type end) const
void operator()(const size_type begin, const size_type end) const
Vectorization_scale(Number *const val, const Number *const v_val)
Vectorization_subtract_v(Number *val, const Number *const v_val)
void operator()(const size_type begin, const size_type end) const
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, const real_type p, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Host >::value, int >=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Default >::value, int >=0)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const ::MemorySpace::MemorySpaceData< Number2, ::MemorySpace::Host > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Host > &data)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, real_type exp, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, Number a, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Default >::value, int >=0)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number s, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &thread_loop_partitioner, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, VectorOperation::values operation, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data, std::enable_if_t< std::is_same< MemorySpace2, ::MemorySpace::Host >::value, int >=0)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number x, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number factor, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const Number b, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &w_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, real_type &sum, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type size, const Number a, const ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &v_data, ::MemorySpace::MemorySpaceData< Number, ::MemorySpace::Default > &data)
static Number mean_value(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_au(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_avpbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_2(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void add_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void scale(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void subtract_vector(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xavbw(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static Number add_and_dot(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void sadd_xav(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void import_elements(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, VectorOperation::values, const ::MemorySpace::MemorySpaceData< Number, MemorySpace2 > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void copy(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const ::MemorySpace::MemorySpaceData< Number2, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_1(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, Number *, Number *)
static void add_av(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void multiply_factor(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void norm_p(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, real_type &, real_type, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void equ_aubv(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, const Number, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, const ::MemorySpace::MemorySpaceData< Number, MemorySpace > &, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static void set(const std::shared_ptr<::parallel::internal::TBBPartitioner > &, const size_type, const Number, ::MemorySpace::MemorySpaceData< Number, MemorySpace > &)
static real_type abs(const number &x)
Definition numbers.h:593
static constexpr real_type abs_square(const number &x)
Definition numbers.h:584