Reference documentation for deal.II version GIT 58febcd5cf 2023-09-30 20:00:01+00:00
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
work_stream.h
Go to the documentation of this file.
1 // ---------------------------------------------------------------------
2 //
3 // Copyright (C) 2008 - 2023 by the deal.II authors
4 //
5 // This file is part of the deal.II library.
6 //
7 // The deal.II library is free software; you can use it, redistribute
8 // it, and/or modify it under the terms of the GNU Lesser General
9 // Public License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
11 // The full text of the license can be found in the file LICENSE.md at
12 // the top level directory of deal.II.
13 //
14 // ---------------------------------------------------------------------
15 
16 #ifndef dealii_work_stream_h
17 # define dealii_work_stream_h
18 
19 
20 # include <deal.II/base/config.h>
21 
25 # include <deal.II/base/parallel.h>
29 
30 # ifdef DEAL_II_WITH_TBB
31 # ifdef DEAL_II_TBB_WITH_ONEAPI
32 # include <tbb/parallel_pipeline.h>
33 # else
34 # include <tbb/pipeline.h>
35 # endif
36 # endif
37 
38 # include <functional>
39 # include <iterator>
40 # include <memory>
41 # include <utility>
42 # include <vector>
43 
45 
46 
47 
160 namespace WorkStream
161 {
166  namespace internal
167  {
168 # ifdef DEAL_II_WITH_TBB
183  namespace tbb_no_coloring
184  {
188  template <typename Iterator, typename ScratchData, typename CopyData>
190  {
191  public:
198  struct ItemType
199  {
206  {
207  std::unique_ptr<ScratchData> scratch_data;
209 
214  : currently_in_use(false)
215  {}
216 
217  ScratchDataObject(ScratchData *p, const bool in_use)
218  : scratch_data(p)
219  , currently_in_use(in_use)
220  {}
221 
222  // Provide a copy constructor that actually doesn't copy the
223  // internal state. This makes handling ScratchAndCopyDataObjects
224  // easier to handle with STL containers.
226  : currently_in_use(false)
227  {}
228 
229  ScratchDataObject(ScratchDataObject &&o) noexcept = default;
230  };
231 
232 
237  using ScratchDataList = std::list<ScratchDataObject>;
238 
243  std::vector<Iterator> iterators;
244 
250  std::vector<CopyData> copy_datas;
251 
257  unsigned int n_iterators;
258 
291 
296  const ScratchData *sample_scratch_data;
297 
303 
304 
310  : n_iterators(0)
311  , scratch_data(nullptr)
312  , sample_scratch_data(nullptr)
313  , currently_in_use(false)
314  {}
315  };
316 
317 
324  const Iterator &end,
325  const unsigned int buffer_size,
326  const unsigned int chunk_size,
327  const ScratchData &sample_scratch_data,
328  const CopyData &sample_copy_data)
330  , item_buffer(buffer_size)
333  {
334  // initialize the elements of the ring buffer
335  for (auto &item : item_buffer)
336  {
337  Assert(item.n_iterators == 0, ExcInternalError());
338 
339  item.iterators.resize(chunk_size,
340  remaining_iterator_range.second);
341  item.scratch_data = &thread_local_scratch;
342  item.sample_scratch_data = &sample_scratch_data;
343  item.copy_datas.resize(chunk_size, sample_copy_data);
344  item.currently_in_use = false;
345  }
346  }
347 
348 
352  ItemType *
354  {
355  // find first unused item. we know that there must be one
356  // because we have set the maximal number of tokens in flight
357  // and have set the ring buffer to have exactly this size. so
358  // if this function is called, we know that less than the
359  // maximal number of items in currently in flight
360  //
361  // note that we need not lock access to this array since
362  // the current stage is run sequentially and we can therefore
363  // enter the following block only once at any given time.
364  // thus, there can be no race condition between checking that
365  // a flag is false and setting it to true. (there may be
366  // another thread where we release items and set 'false'
367  // flags to 'true', but that too does not produce any
368  // problems)
369  ItemType *current_item = nullptr;
370  for (unsigned int i = 0; i < item_buffer.size(); ++i)
371  if (item_buffer[i].currently_in_use == false)
372  {
373  item_buffer[i].currently_in_use = true;
374  current_item = &item_buffer[i];
375  break;
376  }
377  Assert(current_item != nullptr,
378  ExcMessage("This can't be. There must be a free item!"));
379 
380  // initialize the next item. it may
381  // consist of at most chunk_size
382  // elements
383  current_item->n_iterators = 0;
384  while ((remaining_iterator_range.first !=
385  remaining_iterator_range.second) &&
386  (current_item->n_iterators < chunk_size))
387  {
388  current_item->iterators[current_item->n_iterators] =
390 
391  ++remaining_iterator_range.first;
392  ++current_item->n_iterators;
393  }
394 
395  if (current_item->n_iterators == 0)
396  // there were no items
397  // left. terminate the pipeline
398  return nullptr;
399  else
400  return current_item;
401  }
402 
403  private:
408  std::pair<Iterator, Iterator> remaining_iterator_range;
409 
413  std::vector<ItemType> item_buffer;
414 
447 
453  const ScratchData &sample_scratch_data;
454 
461  const unsigned int chunk_size;
462  };
463 
464 
465 
466  template <typename Worker,
467  typename Copier,
468  typename Iterator,
469  typename ScratchData,
470  typename CopyData>
471  void
472  run(const Iterator &begin,
474  Worker worker,
475  Copier copier,
476  const ScratchData &sample_scratch_data,
477  const CopyData &sample_copy_data,
478  const unsigned int queue_length,
479  const unsigned int chunk_size)
480  {
481  using ItemType = typename IteratorRangeToItemStream<Iterator,
482  ScratchData,
483  CopyData>::ItemType;
484 
485  // Create the three stages of the pipeline:
486 
487  //
488  // ----- Stage 1 -----
489  //
490  // The first stage is the one that provides us with chunks of data
491  // to work on (the stream of "items"). This stage runs sequentially.
493  iterator_range_to_item_stream(begin,
494  end,
495  queue_length,
496  chunk_size,
497  sample_scratch_data,
498  sample_copy_data);
499  auto tbb_item_stream_filter = tbb::make_filter<void, ItemType *>(
500 # ifdef DEAL_II_TBB_WITH_ONEAPI
501  tbb::filter_mode::serial_in_order,
502 # else
504 # endif
505  [&](tbb::flow_control &fc) -> ItemType * {
506  if (const auto item = iterator_range_to_item_stream.get_item())
507  return item;
508  else
509  {
510  fc.stop();
511  return nullptr;
512  }
513  });
514 
515  //
516  // ----- Stage 2 -----
517  //
518  // The second stage is the one that does the actual work. This is the
519  // stage that runs in parallel
520  auto tbb_worker_filter = tbb::make_filter<ItemType *, ItemType *>(
521 # ifdef DEAL_II_TBB_WITH_ONEAPI
522  tbb::filter_mode::parallel,
523 # else
524  tbb::filter::parallel,
525 # endif
526  [worker =
527  std::function<void(const Iterator &, ScratchData &, CopyData &)>(
528  worker),
529  copier_exists =
530  static_cast<bool>(std::function<void(const CopyData &)>(copier))](
531  ItemType *current_item) {
532  // we need to find an unused scratch data object in the list that
533  // corresponds to the current thread and then mark it as used. if
534  // we can't find one, create one
535  //
536  // as discussed in the discussion of the documentation of the
537  // IteratorRangeToItemStream::scratch_data variable, there is no
538  // need to synchronize access to this variable using a mutex
539  // as long as we have no yield-point in between. this means that
540  // we can't take an iterator into the list now and expect it to
541  // still be valid after calling the worker, but we at least do
542  // not have to lock the following section
543  ScratchData *scratch_data = nullptr;
544  {
545  // see if there is an unused object. if so, grab it and mark
546  // it as used
547  for (auto &p : current_item->scratch_data->get())
548  if (p.currently_in_use == false)
549  {
550  scratch_data = p.scratch_data.get();
551  p.currently_in_use = true;
552 
553  break;
554  }
555 
556  // if no object was found, create one and mark it as used
557  if (scratch_data == nullptr)
558  {
559  scratch_data =
560  new ScratchData(*current_item->sample_scratch_data);
561  current_item->scratch_data->get().emplace_back(scratch_data,
562  true);
563  }
564  }
565 
566  // then call the worker function on each element of the chunk we
567  // were given. since these worker functions are called on separate
568  // threads, nothing good can happen if they throw an exception and
569  // we are best off catching it and showing an error message
570  for (unsigned int i = 0; i < current_item->n_iterators; ++i)
571  {
572  try
573  {
574  if (worker)
575  worker(current_item->iterators[i],
576  *scratch_data,
577  current_item->copy_datas[i]);
578  }
579  catch (const std::exception &exc)
580  {
582  }
583  catch (...)
584  {
586  }
587  }
588 
589  // finally mark the scratch object as unused again. as above, there
590  // is no need to lock anything here since the object we work on
591  // is thread-local
592  for (auto &p : current_item->scratch_data->get())
593  if (p.scratch_data.get() == scratch_data)
594  {
595  Assert(p.currently_in_use == true, ExcInternalError());
596  p.currently_in_use = false;
597 
598  break;
599  }
600 
601  // if there is no copier, mark current item as usable again
602  if (copier_exists == false)
603  current_item->currently_in_use = false;
604 
605 
606  // Then return the original pointer
607  // to the now modified object. The copier will work on it next.
608  return current_item;
609  });
610 
611 
612  //
613  // ----- Stage 3 -----
614  //
615  // The last stage is the one that copies data from the CopyData objects
616  // to the final destination. This stage runs sequentially again.
617  auto tbb_copier_filter = tbb::make_filter<ItemType *, void>(
618 # ifdef DEAL_II_TBB_WITH_ONEAPI
619  tbb::filter_mode::serial_in_order,
620 # else
622 # endif
623  [copier = std::function<void(const CopyData &)>(copier)](
624  ItemType *current_item) {
625  if (copier)
626  {
627  // Initiate copying data. For the same reasons as in the worker
628  // class above, catch exceptions rather than letting them
629  // propagate into unknown territories:
630  for (unsigned int i = 0; i < current_item->n_iterators; ++i)
631  {
632  try
633  {
634  copier(current_item->copy_datas[i]);
635  }
636  catch (const std::exception &exc)
637  {
639  }
640  catch (...)
641  {
643  }
644  }
645  }
646  // mark current item as usable again
647  current_item->currently_in_use = false;
648  });
649 
650 
651  //
652  // ----- The pipeline -----
653  //
654  // Now create a pipeline from these stages and execute it:
655  tbb::parallel_pipeline(queue_length,
656  tbb_item_stream_filter & tbb_worker_filter &
657  tbb_copier_filter);
658  }
659 
660  } // namespace tbb_no_coloring
661 # endif // DEAL_II_WITH_TBB
662 
663 
670  namespace sequential
671  {
675  template <typename Worker,
676  typename Copier,
677  typename Iterator,
678  typename ScratchData,
679  typename CopyData>
680  void
681  run(const Iterator &begin,
683  Worker worker,
684  Copier copier,
685  const ScratchData &sample_scratch_data,
686  const CopyData &sample_copy_data)
687  {
688  // need to copy the sample since it is marked const
689  ScratchData scratch_data = sample_scratch_data;
690  CopyData copy_data = sample_copy_data; // NOLINT
691 
692  // Optimization: Check if the functions are not the zero function. To
693  // check zero-ness, create a C++ function out of it:
694  const bool have_worker =
695  (static_cast<const std::function<
696  void(const Iterator &, ScratchData &, CopyData &)> &>(worker)) !=
697  nullptr;
698  const bool have_copier =
699  (static_cast<const std::function<void(const CopyData &)> &>(
700  copier)) != nullptr;
701 
702  // Finally loop over all items and perform the necessary work:
703  for (Iterator i = begin; i != end; ++i)
704  {
705  if (have_worker)
706  worker(i, scratch_data, copy_data);
707  if (have_copier)
708  copier(copy_data);
709  }
710  }
711 
712 
713 
717  template <typename Worker,
718  typename Copier,
719  typename Iterator,
720  typename ScratchData,
721  typename CopyData>
722  void
723  run(const std::vector<std::vector<Iterator>> &colored_iterators,
724  Worker worker,
725  Copier copier,
726  const ScratchData &sample_scratch_data,
727  const CopyData &sample_copy_data)
728  {
729  // need to copy the sample since it is marked const
730  ScratchData scratch_data = sample_scratch_data;
731  CopyData copy_data = sample_copy_data; // NOLINT
732 
733  // Optimization: Check if the functions are not the zero function. To
734  // check zero-ness, create a C++ function out of it:
735  const bool have_worker =
736  (static_cast<const std::function<
737  void(const Iterator &, ScratchData &, CopyData &)> &>(worker)) !=
738  nullptr;
739  const bool have_copier =
740  (static_cast<const std::function<void(const CopyData &)> &>(
741  copier)) != nullptr;
742 
743  // Finally loop over all items and perform the necessary work:
744  for (unsigned int color = 0; color < colored_iterators.size(); ++color)
745  if (colored_iterators[color].size() > 0)
746  for (auto &it : colored_iterators[color])
747  {
748  if (have_worker)
749  worker(it, scratch_data, copy_data);
750  if (have_copier)
751  copier(copy_data);
752  }
753  }
754 
755  } // namespace sequential
756 
757 
758 
759 # ifdef DEAL_II_WITH_TBB
767  namespace tbb_colored
768  {
774  template <typename Iterator, typename ScratchData, typename CopyData>
776  {
777  std::unique_ptr<ScratchData> scratch_data;
778  std::unique_ptr<CopyData> copy_data;
780 
785  : currently_in_use(false)
786  {}
787 
788  ScratchAndCopyDataObjects(std::unique_ptr<ScratchData> &&p,
789  std::unique_ptr<CopyData> &&q,
790  const bool in_use)
791  : scratch_data(std::move(p))
792  , copy_data(std::move(q))
793  , currently_in_use(in_use)
794  {}
795 
796  // Provide a copy constructor that actually doesn't copy the
797  // internal state. This makes handling ScratchAndCopyDataObjects
798  // easier to handle with STL containers.
800  : currently_in_use(false)
801  {}
802  };
803 
804 
805 
811  template <typename Iterator, typename ScratchData, typename CopyData>
813  {
814  public:
819  const std::function<void(const Iterator &, ScratchData &, CopyData &)>
820  &worker,
821  const std::function<void(const CopyData &)> &copier,
822  const ScratchData &sample_scratch_data,
823  const CopyData &sample_copy_data)
824  : worker(worker)
825  , copier(copier)
828  {}
829 
830 
835  void
836  operator()(const tbb::blocked_range<
837  typename std::vector<Iterator>::const_iterator> &range)
838  {
839  // we need to find an unused scratch and corresponding copy
840  // data object in the list that corresponds to the current
841  // thread and then mark it as used. If we can't find one,
842  // create one as discussed in the discussion of the documentation
843  // of the IteratorRangeToItemStream::scratch_data variable,
844  // there is no need to synchronize access to this variable
845  // using a mutex as long as we have no yield-point in between.
846  // This means that we can't take an iterator into the list
847  // now and expect it to still be valid after calling the worker,
848  // but we at least do not have to lock the following section.
849  ScratchData *scratch_data = nullptr;
850  CopyData *copy_data = nullptr;
851  {
852  ScratchAndCopyDataList &scratch_and_copy_data_list = data.get();
853 
854  // see if there is an unused object. if so, grab it and mark
855  // it as used
856  for (typename ScratchAndCopyDataList::iterator p =
857  scratch_and_copy_data_list.begin();
858  p != scratch_and_copy_data_list.end();
859  ++p)
860  if (p->currently_in_use == false)
861  {
862  scratch_data = p->scratch_data.get();
863  copy_data = p->copy_data.get();
864  p->currently_in_use = true;
865  break;
866  }
867 
868  // if no element in the list was found, create one and mark it as
869  // used
870  if (scratch_data == nullptr)
871  {
872  Assert(copy_data == nullptr, ExcInternalError());
873 
874  scratch_and_copy_data_list.emplace_back(
875  std::make_unique<ScratchData>(sample_scratch_data),
876  std::make_unique<CopyData>(sample_copy_data),
877  true);
878  scratch_data =
879  scratch_and_copy_data_list.back().scratch_data.get();
880  copy_data = scratch_and_copy_data_list.back().copy_data.get();
881  }
882  }
883 
884  // then call the worker and copier functions on each
885  // element of the chunk we were given.
886  for (typename std::vector<Iterator>::const_iterator p = range.begin();
887  p != range.end();
888  ++p)
889  {
890  try
891  {
892  if (worker)
893  worker(*p, *scratch_data, *copy_data);
894  if (copier)
895  copier(*copy_data);
896  }
897  catch (const std::exception &exc)
898  {
900  }
901  catch (...)
902  {
904  }
905  }
906 
907  // finally mark the scratch object as unused again. as above, there
908  // is no need to lock anything here since the object we work on
909  // is thread-local
910  {
911  ScratchAndCopyDataList &scratch_and_copy_data_list = data.get();
912 
913  for (typename ScratchAndCopyDataList::iterator p =
914  scratch_and_copy_data_list.begin();
915  p != scratch_and_copy_data_list.end();
916  ++p)
917  if (p->scratch_data.get() == scratch_data)
918  {
919  Assert(p->currently_in_use == true, ExcInternalError());
920  p->currently_in_use = false;
921  }
922  }
923  }
924 
925  private:
926  using ScratchAndCopyDataObjects = typename tbb_colored::
927  ScratchAndCopyDataObjects<Iterator, ScratchData, CopyData>;
928 
933  using ScratchAndCopyDataList = std::list<ScratchAndCopyDataObjects>;
934 
936 
941  const std::function<void(const Iterator &, ScratchData &, CopyData &)>
943 
948  const std::function<void(const CopyData &)> copier;
949 
953  const ScratchData &sample_scratch_data;
954  const CopyData &sample_copy_data;
955  };
956 
960  template <typename Worker,
961  typename Copier,
962  typename Iterator,
963  typename ScratchData,
964  typename CopyData>
965  void
966  run(const std::vector<std::vector<Iterator>> &colored_iterators,
967  Worker worker,
968  Copier copier,
969  const ScratchData &sample_scratch_data,
970  const CopyData &sample_copy_data,
971  const unsigned int chunk_size)
972  {
973  // loop over the various colors of what we're given
974  for (unsigned int color = 0; color < colored_iterators.size(); ++color)
975  if (colored_iterators[color].size() > 0)
976  {
977  using WorkerAndCopier = internal::tbb_colored::
978  WorkerAndCopier<Iterator, ScratchData, CopyData>;
979 
980  WorkerAndCopier worker_and_copier(worker,
981  copier,
982  sample_scratch_data,
983  sample_copy_data);
984 
986  colored_iterators[color].begin(),
987  colored_iterators[color].end(),
988  [&worker_and_copier](
989  const tbb::blocked_range<
990  typename std::vector<Iterator>::const_iterator> &range) {
991  worker_and_copier(range);
992  },
993  chunk_size);
994  }
995  }
996 
997  } // namespace tbb_colored
998 # endif // DEAL_II_WITH_TBB
999 
1000 
1001  } // namespace internal
1002 
1003 
1004 
1052  template <typename Worker,
1053  typename Copier,
1054  typename Iterator,
1055  typename ScratchData,
1056  typename CopyData>
1057  void
1058  run(const std::vector<std::vector<Iterator>> &colored_iterators,
1059  Worker worker,
1060  Copier copier,
1061  const ScratchData &sample_scratch_data,
1062  const CopyData &sample_copy_data,
1063  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1064  const unsigned int chunk_size = 8);
1065 
1066 
1116  template <typename Worker,
1117  typename Copier,
1118  typename Iterator,
1119  typename ScratchData,
1120  typename CopyData>
1121  void
1122  run(const Iterator &begin,
1124  Worker worker,
1125  Copier copier,
1126  const ScratchData &sample_scratch_data,
1127  const CopyData &sample_copy_data,
1128  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1129  const unsigned int chunk_size = 8)
1130  {
1131  Assert(queue_length > 0,
1132  ExcMessage("The queue length must be at least one, and preferably "
1133  "larger than the number of processors on this system."));
1134  (void)queue_length; // removes -Wunused-parameter warning in optimized mode
1135  Assert(chunk_size > 0, ExcMessage("The chunk_size must be at least one."));
1136  (void)chunk_size; // removes -Wunused-parameter warning in optimized mode
1137 
1138  // If no work then skip. (only use operator!= for iterators since we may
1139  // not have an equality comparison operator)
1140  if (!(begin != end))
1141  return;
1142 
1143  if (MultithreadInfo::n_threads() > 1)
1144  {
1145 # ifdef DEAL_II_WITH_TBB
1146  if (static_cast<const std::function<void(const CopyData &)> &>(copier))
1147  {
1148  // If we have a copier, run the algorithm:
1150  end,
1151  worker,
1152  copier,
1153  sample_scratch_data,
1154  sample_copy_data,
1155  queue_length,
1156  chunk_size);
1157  }
1158  else
1159  {
1160  // There is no copier function. in this case, we have an
1161  // embarrassingly parallel problem where we can
1162  // essentially apply parallel_for. because parallel_for
1163  // requires subdividing the range for which operator- is
1164  // necessary between iterators, it is often inefficient to
1165  // apply it directly to cell ranges and similar iterator
1166  // types for which operator- is expensive or, in fact,
1167  // nonexistent. rather, in that case, we simply copy the
1168  // iterators into a large array and use operator- on
1169  // iterators to this array of iterators.
1170  //
1171  // instead of duplicating code, this is essentially the
1172  // same situation we have in the colored implementation below, so we
1173  // just defer to that place
1174  std::vector<std::vector<Iterator>> all_iterators(1);
1175  for (Iterator p = begin; p != end; ++p)
1176  all_iterators[0].push_back(p);
1177 
1178  run(all_iterators,
1179  worker,
1180  copier,
1181  sample_scratch_data,
1182  sample_copy_data,
1183  queue_length,
1184  chunk_size);
1185  }
1186 
1187  // exit this function to not run the sequential version below:
1188  return;
1189 # endif
1190  }
1191 
1192  // no TBB installed or we are requested to run sequentially:
1194  begin, end, worker, copier, sample_scratch_data, sample_copy_data);
1195  }
1196 
1197 
1198 
1206  template <typename Worker,
1207  typename Copier,
1208  typename IteratorRangeType,
1209  typename ScratchData,
1210  typename CopyData,
1211  typename = std::enable_if_t<has_begin_and_end<IteratorRangeType>>>
1212  void
1213  run(IteratorRangeType iterator_range,
1214  Worker worker,
1215  Copier copier,
1216  const ScratchData &sample_scratch_data,
1217  const CopyData &sample_copy_data,
1218  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1219  const unsigned int chunk_size = 8)
1220  {
1221  // Call the function above
1222  run(iterator_range.begin(),
1223  iterator_range.end(),
1224  worker,
1225  copier,
1226  sample_scratch_data,
1227  sample_copy_data,
1228  queue_length,
1229  chunk_size);
1230  }
1231 
1232 
1233 
1237  template <typename Worker,
1238  typename Copier,
1239  typename Iterator,
1240  typename ScratchData,
1241  typename CopyData>
1242  void
1243  run(const IteratorRange<Iterator> &iterator_range,
1244  Worker worker,
1245  Copier copier,
1246  const ScratchData &sample_scratch_data,
1247  const CopyData &sample_copy_data,
1248  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1249  const unsigned int chunk_size = 8)
1250  {
1251  // Call the function above
1252  run(iterator_range.begin(),
1253  iterator_range.end(),
1254  worker,
1255  copier,
1256  sample_scratch_data,
1257  sample_copy_data,
1258  queue_length,
1259  chunk_size);
1260  }
1261 
1262 
1263 
1264  template <typename Worker,
1265  typename Copier,
1266  typename Iterator,
1267  typename ScratchData,
1268  typename CopyData>
1269  void
1270  run(const std::vector<std::vector<Iterator>> &colored_iterators,
1271  Worker worker,
1272  Copier copier,
1273  const ScratchData &sample_scratch_data,
1274  const CopyData &sample_copy_data,
1275  const unsigned int queue_length,
1276  const unsigned int chunk_size)
1277  {
1278  Assert(queue_length > 0,
1279  ExcMessage("The queue length must be at least one, and preferably "
1280  "larger than the number of processors on this system."));
1281  (void)queue_length; // removes -Wunused-parameter warning in optimized mode
1282  Assert(chunk_size > 0, ExcMessage("The chunk_size must be at least one."));
1283  (void)chunk_size; // removes -Wunused-parameter warning in optimized mode
1284 
1285 
1286  if (MultithreadInfo::n_threads() > 1)
1287  {
1288 # ifdef DEAL_II_WITH_TBB
1289  internal::tbb_colored::run(colored_iterators,
1290  worker,
1291  copier,
1292  sample_scratch_data,
1293  sample_copy_data,
1294  chunk_size);
1295 
1296  // exit this function to not run the sequential version below:
1297  return;
1298 # endif
1299  }
1300 
1301  // run all colors sequentially:
1302  {
1303  internal::sequential::run(colored_iterators,
1304  worker,
1305  copier,
1306  sample_scratch_data,
1307  sample_copy_data);
1308  }
1309  }
1310 
1311 
1312 
1354  template <typename MainClass,
1355  typename Iterator,
1356  typename ScratchData,
1357  typename CopyData>
1358  void
1359  run(const Iterator &begin,
1361  MainClass &main_object,
1362  void (MainClass::*worker)(const Iterator &, ScratchData &, CopyData &),
1363  void (MainClass::*copier)(const CopyData &),
1364  const ScratchData &sample_scratch_data,
1365  const CopyData &sample_copy_data,
1366  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1367  const unsigned int chunk_size = 8)
1368  {
1369  // forward to the other function
1370  run(
1371  begin,
1372  end,
1373  [&main_object, worker](const Iterator &iterator,
1374  ScratchData &scratch_data,
1375  CopyData &copy_data) {
1376  (main_object.*worker)(iterator, scratch_data, copy_data);
1377  },
1378  [&main_object, copier](const CopyData &copy_data) {
1379  (main_object.*copier)(copy_data);
1380  },
1381  sample_scratch_data,
1382  sample_copy_data,
1383  queue_length,
1384  chunk_size);
1385  }
1386 
1387 
1388  template <typename MainClass,
1389  typename Iterator,
1390  typename ScratchData,
1391  typename CopyData>
1392  void
1395  MainClass &main_object,
1396  void (MainClass::*worker)(const Iterator &, ScratchData &, CopyData &),
1397  void (MainClass::*copier)(const CopyData &),
1398  const ScratchData &sample_scratch_data,
1399  const CopyData &sample_copy_data,
1400  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1401  const unsigned int chunk_size = 8)
1402  {
1403  // forward to the other function
1404  run(
1405  begin,
1406  end,
1407  [&main_object, worker](const Iterator &iterator,
1408  ScratchData &scratch_data,
1409  CopyData &copy_data) {
1410  (main_object.*worker)(iterator, scratch_data, copy_data);
1411  },
1412  [&main_object, copier](const CopyData &copy_data) {
1413  (main_object.*copier)(copy_data);
1414  },
1415  sample_scratch_data,
1416  sample_copy_data,
1417  queue_length,
1418  chunk_size);
1419  }
1420 
1421 
1422 
1430  template <typename MainClass,
1431  typename IteratorRangeType,
1432  typename ScratchData,
1433  typename CopyData,
1434  typename = std::enable_if_t<has_begin_and_end<IteratorRangeType>>>
1435  void
1437  IteratorRangeType iterator_range,
1438  MainClass &main_object,
1439  void (MainClass::*worker)(
1441  ScratchData &,
1442  CopyData &),
1443  void (MainClass::*copier)(const CopyData &),
1444  const ScratchData &sample_scratch_data,
1445  const CopyData &sample_copy_data,
1446  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1447  const unsigned int chunk_size = 8)
1448  {
1449  // Call the function above
1450  run(std::begin(iterator_range),
1451  std::end(iterator_range),
1452  main_object,
1453  worker,
1454  copier,
1455  sample_scratch_data,
1456  sample_copy_data,
1457  queue_length,
1458  chunk_size);
1459  }
1460 
1461 
1462 
1466  template <typename MainClass,
1467  typename Iterator,
1468  typename ScratchData,
1469  typename CopyData>
1470  void
1472  MainClass &main_object,
1473  void (MainClass::*worker)(const Iterator &, ScratchData &, CopyData &),
1474  void (MainClass::*copier)(const CopyData &),
1475  const ScratchData &sample_scratch_data,
1476  const CopyData &sample_copy_data,
1477  const unsigned int queue_length = 2 * MultithreadInfo::n_threads(),
1478  const unsigned int chunk_size = 8)
1479  {
1480  // Call the function above
1481  run(std::begin(iterator_range),
1482  std::end(iterator_range),
1483  main_object,
1484  worker,
1485  copier,
1486  sample_scratch_data,
1487  sample_copy_data,
1488  queue_length,
1489  chunk_size);
1490  }
1491 
1492 } // namespace WorkStream
1493 
1494 
1495 
1497 
1498 
1499 
1500 //---------------------------- work_stream.h ---------------------------
1501 // end of #ifndef dealii_work_stream_h
1502 #endif
1503 //---------------------------- work_stream.h ---------------------------
IteratorOverIterators end() const
IteratorOverIterators begin()
static unsigned int n_threads()
std::list< ScratchAndCopyDataObjects > ScratchAndCopyDataList
Definition: work_stream.h:933
WorkerAndCopier(const std::function< void(const Iterator &, ScratchData &, CopyData &)> &worker, const std::function< void(const CopyData &)> &copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Definition: work_stream.h:818
Threads::ThreadLocalStorage< ScratchAndCopyDataList > data
Definition: work_stream.h:935
void operator()(const tbb::blocked_range< typename std::vector< Iterator >::const_iterator > &range)
Definition: work_stream.h:836
const std::function< void(const Iterator &, ScratchData &, CopyData &)> worker
Definition: work_stream.h:942
typename tbb_colored::ScratchAndCopyDataObjects< Iterator, ScratchData, CopyData > ScratchAndCopyDataObjects
Definition: work_stream.h:927
const std::function< void(const CopyData &)> copier
Definition: work_stream.h:948
IteratorRangeToItemStream(const Iterator &begin, const Iterator &end, const unsigned int buffer_size, const unsigned int chunk_size, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Definition: work_stream.h:323
Threads::ThreadLocalStorage< typename ItemType::ScratchDataList > thread_local_scratch
Definition: work_stream.h:446
#define DEAL_II_NAMESPACE_OPEN
Definition: config.h:477
#define DEAL_II_NAMESPACE_CLOSE
Definition: config.h:478
static ::ExceptionBase & ExcInternalError()
#define Assert(cond, exc)
Definition: exceptions.h:1616
static ::ExceptionBase & ExcMessage(std::string arg1)
constexpr int chunk_size
Definition: cuda_size.h:35
void handle_std_exception(const std::exception &exc)
VectorType::value_type * begin(VectorType &V)
VectorType::value_type * end(VectorType &V)
std::vector< unsigned int > serial(const std::vector< unsigned int > &targets, const std::function< RequestType(const unsigned int)> &create_request, const std::function< AnswerType(const unsigned int, const RequestType &)> &answer_request, const std::function< void(const unsigned int, const AnswerType &)> &process_answer, const MPI_Comm comm)
void run(const Iterator &begin, const std_cxx20::type_identity_t< Iterator > &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Definition: work_stream.h:681
void run(const std::vector< std::vector< Iterator >> &colored_iterators, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int chunk_size)
Definition: work_stream.h:966
void run(const Iterator &begin, const std_cxx20::type_identity_t< Iterator > &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
Definition: work_stream.h:472
void run(const std::vector< std::vector< Iterator >> &colored_iterators, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length=2 *MultithreadInfo::n_threads(), const unsigned int chunk_size=8)
Definition: work_stream.h:1270
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
Definition: parallel.h:84
typename type_identity< T >::type type_identity_t
Definition: type_traits.h:96
ScratchAndCopyDataObjects(const ScratchAndCopyDataObjects &)
Definition: work_stream.h:799
ScratchAndCopyDataObjects(std::unique_ptr< ScratchData > &&p, std::unique_ptr< CopyData > &&q, const bool in_use)
Definition: work_stream.h:788
Threads::ThreadLocalStorage< ScratchDataList > * scratch_data
Definition: work_stream.h:290