15#ifndef dealii_work_stream_h
16# define dealii_work_stream_h
29# ifdef DEAL_II_WITH_TBB
30# ifdef DEAL_II_TBB_WITH_ONEAPI
31# include <tbb/parallel_pipeline.h>
33# include <tbb/pipeline.h>
167# ifdef DEAL_II_WITH_TBB
182 namespace tbb_no_coloring
187 template <
typename Iterator,
typename ScratchData,
typename CopyData>
324 const unsigned int buffer_size,
327 const CopyData &sample_copy_data)
342 item.copy_datas.resize(
chunk_size, sample_copy_data);
343 item.currently_in_use =
false;
369 for (
unsigned int i = 0; i <
item_buffer.size(); ++i)
376 Assert(current_item !=
nullptr,
377 ExcMessage(
"This can't be. There must be a free item!"));
465 template <
typename Worker,
468 typename ScratchData,
471 run(
const Iterator &begin,
475 const ScratchData &sample_scratch_data,
476 const CopyData &sample_copy_data,
477 const unsigned int queue_length,
478 const unsigned int chunk_size)
492 iterator_range_to_item_stream(begin,
498 auto tbb_item_stream_filter = tbb::make_filter<void, ItemType *>(
499# ifdef DEAL_II_TBB_WITH_ONEAPI
500 tbb::filter_mode::serial_in_order,
504 [&](tbb::flow_control &fc) -> ItemType * {
505 if (
const auto item = iterator_range_to_item_stream.
get_item())
519 auto tbb_worker_filter = tbb::make_filter<ItemType *, ItemType *>(
520# ifdef DEAL_II_TBB_WITH_ONEAPI
521 tbb::filter_mode::parallel,
523 tbb::filter::parallel,
526 std::function<
void(
const Iterator &, ScratchData &, CopyData &)>(
529 static_cast<bool>(std::function<
void(
const CopyData &)>(copier))](
530 ItemType *current_item) {
542 ScratchData *scratch_data =
nullptr;
546 for (
auto &p : current_item->scratch_data->get())
547 if (p.currently_in_use ==
false)
549 scratch_data = p.scratch_data.get();
550 p.currently_in_use =
true;
556 if (scratch_data ==
nullptr)
559 new ScratchData(*current_item->sample_scratch_data);
560 current_item->scratch_data->get().emplace_back(scratch_data,
569 for (
unsigned int i = 0; i < current_item->n_iterators; ++i)
574 worker(current_item->iterators[i],
576 current_item->copy_datas[i]);
578 catch (
const std::exception &exc)
591 for (
auto &p : current_item->scratch_data->get())
592 if (p.scratch_data.get() == scratch_data)
595 p.currently_in_use =
false;
601 if (copier_exists ==
false)
602 current_item->currently_in_use =
false;
616 auto tbb_copier_filter = tbb::make_filter<ItemType *, void>(
617# ifdef DEAL_II_TBB_WITH_ONEAPI
618 tbb::filter_mode::serial_in_order,
622 [copier = std::function<
void(
const CopyData &)>(copier)](
623 ItemType *current_item) {
629 for (
unsigned int i = 0; i < current_item->n_iterators; ++i)
633 copier(current_item->copy_datas[i]);
635 catch (
const std::exception &exc)
646 current_item->currently_in_use =
false;
654 tbb::parallel_pipeline(queue_length,
655 tbb_item_stream_filter & tbb_worker_filter &
674 template <
typename Worker,
677 typename ScratchData,
680 run(
const Iterator &begin,
684 const ScratchData &sample_scratch_data,
685 const CopyData &sample_copy_data)
688 ScratchData scratch_data = sample_scratch_data;
689 CopyData copy_data = sample_copy_data;
693 const bool have_worker =
694 (
static_cast<const std::function<
695 void(
const Iterator &, ScratchData &, CopyData &)
> &>(worker)) !=
697 const bool have_copier =
698 (
static_cast<const std::function<
void(
const CopyData &)
> &>(
702 for (Iterator i = begin; i != end; ++i)
705 worker(i, scratch_data, copy_data);
716 template <
typename Worker,
719 typename ScratchData,
722 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
725 const ScratchData &sample_scratch_data,
726 const CopyData &sample_copy_data)
729 ScratchData scratch_data = sample_scratch_data;
730 CopyData copy_data = sample_copy_data;
734 const bool have_worker =
735 (
static_cast<const std::function<
736 void(
const Iterator &, ScratchData &, CopyData &)
> &>(worker)) !=
738 const bool have_copier =
739 (
static_cast<const std::function<
void(
const CopyData &)
> &>(
743 for (
unsigned int color = 0; color < colored_iterators.size(); ++color)
744 if (colored_iterators[color].size() > 0)
745 for (
auto &it : colored_iterators[color])
748 worker(it, scratch_data, copy_data);
758# ifdef DEAL_II_WITH_TBB
766 namespace tbb_colored
773 template <
typename Iterator,
typename ScratchData,
typename CopyData>
788 std::unique_ptr<CopyData> &&q,
810 template <
typename Iterator,
typename ScratchData,
typename CopyData>
818 const std::function<
void(
const Iterator &, ScratchData &, CopyData &)>
820 const std::function<
void(
const CopyData &)> &
copier,
836 typename std::vector<Iterator>::const_iterator> &range)
848 ScratchData *scratch_data =
nullptr;
849 CopyData *copy_data =
nullptr;
855 for (
typename ScratchAndCopyDataList::iterator p =
856 scratch_and_copy_data_list.begin();
857 p != scratch_and_copy_data_list.end();
859 if (p->currently_in_use ==
false)
861 scratch_data = p->scratch_data.get();
862 copy_data = p->copy_data.get();
863 p->currently_in_use =
true;
869 if (scratch_data ==
nullptr)
873 scratch_and_copy_data_list.emplace_back(
878 scratch_and_copy_data_list.back().scratch_data.get();
879 copy_data = scratch_and_copy_data_list.back().copy_data.get();
885 for (
typename std::vector<Iterator>::const_iterator p = range.begin();
892 worker(*p, *scratch_data, *copy_data);
896 catch (
const std::exception &exc)
912 for (
typename ScratchAndCopyDataList::iterator p =
913 scratch_and_copy_data_list.begin();
914 p != scratch_and_copy_data_list.end();
916 if (p->scratch_data.get() == scratch_data)
919 p->currently_in_use =
false;
926 ScratchAndCopyDataObjects<Iterator, ScratchData, CopyData>;
940 const std::function<void(
const Iterator &, ScratchData &, CopyData &)>
947 const std::function<void(
const CopyData &)>
copier;
959 template <
typename Worker,
962 typename ScratchData,
965 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
968 const ScratchData &sample_scratch_data,
969 const CopyData &sample_copy_data,
970 const unsigned int chunk_size)
973 for (
unsigned int color = 0; color < colored_iterators.size(); ++color)
974 if (colored_iterators[color].size() > 0)
977 WorkerAndCopier<Iterator, ScratchData, CopyData>;
985 colored_iterators[color].begin(),
986 colored_iterators[color].end(),
987 [&worker_and_copier](
988 const tbb::blocked_range<
989 typename std::vector<Iterator>::const_iterator> &range) {
990 worker_and_copier(range);
1051 template <
typename Worker,
1054 typename ScratchData,
1057 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
1060 const ScratchData &sample_scratch_data,
1061 const CopyData &sample_copy_data,
1063 const unsigned int chunk_size = 8);
1115 template <
typename Worker,
1118 typename ScratchData,
1125 const ScratchData &sample_scratch_data,
1126 const CopyData &sample_copy_data,
1128 const unsigned int chunk_size = 8)
1131 ExcMessage(
"The queue length must be at least one, and preferably "
1132 "larger than the number of processors on this system."));
1134 Assert(chunk_size > 0,
ExcMessage(
"The chunk_size must be at least one."));
1139 if (!(begin != end))
1144# ifdef DEAL_II_WITH_TBB
1145 if (
static_cast<const std::function<
void(
const CopyData &)
> &>(copier))
1152 sample_scratch_data,
1173 std::vector<std::vector<Iterator>> all_iterators(1);
1174 for (Iterator p = begin; p != end; ++p)
1175 all_iterators[0].push_back(p);
1180 sample_scratch_data,
1193 begin, end, worker, copier, sample_scratch_data, sample_copy_data);
1205 template <
typename Worker,
1207 typename IteratorRangeType,
1208 typename ScratchData,
1210 typename = std::enable_if_t<has_begin_and_end<IteratorRangeType>>>
1212 run(IteratorRangeType iterator_range,
1215 const ScratchData &sample_scratch_data,
1216 const CopyData &sample_copy_data,
1218 const unsigned int chunk_size = 8)
1221 run(iterator_range.begin(),
1222 iterator_range.end(),
1225 sample_scratch_data,
1236 template <
typename Worker,
1239 typename ScratchData,
1245 const ScratchData &sample_scratch_data,
1246 const CopyData &sample_copy_data,
1248 const unsigned int chunk_size = 8)
1252 iterator_range.
end(),
1255 sample_scratch_data,
1263 template <
typename Worker,
1266 typename ScratchData,
1269 run(
const std::vector<std::vector<Iterator>> &colored_iterators,
1272 const ScratchData &sample_scratch_data,
1273 const CopyData &sample_copy_data,
1274 const unsigned int queue_length,
1275 const unsigned int chunk_size)
1278 ExcMessage(
"The queue length must be at least one, and preferably "
1279 "larger than the number of processors on this system."));
1281 Assert(chunk_size > 0,
ExcMessage(
"The chunk_size must be at least one."));
1287# ifdef DEAL_II_WITH_TBB
1291 sample_scratch_data,
1305 sample_scratch_data,
1353 template <
typename MainClass,
1355 typename ScratchData,
1360 MainClass &main_object,
1361 void (MainClass::*worker)(const Iterator &, ScratchData &, CopyData &),
1362 void (MainClass::*copier)(const CopyData &),
1363 const ScratchData &sample_scratch_data,
1364 const CopyData &sample_copy_data,
1366 const unsigned
int chunk_size = 8)
1372 [&main_object, worker](
const Iterator &iterator,
1373 ScratchData &scratch_data,
1374 CopyData ©_data) {
1375 (main_object.*worker)(iterator, scratch_data, copy_data);
1377 [&main_object, copier](
const CopyData ©_data) {
1378 (main_object.*copier)(copy_data);
1380 sample_scratch_data,
1387 template <
typename MainClass,
1389 typename ScratchData,
1394 MainClass &main_object,
1395 void (MainClass::*worker)(const Iterator &, ScratchData &, CopyData &),
1396 void (MainClass::*copier)(const CopyData &),
1397 const ScratchData &sample_scratch_data,
1398 const CopyData &sample_copy_data,
1400 const unsigned
int chunk_size = 8)
1406 [&main_object, worker](
const Iterator &iterator,
1407 ScratchData &scratch_data,
1408 CopyData ©_data) {
1409 (main_object.*worker)(iterator, scratch_data, copy_data);
1411 [&main_object, copier](
const CopyData ©_data) {
1412 (main_object.*copier)(copy_data);
1414 sample_scratch_data,
1429 template <
typename MainClass,
1430 typename IteratorRangeType,
1431 typename ScratchData,
1433 typename = std::enable_if_t<has_begin_and_end<IteratorRangeType>>>
1436 IteratorRangeType iterator_range,
1437 MainClass &main_object,
1438 void (MainClass::*worker)(
1439 const typename
std_cxx20::type_identity_t<IteratorRangeType>::iterator &,
1442 void (MainClass::*copier)(const CopyData &),
1443 const ScratchData &sample_scratch_data,
1444 const CopyData &sample_copy_data,
1446 const unsigned
int chunk_size = 8)
1449 run(std::begin(iterator_range),
1450 std::end(iterator_range),
1454 sample_scratch_data,
1465 template <
typename MainClass,
1467 typename ScratchData,
1471 MainClass &main_object,
1472 void (MainClass::*worker)(const Iterator &, ScratchData &, CopyData &),
1473 void (MainClass::*copier)(const CopyData &),
1474 const ScratchData &sample_scratch_data,
1475 const CopyData &sample_copy_data,
1477 const unsigned
int chunk_size = 8)
1480 run(std::begin(iterator_range),
1481 std::end(iterator_range),
1485 sample_scratch_data,
IteratorOverIterators end() const
IteratorOverIterators begin()
static unsigned int n_threads()
A class that provides a separate storage location on each thread that accesses the object.
std::list< ScratchAndCopyDataObjects > ScratchAndCopyDataList
WorkerAndCopier(const std::function< void(const Iterator &, ScratchData &, CopyData &)> &worker, const std::function< void(const CopyData &)> &copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Threads::ThreadLocalStorage< ScratchAndCopyDataList > data
void operator()(const tbb::blocked_range< typename std::vector< Iterator >::const_iterator > &range)
const std::function< void(const Iterator &, ScratchData &, CopyData &)> worker
typename tbb_colored::ScratchAndCopyDataObjects< Iterator, ScratchData, CopyData > ScratchAndCopyDataObjects
const CopyData & sample_copy_data
const ScratchData & sample_scratch_data
const std::function< void(const CopyData &)> copier
IteratorRangeToItemStream(const Iterator &begin, const Iterator &end, const unsigned int buffer_size, const unsigned int chunk_size, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
Threads::ThreadLocalStorage< typename ItemType::ScratchDataList > thread_local_scratch
std::vector< ItemType > item_buffer
const unsigned int chunk_size
std::pair< Iterator, Iterator > remaining_iterator_range
const ScratchData & sample_scratch_data
#define DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_CLOSE
#define Assert(cond, exc)
static ::ExceptionBase & ExcInternalError()
static ::ExceptionBase & ExcMessage(std::string arg1)
void handle_unknown_exception()
void handle_std_exception(const std::exception &exc)
void run(const Iterator &begin, const std_cxx20::type_identity_t< Iterator > &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data)
void run(const std::vector< std::vector< Iterator > > &colored_iterators, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int chunk_size)
void run(const Iterator &begin, const std_cxx20::type_identity_t< Iterator > &end, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length, const unsigned int chunk_size)
void run(const std::vector< std::vector< Iterator > > &colored_iterators, Worker worker, Copier copier, const ScratchData &sample_scratch_data, const CopyData &sample_copy_data, const unsigned int queue_length=2 *MultithreadInfo::n_threads(), const unsigned int chunk_size=8)
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
typename type_identity< T >::type type_identity_t
ScratchAndCopyDataObjects(const ScratchAndCopyDataObjects &)
ScratchAndCopyDataObjects(std::unique_ptr< ScratchData > &&p, std::unique_ptr< CopyData > &&q, const bool in_use)
std::unique_ptr< ScratchData > scratch_data
ScratchAndCopyDataObjects()
std::unique_ptr< CopyData > copy_data
std::unique_ptr< ScratchData > scratch_data
ScratchDataObject(ScratchDataObject &&o) noexcept=default
ScratchDataObject(const ScratchDataObject &)
ScratchDataObject(ScratchData *p, const bool in_use)
const ScratchData * sample_scratch_data
std::list< ScratchDataObject > ScratchDataList
std::vector< Iterator > iterators
std::vector< CopyData > copy_datas
Threads::ThreadLocalStorage< ScratchDataList > * scratch_data