12#include <fmt/format.h>
13#include <fmt/ranges.h>
14#include <magic_enum/magic_enum.hpp>
17#include <spdlog/spdlog.h>
20#include <taskflow/algorithm/pipeline.hpp>
21#include <taskflow/core/task.hpp>
22#include <taskflow/core/taskflow.hpp>
33 ,
sinks_{ &handle.get_sink_manager_ref() }
34 ,
report_{ &handle.get_app_ref().get_report() }
39 for (
auto _ : std::views::iota(std::size_t{ 0 },
n_lines_))
47 is_pipe_stopped.store(
true);
59 if (not
raw_data_[line_number].is_empty())
71 for (
const auto [line_number, taskflow] : std::views::zip(std::views::iota(0),
taskflow_lines_))
81 const auto& conversion_req_map =
sinks_->generate_conversion_req_map();
82 spdlog::debug(
"Starting taskflow with enabled writers");
83 spdlog::debug(
"Conversion requirements map: \n"
88 fmt::join(conversion_req_map | std::views::transform(
89 [](
const auto conv_req) -> std::string
93 magic_enum::enum_name(conv_req.first),
94 conv_req.second ?
"✅" :
"❌");
102 tf::Pipe{ tf::PipeType::SERIAL, []([[maybe_unused]] tf::Pipeflow& pipeflow) {} },
103 tf::Pipe{ tf::PipeType::PARALLEL,
104 [
this, &buffer_queue]([[maybe_unused]] tf::Pipeflow& pipeflow)
106 if (not
run_task(buffer_queue, pipeflow.line()))
116 tf::Pipe{ tf::PipeType::SERIAL, []([[maybe_unused]] tf::Pipeflow& pipeflow) {} } };
118 auto pipeline_task =
main_taskflow_.composed_of(main_pipeline).name(
"Main pipeline");
119 starting_task.precede(pipeline_task);
126 static constexpr auto SLEEP_TIME = std::chrono::milliseconds(10);
127 std::this_thread::sleep_for(SLEEP_TIME);
129 [](
const std::atomic<bool>& is_pipe_stopped) ->
bool
130 {
return is_pipe_stopped.load(); });
134 template <
typename PrevConverter,
typename ThisTask>
136 std::pair<const PrevConverter&, tf::Task>& prev_task,
137 tf::Taskflow& taskflow,
138 std::size_t line_number)
139 -> std::optional<std::pair<const ThisTask&, tf::Task>>
141 current_task.set_report(
report_);
144 .emplace([line_number, ¤t_task, &prev_converter = prev_task.first]()
145 { [[maybe_unused]] auto res = current_task.run_once(prev_converter, line_number); })
146 .name(current_task.get_name_str());
147 if (not prev_task.second.empty())
149 prev_task.second.precede(task);
151 return std::pair<const ThisTask&, tf::Task>{ current_task, task };
154 template <
typename PrevConverter, ConverterType ThisTask>
156 std::optional<std::pair<const PrevConverter&, tf::Task>>& prev_task,
157 tf::Taskflow& taskflow,
158 std::size_t line_number) -> std::optional<std::pair<const ThisTask&, tf::Task>>
160 if (not prev_task.has_value())
164 auto is_required =
sinks_->is_convert_required(ThisTask::converter_type);
169 if (not current_task)
173 return emplace_to_taskflow(current_task.value(), prev_task.value(), taskflow, line_number);
176 template <
typename PrevConverter, SinkType ThisTask>
178 std::optional<std::pair<const PrevConverter&, tf::Task>>& prev_task,
179 tf::Taskflow& taskflow,
180 std::size_t line_number) -> std::optional<std::pair<const ThisTask&, tf::Task>>
182 if (not prev_task.has_value())
191 auto create_task = [line_number, &taskflow,
this]<
typename PrevConverter,
typename ThisTask>(
192 ThisTask& current_task,
193 std::optional<std::pair<const PrevConverter&, tf::Task>>& prev_task)
195 {
return create_task_imp(current_task, prev_task, taskflow, line_number); };
197 auto empty_task = std::optional{ std::pair<const TaskDiagram&, tf::Task>{ *
this, tf::Task{} } };
211 &proto_delim_serial_task](std::string_view filename,
auto&
sink) ->
void
213 if constexpr (std::remove_cvref_t<
decltype(
sink)>::IsStructType)
215 create_task(
sink, struct_deser_task);
220 const auto convert_mode =
sink.get_required_conversion();
221 switch (convert_mode)
224 create_task(
sink, empty_task);
227 create_task(
sink, raw_delimiter_task);
230 create_task(
sink, proto_serial_task);
233 create_task(
sink, proto_delim_serial_task);
236 spdlog::warn(
"unrecognized conversion {} from the file {}", convert_mode, filename);
237 create_task(
sink, empty_task);
Class to manage the production and consumption of LargeBuffer asynchronously.
auto get_queue_consumer_token() -> BufferQueue::Token
auto get_buffer_queue() const -> const auto &
std::atomic< uint64_t > total_read_data_bytes_
std::optional< process::Struct2ProtoConverter > struct_to_proto_converter_
std::optional< process::ProtoSerializer > proto_serializer_converter_
void stop_time_record(std::size_t line_num)
auto is_taskflow_abort_ready() const -> bool
auto run_task(BufferQueue &data_queue, std::size_t line_number) -> bool
tf::Taskflow main_taskflow_
void construct_taskflow_and_run(BufferQueue &data_queue, const std::atomic< bool > &is_stopped)
Run the taskflow and block the current thread until finished.
gsl::not_null< sink::Manager * > sinks_
std::optional< process::ProtoDelimSerializer > proto_delim_serializer_converter_
void start_time_record(std::size_t line_num)
std::vector< AppReport::TaskStat > stats_
std::vector< std::atomic< bool > > is_pipeline_stopped_
auto emplace_to_taskflow(ThisTask ¤t_task, std::pair< const PrevConverter &, tf::Task > &prev_task, tf::Taskflow &taskflow, std::size_t line_number) -> std::optional< std::pair< const ThisTask &, tf::Task > >
std::vector< LargeBuffer > raw_data_
std::vector< tf::Taskflow > taskflow_lines_
std::optional< process::StructDeserializer > struct_deserializer_converter_
std::vector< TimePoint > last_times_
std::vector< BufferQueue::Token > consumer_tokens_
std::optional< process::Raw2DelimRawConverter > raw_to_delim_raw_converter_
TaskDiagram(AnalysisHandle &data_processor, std::size_t n_lines=1)
Constructor with number of pipelines.
std::atomic< bool > is_done_
tf::Executor tf_executor_
void construct_taskflow_line(tf::Taskflow &taskflow, std::size_t line_number)
auto create_task_imp(std::optional< ThisTask > ¤t_task, std::optional< std::pair< const PrevConverter &, tf::Task > > &prev_task, tf::Taskflow &taskflow, std::size_t line_number) -> std::optional< std::pair< const ThisTask &, tf::Task > >
remove_optional< T >::type remove_optional_t