srs-control 0.2.2
Loading...
Searching...
No Matches
TaskDiagram.cpp
Go to the documentation of this file.
7#include <algorithm>
8#include <atomic>
9#include <cassert>
10#include <chrono>
11#include <cstddef>
12#include <fmt/format.h>
13#include <fmt/ranges.h>
14#include <magic_enum/magic_enum.hpp>
15#include <optional>
16#include <ranges>
17#include <spdlog/spdlog.h>
18#include <string>
19#include <string_view>
20#include <taskflow/algorithm/pipeline.hpp>
21#include <taskflow/core/task.hpp>
22#include <taskflow/core/taskflow.hpp>
23#include <thread>
24#include <type_traits>
25#include <utility>
26#include <vector>
27
28namespace srs::workflow
29{
30 TaskDiagram::TaskDiagram(AnalysisHandle& handle, std::size_t n_lines)
31 : n_lines_{ n_lines }
32 , is_pipeline_stopped_{ std::vector<std::atomic<bool>>(n_lines_) }
33 , sinks_{ &handle.get_sink_manager_ref() }
34 , report_{ &handle.get_app_ref().get_report() }
35 {
36 assert(report_ != nullptr);
37 const auto buffer_size = handle.get_buffer_queue().get_config().buffer_size;
39 for (auto _ : std::views::iota(std::size_t{ 0 }, n_lines_))
40 {
42 raw_data_.emplace_back(buffer_size);
43 }
44
45 for (auto& is_pipe_stopped : is_pipeline_stopped_)
46 {
47 is_pipe_stopped.store(true);
48 }
49
50 stats_.resize(n_lines);
51 last_times_.resize(n_lines);
52 }
53
54 // blocking here with pop
55 auto TaskDiagram::run_task(BufferQueue& buffer_queue, std::size_t line_number) -> bool
56 {
57 // raw_data_[line_number].clear();
58 buffer_queue.dequeue(raw_data_[line_number], consumer_tokens_[line_number]);
59 if (not raw_data_[line_number].is_empty())
60 {
61 total_read_data_bytes_ += raw_data_[line_number].get_size();
62 return true;
63 }
64 return false;
65 }
66
67 void TaskDiagram::construct_taskflow_and_run(BufferQueue& buffer_queue, const std::atomic<bool>&)
68 {
69 const auto _ = ExitLogger{};
71 for (const auto [line_number, taskflow] : std::views::zip(std::views::iota(0), taskflow_lines_))
72 {
73 construct_taskflow_line(taskflow, static_cast<std::size_t>(line_number));
74 }
75
76 auto starting_task =
78 .emplace(
79 [this]()
80 {
81 const auto& conversion_req_map = sinks_->generate_conversion_req_map();
82 spdlog::debug("Starting taskflow with enabled writers");
83 spdlog::debug("Conversion requirements map: \n"
84 "\t|{:^30}|{:^30}|\n"
85 "{}",
86 "Conversions",
87 "Required",
88 fmt::join(conversion_req_map | std::views::transform(
89 [](const auto conv_req) -> std::string
90 {
91 return fmt::format(
92 "\t|{:<30}|{:^29}|",
93 magic_enum::enum_name(conv_req.first),
94 conv_req.second ? "✅" : "❌");
95 }),
96 "\n"));
97 })
98 .name("Starting");
99
100 auto main_pipeline =
101 tf::Pipeline{ n_lines_,
102 tf::Pipe{ tf::PipeType::SERIAL, []([[maybe_unused]] tf::Pipeflow& pipeflow) {} },
103 tf::Pipe{ tf::PipeType::PARALLEL,
104 [this, &buffer_queue]([[maybe_unused]] tf::Pipeflow& pipeflow)
105 {
106 if (not run_task(buffer_queue, pipeflow.line()))
107 {
108 pipeflow.stop();
109 }
110 is_pipeline_stopped_[pipeflow.line()].store(false);
111 start_time_record(pipeflow.line());
112 tf_executor_.corun(taskflow_lines_[pipeflow.line()]);
113 stop_time_record(pipeflow.line());
114 is_pipeline_stopped_[pipeflow.line()].store(true);
115 } },
116 tf::Pipe{ tf::PipeType::SERIAL, []([[maybe_unused]] tf::Pipeflow& pipeflow) {} } };
117
118 auto pipeline_task = main_taskflow_.composed_of(main_pipeline).name("Main pipeline");
119 starting_task.precede(pipeline_task);
120 tf_executor_.run(main_taskflow_).wait();
121 is_done_.store(true);
122 }
123
125 {
126 static constexpr auto SLEEP_TIME = std::chrono::milliseconds(10);
127 std::this_thread::sleep_for(SLEEP_TIME);
128 auto res = std::ranges::all_of(is_pipeline_stopped_,
129 [](const std::atomic<bool>& is_pipe_stopped) -> bool
130 { return is_pipe_stopped.load(); });
131 return res;
132 }
133
134 template <typename PrevConverter, typename ThisTask>
135 auto TaskDiagram::emplace_to_taskflow(ThisTask& current_task,
136 std::pair<const PrevConverter&, tf::Task>& prev_task,
137 tf::Taskflow& taskflow,
138 std::size_t line_number)
139 -> std::optional<std::pair<const ThisTask&, tf::Task>>
140 {
141 current_task.set_report(report_);
142 // NOTE: This is where the previous converter and current task is connected.
143 auto task = taskflow
144 .emplace([line_number, &current_task, &prev_converter = prev_task.first]()
145 { [[maybe_unused]] auto res = current_task.run_once(prev_converter, line_number); })
146 .name(current_task.get_name_str());
147 if (not prev_task.second.empty())
148 {
149 prev_task.second.precede(task);
150 }
151 return std::pair<const ThisTask&, tf::Task>{ current_task, task };
152 }
153
154 template <typename PrevConverter, ConverterType ThisTask>
155 auto TaskDiagram::create_task_imp(std::optional<ThisTask>& current_task,
156 std::optional<std::pair<const PrevConverter&, tf::Task>>& prev_task,
157 tf::Taskflow& taskflow,
158 std::size_t line_number) -> std::optional<std::pair<const ThisTask&, tf::Task>>
159 {
160 if (not prev_task.has_value())
161 {
162 return {};
163 }
164 auto is_required = sinks_->is_convert_required(ThisTask::converter_type);
165 if (not is_required)
166 {
167 return {};
168 }
169 if (not current_task)
170 {
171 current_task.emplace(n_lines_);
172 }
173 return emplace_to_taskflow(current_task.value(), prev_task.value(), taskflow, line_number);
174 }
175
176 template <typename PrevConverter, SinkType ThisTask>
177 auto TaskDiagram::create_task_imp(ThisTask& current_task,
178 std::optional<std::pair<const PrevConverter&, tf::Task>>& prev_task,
179 tf::Taskflow& taskflow,
180 std::size_t line_number) -> std::optional<std::pair<const ThisTask&, tf::Task>>
181 {
182 if (not prev_task.has_value())
183 {
184 return {};
185 }
186 return emplace_to_taskflow(current_task, prev_task.value(), taskflow, line_number);
187 }
188
189 void TaskDiagram::construct_taskflow_line(tf::Taskflow& taskflow, std::size_t line_number)
190 {
191 auto create_task = [line_number, &taskflow, this]<typename PrevConverter, typename ThisTask>(
192 ThisTask& current_task,
193 std::optional<std::pair<const PrevConverter&, tf::Task>>& prev_task)
194 -> std::optional<std::pair<const remove_optional_t<ThisTask>&, tf::Task>>
195 { return create_task_imp(current_task, prev_task, taskflow, line_number); };
196
197 auto empty_task = std::optional{ std::pair<const TaskDiagram&, tf::Task>{ *this, tf::Task{} } };
198 auto raw_delimiter_task = create_task(raw_to_delim_raw_converter_, empty_task);
199 auto struct_deser_task = create_task(struct_deserializer_converter_, empty_task);
200 auto struct_to_proto_task = create_task(struct_to_proto_converter_, struct_deser_task);
201 auto proto_serial_task = create_task(proto_serializer_converter_, struct_to_proto_task);
202 auto proto_delim_serial_task = create_task(proto_delim_serializer_converter_, struct_to_proto_task);
203 // TODO: root_deser
204
205 sinks_->do_for_each_sink(
206 [&create_task,
207 &empty_task,
208 &raw_delimiter_task,
209 &struct_deser_task,
210 &proto_serial_task,
211 &proto_delim_serial_task](std::string_view filename, auto& sink) -> void
212 {
213 if constexpr (std::remove_cvref_t<decltype(sink)>::IsStructType)
214 {
215 create_task(sink, struct_deser_task);
216 }
217 else
218 {
220 const auto convert_mode = sink.get_required_conversion();
221 switch (convert_mode)
222 {
223 case raw:
224 create_task(sink, empty_task);
225 break;
226 case raw_frame:
227 create_task(sink, raw_delimiter_task);
228 break;
229 case proto:
230 create_task(sink, proto_serial_task);
231 break;
232 case proto_frame:
233 create_task(sink, proto_delim_serial_task);
234 break;
235 default:
236 spdlog::warn("unrecognized conversion {} from the file {}", convert_mode, filename);
237 create_task(sink, empty_task);
238 break;
239 }
240 }
241 });
242 }
243} // namespace srs::workflow
Class to manage the production and consumption of LargeBuffer asynchronously.
auto get_queue_consumer_token() -> BufferQueue::Token
auto get_buffer_queue() const -> const auto &
std::atomic< uint64_t > total_read_data_bytes_
std::optional< process::Struct2ProtoConverter > struct_to_proto_converter_
std::optional< process::ProtoSerializer > proto_serializer_converter_
void stop_time_record(std::size_t line_num)
auto is_taskflow_abort_ready() const -> bool
auto run_task(BufferQueue &data_queue, std::size_t line_number) -> bool
void construct_taskflow_and_run(BufferQueue &data_queue, const std::atomic< bool > &is_stopped)
Run the taskflow and block the current thread until finished.
gsl::not_null< sink::Manager * > sinks_
std::optional< process::ProtoDelimSerializer > proto_delim_serializer_converter_
void start_time_record(std::size_t line_num)
std::vector< AppReport::TaskStat > stats_
std::vector< std::atomic< bool > > is_pipeline_stopped_
auto emplace_to_taskflow(ThisTask &current_task, std::pair< const PrevConverter &, tf::Task > &prev_task, tf::Taskflow &taskflow, std::size_t line_number) -> std::optional< std::pair< const ThisTask &, tf::Task > >
std::vector< LargeBuffer > raw_data_
std::vector< tf::Taskflow > taskflow_lines_
std::optional< process::StructDeserializer > struct_deserializer_converter_
std::vector< TimePoint > last_times_
std::vector< BufferQueue::Token > consumer_tokens_
std::optional< process::Raw2DelimRawConverter > raw_to_delim_raw_converter_
TaskDiagram(AnalysisHandle &data_processor, std::size_t n_lines=1)
Constructor with number of pipelines.
std::atomic< bool > is_done_
void construct_taskflow_line(tf::Taskflow &taskflow, std::size_t line_number)
auto create_task_imp(std::optional< ThisTask > &current_task, std::optional< std::pair< const PrevConverter &, tf::Task > > &prev_task, tf::Taskflow &taskflow, std::size_t line_number) -> std::optional< std::pair< const ThisTask &, tf::Task > >
remove_optional< T >::type remove_optional_t