simdjson 5.0.2
Ridiculously Fast JSON
Loading...
Searching...
No Matches
document_stream.h
1#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H
2
3#ifndef SIMDJSON_CONDITIONAL_INCLUDE
4#define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H
5#include "simdjson/generic/ondemand/base.h"
6#include "simdjson/generic/implementation_simdjson_result_base.h"
7#include "simdjson/generic/ondemand/document.h"
8#include "simdjson/generic/ondemand/parser.h"
9#endif // SIMDJSON_CONDITIONAL_INCLUDE
10
11#ifdef SIMDJSON_THREADS_ENABLED
12#include <thread>
13#include <mutex>
14#include <condition_variable>
15#endif
16
17namespace simdjson {
18namespace SIMDJSON_IMPLEMENTATION {
19namespace ondemand {
20
21#ifdef SIMDJSON_THREADS_ENABLED
23struct stage1_worker {
24 stage1_worker() noexcept = default;
25 stage1_worker(const stage1_worker&) = delete;
26 stage1_worker(stage1_worker&&) = delete;
27 stage1_worker operator=(const stage1_worker&) = delete;
28 ~stage1_worker();
33 void start_thread();
38 void run(document_stream * ds, parser * stage1, size_t next_batch_start);
40 void finish();
41
42private:
43
49 void stop_thread();
50
51 std::thread thread{};
53 ondemand::parser * stage1_thread_parser{};
54 size_t _next_batch_start{};
55 document_stream * owner{};
60 bool has_work{false};
61 bool can_work{true};
62
66 std::mutex locking_mutex{};
67 std::condition_variable cond_var{};
68
69 friend class document_stream;
70};
71#endif // SIMDJSON_THREADS_ENABLED
72
80public:
89 simdjson_inline document_stream() noexcept;
91 simdjson_inline document_stream(document_stream &&other) noexcept = default;
93 simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default;
94
95 simdjson_inline ~document_stream() noexcept;
96
100 inline size_t size_in_bytes() const noexcept;
101
132 inline size_t truncated_bytes() const noexcept;
133
134 class iterator {
135 public:
138 using pointer = void;
139 using difference_type = std::ptrdiff_t;
140 using iterator_category = std::input_iterator_tag;
141
145 simdjson_inline iterator() noexcept;
146 simdjson_inline iterator(const iterator &other) noexcept = default;
150 simdjson_inline reference operator*() noexcept;
154 inline iterator& operator++() noexcept;
159 simdjson_inline bool operator!=(const iterator &other) const noexcept;
160 simdjson_inline bool operator==(const iterator &other) const noexcept;
176 simdjson_inline size_t current_index() const noexcept;
177
200 simdjson_inline std::string_view source() const noexcept;
201
205 inline error_code error() const noexcept;
206
210 inline bool at_end() const noexcept;
211
212 private:
213 simdjson_inline iterator(document_stream *s, bool finished) noexcept;
215 document_stream* stream;
217 bool finished;
218
219 friend class document;
220 friend class document_stream;
221 friend class json_iterator;
222 };
224
228 simdjson_inline iterator begin() noexcept;
232 simdjson_inline iterator end() noexcept;
233
234private:
235
236 document_stream &operator=(const document_stream &) = delete; // Disallow copying
237 document_stream(const document_stream &other) = delete; // Disallow copying
238
250 simdjson_inline document_stream(
251 ondemand::parser &parser,
252 const uint8_t *buf,
253 size_t len,
254 size_t batch_size,
255 bool allow_comma_separated,
257 ) noexcept;
258
263 inline void start() noexcept;
264
288 inline void next() noexcept;
289
299 inline void next_document() noexcept;
301 simdjson_inline uint8_t document_delimiter() const noexcept;
306 simdjson_inline bool skip_to_delimiter(uint8_t delimiter) noexcept;
307
309 inline size_t next_batch_start() const noexcept;
310
312 inline error_code run_stage1(ondemand::parser &p, size_t batch_start) noexcept;
313
314 // Fields
315 ondemand::parser *parser;
316 const uint8_t *buf;
317 size_t len;
318 size_t batch_size;
319 bool allow_comma_separated;
320 stream_format format;
326 document doc{};
328 error_code error;
329 size_t batch_start{0};
330 size_t doc_index{};
331
332 #ifdef SIMDJSON_THREADS_ENABLED
334 bool use_thread;
335
336 inline void load_from_stage1_thread() noexcept;
337
339 inline void start_stage1_thread() noexcept;
340
342 inline void finish_stage1_thread() noexcept;
343
345 error_code stage1_thread_error{UNINITIALIZED};
347 std::unique_ptr<stage1_worker> worker{};
352 ondemand::parser stage1_thread_parser{};
353
354 friend struct stage1_worker;
355 #endif // SIMDJSON_THREADS_ENABLED
356
357 friend class parser;
358 friend class document;
359 friend class json_iterator;
360 friend struct simdjson_result<ondemand::document_stream>;
361 friend struct simdjson::internal::simdjson_result_base<ondemand::document_stream>;
362}; // document_stream
363
364} // namespace ondemand
365} // namespace SIMDJSON_IMPLEMENTATION
366} // namespace simdjson
367
368namespace simdjson {
369template<>
370struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_stream> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::document_stream> {
371public:
372 simdjson_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::document_stream &&value) noexcept;
373 simdjson_inline simdjson_result(error_code error) noexcept;
374 simdjson_inline simdjson_result() noexcept = default;
375};
376
377} // namespace simdjson
378
379#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H
simdjson_inline iterator end() noexcept
The end of the stream, for iterator comparison purposes.
simdjson_inline document_stream(document_stream &&other) noexcept=default
Move one document_stream to another.
size_t size_in_bytes() const noexcept
Returns the input size in bytes.
simdjson_inline document_stream & operator=(document_stream &&other) noexcept=default
Move one document_stream to another.
size_t truncated_bytes() const noexcept
After iterating through the stream, this method returns the number of bytes that were not parsed at t...
simdjson_inline iterator begin() noexcept
Start iterating the documents in the stream.
simdjson_inline document_stream() noexcept
Construct an uninitialized document_stream.
simdjson_unused simdjson_inline bool operator==(const raw_json_string &a, std::string_view c) noexcept
Comparisons between raw_json_string and std::string_view instances are potentially unsafe: the user i...
The top level simdjson namespace, containing everything the library provides.
Definition base.h:8
stream_format
Stream format for parse_many/iterate_many.
Definition base.h:61
@ whitespace_delimited
Whitespace-delimited JSON documents (default, includes NDJSON/JSONL)
error_code
All possible errors returned by simdjson.
Definition error.h:19
@ UNINITIALIZED
unknown error, or uninitialized document
Definition error.h:32
The result of a simdjson operation that could fail.
Definition error.h:281