simdjson 5.0.2
Ridiculously Fast JSON
Loading...
Searching...
No Matches
document_stream.h
1#ifndef SIMDJSON_DOCUMENT_STREAM_H
2#define SIMDJSON_DOCUMENT_STREAM_H
3
4#include "simdjson/dom/base.h"
5#include "simdjson/dom/parser.h"
6
7#ifdef SIMDJSON_THREADS_ENABLED
8#include <thread>
9#include <mutex>
10#include <condition_variable>
11#endif
12
13namespace simdjson {
14namespace dom {
15
16#ifdef SIMDJSON_THREADS_ENABLED
18struct stage1_worker {
19 stage1_worker() noexcept = default;
20 stage1_worker(const stage1_worker&) = delete;
21 stage1_worker(stage1_worker&&) = delete;
22 stage1_worker operator=(const stage1_worker&) = delete;
23 ~stage1_worker();
28 void start_thread();
33 void run(document_stream * ds, dom::parser * stage1, size_t next_batch_start);
35 void finish();
36
37private:
38
44 void stop_thread();
45
46 std::thread thread{};
48 dom::parser * stage1_thread_parser{};
49 size_t _next_batch_start{};
50 document_stream * owner{};
55 bool has_work{false};
56 bool can_work{true};
57
61 std::mutex locking_mutex{};
62 std::condition_variable cond_var{};
63};
64#endif
65
73public:
82 simdjson_inline document_stream() noexcept;
84 simdjson_inline document_stream(document_stream &&other) noexcept = default;
86 simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default;
87
88 simdjson_inline ~document_stream() noexcept;
92 inline size_t size_in_bytes() const noexcept;
127 inline size_t truncated_bytes() const noexcept;
131 class iterator {
132 public:
134 using reference = value_type;
135
136 using difference_type = std::ptrdiff_t;
137
138 using iterator_category = std::input_iterator_tag;
139
143 simdjson_inline iterator() noexcept;
147 simdjson_inline reference operator*() noexcept;
151 inline iterator& operator++() noexcept;
156 simdjson_inline bool operator!=(const iterator &other) const noexcept;
172 simdjson_inline size_t current_index() const noexcept;
192 simdjson_inline std::string_view source() const noexcept;
193
194 private:
195 simdjson_inline iterator(document_stream *s, bool finished) noexcept;
197 document_stream* stream;
199 bool finished;
200 friend class document_stream;
201 };
202
206 simdjson_inline iterator begin() noexcept;
210 simdjson_inline iterator end() noexcept;
211
212private:
213
214 document_stream &operator=(const document_stream &) = delete; // Disallow copying
215 document_stream(const document_stream &other) = delete; // Disallow copying
216
227 simdjson_inline document_stream(
228 dom::parser &parser,
229 const uint8_t *buf,
230 size_t len,
231 size_t batch_size,
233 ) noexcept;
234
239 inline void start() noexcept;
240
264 inline void next() noexcept;
265
270 inline void load_batch() noexcept;
271
273 inline size_t next_batch_start() const noexcept;
274
276 inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept;
277
278 dom::parser *parser;
279 const uint8_t *buf;
280 size_t len;
281 size_t batch_size;
283 stream_format format;
285 error_code error;
286 size_t batch_start{0};
287 size_t doc_index{};
288#ifdef SIMDJSON_THREADS_ENABLED
290 bool use_thread;
291
292 inline void load_from_stage1_thread() noexcept;
293
295 inline void start_stage1_thread() noexcept;
296
298 inline void finish_stage1_thread() noexcept;
299
301 error_code stage1_thread_error{UNINITIALIZED};
303 friend struct stage1_worker;
304 std::unique_ptr<stage1_worker> worker{new(std::nothrow) stage1_worker()};
309 dom::parser stage1_thread_parser{};
310#endif // SIMDJSON_THREADS_ENABLED
311
312 friend class dom::parser;
313 friend struct simdjson_result<dom::document_stream>;
314 friend struct internal::simdjson_result_base<dom::document_stream>;
315
316}; // class document_stream
317
318} // namespace dom
319
320template<>
321struct simdjson_result<dom::document_stream> : public internal::simdjson_result_base<dom::document_stream> {
322public:
323 simdjson_inline simdjson_result() noexcept;
324 simdjson_inline simdjson_result(error_code error) noexcept;
325 simdjson_inline simdjson_result(dom::document_stream &&value) noexcept;
326
327#if SIMDJSON_EXCEPTIONS
328 simdjson_inline dom::document_stream::iterator begin() noexcept(false);
329 simdjson_inline dom::document_stream::iterator end() noexcept(false);
330#else // SIMDJSON_EXCEPTIONS
331#ifndef SIMDJSON_DISABLE_DEPRECATED_API
332 [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]]
333 simdjson_inline dom::document_stream::iterator begin() noexcept;
334 [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]]
335 simdjson_inline dom::document_stream::iterator end() noexcept;
336#endif // SIMDJSON_DISABLE_DEPRECATED_API
337#endif // SIMDJSON_EXCEPTIONS
338}; // struct simdjson_result<dom::document_stream>
339
340} // namespace simdjson
341
342#endif // SIMDJSON_DOCUMENT_STREAM_H
An iterator through a forward-only stream of documents.
A forward-only stream of documents.
size_t size_in_bytes() const noexcept
Returns the input size in bytes.
size_t truncated_bytes() const noexcept
After iterating through the stream, this method returns the number of bytes that were not parsed at t...
simdjson_inline iterator begin() noexcept
Start iterating the documents in the stream.
simdjson_inline document_stream(document_stream &&other) noexcept=default
Move one document_stream to another.
simdjson_inline document_stream & operator=(document_stream &&other) noexcept=default
Move one document_stream to another.
simdjson_inline iterator end() noexcept
The end of the stream, for iterator comparison purposes.
simdjson_inline document_stream() noexcept
Construct an uninitialized document_stream.
A persistent document parser.
Definition parser.h:30
The top level simdjson namespace, containing everything the library provides.
Definition base.h:8
stream_format
Stream format for parse_many/iterate_many.
Definition base.h:61
@ whitespace_delimited
Whitespace-delimited JSON documents (default, includes NDJSON/JSONL)
error_code
All possible errors returned by simdjson.
Definition error.h:19
@ UNINITIALIZED
unknown error, or uninitialized document
Definition error.h:32
The result of a simdjson operation that could fail.
Definition error.h:281