GridFormat 0.6.0
I/O-Library for grid-like data structures
Loading...
Searching...
No Matches
xml.hpp
Go to the documentation of this file.
1// SPDX-FileCopyrightText: 2022-2023 Dennis Gläser <dennis.glaeser@iws.uni-stuttgart.de>
2// SPDX-License-Identifier: MIT
8#ifndef GRIDFORMAT_VTK_XML_HPP_
9#define GRIDFORMAT_VTK_XML_HPP_
10
11#include <bit>
12#include <string>
13#include <ranges>
14#include <utility>
15#include <type_traits>
16#include <functional>
17#include <optional>
18#include <iterator>
19#include <string_view>
20#include <concepts>
21#include <limits>
22
23#include <gridformat/common/detail/crtp.hpp>
24#include <gridformat/common/callable_overload_set.hpp>
25#include <gridformat/common/optional_reference.hpp>
26#include <gridformat/common/exceptions.hpp>
27#include <gridformat/common/type_traits.hpp>
28#include <gridformat/common/variant.hpp>
29#include <gridformat/common/precision.hpp>
30#include <gridformat/common/logging.hpp>
32#include <gridformat/common/multi_string.hpp>
33#include <gridformat/common/lazy_field.hpp>
34#include <gridformat/common/path.hpp>
35
39
43
46#include <gridformat/grid/grid.hpp>
47
48#include <gridformat/xml/element.hpp>
49#include <gridformat/xml/parser.hpp>
50
55
56namespace GridFormat::VTK {
57
58#ifndef DOXYGEN
59namespace XMLDetail {
60 using LZMACompressor = std::conditional_t<Compression::Detail::_have_lzma, Compression::LZMA, None>;
61 using ZLIBCompressor = std::conditional_t<Compression::Detail::_have_zlib, Compression::ZLIB, None>;
62 using LZ4Compressor = std::conditional_t<Compression::Detail::_have_lz4, Compression::LZ4, None>;
63 using Compressor = UniqueVariant<LZ4Compressor, ZLIBCompressor, LZMACompressor>;
64} // namespace XMLDetail
65
66namespace XML {
67
68using HeaderPrecision = std::variant<UInt32, UInt64>;
69using CoordinatePrecision = std::variant<Float32, Float64>;
70using Compressor = ExtendedVariant<XMLDetail::Compressor, None>;
71using Encoder = std::variant<Encoding::Ascii, Encoding::Base64, Encoding::RawBinary>;
72using DataFormat = std::variant<VTK::DataFormat::Inlined, VTK::DataFormat::Appended>;
73using DefaultCompressor = std::variant_alternative_t<0, XMLDetail::Compressor>;
74
75} // namespace XML
76#endif // DOXYGEN
77
102 using EncoderOption = ExtendedVariant<XML::Encoder, Automatic>;
103 using CompressorOption = ExtendedVariant<XML::Compressor, Automatic>;
104 using DataFormatOption = ExtendedVariant<XML::DataFormat, Automatic>;
105 using CoordinatePrecisionOption = ExtendedVariant<XML::CoordinatePrecision, Automatic>;
106 EncoderOption encoder = automatic;
107 CompressorOption compressor = automatic;
108 DataFormatOption data_format = automatic;
109 CoordinatePrecisionOption coordinate_precision = automatic;
110 XML::HeaderPrecision header_precision = _from_size_t();
111
112 private:
113 static constexpr XML::HeaderPrecision _from_size_t() {
114 if constexpr(sizeof(std::size_t) == 8) return uint64;
115 else return uint32;
116 }
117};
118
119
120#ifndef DOXYGEN
121namespace XMLDetail {
122
123 struct XMLSettings {
124 XML::Encoder encoder;
125 XML::Compressor compressor;
126 XML::DataFormat data_format;
127 XML::CoordinatePrecision coordinate_precision;
128 XML::HeaderPrecision header_precision;
129
130 template<typename GridCoordinateType>
131 static XMLSettings from(const XMLOptions& opts) {
132 static_assert(
133 std::is_same_v<GridCoordinateType, float> || std::is_same_v<GridCoordinateType, double>,
134 "Only float/double coordinate types are supported by VTK"
135 );
136 const auto _enc = _make_encoder(opts.encoder);
137 const auto _format = _make_data_format(_enc, opts.data_format);
138 const auto _comp = _make_compressor(_enc, opts.compressor);
139 return {
140 .encoder = _enc,
141 .compressor = _comp,
142 .data_format = _format,
143 .coordinate_precision =
144 Variant::is<Automatic>(opts.coordinate_precision) ?
145 XML::CoordinatePrecision{Precision<GridCoordinateType>{}} :
146 Variant::without<Automatic>(opts.coordinate_precision),
147 .header_precision = opts.header_precision
148 };
149 }
150
151 private:
152 static XML::Encoder _make_encoder(const typename XMLOptions::EncoderOption& enc) {
153 if (Variant::is<Automatic>(enc))
154 return Encoding::base64;
155 return Variant::without<Automatic>(enc);
156 }
157
158 static XML::DataFormat _make_data_format(const typename XML::Encoder& enc,
159 const typename XMLOptions::DataFormatOption& data_format) {
160 if (Variant::is<Automatic>(data_format))
161 return Variant::is<Encoding::Ascii>(enc)
162 ? XML::DataFormat{VTK::DataFormat::inlined}
163 : XML::DataFormat{VTK::DataFormat::appended};
164 return Variant::without<Automatic>(data_format);
165 }
166
167 static XML::Compressor _make_compressor(const typename XML::Encoder& enc,
168 const typename XMLOptions::CompressorOption& compressor) {
169 if (Variant::is<Automatic>(compressor))
170 return Variant::is<Encoding::Ascii>(enc)
171 ? XML::Compressor{none}
172 : XML::Compressor{XML::DefaultCompressor{}};
173 if (Variant::is<Encoding::Ascii>(enc) && !Variant::is<None>(compressor)) {
174 log_warning("Ascii output cannot be compressed. Ignoring chosen compressor...");
175 return none;
176 }
177 return Variant::without<Automatic>(compressor);
178 }
179 };
180
181} // namespace XMLDetail
182#endif // DOXYGEN
183
184
189template<Concepts::Grid G, typename Impl>
190class XMLWriterBase
191: public GridWriter<G>
192, public GridFormat::Detail::CRTPBase<Impl> {
193 using ParentType = GridWriter<G>;
194 using GridCoordinateType = CoordinateType<G>;
195
196 public:
198 using Grid = G;
199
200 virtual ~XMLWriterBase() = default;
201 explicit XMLWriterBase(const Grid& grid,
202 std::string extension,
203 bool use_structured_grid_ordering,
204 XMLOptions xml_opts = {})
205 : ParentType(grid, std::move(extension), WriterOptions{use_structured_grid_ordering, true})
206 , _xml_opts{std::move(xml_opts)}
207 , _xml_settings{XMLDetail::XMLSettings::from<GridCoordinateType>(_xml_opts)}
208 {}
209
210 XMLWriterBase(XMLWriterBase&&) = default;
211 XMLWriterBase(const XMLWriterBase&) = delete;
212 // not assignable since the grid is stored by reference
213 XMLWriterBase& operator=(XMLWriterBase&&) = delete;
214 XMLWriterBase& operator=(const XMLWriterBase&) = delete;
215
216 Impl with(XMLOptions opts) const {
217 auto result = _with(std::move(opts));
218 this->copy_fields(result);
219 return result;
220 }
221
222 Impl with_data_format(const XML::DataFormat& format) const {
223 auto opts = _xml_opts;
224 Variant::unwrap_to(opts.data_format, format);
225 return with(std::move(opts));
226 }
227
228 Impl with_compression(const XML::Compressor& compressor) const {
229 auto opts = _xml_opts;
230 Variant::unwrap_to(opts.compressor, compressor);
231 return with(std::move(opts));
232 }
233
234 Impl with_encoding(const XML::Encoder& encoder) const {
235 auto opts = _xml_opts;
236 Variant::unwrap_to(opts.encoder, encoder);
237 return with(std::move(opts));
238 }
239
240 Impl with_coordinate_precision(const XML::CoordinatePrecision& prec) const {
241 auto opts = _xml_opts;
242 Variant::unwrap_to(opts.coordinate_precision, prec);
243 return with(std::move(opts));
244 }
245
246 Impl with_header_precision(const XML::HeaderPrecision& prec) const {
247 auto opts = _xml_opts;
248 opts.header_precision = prec;
249 return with(std::move(opts));
250 }
251
252 private:
253 virtual Impl _with(XMLOptions opts) const = 0;
254
255 protected:
256 XMLOptions _xml_opts;
257 XMLDetail::XMLSettings _xml_settings;
258
260 std::string vtk_grid_type;
261 XMLElement xml_representation;
262 Appendix appendix;
263 };
264
265 WriteContext _get_write_context(std::string vtk_grid_type) const {
266 return std::visit([&] (const auto& compressor) {
267 return std::visit([&] (const auto& header_precision) {
268 XMLElement xml("VTKFile");
269 xml.set_attribute("type", vtk_grid_type);
270 xml.set_attribute("version", "2.2");
271 xml.set_attribute("byte_order", attribute_name(std::endian::native));
272 xml.set_attribute("header_type", attribute_name(DynamicPrecision{header_precision}));
273 if constexpr (!is_none<decltype(compressor)>)
274 xml.set_attribute("compressor", attribute_name(compressor));
275 xml.add_child(vtk_grid_type);
276
277 WriteContext context{
278 .vtk_grid_type = std::move(vtk_grid_type),
279 .xml_representation = std::move(xml),
280 .appendix = {}
281 };
282 _add_meta_data_fields(context);
283 return context;
284 }, _xml_settings.header_precision);
285 }, _xml_settings.compressor);
286 }
287
288 void _add_meta_data_fields(WriteContext& context) const {
289 const auto& names = this->_meta_data_field_names();
290 if (std::ranges::empty(names))
291 return;
292
293 XMLElement& field_data = context.xml_representation
294 .get_child(context.vtk_grid_type)
295 .add_child("FieldData");
296 std::visit([&] (const auto& encoder) {
297 std::visit([&] (const auto& compressor) {
298 std::visit([&] (const auto& data_format) {
299 std::visit([&] (const auto& header_precision) {
300 std::ranges::for_each(names, [&] (const std::string& name) {
301 const auto& field = this->_get_meta_data_field(name);
302 const auto layout = field.layout();
303 const auto precision = field.precision();
304
305 auto& array = field_data.add_child("DataArray");
306 array.set_attribute("Name", name);
307 array.set_attribute("format", data_format_name(encoder, data_format));
308 if (precision.template is<char>() && layout.dimension() == 1) {
309 array.set_attribute("type", "String");
310 const auto strings = field.template export_to<MultiString>();
311 array.set_attribute("NumberOfTuples", strings.slices().size());
312 } else {
313 array.set_attribute("NumberOfTuples", layout.extent(0));
314 array.set_attribute("type", attribute_name(precision));
315 array.set_attribute(
316 "NumberOfComponents",
317 layout.dimension() > 1
318 ? layout.sub_layout(1).number_of_entries()
319 : 1
320 );
321 }
322 DataArray content{field, encoder, compressor, header_precision};
323 _set_data_array_content(data_format, array, context.appendix, std::move(content));
324 });
325 }, _xml_settings.header_precision);
326 }, _xml_settings.data_format);
327 }, _xml_settings.compressor);
328 }, _xml_settings.encoder);
329 }
330
331 template<typename ValueType>
332 void _set_attribute(WriteContext& context,
333 std::string_view xml_group,
334 const std::string& attr_name,
335 const ValueType& attr_value) const {
336 _access_at(xml_group, context).set_attribute(attr_name, attr_value);
337 }
338
339 void _set_data_array(WriteContext& context,
340 std::string_view xml_group,
341 std::string data_array_name,
342 const Field& field) const {
343 const auto layout = field.layout();
344 XMLElement& da = _access_at(xml_group, context).add_child("DataArray");
345 da.set_attribute("Name", std::move(data_array_name));
346 da.set_attribute("type", attribute_name(field.precision()));
347 da.set_attribute("NumberOfComponents", (layout.dimension() == 1 ? 1 : layout.number_of_entries(1)));
348 std::visit([&] (const auto& encoder) {
349 std::visit([&] (const auto& compressor) {
350 std::visit([&] (const auto& data_format) {
351 std::visit([&] (const auto& header_prec) {
352 da.set_attribute("format", data_format_name(encoder, data_format));
353 DataArray content{field, encoder, compressor, header_prec};
354 _set_data_array_content(data_format, da, context.appendix, std::move(content));
355 }, _xml_settings.header_precision);
356 }, _xml_settings.data_format);
357 }, _xml_settings.compressor);
358 }, _xml_settings.encoder);
359 }
360
361 template<typename DataFormat, typename Appendix, typename Content>
362 requires(!std::is_lvalue_reference_v<Content>)
363 void _set_data_array_content(const DataFormat&,
364 XMLElement& e,
365 Appendix& app,
366 Content&& c) const {
367 static constexpr bool is_inlined = std::is_same_v<DataFormat, VTK::DataFormat::Inlined>;
368 static constexpr bool is_appended = std::is_same_v<DataFormat, VTK::DataFormat::Appended>;
369 static_assert(is_inlined || is_appended, "Unknown data format");
370
371 if constexpr (is_inlined)
372 e.set_content(std::move(c));
373 else
374 app.add(std::move(c));
375 }
376
377 XMLElement& _access_at(std::string_view path, WriteContext& context) const {
378 return access_or_create_at(path, context.xml_representation.get_child(context.vtk_grid_type));
379 }
380
381 void _write_xml(WriteContext&& context, std::ostream& s) const {
382 Indentation indentation{{.width = 2}};
383 _set_default_active_fields(context.xml_representation.get_child(context.vtk_grid_type));
384 std::visit([&] (const auto& encoder) {
385 std::visit([&] <typename DataFormat> (const DataFormat&) {
386 if constexpr (std::is_same_v<DataFormat, VTK::DataFormat::Inlined>)
387 write_xml_with_version_header(context.xml_representation, s, indentation);
388 else
389 XML::Detail::write_with_appendix(std::move(context), s, encoder, indentation);
390 }, this->_xml_settings.data_format);
391 }, this->_xml_settings.encoder);
392 }
393
394 void _set_default_active_fields(XMLElement& xml) const {
395 const auto set = [&] (std::string_view group, const auto& attr, const auto& name) -> void {
396 auto group_element = access_at(group, xml);
397 if (!group_element)
398 return;
399 group_element.unwrap().set_attribute(attr, name);
400 };
401
402 // discard vectors with more than 3 elements for active arrays
403 const auto get_vector_filter = [] (unsigned int rank) {
404 return [r=rank] (const auto& name_field_ptr_pair) {
405 if (r == 1)
406 return name_field_ptr_pair.second->layout().extent(1) <= 3;
407 return true;
408 };
409 };
410
411 for (std::string_view group : {"Piece/PointData", "PPointData"})
412 for (unsigned int i = 0; i <= 2; ++i)
413 for (const auto& [n, _] : point_fields_of_rank(i, *this)
414 | std::views::filter(get_vector_filter(i))
415 | std::views::take(1))
416 set(group, active_array_attribute_for_rank(i), n);
417
418 for (std::string_view group : {"Piece/CellData", "PCellData"})
419 for (unsigned int i = 0; i <= 2; ++i)
420 for (const auto& [n, _] : cell_fields_of_rank(i, *this)
421 | std::views::filter(get_vector_filter(i))
422 | std::views::take(1))
423 set(group, active_array_attribute_for_rank(i), n);
424 }
425};
426
427
428#ifndef DOXYGEN
429namespace XMLDetail {
430
431 struct DataArrayStreamLocation {
432 std::streamsize begin; // location where data begins
433 std::optional<std::streamsize> offset = {}; // used for appended data
434 };
435
436 template<std::integral I, std::integral O>
437 void _move_to_appendix_position(std::istream& stream, I appendix_begin, O offset_in_appendix) {
438 InputStreamHelper helper{stream};
439 helper.seek_position(appendix_begin);
440 helper.shift_whitespace();
441 if (helper.read_chunk(1) != "_")
442 throw IOError("VTK-XML appendix must start with '_'");
443 helper.shift_by(offset_in_appendix);
444 }
445
446 inline void _move_to_data(const DataArrayStreamLocation& location, std::istream& s) {
447 if (location.offset)
448 _move_to_appendix_position(s, location.begin, location.offset.value());
449 else {
450 InputStreamHelper helper{s};
451 helper.seek_position(location.begin);
452 helper.shift_whitespace();
453 }
454 }
455
457 template<std::unsigned_integral T>
458 T _number_of_uncompressed_bytes(T number_of_blocks, T block_size, T residual_block_size) {
459 if (number_of_blocks == 0) {
460 if (residual_block_size > 0)
461 throw IOError("Invalid compression header: residual block size given without any blocks");
462 return 0;
463 }
464 if (block_size == 0 || residual_block_size > block_size)
465 throw IOError("Invalid compression header: inconsistent block sizes");
466 const T last_block_size = residual_block_size > 0 ? residual_block_size : block_size;
467 if (number_of_blocks - 1 > (std::numeric_limits<T>::max() - last_block_size)/block_size)
468 throw IOError("Invalid compression header: number of bytes exceeds the range of the header type");
469 return block_size*(number_of_blocks - 1) + last_block_size;
470 }
471
473 template<std::invocable Action>
474 decltype(auto) _read_data_array(const std::string& name, Action&& action) {
475 try {
476 return action();
477 } catch (const std::exception& e) {
478 throw IOError("Could not read data array '" + name + "': " + e.what());
479 }
480 }
481
482 template<typename HeaderType>
483 void _decompress_with(const std::string& vtk_compressor,
484 [[maybe_unused]] Serialization& data,
485 [[maybe_unused]] const Compression::CompressedBlocks<HeaderType>& blocks) {
486 if (vtk_compressor == "vtkLZ4DataCompressor") {
487#if GRIDFORMAT_HAVE_LZ4
488 LZ4Compressor{}.decompress(data, blocks);
489#else
490 throw InvalidState("Need LZ4 to decompress the data");
491#endif
492 } else if (vtk_compressor == "vtkLZMADataCompressor") {
493#if GRIDFORMAT_HAVE_LZMA
494 LZMACompressor{}.decompress(data, blocks);
495#else
496 throw InvalidState("Need LZMA to decompress the data");
497#endif
498 } else if (vtk_compressor == "vtkZLibDataCompressor") {
499#if GRIDFORMAT_HAVE_ZLIB
500 ZLIBCompressor{}.decompress(data, blocks);
501#else
502 throw InvalidState("Need ZLib to decompress the data");
503#endif
504 } else {
505 throw NotImplemented("Unsupported vtk compressor '" + vtk_compressor + "'");
506 }
507 }
508
509 template<Concepts::Scalar TargetType, Concepts::Scalar HeaderType = std::size_t>
510 class DataArrayReader {
511 static constexpr Precision<TargetType> target_precision{};
512 static constexpr Precision<HeaderType> header_precision{};
513
514 static constexpr bool is_too_small_integral = std::integral<TargetType> && sizeof(TargetType) < 4;
515 static constexpr bool is_signed_integral = std::signed_integral<TargetType>;
516 using BufferedType = std::conditional_t<is_signed_integral, short, unsigned short>;
517
518 public:
519 using Header = std::vector<HeaderType>;
520
521 DataArrayReader(std::istream& s,
522 std::endian e = std::endian::native,
523 std::string compressor = "")
524 : _stream{s}
525 , _endian{e}
526 , _compressor{compressor}
527 {}
528
529 void read_ascii(std::size_t number_of_values, Serialization& out_values) {
530 out_values.resize(number_of_values*sizeof(TargetType));
531 std::span<TargetType> out_span = out_values.as_span_of(target_precision);
532
533 // istream_view uses operator>>, which seems to do weird stuff for e.g. uint8_t.
534 // The smallest supported integral type seems to be short. In case we deal with
535 // small integral types, we first read into a vector and then cast into the desired type.
536 if constexpr (is_too_small_integral) {
537 std::vector<BufferedType> buffer(number_of_values);
538 _read_ascii_to(std::span{buffer}, number_of_values);
539 std::ranges::copy(buffer | std::views::transform([] <typename T> (const T& value) {
540 return static_cast<TargetType>(value);
541 }), out_span.begin());
542 } else {
543 _read_ascii_to(out_span, number_of_values);
544 }
545 }
546
547 template<Concepts::Decoder Decoder>
548 void read_binary(const Decoder& decoder,
549 OptionalReference<Header> header = {},
550 OptionalReference<Serialization> values = {}) {
551 if constexpr (std::unsigned_integral<HeaderType> && sizeof(HeaderType) >= 4) {
552 if (_compressor.empty())
553 return _read_encoded(decoder, header, values);
554 else
555 return _read_encoded_compressed(decoder, header, values);
556 } else {
557 throw IOError("Unsupported header type");
558 }
559 }
560
561 private:
562 template<typename T, std::size_t size>
563 void _read_ascii_to(std::span<T, size> buffer, std::size_t expected_num_values) const {
564 const auto [_, out] = std::ranges::copy(
565 std::ranges::istream_view<T>{_stream}
566 | std::views::take(expected_num_values),
567 buffer.begin()
568 );
569 const auto number_of_values_read = std::ranges::distance(buffer.begin(), out);
570 if (number_of_values_read < 0 || static_cast<std::size_t>(number_of_values_read) < expected_num_values) {
571 if (_stream.fail())
572 throw IOError("A read value could not be converted to type: " + std::string{typeid(TargetType).name()});
573 throw SizeError("Could not read the requested number of values from the stream");
574 }
575 }
576
577 template<typename Decoder>
578 void _read_encoded(const Decoder& decoder,
579 OptionalReference<Header> out_header = {},
580 OptionalReference<Serialization> out_values = {}) {
581 const auto pos = _stream.tellg();
582 Serialization header = decoder.decode_from(_stream, sizeof(HeaderType));
583
584 if (header.size() != sizeof(HeaderType)) { // no padding - header & values are encoded together
585 if (header.size() < sizeof(HeaderType))
586 throw SizeError("Could not read header");
587
588 header.resize(sizeof(HeaderType));
589 change_byte_order(header.as_span_of(header_precision), {.from = _endian});
590
591 if (out_header)
592 std::ranges::copy(header.as_span_of(header_precision), std::back_inserter(out_header.unwrap()));
593
594 if (out_values) {
595 _stream.seekg(pos);
596 const auto number_of_bytes = header.as_span_of(header_precision)[0];
597 _check_available(number_of_bytes, "data array");
598 const auto number_of_bytes_with_header = number_of_bytes + sizeof(HeaderType);
599 Serialization& header_and_values = out_values.unwrap();
600 header_and_values = decoder.decode_from(_stream, number_of_bytes_with_header);
601 header_and_values.cut_front(sizeof(HeaderType));
602 change_byte_order(header_and_values.as_span_of(header_precision), {.from = _endian});
603 }
604 } else { // values are encoded separately
605 change_byte_order(header.as_span_of(header_precision), {.from = _endian});
606
607 if (out_header)
608 std::ranges::copy(header.as_span_of(header_precision), std::back_inserter(out_header.unwrap()));
609
610 if (out_values) {
611 const auto number_of_bytes = header.as_span_of(header_precision)[0];
612 _check_available(number_of_bytes, "data array");
613 Serialization& values = out_values.unwrap();
614 values = decoder.decode_from(_stream, number_of_bytes);
615 change_byte_order(values.as_span_of(Precision<TargetType>{}), {.from = _endian});
616 }
617 }
618 }
619
620 template<typename Decoder>
621 void _read_encoded_compressed(const Decoder& decoder,
622 OptionalReference<Header> out_header = {},
623 OptionalReference<Serialization> out_values = {}) {
624 const auto begin_pos = _stream.tellg();
625 const auto header_bytes = sizeof(HeaderType)*3;
626 Serialization header = decoder.decode_from(_stream, header_bytes);
627
628 // if the decoded header is larger than requested, then there is no padding,
629 // which means that we'll have to decode the header together with the values.
630 const bool decode_blocks_with_header = header.size() != header_bytes;
631 if (decode_blocks_with_header)
632 header.resize(header_bytes);
633
634 auto header_data = header.as_span_of(header_precision);
635 if (header_data.size() < 3)
636 throw SizeError("Could not read data array header");
637
638 change_byte_order(header_data, {.from = _endian});
639 const auto number_of_blocks = header_data[0];
640 const auto full_block_size = header_data[1];
641 const auto residual_block_size = header_data[2];
642 const HeaderType number_of_raw_bytes = _number_of_uncompressed_bytes(
643 number_of_blocks, full_block_size, residual_block_size
644 );
645 if (number_of_blocks > _remaining_bytes()/sizeof(HeaderType))
646 throw IOError("Number of compressed blocks stated in the header exceeds the remaining file size");
647
648 Serialization block_sizes;
649 const std::size_t block_sizes_bytes = sizeof(HeaderType)*number_of_blocks;
650 if (decode_blocks_with_header) {
651 _stream.seekg(begin_pos);
652 block_sizes = decoder.decode_from(_stream, header_bytes + block_sizes_bytes);
653 block_sizes.cut_front(sizeof(HeaderType)*3);
654 } else {
655 block_sizes = decoder.decode_from(_stream, block_sizes_bytes);
656 }
657
658 if (block_sizes.size() < block_sizes_bytes)
659 throw IOError("Could not read the compressed block sizes");
660 block_sizes.resize(block_sizes_bytes);
661
662 std::vector<HeaderType> compressed_block_sizes(number_of_blocks);
663 change_byte_order(block_sizes.as_span_of(header_precision), {.from = _endian});
664 std::ranges::copy(
665 block_sizes.as_span_of(header_precision),
666 compressed_block_sizes.begin()
667 );
668
669 if (out_header) {
670 std::ranges::copy(header_data, std::back_inserter(out_header.unwrap()));
671 std::ranges::copy(compressed_block_sizes, std::back_inserter(out_header.unwrap()));
672 }
673
674 if (out_values) {
675 const std::size_t remaining_bytes = _remaining_bytes();
676 std::size_t compressed_bytes = 0;
677 for (const auto bytes : compressed_block_sizes) {
678 if (bytes > remaining_bytes - compressed_bytes)
679 throw IOError("Size of the compressed data stated in the header exceeds the remaining file size");
680 compressed_bytes += bytes;
681 }
682
683 Serialization& values = out_values.unwrap();
684 if (number_of_raw_bytes == 0) {
685 values = Serialization{};
686 return;
687 }
688
689 values = decoder.decode_from(_stream, compressed_bytes);
690 if (values.size() < compressed_bytes)
691 throw IOError("Could not read the compressed data");
692 _decompress_with(_compressor, values, Compression::CompressedBlocks{
693 {number_of_raw_bytes, full_block_size},
694 std::move(compressed_block_sizes)
695 });
696 change_byte_order(values.as_span_of(target_precision), {.from = _endian});
697 }
698 }
699
700 // return the number of bytes left in the stream (or the maximum value if this cannot be determined)
701 std::size_t _remaining_bytes() const {
702 const auto pos = _stream.tellg();
703 if (pos < 0)
704 return std::numeric_limits<std::size_t>::max();
705 _stream.seekg(0, std::ios::end);
706 const auto end = _stream.tellg();
707 _stream.seekg(pos);
708 return end >= pos ? static_cast<std::size_t>(end - pos) : std::numeric_limits<std::size_t>::max();
709 }
710
711 // throw if the given number of bytes (as read from a header) exceeds the remaining stream size
712 void _check_available(std::size_t number_of_bytes, std::string_view what) const {
713 if (number_of_bytes > _remaining_bytes())
714 throw IOError(
715 "Size of the " + std::string{what} + " (" + std::to_string(number_of_bytes) + " bytes) "
716 + "stated in the header exceeds the remaining file size"
717 );
718 }
719
720 std::istream& _stream;
721 std::endian _endian;
722 std::string _compressor;
723 };
724
725} // namespace XMLDetail
726#endif // DOXYGEN
727
728
729namespace XML {
730
735inline std::ranges::range auto data_arrays(const XMLElement& e) {
736 return children(e) | std::views::filter([] (const XMLElement& child) {
737 return child.name() == "DataArray";
738 });
739}
740
745inline std::ranges::range auto data_array_names(const XMLElement& e) {
746 return data_arrays(e) | std::views::transform([] (const XMLElement& data_array) {
747 return data_array.get_attribute("Name");
748 });
749}
750
755inline const XMLElement& get_data_array(std::string_view name, const XMLElement& section) {
756 for (const auto& da
757 : data_arrays(section)
758 | std::views::filter([&] (const auto& e) { return e.get_attribute("Name") == name; }))
759 return da;
760 throw ValueError(
761 "Could not find data array with name '" + std::string{name}
762 + "' in section '" + std::string{section.name()} + "'"
763 );
764}
765
766} // namespace XML
767
768
769#ifndef DOXYGEN
770namespace XMLDetail {
771
772 // can be reused by readers to copy read field names into the storage container
773 template<typename FieldNameContainer>
774 void copy_field_names_from(const XMLElement& vtk_grid, FieldNameContainer& names) {
775 if (vtk_grid.has_child("Piece")) {
776 const XMLElement& piece = vtk_grid.get_child("Piece");
777 if (piece.has_child("PointData"))
778 std::ranges::copy(
779 XML::data_array_names(piece.get_child("PointData")),
780 std::back_inserter(names.point_fields)
781 );
782 if (piece.has_child("CellData"))
783 std::ranges::copy(
784 XML::data_array_names(piece.get_child("CellData")),
785 std::back_inserter(names.cell_fields)
786 );
787 }
788 if (vtk_grid.has_child("FieldData"))
789 std::ranges::copy(
790 XML::data_array_names(vtk_grid.get_child("FieldData")),
791 std::back_inserter(names.meta_data_fields)
792 );
793 }
794
795} // namespace XMLDetail
796#endif // DOXYGEN
797
798
803class XMLReaderHelper {
804 using DataArrayStreamLocation = XMLDetail::DataArrayStreamLocation;
805
806 public:
807 explicit XMLReaderHelper(const std::string& filename)
808 : _filename{filename}
809 , _parser{filename, "ROOT", [] (const XMLElement& e) { return e.name() == "AppendedData"; }} {
810 if (!_element().has_child("VTKFile"))
811 throw IOError("Could not read " + filename + " as vtk-xml file. No root element <VTKFile> found.");
812 }
813
814 static XMLReaderHelper make_from(const std::string& filename, std::string_view vtk_type) {
815 if (!Path::exists(filename))
816 throw IOError("File '" + filename + "' does not exist.");
817 if (!Path::is_file(filename))
818 throw IOError("Given path '" + filename + "' is not a file.");
819
820 std::optional<XMLReaderHelper> helper;
821 try {
822 helper.emplace(XMLReaderHelper{filename});
823 } catch (const std::exception& e) {
824 throw IOError("Could not parse '" + filename + "' as xml file. Error: " + e.what());
825 }
826
827 if (!helper->get().has_attribute("type"))
828 throw IOError("'type' attribute missing in VTKFile root element.");
829 if (helper->get().get_attribute("type") != vtk_type)
830 throw IOError(
831 "Given vtk-xml file has type '"
832 + helper->get().get_attribute("type")
833 + "', expected '" + std::string{vtk_type} + "'"
834 );
835
836 return std::move(helper).value();
837 }
838
839 const XMLElement& get(std::string_view path = "") const {
840 OptionalReference opt_ref = access_at(path, _element().get_child("VTKFile"));
841 if (!opt_ref)
842 throw ValueError("The given path '" + std::string{path} + "' could not be found.");
843 return opt_ref.unwrap();
844 }
845
847 FieldPtr make_points_field(std::string_view section_path, std::size_t num_expected_points) const {
848 std::size_t visited = 0;
849 FieldPtr result{nullptr};
850 for (const auto& data_array : XML::data_arrays(get(section_path))) {
851 if (!result)
852 result = make_data_array_field(data_array.get_attribute("Name"), section_path, num_expected_points);
853 else {
854 log_warning("Points section contains more than one data array, using first one as point coordinates");
855 break;
856 }
857 visited++;
858 }
859 if (visited == 0)
860 throw ValueError("Points section does not contain a data array element");
861 return result;
862 }
863
865 FieldPtr make_data_array_field(std::string_view name,
866 std::string_view section_path,
867 const std::optional<std::size_t> number_of_tuples = {}) const {
868 return make_data_array_field(XML::get_data_array(name, get(section_path)), number_of_tuples);
869 }
870
872 FieldPtr make_data_array_field(const XMLElement& element,
873 const std::optional<std::size_t> number_of_tuples = {}) const {
874 if (element.name() != "DataArray")
875 throw ValueError("Given path is not a DataArray element");
876 if (!element.has_attribute("type"))
877 throw ValueError("DataArray element does not specify the data type (`type` attribute)");
878 if (!element.has_attribute("format"))
879 throw ValueError("Data array element does not specify its format (e.g. ascii/binary)");
880 if (number_of_tuples.has_value())
881 return _make_data_array_field(element, number_of_tuples.value());
882 const auto name = element.get_attribute_or(std::string{""}, "Name");
883 return _make_data_array_field(element, XMLDetail::_read_data_array(name, [&] () {
884 return _number_of_tuples(element);
885 }));
886 }
887
888 private:
889 const XMLElement& _element() const {
890 return _parser.get_xml();
891 }
892
893 const XMLElement& _appendix() const {
894 if (!_element().get_child("VTKFile").has_child("AppendedData"))
895 throw ValueError("Read vtk file has no appendix");
896 return _element().get_child("VTKFile").get_child("AppendedData");
897 }
898
899 FieldPtr _make_data_array_field(const XMLElement& element,
900 const std::size_t number_of_tuples) const {
901 if (element.name() != "DataArray")
902 throw ValueError("Given xml element does not describe a data array");
903 if (!element.has_attribute("format"))
904 throw ValueError("Data array element does not specify its format (e.g. ascii/binary)");
905 if (!element.has_attribute("type"))
906 throw ValueError("Data array element does not specify the data type");
907 if (element.get_attribute("format") == "appended" && !element.has_attribute("offset"))
908 throw ValueError("Data array element specifies to use appended data but does not specify offset");
909 return element.get_attribute("format") == "ascii"
910 ? _make_ascii_data_array_field(element, number_of_tuples)
911 : _make_binary_data_array_field(element, number_of_tuples);
912 }
913
914 FieldPtr _make_ascii_data_array_field(const XMLElement& e, const std::size_t num_tuples) const {
915 MDLayout expected_layout = _expected_layout(e, num_tuples);
916 auto num_values = expected_layout.number_of_entries();
917 return from_precision_attribute(e.get_attribute("type")).visit([&] <typename T> (const Precision<T>& prec) {
918 return make_field_ptr(LazyField{
919 std::string{_filename},
920 std::move(expected_layout),
921 prec,
922 [
923 _nv=num_values,
924 _begin=_parser.get_content_bounds(e).begin_pos,
925 _name=e.get_attribute_or(std::string{""}, "Name")
926 ] (std::string filename) {
927 return XMLDetail::_read_data_array(_name, [&] () {
928 std::ifstream file{filename};
929 file.seekg(_begin);
930 Serialization result{_nv*sizeof(T)};
931 XMLDetail::DataArrayReader<T>{file}.read_ascii(_nv, result);
932 return result;
933 });
934 }
935 });
936 });
937 }
938
939 FieldPtr _make_binary_data_array_field(const XMLElement& e, const std::size_t num_tuples) const {
940 auto expected_layout = _expected_layout(e, num_tuples);
941 return from_precision_attribute(e.get_attribute("type")).visit([&] <typename T> (const Precision<T>& prec) {
942 FieldPtr result;
943 _apply_decoder_for(e, [&] (auto&& decoder) {
944 result = make_field_ptr(LazyField{
945 std::string{_filename},
946 std::move(expected_layout),
947 prec,
948 [
949 _loc=_stream_location_for(e),
950 _header_prec=_header_precision(),
951 _endian=from_endian_attribute(get().get_attribute("byte_order")),
952 _comp=get().get_attribute_or(std::string{""}, "compressor"),
953 _decoder=std::move(decoder),
954 _name=e.get_attribute_or(std::string{""}, "Name")
955 ] (std::string filename) {
956 return XMLDetail::_read_data_array(_name, [&] () {
957 std::ifstream file{filename};
958 XMLDetail::_move_to_data(_loc, file);
959 return _header_prec.visit([&] <typename H> (const Precision<H>&) {
960 Serialization result;
961 XMLDetail::DataArrayReader<T, H>{file, _endian, _comp}.read_binary(_decoder, {}, result);
962 return result;
963 });
964 });
965 }
966 });
967 });
968 return result;
969 });
970 }
971
972 DynamicPrecision _header_precision() const {
973 if (get().has_attribute("header_type"))
974 return from_precision_attribute(get().get_attribute("header_type"));
975 if (get().get_attribute("version") == "0.1")
976 return uint32;
977 throw IOError("Header type is not specified");
978 }
979
980 MDLayout _expected_layout(const XMLElement& e, const std::size_t num_tuples) const {
981 const auto num_comps = e.get_attribute_or(std::size_t{1}, "NumberOfComponents");
982 if (num_comps > 1)
983 return MDLayout{{num_tuples, num_comps}};
984 return MDLayout{{num_tuples}};
985 }
986
987 std::size_t _number_of_tuples(const XMLElement& element) const {
988 const auto number_of_components = element.get_attribute_or(std::size_t{1}, "NumberOfComponents");
989 const auto number_of_values = [&] () {
990 // for strings, the tuples are the individual strings, but we read all characters
991 if (element.get_attribute("type") != "String" && element.has_attribute("NumberOfTuples"))
992 return from_string<std::size_t>(element.get_attribute("NumberOfTuples"));
993 return _deduce_number_of_values(element);
994 } ();
995
996 if (number_of_values%number_of_components != 0)
997 throw ValueError(
998 "The number of components of data array '" + element.name() + "' "
999 + "(" + as_string(number_of_components) + ") "
1000 + "is incompatible with the number of values it contains "
1001 + "(" + as_string(number_of_values) + ")"
1002 );
1003 return number_of_values/number_of_components;
1004 }
1005
1006 std::size_t _deduce_number_of_values(const XMLElement& element) const {
1007 std::ifstream file{_filename};
1008 XMLDetail::_move_to_data(_stream_location_for(element), file);
1009
1010 if (element.get_attribute("format") == "ascii") {
1011 const auto precision = from_precision_attribute(element.get_attribute("type"));
1012 return precision.visit([&] <typename T> (const Precision<T>&) {
1013 using _T = std::conditional_t<
1014 std::integral<T> && sizeof(T) < 4, // see XMLDetail::DataArrayReader::read_ascii
1015 int,
1016 T
1017 >;
1018 return std::ranges::distance(std::ranges::istream_view<_T>{file});
1019 });
1020 }
1021
1022 InputStreamHelper helper{file};
1023 const auto header = _read_binary_data_array_header(helper, element);
1024 if (get().has_attribute("compressor") && header.size() < 3)
1025 throw ValueError("Could not read compression header");
1026 const std::size_t number_of_bytes = [&] () {
1027 if (get().has_attribute("compressor"))
1028 return XMLDetail::_number_of_uncompressed_bytes(header.at(0), header.at(1), header.at(2));
1029 return header.at(0);
1030 } ();
1031 const std::size_t value_type_number_of_bytes = from_precision_attribute(
1032 element.get_attribute("type")
1033 ).size_in_bytes();
1034
1035 if (number_of_bytes%value_type_number_of_bytes != 0)
1036 throw ValueError(
1037 "The length of the data array '" + element.name() + "' "
1038 + "is incompatible with the data type '" + element.get_attribute("type") + "'"
1039 );
1040
1041 return number_of_bytes/value_type_number_of_bytes;
1042 }
1043
1044 DataArrayStreamLocation _stream_location_for(const XMLElement& element) const {
1045 if (element.get_attribute("format") == "appended")
1046 return DataArrayStreamLocation{
1047 .begin = _parser.get_content_bounds(_appendix()).begin_pos,
1048 .offset = from_string<std::size_t>(element.get_attribute("offset"))
1049 };
1050 return DataArrayStreamLocation{.begin = _parser.get_content_bounds(element).begin_pos};
1051 }
1052
1053 std::vector<std::size_t> _read_binary_data_array_header(InputStreamHelper& stream,
1054 const XMLElement& element) const {
1055 const std::string compressor = get().get_attribute_or(std::string{""}, "compressor");
1056 const std::endian endian = from_endian_attribute(get().get_attribute("byte_order"));
1057 const auto header_prec = _header_precision();
1058 const auto values_prec = from_precision_attribute(element.get_attribute("type"));
1059
1060 return header_prec.visit([&] <typename HT> (const Precision<HT>&) {
1061 return values_prec.visit([&] <typename VT> (const Precision<VT>&) {
1062 std::vector<HT> _header;
1063 XMLDetail::DataArrayReader<VT, HT> reader{stream, endian, compressor};
1064 _apply_decoder_for(element, [&] (const auto& decoder) {
1065 reader.read_binary(decoder, _header);
1066 });
1067
1068 if (_header.empty())
1069 throw IOError("Could not read header for data array '" + element.get_attribute("Name") + "'");
1070
1071 std::vector<std::size_t> result;
1072 std::ranges::for_each(_header, [&] <typename T> (const T& value) {
1073 result.push_back(static_cast<std::size_t>(value));
1074 });
1075 return result;
1076 });
1077 });
1078 }
1079
1080 template<typename Action>
1081 void _apply_decoder_for(const XMLElement& data_array, const Action& action) const {
1082 if (data_array.get_attribute("format") == "binary")
1083 return action(Base64Decoder{});
1084 else if (data_array.get_attribute("format") == "appended")
1085 return _appendix().get_attribute("encoding") == "base64"
1086 ? action(Base64Decoder{})
1087 : action(RawDecoder{});
1088 throw InvalidState("Unknown data format");
1089 }
1090
1091 std::string _filename;
1092 XMLParser _parser;
1093};
1094
1095} // namespace GridFormat::VTK
1096
1097#endif // GRIDFORMAT_VTK_XML_HPP_
Helper classes for writing VTK appendices of xml formats.
Encoder and stream using ascii.
Helper functions to get the VTK-specific names of things.
Encoder and stream using base64.
friend Concepts::RangeOf< std::pair< std::string, FieldPtr > > auto cell_fields_of_rank(unsigned int rank, const GridWriterBase &writer)
Return a range over the fields with the given rank (0=scalars, 1=vectors, 2=tensors).
Definition writer.hpp:220
friend Concepts::RangeOf< std::pair< std::string, FieldPtr > > auto point_fields_of_rank(unsigned int rank, const GridWriterBase &writer)
Return a range over the fields with the given rank (0=scalars, 1=vectors, 2=tensors).
Definition writer.hpp:207
Stores vtk data arrays to be exported as vtk-xml appendix.
Definition appendix.hpp:53
FieldPtr make_data_array_field(const XMLElement &element, const std::optional< std::size_t > number_of_tuples={}) const
Returns a field which draws the actual field values from the file upon request.
Definition xml.hpp:872
FieldPtr make_points_field(std::string_view section_path, std::size_t num_expected_points) const
Returns the field representing the points of the grid.
Definition xml.hpp:847
FieldPtr make_data_array_field(std::string_view name, std::string_view section_path, const std::optional< std::size_t > number_of_tuples={}) const
Returns a field which draws the actual field values from the file upon request.
Definition xml.hpp:865
Wraps a field and exposes it as VTK data array. Essentially, this implements the operator<< to stream...
Abstract interface for fields of values that is used by writers/readers to store fields.
Grid concepts.
Base classes for grid data writers.
FieldPtr make_field_ptr(F &&f)
Factory function for field pointers.
Definition field.hpp:211
std::shared_ptr< const Field > FieldPtr
Pointer type used by writers/readers for fields.
Definition field.hpp:205
std::ranges::range auto data_arrays(const XMLElement &e)
Return a range over all data array elements in the given xml section.
Definition xml.hpp:735
std::ranges::range auto data_array_names(const XMLElement &e)
Return a range over the names of all data array elements in the given xml section.
Definition xml.hpp:745
const XMLElement & get_data_array(std::string_view name, const XMLElement &section)
Return the data array element with the given name within the given xml section.
Definition xml.hpp:755
Compressor using the LZ4 library.
Compressor using the LZMA library.
Encoder and stream for raw binary output.
Options for VTK-XML files for setting the desired encoding, data format and compression.
Definition xml.hpp:101
Options that writer implementations can pass to the base class.
Definition writer.hpp:60
Common functionality for VTK writers.
Compressor using the ZLIB library.