GridFormat 0.2.1
I/O-Library for grid-like data structures
Loading...
Searching...
No Matches
xml.hpp
Go to the documentation of this file.
1// SPDX-FileCopyrightText: 2022-2023 Dennis Gläser <dennis.glaeser@iws.uni-stuttgart.de>
2// SPDX-License-Identifier: MIT
8#ifndef GRIDFORMAT_VTK_XML_HPP_
9#define GRIDFORMAT_VTK_XML_HPP_
10
11#include <bit>
12#include <string>
13#include <ranges>
14#include <utility>
15#include <type_traits>
16#include <functional>
17#include <optional>
18#include <iterator>
19#include <string_view>
20#include <concepts>
21
22#include <gridformat/common/detail/crtp.hpp>
23#include <gridformat/common/callable_overload_set.hpp>
24#include <gridformat/common/optional_reference.hpp>
25#include <gridformat/common/exceptions.hpp>
26#include <gridformat/common/type_traits.hpp>
27#include <gridformat/common/variant.hpp>
28#include <gridformat/common/precision.hpp>
29#include <gridformat/common/logging.hpp>
31#include <gridformat/common/lazy_field.hpp>
32#include <gridformat/common/path.hpp>
33
37
41
44#include <gridformat/grid/grid.hpp>
45
46#include <gridformat/xml/element.hpp>
47#include <gridformat/xml/parser.hpp>
48
53
54namespace GridFormat::VTK {
55
56#ifndef DOXYGEN
57namespace XMLDetail {
58 using LZMACompressor = std::conditional_t<Compression::Detail::_have_lzma, Compression::LZMA, None>;
59 using ZLIBCompressor = std::conditional_t<Compression::Detail::_have_zlib, Compression::ZLIB, None>;
60 using LZ4Compressor = std::conditional_t<Compression::Detail::_have_lz4, Compression::LZ4, None>;
61 using Compressor = UniqueVariant<LZ4Compressor, ZLIBCompressor, LZMACompressor>;
62} // namespace XMLDetail
63
64namespace XML {
65
66using HeaderPrecision = std::variant<UInt32, UInt64>;
67using CoordinatePrecision = std::variant<Float32, Float64>;
68using Compressor = ExtendedVariant<XMLDetail::Compressor, None>;
69using Encoder = std::variant<Encoding::Ascii, Encoding::Base64, Encoding::RawBinary>;
70using DataFormat = std::variant<VTK::DataFormat::Inlined, VTK::DataFormat::Appended>;
71using DefaultCompressor = std::variant_alternative_t<0, XMLDetail::Compressor>;
72
73} // namespace XML
74#endif // DOXYGEN
75
99struct XMLOptions {
100 using EncoderOption = ExtendedVariant<XML::Encoder, Automatic>;
101 using CompressorOption = ExtendedVariant<XML::Compressor, Automatic>;
102 using DataFormatOption = ExtendedVariant<XML::DataFormat, Automatic>;
103 using CoordinatePrecisionOption = ExtendedVariant<XML::CoordinatePrecision, Automatic>;
104 EncoderOption encoder = automatic;
105 CompressorOption compressor = automatic;
106 DataFormatOption data_format = automatic;
107 CoordinatePrecisionOption coordinate_precision = automatic;
108 XML::HeaderPrecision header_precision = _from_size_t();
109
110 private:
111 static constexpr XML::HeaderPrecision _from_size_t() {
112 if constexpr(sizeof(std::size_t) == 8) return uint64;
113 else return uint32;
114 }
115};
116
117
118#ifndef DOXYGEN
119namespace XMLDetail {
120
121 struct XMLSettings {
122 XML::Encoder encoder;
123 XML::Compressor compressor;
124 XML::DataFormat data_format;
125 XML::CoordinatePrecision coordinate_precision;
126 XML::HeaderPrecision header_precision;
127
128 template<typename GridCoordinateType>
129 static XMLSettings from(const XMLOptions& opts) {
130 static_assert(
131 std::is_same_v<GridCoordinateType, float> || std::is_same_v<GridCoordinateType, double>,
132 "Only float/double coordinate types are supported by VTK"
133 );
134 const auto _enc = _make_encoder(opts.encoder);
135 const auto _format = _make_data_format(_enc, opts.data_format);
136 const auto _comp = _make_compressor(_enc, opts.compressor);
137 return {
138 .encoder = _enc,
139 .compressor = _comp,
140 .data_format = _format,
141 .coordinate_precision =
142 Variant::is<Automatic>(opts.coordinate_precision) ?
143 XML::CoordinatePrecision{Precision<GridCoordinateType>{}} :
144 Variant::without<Automatic>(opts.coordinate_precision),
145 .header_precision = opts.header_precision
146 };
147 }
148
149 private:
150 static XML::Encoder _make_encoder(const typename XMLOptions::EncoderOption& enc) {
151 if (Variant::is<Automatic>(enc))
152 return Encoding::base64;
153 return Variant::without<Automatic>(enc);
154 }
155
156 static XML::DataFormat _make_data_format(const typename XML::Encoder& enc,
157 const typename XMLOptions::DataFormatOption& data_format) {
158 if (Variant::is<Automatic>(data_format))
159 return Variant::is<Encoding::Ascii>(enc)
160 ? XML::DataFormat{VTK::DataFormat::inlined}
161 : XML::DataFormat{VTK::DataFormat::appended};
162 return Variant::without<Automatic>(data_format);
163 }
164
165 static XML::Compressor _make_compressor(const typename XML::Encoder& enc,
166 const typename XMLOptions::CompressorOption& compressor) {
167 if (Variant::is<Automatic>(compressor))
168 return Variant::is<Encoding::Ascii>(enc)
169 ? XML::Compressor{none}
170 : XML::Compressor{XML::DefaultCompressor{}};
171 if (Variant::is<Encoding::Ascii>(enc) && !Variant::is<None>(compressor)) {
172 log_warning("Ascii output cannot be compressed. Ignoring chosen compressor...");
173 return none;
174 }
175 return Variant::without<Automatic>(compressor);
176 }
177 };
178
179} // namespace XMLDetail
180#endif // DOXYGEN
181
182
187template<Concepts::Grid G, typename Impl>
189: public GridWriter<G>
190, public GridFormat::Detail::CRTPBase<Impl> {
192 using GridCoordinateType = CoordinateType<G>;
193
194 public:
196 using Grid = G;
197
198 virtual ~XMLWriterBase() = default;
199 explicit XMLWriterBase(const Grid& grid,
200 std::string extension,
201 bool use_structured_grid_ordering,
202 XMLOptions xml_opts = {})
203 : ParentType(grid, std::move(extension), WriterOptions{use_structured_grid_ordering, true})
204 , _xml_opts{std::move(xml_opts)}
205 , _xml_settings{XMLDetail::XMLSettings::from<GridCoordinateType>(_xml_opts)}
206 {}
207
208 XMLWriterBase() = default;
209 XMLWriterBase(XMLWriterBase&&) = default;
210 XMLWriterBase(const XMLWriterBase&) = delete;
211 XMLWriterBase& operator=(XMLWriterBase&&) = default;
212 XMLWriterBase& operator=(const XMLWriterBase&) = delete;
213
214 Impl with(XMLOptions opts) const {
215 auto result = _with(std::move(opts));
216 this->copy_fields(result);
217 return result;
218 }
219
220 Impl with_data_format(const XML::DataFormat& format) const {
221 auto opts = _xml_opts;
222 Variant::unwrap_to(opts.data_format, format);
223 return with(std::move(opts));
224 }
225
226 Impl with_compression(const XML::Compressor& compressor) const {
227 auto opts = _xml_opts;
228 Variant::unwrap_to(opts.compressor, compressor);
229 return with(std::move(opts));
230 }
231
232 Impl with_encoding(const XML::Encoder& encoder) const {
233 auto opts = _xml_opts;
234 Variant::unwrap_to(opts.encoder, encoder);
235 return with(std::move(opts));
236 }
237
238 Impl with_coordinate_precision(const XML::CoordinatePrecision& prec) const {
239 auto opts = _xml_opts;
240 Variant::unwrap_to(opts.coordinate_precision, prec);
241 return with(std::move(opts));
242 }
243
244 Impl with_header_precision(const XML::HeaderPrecision& prec) const {
245 auto opts = _xml_opts;
246 opts.header_precision = prec;
247 return with(std::move(opts));
248 }
249
250 private:
251 virtual Impl _with(XMLOptions opts) const = 0;
252
253 protected:
254 XMLOptions _xml_opts;
255 XMLDetail::XMLSettings _xml_settings;
256
258 std::string vtk_grid_type;
259 XMLElement xml_representation;
260 Appendix appendix;
261 };
262
263 WriteContext _get_write_context(std::string vtk_grid_type) const {
264 return std::visit([&] (const auto& compressor) {
265 return std::visit([&] (const auto& header_precision) {
266 XMLElement xml("VTKFile");
267 xml.set_attribute("type", vtk_grid_type);
268 xml.set_attribute("version", "2.2");
269 xml.set_attribute("byte_order", attribute_name(std::endian::native));
270 xml.set_attribute("header_type", attribute_name(DynamicPrecision{header_precision}));
271 if constexpr (!is_none<decltype(compressor)>)
272 xml.set_attribute("compressor", attribute_name(compressor));
273
274 xml.add_child(vtk_grid_type).add_child("FieldData");
275 WriteContext context{
276 .vtk_grid_type = std::move(vtk_grid_type),
277 .xml_representation = std::move(xml),
278 .appendix = {}
279 };
280 _add_meta_data_fields(context);
281 return context;
282 }, _xml_settings.header_precision);
283 }, _xml_settings.compressor);
284 }
285
286 void _add_meta_data_fields(WriteContext& context) const {
287 XMLElement& field_data = context.xml_representation
288 .get_child(context.vtk_grid_type)
289 .get_child("FieldData");
290 std::visit([&] (const auto& encoder) {
291 std::visit([&] (const auto& compressor) {
292 std::visit([&] (const auto& data_format) {
293 std::visit([&] (const auto& header_precision) {
294 const auto& names = this->_meta_data_field_names();
295 std::ranges::for_each(names, [&] (const std::string& name) {
296 const auto& field = this->_get_meta_data_field(name);
297 const auto layout = field.layout();
298 const auto precision = field.precision();
299
300 auto& array = field_data.add_child("DataArray");
301 array.set_attribute("Name", name);
302 array.set_attribute("format", data_format_name(encoder, data_format));
303 if (precision.template is<char>() && layout.dimension() == 1) {
304 array.set_attribute("type", "String");
305 array.set_attribute("NumberOfTuples", 1);
306 } else {
307 array.set_attribute("NumberOfTuples", layout.extent(0));
308 array.set_attribute("type", attribute_name(precision));
309 array.set_attribute(
310 "NumberOfComponents",
311 layout.dimension() > 1
312 ? layout.sub_layout(1).number_of_entries()
313 : 1
314 );
315 }
316 DataArray content{field, encoder, compressor, header_precision};
317 _set_data_array_content(data_format, array, context.appendix, std::move(content));
318 });
319 }, _xml_settings.header_precision);
320 }, _xml_settings.data_format);
321 }, _xml_settings.compressor);
322 }, _xml_settings.encoder);
323 }
324
325 template<typename ValueType>
326 void _set_attribute(WriteContext& context,
327 std::string_view xml_group,
328 const std::string& attr_name,
329 const ValueType& attr_value) const {
330 _access_at(xml_group, context).set_attribute(attr_name, attr_value);
331 }
332
333 void _set_data_array(WriteContext& context,
334 std::string_view xml_group,
335 std::string data_array_name,
336 const Field& field) const {
337 const auto layout = field.layout();
338 XMLElement& da = _access_at(xml_group, context).add_child("DataArray");
339 da.set_attribute("Name", std::move(data_array_name));
340 da.set_attribute("type", attribute_name(field.precision()));
341 da.set_attribute("NumberOfComponents", (layout.dimension() == 1 ? 1 : layout.number_of_entries(1)));
342 std::visit([&] (const auto& encoder) {
343 std::visit([&] (const auto& compressor) {
344 std::visit([&] (const auto& data_format) {
345 std::visit([&] (const auto& header_prec) {
346 da.set_attribute("format", data_format_name(encoder, data_format));
347 DataArray content{field, encoder, compressor, header_prec};
348 _set_data_array_content(data_format, da, context.appendix, std::move(content));
349 }, _xml_settings.header_precision);
350 }, _xml_settings.data_format);
351 }, _xml_settings.compressor);
352 }, _xml_settings.encoder);
353 }
354
355 template<typename DataFormat, typename Appendix, typename Content>
356 requires(!std::is_lvalue_reference_v<Content>)
357 void _set_data_array_content(const DataFormat&,
358 XMLElement& e,
359 Appendix& app,
360 Content&& c) const {
361 static constexpr bool is_inlined = std::is_same_v<DataFormat, VTK::DataFormat::Inlined>;
362 static constexpr bool is_appended = std::is_same_v<DataFormat, VTK::DataFormat::Appended>;
363 static_assert(is_inlined || is_appended, "Unknown data format");
364
365 if constexpr (is_inlined)
366 e.set_content(std::move(c));
367 else
368 app.add(std::move(c));
369 }
370
371 XMLElement& _access_at(std::string_view path, WriteContext& context) const {
372 return access_or_create_at(path, context.xml_representation.get_child(context.vtk_grid_type));
373 }
374
375 void _write_xml(WriteContext&& context, std::ostream& s) const {
376 Indentation indentation{{.width = 2}};
377 _set_default_active_fields(context.xml_representation.get_child(context.vtk_grid_type));
378 std::visit([&] (const auto& encoder) {
379 std::visit([&] <typename DataFormat> (const DataFormat&) {
380 if constexpr (std::is_same_v<DataFormat, VTK::DataFormat::Inlined>)
381 write_xml_with_version_header(context.xml_representation, s, indentation);
382 else
383 XML::Detail::write_with_appendix(std::move(context), s, encoder, indentation);
384 }, this->_xml_settings.data_format);
385 }, this->_xml_settings.encoder);
386 }
387
388 void _set_default_active_fields(XMLElement& xml) const {
389 const auto set = [&] (std::string_view group, const auto& attr, const auto& name) -> void {
390 auto group_element = access_at(group, xml);
391 if (!group_element)
392 return;
393 group_element.unwrap().set_attribute(attr, name);
394 };
395
396 // discard vectors with more than 3 elements for active arrays
397 const auto get_vector_filter = [] (unsigned int rank) {
398 return [r=rank] (const auto& name_field_ptr_pair) {
399 if (r == 1)
400 return name_field_ptr_pair.second->layout().extent(1) <= 3;
401 return true;
402 };
403 };
404
405 for (std::string_view group : {"Piece/PointData", "PPointData"})
406 for (unsigned int i = 0; i <= 2; ++i)
407 for (const auto& [n, _] : point_fields_of_rank(i, *this)
408 | std::views::filter(get_vector_filter(i))
409 | std::views::take(1))
410 set(group, active_array_attribute_for_rank(i), n);
411
412 for (std::string_view group : {"Piece/CellData", "PCellData"})
413 for (unsigned int i = 0; i <= 2; ++i)
414 for (const auto& [n, _] : cell_fields_of_rank(i, *this)
415 | std::views::filter(get_vector_filter(i))
416 | std::views::take(1))
417 set(group, active_array_attribute_for_rank(i), n);
418 }
419};
420
421
422#ifndef DOXYGEN
423namespace XMLDetail {
424
425 struct DataArrayStreamLocation {
426 std::streamsize begin; // location where data begins
427 std::optional<std::streamsize> offset = {}; // used for appended data
428 };
429
430 template<std::integral I, std::integral O>
431 void _move_to_appendix_position(std::istream& stream, I appendix_begin, O offset_in_appendix) {
432 InputStreamHelper helper{stream};
433 helper.seek_position(appendix_begin);
434 helper.shift_until_not_any_of(" \n\t");
435 if (helper.read_chunk(1) != "_")
436 throw IOError("VTK-XML appendix must start with '_'");
437 helper.shift_by(offset_in_appendix);
438 }
439
440 void _move_to_data(const DataArrayStreamLocation& location, std::istream& s) {
441 if (location.offset)
442 _move_to_appendix_position(s, location.begin, location.offset.value());
443 else {
444 InputStreamHelper helper{s};
445 helper.seek_position(location.begin);
446 helper.shift_whitespace();
447 }
448 }
449
450 template<typename HeaderType>
451 void _decompress_with(const std::string& vtk_compressor,
452 [[maybe_unused]] Serialization& data,
453 [[maybe_unused]] const Compression::CompressedBlocks<HeaderType>& blocks) {
454 if (vtk_compressor == "vtkLZ4DataCompressor") {
455#if GRIDFORMAT_HAVE_LZ4
456 LZ4Compressor{}.decompress(data, blocks);
457#else
458 throw InvalidState("Need LZ4 to decompress the data");
459#endif
460 } else if (vtk_compressor == "vtkLZMADataCompressor") {
461#if GRIDFORMAT_HAVE_LZMA
462 LZMACompressor{}.decompress(data, blocks);
463#else
464 throw InvalidState("Need LZMA to decompress the data");
465#endif
466 } else if (vtk_compressor == "vtkZLibDataCompressor") {
467#if GRIDFORMAT_HAVE_ZLIB
468 ZLIBCompressor{}.decompress(data, blocks);
469#else
470 throw InvalidState("Need ZLib to decompress the data");
471#endif
472 } else {
473 throw NotImplemented("Unsupported vtk compressor '" + vtk_compressor + "'");
474 }
475 }
476
477 template<Concepts::Scalar TargetType, Concepts::Scalar HeaderType = std::size_t>
478 class DataArrayReader {
479 static constexpr Precision<TargetType> target_precision{};
480 static constexpr Precision<HeaderType> header_precision{};
481
482 static constexpr bool is_too_small_integral = std::integral<TargetType> && sizeof(TargetType) < 4;
483 static constexpr bool is_signed_integral = std::signed_integral<TargetType>;
484 using BufferedType = std::conditional_t<is_signed_integral, short, unsigned short>;
485
486 public:
487 using Header = std::vector<HeaderType>;
488
489 DataArrayReader(std::istream& s,
490 std::endian e = std::endian::native,
491 std::string compressor = "")
492 : _stream{s}
493 , _endian{e}
494 , _compressor{compressor}
495 {}
496
497 void read_ascii(std::size_t number_of_values, Serialization& out_values) {
498 out_values.resize(number_of_values*sizeof(TargetType));
499 std::span<TargetType> out_span = out_values.as_span_of(target_precision);
500
501 // istream_view uses operator>>, which seems to do weird stuff for e.g. uint8_t.
502 // The smallest supported integral type seems to be short. In case we deal with
503 // small integral types, we first read into a vector and then cast into the desired type.
504 if constexpr (is_too_small_integral) {
505 std::vector<BufferedType> buffer(number_of_values);
506 _read_ascii_to(std::span{buffer}, number_of_values);
507 std::ranges::copy(buffer | std::views::transform([] <typename T> (const T& value) {
508 return static_cast<TargetType>(value);
509 }), out_span.begin());
510 } else {
511 _read_ascii_to(out_span, number_of_values);
512 }
513 }
514
515 template<Concepts::Decoder Decoder>
516 void read_binary(const Decoder& decoder,
517 OptionalReference<Header> header = {},
518 OptionalReference<Serialization> values = {}) {
519 if constexpr (std::unsigned_integral<HeaderType> && sizeof(HeaderType) >= 4) {
520 if (_compressor.empty())
521 return _read_encoded(decoder, header, values);
522 else
523 return _read_encoded_compressed(decoder, header, values);
524 } else {
525 throw IOError("Unsupported header type");
526 }
527 }
528
529 private:
530 template<typename T, std::size_t size>
531 void _read_ascii_to(std::span<T, size> buffer, std::size_t expected_num_values) const {
532 const auto [_, out] = std::ranges::copy(
533 std::ranges::istream_view<T>{_stream}
534 | std::views::take(expected_num_values),
535 buffer.begin()
536 );
537 const auto number_of_values_read = std::ranges::distance(buffer.begin(), out);
538 if (number_of_values_read < 0 || static_cast<std::size_t>(number_of_values_read) < expected_num_values)
539 throw SizeError("Could not read the requested number of values from the stream");
540 }
541
542 template<typename Decoder>
543 void _read_encoded(const Decoder& decoder,
544 OptionalReference<Header> out_header = {},
545 OptionalReference<Serialization> out_values = {}) {
546 const auto pos = _stream.tellg();
547 Serialization header = decoder.decode_from(_stream, sizeof(HeaderType));
548
549 if (header.size() != sizeof(HeaderType)) { // no padding - header & values are encoded together
550 if (header.size() < sizeof(HeaderType))
551 throw SizeError("Could not read header");
552
553 header.resize(sizeof(HeaderType));
554 change_byte_order(header.as_span_of(header_precision), {.from = _endian});
555
556 if (out_header)
557 std::ranges::copy(header.as_span_of(header_precision), std::back_inserter(out_header.unwrap()));
558
559 if (out_values) {
560 _stream.seekg(pos);
561 const auto number_of_bytes = header.as_span_of(header_precision)[0];
562 const auto number_of_bytes_with_header = number_of_bytes + sizeof(HeaderType);
563 Serialization& header_and_values = out_values.unwrap();
564 header_and_values = decoder.decode_from(_stream, number_of_bytes_with_header);
565 header_and_values.cut_front(sizeof(HeaderType));
566 change_byte_order(header_and_values.as_span_of(header_precision), {.from = _endian});
567 }
568 } else { // values are encoded separately
569 change_byte_order(header.as_span_of(header_precision), {.from = _endian});
570
571 if (out_header)
572 std::ranges::copy(header.as_span_of(header_precision), std::back_inserter(out_header.unwrap()));
573
574 if (out_values) {
575 const auto number_of_bytes = header.as_span_of(header_precision)[0];
576 Serialization& values = out_values.unwrap();
577 values = decoder.decode_from(_stream, number_of_bytes);
578 change_byte_order(values.as_span_of(Precision<TargetType>{}), {.from = _endian});
579 }
580 }
581 }
582
583 template<typename Decoder>
584 void _read_encoded_compressed(const Decoder& decoder,
585 OptionalReference<Header> out_header = {},
586 OptionalReference<Serialization> out_values = {}) {
587 const auto begin_pos = _stream.tellg();
588 const auto header_bytes = sizeof(HeaderType)*3;
589 Serialization header = decoder.decode_from(_stream, header_bytes);
590
591 // if the decoded header is larger than requested, then there is no padding,
592 // which means that we'll have to decode the header together with the values.
593 const bool decode_blocks_with_header = header.size() != header_bytes;
594 if (decode_blocks_with_header)
595 header.resize(header_bytes);
596
597 auto header_data = header.as_span_of(header_precision);
598 if (header_data.size() < 3)
599 throw SizeError("Could not read data array header");
600
601 change_byte_order(header_data, {.from = _endian});
602 const auto number_of_blocks = header_data[0];
603 const auto full_block_size = header_data[1];
604 const auto residual_block_size = header_data[2];
605 const HeaderType number_of_raw_bytes = residual_block_size > 0
606 ? full_block_size*(number_of_blocks-1) + residual_block_size
607 : full_block_size*number_of_blocks;
608
609 Serialization block_sizes;
610 const std::size_t block_sizes_bytes = sizeof(HeaderType)*number_of_blocks;
611 if (decode_blocks_with_header) {
612 _stream.seekg(begin_pos);
613 block_sizes = decoder.decode_from(_stream, header_bytes + block_sizes_bytes);
614 block_sizes.cut_front(sizeof(HeaderType)*3);
615 } else {
616 block_sizes = decoder.decode_from(_stream, block_sizes_bytes);
617 }
618
619 std::vector<HeaderType> compressed_block_sizes(number_of_blocks);
620 change_byte_order(block_sizes.as_span_of(header_precision), {.from = _endian});
621 std::ranges::copy(
622 block_sizes.as_span_of(header_precision),
623 compressed_block_sizes.begin()
624 );
625
626 if (out_header) {
627 std::ranges::copy(header_data, std::back_inserter(out_header.unwrap()));
628 std::ranges::copy(compressed_block_sizes, std::back_inserter(out_header.unwrap()));
629 }
630
631 if (out_values) {
632 Serialization& values = out_values.unwrap();
633 values = decoder.decode_from(_stream, std::accumulate(
634 compressed_block_sizes.begin(),
635 compressed_block_sizes.end(),
636 HeaderType{0}
637 ));
638
639 _decompress_with(_compressor, values, Compression::CompressedBlocks{
640 {number_of_raw_bytes, full_block_size},
641 std::move(compressed_block_sizes)
642 });
643 change_byte_order(values.as_span_of(target_precision), {.from = _endian});
644 }
645 }
646
647 std::istream& _stream;
648 std::endian _endian;
649 std::string _compressor;
650 };
651
652} // namespace XMLDetail
653#endif // DOXYGEN
654
655
656namespace XML {
657
662std::ranges::range auto data_arrays(const XMLElement& e) {
663 return children(e) | std::views::filter([] (const XMLElement& child) {
664 return child.name() == "DataArray";
665 });
666}
667
672std::ranges::range auto data_array_names(const XMLElement& e) {
673 return data_arrays(e) | std::views::transform([] (const XMLElement& data_array) {
674 return data_array.get_attribute("Name");
675 });
676}
677
682const XMLElement& get_data_array(std::string_view name, const XMLElement& section) {
683 for (const auto& da
684 : data_arrays(section)
685 | std::views::filter([&] (const auto& e) { return e.get_attribute("Name") == name; }))
686 return da;
687 throw ValueError(
688 "Could not find data array with name '" + std::string{name}
689 + "' in section '" + std::string{section.name()} + "'"
690 );
691}
692
693} // namespace XML
694
695
696#ifndef DOXYGEN
697namespace XMLDetail {
698
699 // can be reused by readers to copy read field names into the storage container
700 template<typename FieldNameContainer>
701 void copy_field_names_from(const XMLElement& vtk_grid, FieldNameContainer& names) {
702 if (vtk_grid.has_child("Piece")) {
703 const XMLElement& piece = vtk_grid.get_child("Piece");
704 if (piece.has_child("PointData"))
705 std::ranges::copy(
706 XML::data_array_names(piece.get_child("PointData")),
707 std::back_inserter(names.point_fields)
708 );
709 if (piece.has_child("CellData"))
710 std::ranges::copy(
711 XML::data_array_names(piece.get_child("CellData")),
712 std::back_inserter(names.cell_fields)
713 );
714 }
715 if (vtk_grid.has_child("FieldData"))
716 std::ranges::copy(
717 XML::data_array_names(vtk_grid.get_child("FieldData")),
718 std::back_inserter(names.meta_data_fields)
719 );
720 }
721
722} // namespace XMLDetail
723#endif // DOXYGEN
724
725
731 using DataArrayStreamLocation = XMLDetail::DataArrayStreamLocation;
732
733 public:
734 explicit XMLReaderHelper(const std::string& filename)
735 : _filename{filename}
736 , _parser{filename, "ROOT", [] (const XMLElement& e) { return e.name() == "AppendedData"; }} {
737 if (!_element().has_child("VTKFile"))
738 throw IOError("Could not read " + filename + " as vtk-xml file. No root element <VTKFile> found.");
739 }
740
741 static XMLReaderHelper make_from(const std::string& filename, std::string_view vtk_type) {
742 if (!Path::exists(filename))
743 throw IOError("File '" + filename + "' does not exist.");
744 if (!Path::is_file(filename))
745 throw IOError("Given path '" + filename + "' is not a file.");
746
747 std::optional<XMLReaderHelper> helper;
748 try {
749 helper.emplace(XMLReaderHelper{filename});
750 } catch (const std::exception& e) {
751 throw IOError("Could not parse '" + filename + "' as xml file. Error: " + e.what());
752 }
753
754 if (!helper->get().has_attribute("type"))
755 throw IOError("'type' attribute missing in VTKFile root element.");
756 if (helper->get().get_attribute("type") != vtk_type)
757 throw IOError(
758 "Given vtk-xml file has type '"
759 + helper->get().get_attribute("type")
760 + "', expected '" + std::string{vtk_type} + "'"
761 );
762
763 return std::move(helper).value();
764 }
765
766 const XMLElement& get(std::string_view path = "") const {
767 OptionalReference opt_ref = access_at(path, _element().get_child("VTKFile"));
768 if (!opt_ref)
769 throw ValueError("The given path '" + std::string{path} + "' could not be found.");
770 return opt_ref.unwrap();
771 }
772
774 FieldPtr make_points_field(std::string_view section_path, std::size_t num_expected_points) const {
775 std::size_t visited = 0;
776 FieldPtr result{nullptr};
777 for (const auto& data_array : XML::data_arrays(get(section_path))) {
778 if (!result)
779 result = make_data_array_field(data_array.get_attribute("Name"), section_path, num_expected_points);
780 else {
781 log_warning("Points section contains more than one data array, using first one as point coordinates");
782 break;
783 }
784 visited++;
785 }
786 if (visited == 0)
787 throw ValueError("Points section does not contain a data array element");
788 return result;
789 }
790
792 FieldPtr make_data_array_field(std::string_view name,
793 std::string_view section_path,
794 const std::optional<std::size_t> number_of_tuples = {}) const {
795 return make_data_array_field(XML::get_data_array(name, get(section_path)), number_of_tuples);
796 }
797
799 FieldPtr make_data_array_field(const XMLElement& element,
800 const std::optional<std::size_t> number_of_tuples = {}) const {
801 if (element.name() != "DataArray")
802 throw ValueError("Given path is not a DataArray element");
803 if (!element.has_attribute("type"))
804 throw ValueError("DataArray element does not specify the data type (`type` attribute)");
805 if (!element.has_attribute("format"))
806 throw ValueError("Data array element does not specify its format (e.g. ascii/binary)");
807 if (number_of_tuples.has_value())
808 return _make_data_array_field(element, number_of_tuples.value());
809 return _make_data_array_field(element, _number_of_tuples(element));
810 }
811
812 private:
813 const XMLElement& _element() const {
814 return _parser.get_xml();
815 }
816
817 const XMLElement& _appendix() const {
818 if (!_element().get_child("VTKFile").has_child("AppendedData"))
819 throw ValueError("Read vtk file has no appendix");
820 return _element().get_child("VTKFile").get_child("AppendedData");
821 }
822
823 FieldPtr _make_data_array_field(const XMLElement& element,
824 const std::size_t number_of_tuples) const {
825 if (element.name() != "DataArray")
826 throw ValueError("Given xml element does not describe a data array");
827 if (!element.has_attribute("format"))
828 throw ValueError("Data array element does not specify its format (e.g. ascii/binary)");
829 if (!element.has_attribute("type"))
830 throw ValueError("Data array element does not specify the data type");
831 if (element.get_attribute("format") == "appended" && !element.has_attribute("offset"))
832 throw ValueError("Data array element specifies to use appended data but does not specify offset");
833 return element.get_attribute("format") == "ascii"
834 ? _make_ascii_data_array_field(element, number_of_tuples)
835 : _make_binary_data_array_field(element, number_of_tuples);
836 }
837
838 FieldPtr _make_ascii_data_array_field(const XMLElement& e, const std::size_t num_tuples) const {
839 MDLayout expected_layout = _expected_layout(e, num_tuples);
840 auto num_values = expected_layout.number_of_entries();
841 return from_precision_attribute(e.get_attribute("type")).visit([&] <typename T> (const Precision<T>& prec) {
842 return make_field_ptr(LazyField{
843 std::string{_filename},
844 std::move(expected_layout),
845 prec,
846 [_nv=num_values, _begin=_parser.get_content_bounds(e).begin_pos] (std::string filename) {
847 std::ifstream file{filename};
848 file.seekg(_begin);
849 Serialization result{_nv*sizeof(T)};
850 XMLDetail::DataArrayReader<T>{file}.read_ascii(_nv, result);
851 return result;
852 }
853 });
854 });
855 }
856
857 FieldPtr _make_binary_data_array_field(const XMLElement& e, const std::size_t num_tuples) const {
858 auto expected_layout = _expected_layout(e, num_tuples);
859 return from_precision_attribute(e.get_attribute("type")).visit([&] <typename T> (const Precision<T>& prec) {
860 FieldPtr result;
861 _apply_decoder_for(e, [&] (auto&& decoder) {
862 result = make_field_ptr(LazyField{
863 std::string{_filename},
864 std::move(expected_layout),
865 prec,
866 [
867 _loc=_stream_location_for(e),
868 _header_prec=_header_precision(),
869 _endian=from_endian_attribute(get().get_attribute("byte_order")),
870 _comp=get().get_attribute_or(std::string{""}, "compressor"),
871 _decoder=std::move(decoder)
872 ] (std::string filename) {
873 std::ifstream file{filename};
874 XMLDetail::_move_to_data(_loc, file);
875 return _header_prec.visit([&] <typename H> (const Precision<H>&) {
876 Serialization result;
877 XMLDetail::DataArrayReader<T, H>{file, _endian, _comp}.read_binary(_decoder, {}, result);
878 return result;
879 });
880 }
881 });
882 });
883 return result;
884 });
885 }
886
887 DynamicPrecision _header_precision() const {
888 if (get().has_attribute("header_type"))
889 return from_precision_attribute(get().get_attribute("header_type"));
890 if (get().get_attribute("version") == "0.1")
891 return uint32;
892 throw IOError("Header type is not specified");
893 }
894
895 MDLayout _expected_layout(const XMLElement& e, const std::size_t num_tuples) const {
896 const auto num_comps = e.get_attribute_or(std::size_t{1}, "NumberOfComponents");
897 if (num_comps > 1)
898 return MDLayout{{num_tuples, num_comps}};
899 return MDLayout{{num_tuples}};
900 }
901
902 std::size_t _number_of_tuples(const XMLElement& element) const {
903 const auto number_of_components = element.get_attribute_or(std::size_t{1}, "NumberOfComponents");
904 const auto number_of_values = [&] () {
905 if (element.get_attribute("type") != "String") {
906 if (element.has_attribute("NumberOfTuples"))
907 return from_string<std::size_t>(element.get_attribute("NumberOfTuples"));
908 } else if (element.has_attribute("NumberOfTuples")) {
909 const auto num_values = from_string<std::size_t>(element.get_attribute("NumberOfTuples"));
910 if (num_values > 1)
911 throw ValueError("Cannot read string data arrays with more than one tuple");
912 }
913 return _deduce_number_of_values(element);
914 } ();
915
916 if (number_of_values%number_of_components != 0)
917 throw ValueError(
918 "The number of components of data array '" + element.name() + "' "
919 + "(" + as_string(number_of_components) + ") "
920 + "is incompatible with the number of values it contains "
921 + "(" + as_string(number_of_values) + ")"
922 );
923 return number_of_values/number_of_components;
924 }
925
926 std::size_t _deduce_number_of_values(const XMLElement& element) const {
927 std::ifstream file{_filename};
928 XMLDetail::_move_to_data(_stream_location_for(element), file);
929
930 if (element.get_attribute("format") == "ascii") {
931 const auto precision = from_precision_attribute(element.get_attribute("type"));
932 return precision.visit([&] <typename T> (const Precision<T>&) {
933 using _T = std::conditional_t<
934 std::integral<T> && sizeof(T) < 4, // see XMLDetail::DataArrayReader::read_ascii
935 int,
936 T
937 >;
938 return std::ranges::distance(std::ranges::istream_view<_T>{file});
939 });
940 }
941
942 InputStreamHelper helper{file};
943 const auto header = _read_binary_data_array_header(helper, element);
944 if (get().has_attribute("compressor") && header.size() < 3)
945 throw ValueError("Could not read compression header");
946 const std::size_t number_of_bytes = [&] () {
947 if (get().has_attribute("compressor")) {
948 const auto num_full_blocks = header.at(0);
949 const auto full_block_size = header.at(1);
950 const auto residual_block_size = header.at(2);
951 return residual_block_size > 0
952 ? full_block_size*(num_full_blocks - 1) + residual_block_size
953 : full_block_size*num_full_blocks;
954 }
955 return header.at(0);
956 } ();
957 const std::size_t value_type_number_of_bytes = from_precision_attribute(
958 element.get_attribute("type")
959 ).size_in_bytes();
960
961 if (number_of_bytes%value_type_number_of_bytes != 0)
962 throw ValueError(
963 "The length of the data array '" + element.name() + "' "
964 + "is incompatible with the data type '" + element.get_attribute("type") + "'"
965 );
966
967 return number_of_bytes/value_type_number_of_bytes;
968 }
969
970 DataArrayStreamLocation _stream_location_for(const XMLElement& element) const {
971 if (element.get_attribute("format") == "appended")
972 return DataArrayStreamLocation{
973 .begin = _parser.get_content_bounds(_appendix()).begin_pos,
974 .offset = from_string<std::size_t>(element.get_attribute("offset"))
975 };
976 return DataArrayStreamLocation{.begin = _parser.get_content_bounds(element).begin_pos};
977 }
978
979 std::vector<std::size_t> _read_binary_data_array_header(InputStreamHelper& stream,
980 const XMLElement& element) const {
981 const std::string compressor = get().get_attribute_or(std::string{""}, "compressor");
982 const std::endian endian = from_endian_attribute(get().get_attribute("byte_order"));
983 const auto header_prec = _header_precision();
984 const auto values_prec = from_precision_attribute(element.get_attribute("type"));
985
986 return header_prec.visit([&] <typename HT> (const Precision<HT>&) {
987 return values_prec.visit([&] <typename VT> (const Precision<VT>&) {
988 std::vector<HT> _header;
989 XMLDetail::DataArrayReader<VT, HT> reader{stream, endian, compressor};
990 _apply_decoder_for(element, [&] (const auto& decoder) {
991 reader.read_binary(decoder, _header);
992 });
993
994 if (_header.empty())
995 throw IOError("Could not read header for data array '" + element.get_attribute("Name") + "'");
996
997 std::vector<std::size_t> result;
998 std::ranges::for_each(_header, [&] <typename T> (const T& value) {
999 result.push_back(static_cast<std::size_t>(value));
1000 });
1001 return result;
1002 });
1003 });
1004 }
1005
1006 template<typename Action>
1007 void _apply_decoder_for(const XMLElement& data_array, const Action& action) const {
1008 if (data_array.get_attribute("format") == "binary")
1009 return action(Base64Decoder{});
1010 else if (data_array.get_attribute("format") == "appended")
1011 return _appendix().get_attribute("encoding") == "base64"
1012 ? action(Base64Decoder{})
1013 : action(RawDecoder{});
1014 throw InvalidState("Unknown data format");
1015 }
1016
1017 std::string _filename;
1018 XMLParser _parser;
1019};
1020
1021} // namespace GridFormat::VTK
1022
1023#endif // GRIDFORMAT_VTK_XML_HPP_
Helper classes for writing VTK appendices of xml formats.
Encoder and stream using ascii.
Helper functions to get the VTK-specific names of things.
Encoder and stream using base64.
friend Concepts::RangeOf< std::pair< std::string, FieldPtr > > auto cell_fields_of_rank(unsigned int rank, const GridWriterBase &writer)
Return a range over the fields with the given rank (0=scalars, 1=vectors, 2=tensors)
Definition: writer.hpp:200
friend Concepts::RangeOf< std::pair< std::string, FieldPtr > > auto point_fields_of_rank(unsigned int rank, const GridWriterBase &writer)
Return a range over the fields with the given rank (0=scalars, 1=vectors, 2=tensors)
Definition: writer.hpp:187
Abstract base class for grid file writers.
Definition: writer.hpp:303
Stores vtk data arrays to be exported as vtk-xml appendix.
Definition: appendix.hpp:53
Helper class for VTK-XML readers to use.
Definition: xml.hpp:730
FieldPtr make_data_array_field(const XMLElement &element, const std::optional< std::size_t > number_of_tuples={}) const
Returns a field which draws the actual field values from the file upon request.
Definition: xml.hpp:799
FieldPtr make_points_field(std::string_view section_path, std::size_t num_expected_points) const
Returns the field representing the points of the grid.
Definition: xml.hpp:774
FieldPtr make_data_array_field(std::string_view name, std::string_view section_path, const std::optional< std::size_t > number_of_tuples={}) const
Returns a field which draws the actual field values from the file upon request.
Definition: xml.hpp:792
Base class for VTK-XML Writer implementations.
Definition: xml.hpp:190
G Grid
Export underlying grid type.
Definition: xml.hpp:196
Grid concepts.
Base classes for grid data writers.
std::shared_ptr< const Field > FieldPtr
Pointer type used by writers/readers for fields.
Definition: field.hpp:186
FieldPtr make_field_ptr(F &&f)
Factory function for field pointers.
Definition: field.hpp:192
std::ranges::range auto data_arrays(const XMLElement &e)
Return a range over all data array elements in the given xml section.
Definition: xml.hpp:662
std::ranges::range auto data_array_names(const XMLElement &e)
Return a range over the names of all data array elements in the given xml section.
Definition: xml.hpp:672
const XMLElement & get_data_array(std::string_view name, const XMLElement &section)
Return the data array element with the given name within the given xml section.
Definition: xml.hpp:682
Compressor using the LZ4 library.
Compressor using the LZMA library.
Encoder and stream for raw binary output.
Options for VTK-XML files for setting the desired encoding, data format and compression.
Definition: xml.hpp:99
Options that writer implementations can pass to the base class.
Definition: writer.hpp:59
Common functionality for VTK writers.
Compressor using the ZLIB library.