9#ifndef GRIDFORMAT_COMPRESSION_LZMA_HPP_
10#define GRIDFORMAT_COMPRESSION_LZMA_HPP_
11#if GRIDFORMAT_HAVE_LZMA
23#include <gridformat/common/exceptions.hpp>
24#include <gridformat/common/serialization.hpp>
25#include <gridformat/common/logging.hpp>
30namespace GridFormat::Compression {
38 std::uint32_t compression_level = LZMA_PRESET_DEFAULT;
43 using LZMAByte = std::uint8_t;
44 static_assert(
sizeof(
typename Serialization::Byte) ==
sizeof(LZMAByte));
46 struct BlockDecompressor {
47 using ByteType = LZMAByte;
49 void operator()(std::span<const ByteType> in, std::span<ByteType> out)
const {
52 uint64_t memlim = UINT64_MAX;
53 if (lzma_stream_buffer_decode(
57 in.data(), &in_pos, in.size(),
58 out.data(), &out_pos, out.size()) != LZMA_OK)
59 throw IOError(
"(LZMACompressor) Error upon decompression");
67 : _opts(std::move(opts))
70 template<std::
integral HeaderType = std::
size_t>
72 static_assert(
sizeof(
typename Serialization::Byte) ==
sizeof(LZMAByte));
73 if (std::numeric_limits<HeaderType>::max() < in.size())
74 throw TypeError(
"Chosen HeaderType is too small for given number of bytes");
75 if (std::numeric_limits<HeaderType>::max() < _opts.block_size)
76 throw TypeError(
"Chosen HeaderType is too small for given block size");
78 auto [blocks, out] = _compress<HeaderType>(in.template as_span_of<const LZMAByte>());
80 in.resize(blocks.compressed_size());
84 template<std::
integral HeaderType>
90 return LZMA{std::move(opts)};
94 template<std::
integral HeaderType>
95 auto _compress(std::span<const LZMAByte> in)
const {
96 HeaderType block_size =
static_cast<HeaderType
>(_opts.block_size);
97 HeaderType size_in_bytes =
static_cast<HeaderType
>(in.size());
100 Serialization compressed;
101 std::vector<LZMAByte> block_buffer;
102 std::vector<HeaderType> compressed_block_sizes;
103 block_buffer.reserve(lzma_stream_buffer_bound(_opts.block_size));
104 compressed_block_sizes.reserve(blocks.number_of_blocks);
105 compressed.resize(block_buffer.capacity()*blocks.number_of_blocks);
107 HeaderType cur_in = 0;
108 HeaderType cur_out = 0;
109 auto out = compressed.template as_span_of<LZMAByte>();
110 while (cur_in < size_in_bytes) {
112 const HeaderType cur_block_size = min(block_size, size_in_bytes - cur_in);
113 assert(cur_in + cur_block_size <= size_in_bytes);
115 std::size_t out_pos = 0;
116 const auto lzma_ret = lzma_easy_buffer_encode(
117 _opts.compression_level, LZMA_CHECK_CRC32,
nullptr,
118 in.data() + cur_in, cur_block_size,
119 block_buffer.data(), &out_pos, block_buffer.capacity()
121 if (lzma_ret != LZMA_OK)
122 throw InvalidState(as_error(
"(LZMACompressor) Error upon compression"));
124 assert(cur_out + out_pos <= out.size());
125 std::copy_n(block_buffer.data(),
127 out.data() + cur_out);
128 cur_in += cur_block_size;
130 compressed_block_sizes.push_back(
static_cast<HeaderType
>(out_pos));
133 if (cur_in != size_in_bytes)
134 throw InvalidState(as_error(
"(LZMACompressor) unexpected number of bytes processed"));
136 return std::make_tuple(
148namespace Detail {
inline constexpr bool _have_lzma =
true; }
157namespace GridFormat::Compression {
159namespace Detail {
inline constexpr bool _have_lzma =
false; }
162 template<
bool b =
false,
typename... Args>
163 explicit LZMA(Args&&...) {
static_assert(b,
"LZMA compressor requires the LZMA library."); }
Common classes used in the context of data compression.
Decompress compressed data.
void decompress(Serialization &in, const CompressedBlocks< HeaderType > &blocks, const Decompressor &block_decompressor)
Decompress compressed data.
Definition: decompress.hpp:26
constexpr LZMA lzma
Instance of the lzma compressor.
Definition: lzma.hpp:145
constexpr std::size_t default_block_size
as in VTK (https://gitlab.kitware.com/vtk/vtk/-/blob/65fc526a83ac829628a9462f61fa57f1801e2c7e/IO/XML/...
Definition: common.hpp:23