Libosmium  2.16.0
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2021 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
50 #include <osmium/util/file.hpp>
51 
52 #include <bzlib.h>
53 
54 #include <cassert>
55 #include <cerrno>
56 #include <cstdio>
57 #include <limits>
58 #include <string>
59 #include <system_error>
60 
61 #ifndef _MSC_VER
62 # include <unistd.h>
63 #endif
64 
65 namespace osmium {
66 
71  struct bzip2_error : public io_error {
72 
74  int system_errno = 0;
75 
76  bzip2_error(const std::string& what, const int error_code) :
77  io_error(what),
78  bzip2_error_code(error_code) {
79  if (error_code == BZ_IO_ERROR) {
80  system_errno = errno;
81  }
82  }
83 
84  }; // struct bzip2_error
85 
86  namespace io {
87 
88  namespace detail {
89 
90  [[noreturn]] inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, const int bzlib_error) {
91  std::string error{"bzip2 error: "};
92  error += msg;
93  error += ": ";
94  int errnum = bzlib_error;
95  if (bzlib_error) {
96  error += std::to_string(bzlib_error);
97  } else if (bzfile) {
98  error += ::BZ2_bzerror(bzfile, &errnum);
99  }
100  throw osmium::bzip2_error{error, errnum};
101  }
102 
103  class file_wrapper {
104 
105  FILE* m_file = nullptr;
106 
107  public:
108 
109  file_wrapper() noexcept = default;
110 
111  file_wrapper(const int fd, const char* mode) {
112 #ifdef _MSC_VER
113  osmium::detail::disable_invalid_parameter_handler diph;
114 #endif
115  m_file = fdopen(fd, mode);
116  if (!m_file) {
117 
118  // Do not close stdout
119  if (fd != 1) {
120  ::close(fd);
121  }
122  throw std::system_error{errno, std::system_category(), "fdopen failed"};
123  }
124  }
125 
126  file_wrapper(const file_wrapper&) = delete;
127  file_wrapper& operator=(const file_wrapper&) = delete;
128 
129  file_wrapper(file_wrapper&&) = delete;
130  file_wrapper& operator=(file_wrapper&&) = delete;
131 
132  ~file_wrapper() noexcept {
133 #ifdef _MSC_VER
134  osmium::detail::disable_invalid_parameter_handler diph;
135 #endif
136  if (m_file) {
137  fclose(m_file);
138  }
139  }
140 
141  FILE* file() const noexcept {
142  return m_file;
143  }
144 
145  void close() {
146 #ifdef _MSC_VER
147  osmium::detail::disable_invalid_parameter_handler diph;
148 #endif
149  if (m_file) {
150  FILE* file = m_file;
151  m_file = nullptr;
152 
153  // Do not close stdout
154  if (fileno(file) == 1) {
155  return;
156  }
157 
158  if (fclose(file) != 0) {
159  throw std::system_error{errno, std::system_category(), "fclose failed"};
160  }
161  }
162  }
163 
164  }; // class file_wrapper
165 
166  } // namespace detail
167 
168  class Bzip2Compressor final : public Compressor {
169 
170  std::size_t m_file_size = 0;
171  detail::file_wrapper m_file;
172  BZFILE* m_bzfile = nullptr;
173 
174  public:
175 
176  explicit Bzip2Compressor(const int fd, const fsync sync) :
177  Compressor(sync),
178  m_file(fd, "wb") {
179 #ifdef _MSC_VER
180  osmium::detail::disable_invalid_parameter_handler diph;
181 #endif
182  int bzerror = BZ_OK;
183  m_bzfile = ::BZ2_bzWriteOpen(&bzerror, m_file.file(), 6, 0, 0);
184  if (!m_bzfile) {
185  throw bzip2_error{"bzip2 error: write open failed", bzerror};
186  }
187  }
188 
191 
194 
195  ~Bzip2Compressor() noexcept override {
196  try {
197  close();
198  } catch (...) {
199  // Ignore any exceptions because destructor must not throw.
200  }
201  }
202 
203  void write(const std::string& data) override {
204  assert(data.size() < std::numeric_limits<int>::max());
205  assert(m_bzfile);
206 #ifdef _MSC_VER
207  osmium::detail::disable_invalid_parameter_handler diph;
208 #endif
209  int bzerror = BZ_OK;
210  ::BZ2_bzWrite(&bzerror, m_bzfile, const_cast<char*>(data.data()), static_cast<int>(data.size()));
211  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
212  detail::throw_bzip2_error(m_bzfile, "write failed", bzerror);
213  }
214  }
215 
216  void close() override {
217  if (m_bzfile) {
218 #ifdef _MSC_VER
219  osmium::detail::disable_invalid_parameter_handler diph;
220 #endif
221  int bzerror = BZ_OK;
222  unsigned int nbytes_out_lo32 = 0;
223  unsigned int nbytes_out_hi32 = 0;
224  ::BZ2_bzWriteClose64(&bzerror, m_bzfile, 0, nullptr, nullptr, &nbytes_out_lo32, &nbytes_out_hi32);
225  m_bzfile = nullptr;
226  if (do_fsync() && m_file.file()) {
227  osmium::io::detail::reliable_fsync(fileno(m_file.file()));
228  }
229  m_file.close();
230  if (bzerror != BZ_OK) {
231  throw bzip2_error{"bzip2 error: write close failed", bzerror};
232  }
233  m_file_size = static_cast<std::size_t>(nbytes_out_hi32) << 32U | nbytes_out_lo32;
234  }
235  }
236 
237  std::size_t file_size() const override {
238  return m_file_size;
239  }
240 
241  }; // class Bzip2Compressor
242 
243  class Bzip2Decompressor final : public Decompressor {
244 
245  detail::file_wrapper m_file;
246  BZFILE* m_bzfile = nullptr;
247  bool m_stream_end = false;
248 
249  public:
250 
251  explicit Bzip2Decompressor(const int fd) :
252  m_file(fd, "rb") {
253 #ifdef _MSC_VER
254  osmium::detail::disable_invalid_parameter_handler diph;
255 #endif
256  int bzerror = BZ_OK;
257  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, nullptr, 0);
258  if (!m_bzfile) {
259  throw bzip2_error{"bzip2 error: read open failed", bzerror};
260  }
261  }
262 
265 
268 
269  ~Bzip2Decompressor() noexcept override {
270  try {
271  close();
272  } catch (...) {
273  // Ignore any exceptions because destructor must not throw.
274  }
275  }
276 
277  std::string read() override {
278 #ifdef _MSC_VER
279  osmium::detail::disable_invalid_parameter_handler diph;
280 #endif
281  assert(m_bzfile);
282  std::string buffer;
283 
284  if (!m_stream_end) {
286  int bzerror = BZ_OK;
287  assert(buffer.size() < std::numeric_limits<int>::max());
288  const int nread = ::BZ2_bzRead(&bzerror, m_bzfile, &*buffer.begin(), static_cast<int>(buffer.size()));
289  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
290  detail::throw_bzip2_error(m_bzfile, "read failed", bzerror);
291  }
292  if (bzerror == BZ_STREAM_END) {
293  void* unused = nullptr;
294  int nunused = 0;
295  if (!feof(m_file.file())) {
296  ::BZ2_bzReadGetUnused(&bzerror, m_bzfile, &unused, &nunused);
297  if (bzerror != BZ_OK) {
298  detail::throw_bzip2_error(m_bzfile, "get unused failed", bzerror);
299  }
300  std::string unused_data{static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused)};
301  ::BZ2_bzReadClose(&bzerror, m_bzfile);
302  if (bzerror != BZ_OK) {
303  throw bzip2_error{"bzip2 error: read close failed", bzerror};
304  }
305  assert(unused_data.size() < std::numeric_limits<int>::max());
306  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, &*unused_data.begin(), static_cast<int>(unused_data.size()));
307  if (!m_bzfile) {
308  throw bzip2_error{"bzip2 error: read open failed", bzerror};
309  }
310  } else {
311  m_stream_end = true;
312  }
313  }
314  buffer.resize(static_cast<std::string::size_type>(nread));
315  }
316 
317  set_offset(static_cast<std::size_t>(ftell(m_file.file())));
318 
319  return buffer;
320  }
321 
322  void close() override {
323  if (m_bzfile) {
324 #ifdef _MSC_VER
325  osmium::detail::disable_invalid_parameter_handler diph;
326 #endif
327  int bzerror = BZ_OK;
328  ::BZ2_bzReadClose(&bzerror, m_bzfile);
329  m_bzfile = nullptr;
330  m_file.close();
331  if (bzerror != BZ_OK) {
332  throw bzip2_error{"bzip2 error: read close failed", bzerror};
333  }
334  }
335  }
336 
337  }; // class Bzip2Decompressor
338 
339  class Bzip2BufferDecompressor final : public Decompressor {
340 
341  const char* m_buffer;
342  std::size_t m_buffer_size;
343  bz_stream m_bzstream;
344 
345  public:
346 
347  Bzip2BufferDecompressor(const char* buffer, const std::size_t size) :
348  m_buffer(buffer),
349  m_buffer_size(size),
350  m_bzstream() {
351  m_bzstream.next_in = const_cast<char*>(buffer);
352  assert(size < std::numeric_limits<unsigned int>::max());
353  m_bzstream.avail_in = static_cast<unsigned int>(size);
354  const int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
355  if (result != BZ_OK) {
356  throw bzip2_error{"bzip2 error: decompression init failed: ", result};
357  }
358  }
359 
362 
365 
366  ~Bzip2BufferDecompressor() noexcept override {
367  try {
368  close();
369  } catch (...) {
370  // Ignore any exceptions because destructor must not throw.
371  }
372  }
373 
374  std::string read() override {
375  std::string output;
376 
377  if (m_buffer) {
378  const std::size_t buffer_size = 10240;
379  output.resize(buffer_size);
380  m_bzstream.next_out = &*output.begin();
381  m_bzstream.avail_out = buffer_size;
382  const int result = BZ2_bzDecompress(&m_bzstream);
383 
384  if (result != BZ_OK) {
385  m_buffer = nullptr;
386  m_buffer_size = 0;
387  }
388 
389  if (result != BZ_OK && result != BZ_STREAM_END) {
390  throw bzip2_error{"bzip2 error: decompress failed: ", result};
391  }
392 
393  output.resize(static_cast<std::size_t>(m_bzstream.next_out - output.data()));
394  }
395 
396  return output;
397  }
398 
399  void close() override {
400  BZ2_bzDecompressEnd(&m_bzstream);
401  }
402 
403  }; // class Bzip2BufferDecompressor
404 
405  namespace detail {
406 
407  // we want the register_compression() function to run, setting
408  // the variable is only a side-effect, it will never be used
410  [](const int fd, const fsync sync) { return new osmium::io::Bzip2Compressor{fd, sync}; },
411  [](const int fd) { return new osmium::io::Bzip2Decompressor{fd}; },
412  [](const char* buffer, const std::size_t size) { return new osmium::io::Bzip2BufferDecompressor{buffer, size}; }
413  );
414 
415  // dummy function to silence the unused variable warning from above
416  inline bool get_registered_bzip2_compression() noexcept {
417  return registered_bzip2_compression;
418  }
419 
420  } // namespace detail
421 
422  } // namespace io
423 
424 } // namespace osmium
425 
426 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
osmium::io::Bzip2Decompressor::Bzip2Decompressor
Bzip2Decompressor(const int fd)
Definition: bzip2_compression.hpp:251
osmium::io::Bzip2Decompressor::operator=
Bzip2Decompressor & operator=(Bzip2Decompressor &&)=delete
osmium::io::Bzip2Compressor::Bzip2Compressor
Bzip2Compressor(const Bzip2Compressor &)=delete
writer_options.hpp
osmium::io::Bzip2Decompressor::close
void close() override
Definition: bzip2_compression.hpp:322
osmium::io::Bzip2Decompressor::m_bzfile
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:246
osmium::io::Bzip2Compressor::Bzip2Compressor
Bzip2Compressor(const int fd, const fsync sync)
Definition: bzip2_compression.hpp:176
osmium::io::Bzip2BufferDecompressor::~Bzip2BufferDecompressor
~Bzip2BufferDecompressor() noexcept override
Definition: bzip2_compression.hpp:366
osmium::io::CompressionFactory::instance
static CompressionFactory & instance()
Definition: compression.hpp:184
osmium::io::Bzip2Decompressor::m_stream_end
bool m_stream_end
Definition: bzip2_compression.hpp:247
osmium::io::Bzip2Compressor::file_size
std::size_t file_size() const override
Definition: bzip2_compression.hpp:237
osmium::io::Bzip2BufferDecompressor::Bzip2BufferDecompressor
Bzip2BufferDecompressor(Bzip2BufferDecompressor &&)=delete
osmium::io::Bzip2Decompressor
Definition: bzip2_compression.hpp:243
compression.hpp
osmium::io::Bzip2Decompressor::m_file
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:245
osmium::bzip2_error
Definition: bzip2_compression.hpp:71
detail
Definition: attr.hpp:342
osmium::bzip2_error::system_errno
int system_errno
Definition: bzip2_compression.hpp:74
osmium::io::Bzip2Compressor::~Bzip2Compressor
~Bzip2Compressor() noexcept override
Definition: bzip2_compression.hpp:195
osmium::io::Compressor
Definition: compression.hpp:57
osmium::io::Bzip2Compressor::m_file
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:171
osmium::io::Decompressor
Definition: compression.hpp:91
osmium::io::Bzip2BufferDecompressor::operator=
Bzip2BufferDecompressor & operator=(Bzip2BufferDecompressor &&)=delete
osmium::io::Bzip2Compressor::m_bzfile
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:172
osmium::io::Bzip2BufferDecompressor::operator=
Bzip2BufferDecompressor & operator=(const Bzip2BufferDecompressor &)=delete
osmium::io::Bzip2Decompressor::read
std::string read() override
Definition: bzip2_compression.hpp:277
osmium::io::Bzip2BufferDecompressor::Bzip2BufferDecompressor
Bzip2BufferDecompressor(const Bzip2BufferDecompressor &)=delete
osmium::io::Bzip2Decompressor::~Bzip2Decompressor
~Bzip2Decompressor() noexcept override
Definition: bzip2_compression.hpp:269
osmium::io::Bzip2Compressor::m_file_size
std::size_t m_file_size
Definition: bzip2_compression.hpp:170
osmium::io::Bzip2Compressor::close
void close() override
Definition: bzip2_compression.hpp:216
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
osmium::io::Bzip2Compressor::Bzip2Compressor
Bzip2Compressor(Bzip2Compressor &&)=delete
osmium::bzip2_error::bzip2_error
bzip2_error(const std::string &what, const int error_code)
Definition: bzip2_compression.hpp:76
osmium::io::Decompressor::input_buffer_size
@ input_buffer_size
Definition: compression.hpp:99
osmium::io::file_compression::none
@ none
osmium::io::Bzip2Compressor::operator=
Bzip2Compressor & operator=(Bzip2Compressor &&)=delete
osmium::io_error
Definition: error.hpp:44
osmium::io::Bzip2BufferDecompressor::Bzip2BufferDecompressor
Bzip2BufferDecompressor(const char *buffer, const std::size_t size)
Definition: bzip2_compression.hpp:347
osmium::io::Bzip2BufferDecompressor::close
void close() override
Definition: bzip2_compression.hpp:399
osmium::io::Bzip2BufferDecompressor
Definition: bzip2_compression.hpp:339
osmium::io::fsync
fsync
Definition: writer_options.hpp:51
osmium::io::Bzip2BufferDecompressor::m_buffer_size
std::size_t m_buffer_size
Definition: bzip2_compression.hpp:342
osmium::io::Bzip2Decompressor::Bzip2Decompressor
Bzip2Decompressor(Bzip2Decompressor &&)=delete
osmium::io::Bzip2BufferDecompressor::m_bzstream
bz_stream m_bzstream
Definition: bzip2_compression.hpp:343
osmium::io::Bzip2BufferDecompressor::m_buffer
const char * m_buffer
Definition: bzip2_compression.hpp:341
osmium::bzip2_error::bzip2_error_code
int bzip2_error_code
Definition: bzip2_compression.hpp:73
file.hpp
file_compression.hpp
osmium::io::Decompressor::set_offset
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:128
osmium::io::CompressionFactory::register_compression
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:189
osmium::io::Bzip2BufferDecompressor::read
std::string read() override
Definition: bzip2_compression.hpp:374
osmium::io::Compressor::do_fsync
bool do_fsync() const noexcept
Definition: compression.hpp:63
osmium::io::Bzip2Decompressor::Bzip2Decompressor
Bzip2Decompressor(const Bzip2Decompressor &)=delete
osmium::io::Bzip2Compressor::write
void write(const std::string &data) override
Definition: bzip2_compression.hpp:203
osmium::io::Bzip2Compressor::operator=
Bzip2Compressor & operator=(const Bzip2Compressor &)=delete
osmium::io::Bzip2Decompressor::operator=
Bzip2Decompressor & operator=(const Bzip2Decompressor &)=delete
error.hpp
osmium::io::Bzip2Compressor
Definition: bzip2_compression.hpp:168