NCBI C++ ToolKit
zlib_cloudflare.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef UTIL_COMPRESS__ZLIB_CLOUDFLARE__HPP
2 #define UTIL_COMPRESS__ZLIB_CLOUDFLARE__HPP
3 
4 /* $Id: zlib_cloudflare.hpp 101434 2023-12-13 16:37:07Z ivanov $
5  * ===========================================================================
6  *
7  * PUBLIC DOMAIN NOTICE
8  * National Center for Biotechnology Information
9  *
10  * This software/database is a "United States Government Work" under the
11  * terms of the United States Copyright Act. It was written as part of
12  * the author's official duties as a United States Government employee and
13  * thus cannot be copyrighted. This software/database is freely available
14  * to the public for use. The National Library of Medicine and the U.S.
15  * Government have not placed any restriction on its use or reproduction.
16  *
17  * Although all reasonable efforts have been taken to ensure the accuracy
18  * and reliability of the software and data, the NLM and the U.S.
19  * Government do not and cannot warrant the performance or results that
20  * may be obtained by using this software or data. The NLM and the U.S.
21  * Government disclaim all warranties, express or implied, including
22  * warranties of performance, merchantability or fitness for any particular
23  * purpose.
24  *
25  * Please cite the author in any work or product based on this material.
26  *
27  * ===========================================================================
28  *
29  * Author: Vladimir Ivanov
30  *
31  */
32 
33 /// @file zlib_cloudflare.hpp
34 ///
35 /// ZLib Compression API - Cloudflare ZLIB wrapper (https://github.com/cloudflare/zlib)
36 ///
37 /// CZipCloudflareCompression - base methods for compression/decompression
38 /// memory buffers and files.
39 /// CZipCloudflareCompressionFile - allow read/write operations on files in
40 /// zlib or gzip (.gz) format.
41 /// CZipCloudflareCompressor - zlib based compressor
42 /// (used in CZipCloudflareStreamCompressor).
43 /// CZipCloudflareDecompressor - zlib based decompressor
44 /// (used in CZipCloudflareStreamDecompressor).
45 /// CZipCloudflareStreamCompressor - zlib based compression stream processor
46 /// (see util/compress/stream.hpp for details).
47 /// CZipCloudflareStreamDecompressor - zlib based decompression stream processor
48 /// (see util/compress/stream.hpp for details).
49 ///
50 /// The zlib documentation can be found here:
51 /// http://zlib.org, or
52 /// http://www.gzip.org/zlib/manual.html
53 
54 
55 #include <util/compress/stream.hpp>
56 #include <util/compress/zlib.hpp> // for kZlibDefault* constants
57 
58 /** @addtogroup Compression
59  *
60  * @{
61  */
62 
64 
65 
66 
67 /////////////////////////////////////////////////////////////////////////////
68 ///
69 /// CZipCloudflareCompression --
70 ///
71 /// Define a base methods for compression/decompression memory buffers
72 /// and files.
73 
75 {
76 public:
77  /// Initialize compression library (for API compatibility, zlib don't need it).
78  static bool Initialize(void) { return true; };
79 
80  /// Compression/decompression flags.
81  enum EFlags {
82  /// Allow transparent reading data from buffer/file/stream
83  /// regardless is it compressed or not. But be aware,
84  /// if data source contains broken data and API cannot detect that
85  /// it is compressed data, that you can get binary instead of
86  /// decompressed data. By default this flag is OFF.
87  /// Note: zlib v1.1.4 and earlier have a bug in decoding.
88  /// In some cases decompressor can produce output data on invalid
89  /// compressed data. So, it is not recommended to use this flag
90  /// with old zlib versions.
91  fAllowTransparentRead = (1<<0),
92  /// Allow to "compress/decompress" empty data. Buffer compression
93  /// functions starts to return TRUE instead of FALSE for zero-length
94  /// input. And, if this flag is used together with fWriteGZipFormat
95  /// than the output will have gzip header and footer only.
96  fAllowEmptyData = (1<<1),
97  /// Check (and skip) gzip file header on decompression stage
98  fCheckFileHeader = (1<<2),
99  /// Use gzip (.gz) file format to write into compression stream
100  /// (the archive also can store file name and file modification
101  /// date in this format). Note: gzip file header and footer will be
102  /// omitted by default if no input data is provided, and you will
103  /// have empty output, that may not be acceptable to tools like
104  /// gunzip and etc -- in this case use fAllowEmptyData.
105  fWriteGZipFormat = (1<<3),
106  /// Allow concatenated gzip files.
107  /// Multiple compressed files can be concatenated into one file.
108  /// In this case, decompressor will try to extract all members
109  /// at once. But note, that better compression can be usually
110  /// obtained if all members are decompressed and then recompressed
111  /// in a single step.
112  fAllowConcatenatedGZip = (1<<4),
113  /// Set of flags for gzip file support. See each flag description above.
114  fGZip = fCheckFileHeader | fWriteGZipFormat | fAllowConcatenatedGZip,
115  /// This flag can be used only with DecompressFile[IntoDir]().
116  /// It allow to restore the original file name and/or time stamp stored
117  /// in the file header, if present.
118  /// @sa DecompressFile, DecompressFileIntoDir
119  fRestoreFileAttr = (1<<5)
120  };
121  typedef CZipCloudflareCompression::TFlags TZipFlags; ///< Bitwise OR of EFlags
122 
123  /// Constructor.
124  /// @note
125  /// For setting up advanced compression parameters see Set*() methods.
126  CZipCloudflareCompression(ELevel level = eLevel_Default);
127 
128  /// Destructor.
130 
131  /// Return name and version of the compression library.
132  virtual CVersionInfo GetVersion(void) const;
133 
134  /// Returns default compression level for a compression algorithm.
135  virtual ELevel GetDefaultLevel(void) const
136  { return ELevel(eLevel_Default); };
137 
138  /// Check if compression have support for a specified feature
139  virtual bool HaveSupport(ESupportFeature feature);
140 
141 
142  //=======================================================================
143  // Utility functions
144  //=======================================================================
145 
146  /// Compress data in the buffer.
147  ///
148  /// @param src_buf
149  /// Source buffer.
150  /// @param src_len
151  /// Size of data in source buffer.
152  /// @param dst_buf
153  /// Destination buffer.
154  /// @param dst_size
155  /// Size of destination buffer.
156  /// In some cases, small source data or bad compressed data for example,
157  /// it should be a little more then size of the source buffer.
158  /// @param dst_len
159  /// Size of compressed data in destination buffer.
160  /// @return
161  /// Return TRUE if operation was successfully or FALSE otherwise.
162  /// On success, 'dst_buf' contains compressed data of 'dst_len' size.
163  /// @sa
164  /// EstimateCompressionBufferSize, DecompressBuffer
165  virtual bool CompressBuffer(
166  const void* src_buf, size_t src_len,
167  void* dst_buf, size_t dst_size,
168  /* out */ size_t* dst_len
169  );
170 
171  /// Decompress data in the buffer.
172  ///
173  /// @note
174  /// The decompressor stops and returns TRUE, if it find logical
175  /// end in the compressed data, even not all compressed data was processed.
176  /// Only for case of decompressing concatenated gzip files in memory
177  /// it try to decompress data behind of logical end of recurrent gzip chunk,
178  /// to check on next portion of data. See fCheckFileHeader,
179  /// fAllowConcatenatedGZip and fGZip flags description.
180  /// @param src_buf
181  /// Source buffer.
182  /// @param src_len
183  /// Size of data in source buffer.
184  /// @param dst_buf
185  /// Destination buffer.
186  /// It must be large enough to hold all of the uncompressed data for the operation to complete.
187  /// @param dst_size
188  /// Size of destination buffer.
189  /// @param dst_len
190  /// Size of decompressed data in destination buffer.
191  /// @return
192  /// Return TRUE if operation was successfully or FALSE otherwise.
193  /// On success, 'dst_buf' contains decompressed data of dst_len size.
194  /// @sa
195  /// CompressBuffer, EFlags
196  virtual bool DecompressBuffer(
197  const void* src_buf, size_t src_len,
198  void* dst_buf, size_t dst_size,
199  /* out */ size_t* dst_len
200  );
201 
202  /// Estimate buffer size for data compression.
203  ///
204  /// The function shall estimate the size of buffer required to compress
205  /// specified number of bytes of data using the CompressBuffer() function.
206  /// This function may return a conservative value that may be larger
207  /// than 'src_len'.
208  ///
209  /// @param src_len
210  /// Size of data in source buffer.
211  /// @return
212  /// Estimated buffer size. 0 if unable to determine.
213  /// @note
214  /// This method ignores used dictionary.
215  /// @sa
216  /// CompressBuffer
217  virtual size_t EstimateCompressionBufferSize(size_t src_len);
218 
219  /// Get recommended buffer sizes for stream/file I/O.
220  ///
221  /// These buffer sizes are softly recommended. They are not required, (de)compression
222  /// streams accepts any reasonable buffer size, for both input and output.
223  /// Respecting the recommended size just makes it a bit easier for (de)compressor,
224  /// reducing the amount of memory shuffling and buffering, resulting in minor
225  /// performance savings. If compression library doesn't have preferences about
226  /// I/O buffer sizes, kCompressionDefaultBufSize will be used.
227  /// @param round_up_by
228  /// If specified, round up a returned value by specified amount.
229  /// Useful for better memory management. For example you can round up to virtual
230  /// memory page size.
231  /// @return
232  /// Structure with recommended buffer sizes.
233  /// @note
234  /// Applicable for streaming/file operations.
235  /// @sa
236  /// kCompressionDefaultBufSize, CSystemInfo::GetVirtualMemoryPageSize()
237  ///
239 
240  /// Compress file.
241  ///
242  /// @param src_file
243  /// File name of source file.
244  /// @param dst_file
245  /// File name of result file.
246  /// @param file_io_bufsize
247  /// Size of the buffer used to read from a source file.
248  /// Writing happens immediately on receiving some data from a compressor.
249  /// @param compression_in_bufsize
250  /// Size of the internal buffer holding input data to be compressed.
251  /// It can be different from 'file_io_bufsize' depending on a using
252  /// compression method, OS and file system.
253  /// @param compression_out_bufsize
254  /// Size of the internal buffer to receive data from a compressor.
255  /// @return
256  /// Return TRUE on success, FALSE on error.
257  /// @note
258  /// This method, as well as some gzip utilities, always keeps the original
259  /// file name and timestamp in the compressed file. On this moment
260  /// DecompressFile() method do not use original file name at all,
261  /// but be aware... If you assign different base name to destination
262  /// compressed file, that behavior of decompression utilities
263  /// on different platforms may differ. For example, WinZip on MS Windows
264  /// always restore original file name and timestamp stored in the file.
265  /// UNIX gunzip have -N option for this, but by default do not use it,
266  /// and just creates a decompressed file with the name of the compressed
267  /// file without .gz extension.
268  /// @sa
269  /// DecompressFile, DecompressFileIntoDir, GetRecommendedBufferSizes,
270  /// CZipCloudflareCompressionFile
271  ///
272  virtual bool CompressFile(
273  const string& src_file,
274  const string& dst_file,
275  size_t file_io_bufsize = kCompressionDefaultBufSize,
276  size_t compression_in_bufsize = kCompressionDefaultBufSize,
277  size_t compression_out_bufsize = kCompressionDefaultBufSize
278  );
279 
280  /// Decompress file.
281  ///
282  /// @param src_file
283  /// File name of source file.
284  /// @param dst_file
285  /// File name of result file.
286  /// @param file_io_bufsize
287  /// Size of the buffer used to read from a source file.
288  /// Writing happens immediately on receiving some data from a decompressor.
289  /// @param decompression_in_bufsize
290  /// Size of the internal buffer holding input data to be decompressed.
291  /// It can be different from 'file_io_bufsize' depending on a using
292  /// compression method, OS and file system.
293  /// @param decompression_out_bufsize
294  /// Size of the internal buffer to receive data from a decompressor.
295  /// @return
296  /// Return TRUE on success, FALSE on error.
297  /// @sa
298  /// CompressFile, DecompressFileIntoDir, GetRecommendedBufferSizes, CZipCloudflareCompressionFile
299  /// @note
300  /// CompressFile() method, as well as some gzip utilities, always keeps
301  /// the original file name and timestamp in the compressed file.
302  /// If fRestoreFileAttr flag is set, that timestamp, stored in the file
303  /// header will be restored. The original file name cannot be restored here,
304  /// see DecompressFileIntoDir().
305  ///
306  virtual bool DecompressFile(
307  const string& src_file,
308  const string& dst_file,
309  size_t file_io_bufsize = kCompressionDefaultBufSize,
310  size_t decompression_in_bufsize = kCompressionDefaultBufSize,
311  size_t decompression_out_bufsize = kCompressionDefaultBufSize
312  );
313 
314  /// Decompress file into specified directory.
315  ///
316  /// @param src_file
317  /// File name of source file.
318  /// @param dst_dir
319  /// Destination directory.
320  /// @param file_io_bufsize
321  /// Size of the buffer used to read from a source file.
322  /// Writing happens immediately on receiving some data from a decompressor.
323  /// @param decompression_in_bufsize
324  /// Size of the internal buffer holding input data to be decompressed.
325  /// It can be different from 'file_io_bufsize' depending on a using
326  /// compression method, OS and file system.
327  /// @param decompression_out_bufsize
328  /// Size of the internal buffer to receive data from a decompressor.
329  /// @return
330  /// Return TRUE on success, FALSE on error.
331  /// @sa
332  /// CompressFile, DecompressFile, GetRecommendedBufferSizes, CZipCloudflareCompressionFile
333  /// @note
334  /// CompressFile() method, as well as some gzip utilities, always keeps
335  /// the original file name and timestamp in the compressed file.
336  /// If fRestoreFileAttr flag is set, that original file name and timestamp,
337  /// stored in the file header will be restored. If not, that destination
338  /// file will be named as archive name without extension.
339  ///
340  virtual bool DecompressFileIntoDir(
341  const string& src_file,
342  const string& dst_dir,
343  size_t file_io_bufsize = kCompressionDefaultBufSize,
344  size_t decompression_in_bufsize = kCompressionDefaultBufSize,
345  size_t decompression_out_bufsize = kCompressionDefaultBufSize
346  );
347 
348  /// Structure to keep compressed file information.
349  struct SFileInfo {
350  string name;
351  string comment;
352  time_t mtime;
353  SFileInfo(void) : mtime(0) {};
354  };
355 
356  /// Set a dictionary for all compression/decompression operations.
357  ///
358  /// Using dictionary can significantly reduce the size of the compressed data.
359  /// Refer to the C++ documentation how to choose/prepare a dictionary.
360  ///
361  /// @param dict
362  /// Dictionary to use. New dictionary will be used for all subsequent
363  /// compression/decompression buffer and file operations. NULL value
364  /// invalidates previous dictionary, meaning "return to no-dictionary mode".
365  /// @param own
366  /// If set to eTakeOwnership the dictionary will be owned by CCompression and
367  /// automatically deleted when necessary.
368  /// @return
369  /// Return TRUE on success, FALSE on error.
370  /// FALSE usually mean that dictionaries are not supported for a current compression.
371  /// @note
372  /// Each compression algorithm have its own dictionary format and cannot
373  /// be reused by some other compression algorithm.
374  /// @note
375  /// Same dictionary should be used to compress and decompress data.
376  /// @note
377  /// .gz files don't store a dictionary inside, so you will be unable to decompress
378  /// files created using CompressFile() or streams with an active dictionary using
379  /// any external utilities like 'gunzip'. But they will be still decompressible
380  /// with DecompressFile() using the same dictionary.
381  /// @note
382  /// If decompressed data have concatenated gzip files, and it is allowed to process
383  /// them all, each gzip file should be compressed with the same dictionary.
384  /// It is allowed to mix dictionary and non-dictionary compressed gzip files,
385  /// the dictionary will be applied only when necessary.
386  /// @sa
387  /// CompressBuffer, DecompressBuffer, CompressFile, DecompressFile
388  ///
389  virtual bool SetDictionary(
390  CCompressionDictionary& dict,
392  );
393 
394  //=======================================================================
395  // Advanced compression-specific parameters
396  //=======================================================================
397  // Allow to tune up (de)compression for a specific needs.
398  //
399  // - Pin down compression parameters to some specific values, so these
400  // values are no longer dynamically selected by the compressor.
401  // - All setting parameters should be in the range [min,max],
402  // or equal to default.
403  // - All parameters should be set before starting (de)compression,
404  // or it will be ignored for current operation.
405  //=======================================================================
406  // You can use listed Z_* parameters after #include <zlib.h>.
407 
408  /// Compression strategy.
409  ///
410  /// The strategy parameter is used to tune the compression algorithm.
411  /// - Z_DEFAULT_STRATEGY
412  /// for normal data;
413  /// - Z_HUFFMAN_ONLY
414  /// to force Huffman encoding only (no string match);
415  /// - Z_RLE
416  /// run-length encoding (RLE) compression;
417  /// - Z_FILTERED
418  /// for data produced by a filter (or predictor);
419  /// Filtered data consists mostly of small values with a somewhat
420  /// random distribution. In this case, the compression algorithm
421  /// is tuned to compress them better. The effect of Z_FILTERED is
422  /// to force more Huffman coding and less string matching;
423  /// it is somewhat intermediate between Z_DEFAULT and Z_HUFFMAN_ONLY.
424  /// - Z_FIXED
425  /// prevents the use of dynamic Huffman codes, allowing for a simpler
426  /// decoder for special applications;
427  ///
428  /// The strategy parameter only affects the compression ratio but not the
429  /// correctness of the compressed output even if it is not set appropriately.
430  /// Used for compression only.
431  ///
432  void SetStrategy(int strategy) {
433  m_c_Strategy = (strategy == kZlibDefaultStrategy ) ? GetStrategyDefault() : strategy;
434  }
435  int GetStrategy(void) const { return m_c_Strategy; }
436  static int GetStrategyDefault(void);
437  static int GetStrategyMin(void);
438  static int GetStrategyMax(void);
439 
440  /// Memory level.
441  ///
442  /// The "mem_level" parameter specifies how much memory should be
443  /// allocated for the internal compression state. Low levels uses
444  /// less memory but are slow and reduces compression ratio; maximum level
445  /// uses maximum memory for optimal speed. See zconf.h for total memory usage
446  /// as a function of windowBits and memLevel.
447  ///
448  void SetMemoryLevel(int mem_level) {
449  m_c_MemLevel = (mem_level == kZlibDefaultMemLevel) ? GetMemoryLevelDefault() : mem_level;
450  }
451  int GetMemoryLevel(void) const { return m_c_MemLevel; }
452  static int GetMemoryLevelDefault(void);
453  static int GetMemoryLevelMin(void);
454  static int GetMemoryLevelMax(void);
455 
456  /// Window bits.
457  ///
458  /// This parameter is the base two logarithm of the window size
459  /// (the size of the history buffer). Larger values of this parameter result
460  /// in better compression at the expense of memory usage.
461  /// Used for compression and decompression. By default it is set to a maximum
462  /// allowed values. Reducing windows bits from default can make to it unable
463  /// to extract .gz files created by gzip.
464  /// @note
465  /// API support positive values for this parameters only. RAW deflate
466  /// data is processed by default, for gzip format we have a apecial
467  /// flags, see description for: fGZip, fCheckFileHeader, fWriteGZipFormat.
468  ///
469  void SetWindowBits(int window_bits) {
470  m_cd_WindowBits = (window_bits == kZlibDefaultWbits) ? GetWindowBitsDefault() : window_bits;
471  }
472  int GetWindowBits(void) const { return m_cd_WindowBits; }
473  static int GetWindowBitsDefault(void);
474  static int GetWindowBitsMin(void);
475  static int GetWindowBitsMax(void);
476 
477 protected:
478  /// Format string with last error description.
479  /// If pos == 0, that use internal m_Stream's position to report.
480  string FormatErrorMessage(string where, size_t pos = 0) const;
481 
482 protected:
483  void* m_Stream; ///< Compressor stream.
484  int m_cd_WindowBits; ///< The base two logarithm of the window size.
485  int m_c_MemLevel; ///< The allocation memory level for the compression.
486  int m_c_Strategy; ///< The parameter to tune up a compression algorithm.
487 
488 private:
489  /// Private copy constructor to prohibit copy.
491  /// Private assignment operator to prohibit assignment.
493 };
494 
495 
496 /////////////////////////////////////////////////////////////////////////////
497 ///
498 /// CZipCloudflareCompressionFile --
499 ///
500 /// Allow read/write operations on files in zlib or gzip (.gz) formats.
501 /// Throw exceptions on critical errors.
502 
504  public CCompressionFile
505 {
506 public:
507  /// Constructor.
508  ///
509  /// Automatically calls Open() with given file name, mode and compression level.
510  /// @note
511  /// This constructor don't allow to use any advanced compression parameters
512  /// or a dictionary. If you need to set any of them, please use simplified
513  /// conventional constructor, set advanced parameters and use Open().
514  ///
516  const string& file_name,
517  EMode mode,
518  ELevel level = eLevel_Default,
519  size_t compression_in_bufsize = kCompressionDefaultBufSize,
520  size_t compression_out_bufsize = kCompressionDefaultBufSize
521  );
522  /// Conventional constructor.
524  ELevel level = eLevel_Default
525  );
526 
527  /// Destructor
529 
530  /// Opens a compressed file for reading or writing.
531  ///
532  /// @param file_name
533  /// File name of the file to open.
534  /// @param mode
535  /// File open mode.
536  /// @param compression_in_bufsize
537  /// Size of the internal buffer holding input data to be (de)compressed.
538  /// @param compression_out_bufsize
539  /// Size of the internal buffer to receive data from a (de)compressor.
540  /// @return
541  /// TRUE if file was opened successfully or FALSE otherwise.
542  /// @sa
543  /// CZipCloudflareCompression, Read, Write, Close
544  /// @note
545  /// All advanced compression parameters or a dictionary should be set before
546  /// Open() method, otherwise they will not have any effect.
547  ///
548  virtual bool Open(
549  const string& file_name,
550  EMode mode,
551  size_t compression_in_bufsize = kCompressionDefaultBufSize,
552  size_t compression_out_bufsize = kCompressionDefaultBufSize
553  );
554 
555  /// Opens a compressed file for reading or writing.
556  ///
557  /// Do the same as standard Open(), but can also get/set file info.
558  /// @param file_name
559  /// File name of the file to open.
560  /// @param mode
561  /// File open mode.
562  /// @param info
563  /// Pointer to file information structure. If it is not NULL,
564  /// that it will be used to get information about compressed file
565  /// in the read mode, and set it in the write mode for gzip files.
566  /// @param compression_in_bufsize
567  /// Size of the internal buffer holding input data to be (de)compressed.
568  /// @param compression_out_bufsize
569  /// Size of the internal buffer to receive data from a (de)compressor.
570  /// @return
571  /// TRUE if file was opened successfully or FALSE otherwise.
572  /// @sa
573  /// CZipCloudflareCompression, Read, Write, Close
574  ///
575  virtual bool Open(
576  const string& file_name,
577  EMode mode,
578  SFileInfo* info,
579  size_t compression_in_bufsize = kCompressionDefaultBufSize,
580  size_t compression_out_bufsize = kCompressionDefaultBufSize
581  );
582 
583  /// Read data from compressed file.
584  ///
585  /// Read up to "len" uncompressed bytes from the compressed file "file"
586  /// into the buffer "buf".
587  /// @param buf
588  /// Buffer for requested data.
589  /// @param len
590  /// Number of bytes to read.
591  /// @return
592  /// Number of bytes actually read (0 for end of file, -1 for error).
593  /// The number of really read bytes can be less than requested.
594  /// @sa
595  /// Open, Write, Close
596  ///
597  virtual long Read(void* buf, size_t len);
598 
599  /// Write data to compressed file.
600  ///
601  /// Writes the given number of uncompressed bytes from the buffer
602  /// into the compressed file.
603  /// @param buf
604  /// Buffer with written data.
605  /// @param len
606  /// Number of bytes to write.
607  /// @return
608  /// Number of bytes actually written or -1 for error.
609  /// Returned value can be less than "len".
610  /// @sa
611  /// Open, Read, Close
612  ///
613  virtual long Write(const void* buf, size_t len);
614 
615  /// Close compressed file.
616  ///
617  /// Flushes all pending output if necessary, closes the compressed file.
618  /// @return
619  /// TRUE on success, FALSE on error.
620  /// @sa
621  /// Open, Read, Write
622  ///
623  virtual bool Close(void);
624 
625 protected:
626  /// Get error code/description of last stream operation (m_Stream).
627  /// It can be received using GetErrorCode()/GetErrorDescription() methods.
628  void GetStreamError(void);
629 
630 protected:
631  EMode m_Mode; ///< I/O mode (read/write).
632  CNcbiFstream* m_File; ///< File stream.
633  CCompressionIOStream* m_Stream; ///< [De]comression stream.
634 
635 private:
636  /// Private copy constructor to prohibit copy.
638  /// Private assignment operator to prohibit assignment.
640 };
641 
642 
643 /////////////////////////////////////////////////////////////////////////////
644 ///
645 /// CZipCloudflareCompressor -- zlib based compressor
646 ///
647 /// Used in CZipCloudflareStreamCompressor.
648 /// @sa CZipCloudflareStreamCompressor, CZipCloudflareCompression, CCompressionProcessor
649 
651  public CCompressionProcessor
652 {
653 public:
654  /// Constructor.
656  ELevel level = eLevel_Default,
657  TZipFlags flags = 0
658  );
659 
660  /// Destructor.
662 
663  /// Set information about compressed file.
664  ///
665  /// Used for compression of gzip files.
666  void SetFileInfo(const SFileInfo& info);
667 
668  /// Return TRUE if fAllowEmptyData flag is set.
669  /// @note
670  /// Used by stream buffer, that don't have access to specific
671  /// compression implementation flags.
672  virtual bool AllowEmptyData() const
673  { return (GetFlags() & fAllowEmptyData) == fAllowEmptyData; }
674 
675 protected:
676  virtual EStatus Init (void);
677  virtual EStatus Process(const char* in_buf, size_t in_len,
678  char* out_buf, size_t out_size,
679  /* out */ size_t* in_avail,
680  /* out */ size_t* out_avail);
681  virtual EStatus Flush (char* out_buf, size_t out_size,
682  /* out */ size_t* out_avail);
683  virtual EStatus Finish (char* out_buf, size_t out_size,
684  /* out */ size_t* out_avail);
685  virtual EStatus End (int abandon = 0);
686 
687 private:
688  unsigned long m_CRC32; ///< CRC32 for compressed data.
689  string m_Cache; ///< Buffer to cache small pieces of data.
691  ///< Is true if needed to write a file header.
692  SFileInfo m_FileInfo; ///< Compressed file info.
693 };
694 
695 
696 
697 /////////////////////////////////////////////////////////////////////////////
698 ///
699 /// CZipCloudflareDecompressor -- zlib based decompressor
700 ///
701 /// Used in CZipCloudflareStreamDecompressor.
702 /// @sa CZipCloudflareStreamDecompressor, CZipCloudflareCompression, CCompressionProcessor
703 
705  public CCompressionProcessor
706 {
707 public:
708  /// Constructor.
710 
711  /// Destructor.
713 
714  /// Return TRUE if fAllowEmptyData flag is set.
715  /// @note
716  /// Used by stream buffer, that don't have access to specific
717  /// compression implementation flags.
718  virtual bool AllowEmptyData() const
719  { return (GetFlags() & fAllowEmptyData) == fAllowEmptyData; }
720 
721 protected:
722  virtual EStatus Init (void);
723  virtual EStatus Process(const char* in_buf, size_t in_len,
724  char* out_buf, size_t out_size,
725  /* out */ size_t* in_avail,
726  /* out */ size_t* out_avail);
727  virtual EStatus Flush (char* out_buf, size_t out_size,
728  /* out */ size_t* out_avail);
729  virtual EStatus Finish (char* out_buf, size_t out_size,
730  /* out */ size_t* out_avail);
731  virtual EStatus End (int abandon = 0);
732 
733 private:
734  bool m_NeedCheckHeader; ///< TRUE if needed to check to file header.
735  bool m_IsGZ; ///< TRUE if data have gzip format.
736  size_t m_SkipInput; ///< Number of bytes to skip from input stream.
737  ///< Used to process concatenated .gz files.
738  string m_Cache; ///< Buffer to cache small pieces of data.
739 };
740 
741 
742 
743 /////////////////////////////////////////////////////////////////////////////
744 ///
745 /// CZipCloudflareStreamCompressor -- zlib based compression stream processor
746 ///
747 /// See util/compress/stream.hpp for details of stream processing.
748 /// @note
749 /// Compression/decompression flags (CZipCloudflareCompression:EFlags) can greatly
750 /// affect CZipCloudflareStreamCompressor behavior. By default, compressor
751 /// produce plain zip data, that is not compatible with gzip/gunzip utility.
752 /// Please use appropriate flags in constructor to change default behavior.
753 /// @sa CCompressionStreamProcessor
754 
757 {
758 public:
759  /// Full constructor
762  streamsize in_bufsize,
763  streamsize out_bufsize,
765  )
767  new CZipCloudflareCompressor(level, flags), eDelete, in_bufsize, out_bufsize)
768  {}
769 
770  /// Conventional constructor.
771  /// Uses default buffer sizes for I/O, that can be not ideal for some scenarios.
775  )
777  new CZipCloudflareCompressor(level, flags),
779  {}
780 
781  /// Conventional constructor.
782  /// Uses default buffer sizes for I/O, that can be not ideal for some scenarios.
787  {}
788 
789  /// Return a pointer to compressor.
790  /// Can be used mostly for setting an advanced compression-specific parameters.
792  return dynamic_cast<CZipCloudflareCompressor*>(GetProcessor());
793  }
794 };
795 
796 
797 /////////////////////////////////////////////////////////////////////////////
798 ///
799 /// CZipCloudflareStreamDecompressor -- zlib based decompression stream processor
800 ///
801 /// See util/compress/stream.hpp for details of stream processing.
802 /// @note
803 /// Compression/decompression flags (CZipCloudflareCompression:EFlags) can greatly
804 /// affect CZipCloudflareStreamDecompressor behavior. By default, decompressor
805 /// do not allow data in gzip format. Please use appropriate flags
806 /// in constructor to change default behavior.
807 /// @sa CCompressionStreamProcessor
808 
811 {
812 public:
813  /// Full constructor
815  streamsize in_bufsize,
816  streamsize out_bufsize,
818  )
820  new CZipCloudflareDecompressor(flags), eDelete, in_bufsize, out_bufsize)
821  {}
822 
823  /// Conventional constructor.
824  /// Uses default buffer sizes for I/O, that can be not ideal for some scenarios.
829  {}
830 
831  /// Return a pointer to decompressor.
832  /// Can be used mostly for setting an advanced compression-specific parameters.
834  return dynamic_cast<CZipCloudflareDecompressor*>(GetProcessor());
835  }
836 };
837 
838 
839 //////////////////////////////////////////////////////////////////////////////
840 //
841 // Global functions
842 //
843 
844 /// Get list of positions of separate gzip files in the concatenated gzip file.
845 /// Return results via user defined handler.
846 /// Throw CCoreException/CCompressionException on error.
847 ///
848 /// @param is
849 /// Opened input stream to scan (should be opened in binary mode).
850 /// @param handler
851 /// Call handler's IChunkHandler::OnChunk() method and pass position
852 /// of each new gzip file inside a stream and size of uncompressed data
853 /// on that moment.
854 /// @note
855 /// This method don't support concatenated .gz files compressed with a dictionary.
856 ///
859 
860 
862 
863 
864 /* @} */
865 
866 #endif /* UTIL_COMPRESS__ZLIB_CLOUDFLARE__HPP */
CVersionInfo –.
CZipCloudflareCompressionFile –.
CZipCloudflareCompression –.
CZipCloudflareCompressor – zlib based compressor.
CZipCloudflareDecompressor – zlib based decompressor.
CZipCloudflareStreamCompressor – zlib based compression stream processor.
CZipCloudflareStreamDecompressor – zlib based decompression stream processor.
Interface class to scan data source for seekable data chunks.
Definition: compress.hpp:631
void(*)(CSeq_entry_Handle seh, IWorkbench *wb, const CSerialObject &obj) handler
static uch flags
const char * file_name[]
@ eNoOwnership
No ownership is assumed.
Definition: ncbi_types.h:135
CZipCloudflareCompressor * GetCompressor(void) const
Return a pointer to compressor.
static int GetWindowBitsDefault(void)
CZipCloudflareCompressionFile(ELevel level=eLevel_Default)
Conventional constructor.
void * m_Stream
Compressor stream.
void SetStrategy(int strategy)
Compression strategy.
CZipCloudflareCompressor(ELevel level=eLevel_Default, TZipFlags flags=0)
Constructor.
virtual EStatus Finish(char *out_buf, size_t out_size, size_t *out_avail)
Finish the compression/decompression process.
static int GetMemoryLevelDefault(void)
virtual EStatus Finish(char *out_buf, size_t out_size, size_t *out_avail)
Finish the compression/decompression process.
int m_c_MemLevel
The allocation memory level for the compression.
bool m_NeedWriteHeader
Is true if needed to write a file header.
virtual ~CZipCloudflareCompression(void)
Destructor.
CZipCloudflareCompressionFile(const string &file_name, EMode mode, ELevel level=eLevel_Default, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Constructor.
virtual bool CompressFile(const string &src_file, const string &dst_file, size_t file_io_bufsize=kCompressionDefaultBufSize, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Compress file.
CZipCloudflareCompression(const CZipCloudflareCompression &)
Private copy constructor to prohibit copy.
CZipCloudflareCompression(ELevel level=eLevel_Default)
Constructor.
virtual CVersionInfo GetVersion(void) const
Return name and version of the compression library.
CZipCloudflareStreamDecompressor(streamsize in_bufsize, streamsize out_bufsize, CZipCloudflareCompression::TZipFlags flags=0)
Full constructor.
CZipCloudflareCompression::TFlags TZipFlags
Bitwise OR of EFlags.
CZipCloudflareStreamDecompressor(CZipCloudflareCompression::TZipFlags flags=0)
Conventional constructor.
SFileInfo m_FileInfo
Compressed file info.
static int GetWindowBitsMin(void)
void SetWindowBits(int window_bits)
Window bits.
virtual TFlags GetFlags(void) const
Get/set flags.
Definition: compress.cpp:105
const int kZlibDefaultStrategy
Definition: zlib.hpp:69
virtual bool DecompressBuffer(const void *src_buf, size_t src_len, void *dst_buf, size_t dst_size, size_t *dst_len)
Decompress data in the buffer.
virtual ~CZipCloudflareCompressor(void)
Destructor.
virtual bool Open(const string &file_name, EMode mode, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Opens a compressed file for reading or writing.
virtual bool Open(const string &file_name, EMode mode, SFileInfo *info, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Opens a compressed file for reading or writing.
CNcbiFstream * m_File
File stream.
ELevel
Compression level.
Definition: compress.hpp:142
CZipCloudflareDecompressor(TZipFlags flags=0)
Constructor.
virtual EStatus Init(void)
Initialize the internal stream state for compression/decompression.
CZipCloudflareStreamCompressor(CZipCloudflareCompression::ELevel level, streamsize in_bufsize, streamsize out_bufsize, CZipCloudflareCompression::TZipFlags flags=0)
Full constructor.
static int GetWindowBitsMax(void)
void GetStreamError(void)
Get error code/description of last stream operation (m_Stream).
virtual bool AllowEmptyData() const
Return TRUE if fAllowEmptyData flag is set.
virtual long Read(void *buf, size_t len)
Read data from compressed file.
virtual long Write(const void *buf, size_t len)
Write data to compressed file.
const streamsize kCompressionDefaultBufSize
Default compression I/O stream buffer size.
Definition: compress.hpp:111
string m_Cache
Buffer to cache small pieces of data.
string FormatErrorMessage(string where, size_t pos=0) const
Format string with last error description.
void SetFileInfo(const SFileInfo &info)
Set information about compressed file.
unsigned long m_CRC32
CRC32 for compressed data.
const int kZlibDefaultWbits
Definition: zlib.hpp:67
virtual bool HaveSupport(ESupportFeature feature)
Check if compression have support for a specified feature.
virtual bool Close(void)
Close compressed file.
virtual bool CompressBuffer(const void *src_buf, size_t src_len, void *dst_buf, size_t dst_size, size_t *dst_len)
Compress data in the buffer.
static int GetMemoryLevelMin(void)
virtual EStatus Flush(char *out_buf, size_t out_size, size_t *out_avail)
Flush compressed/decompressed data from the output buffer.
~CZipCloudflareCompressionFile(void)
Destructor.
unsigned int TFlags
Compression flags.
Definition: compress.hpp:160
CZipCloudflareStreamCompressor(CZipCloudflareCompression::TZipFlags flags=0)
Conventional constructor.
int m_c_Strategy
The parameter to tune up a compression algorithm.
static SRecommendedBufferSizes GetRecommendedBufferSizes(size_t round_up=0)
Get recommended buffer sizes for stream/file I/O.
bool m_IsGZ
TRUE if data have gzip format.
virtual bool SetDictionary(CCompressionDictionary &dict, ENcbiOwnership own=eNoOwnership)
Set a dictionary for all compression/decompression operations.
virtual bool AllowEmptyData() const
Return TRUE if fAllowEmptyData flag is set.
const int kZlibDefaultMemLevel
Definition: zlib.hpp:68
virtual bool DecompressFileIntoDir(const string &src_file, const string &dst_dir, size_t file_io_bufsize=kCompressionDefaultBufSize, size_t decompression_in_bufsize=kCompressionDefaultBufSize, size_t decompression_out_bufsize=kCompressionDefaultBufSize)
Decompress file into specified directory.
virtual size_t EstimateCompressionBufferSize(size_t src_len)
Estimate buffer size for data compression.
virtual EStatus End(int abandon=0)
Free all dynamically allocated data structures.
bool m_NeedCheckHeader
TRUE if needed to check to file header.
int m_cd_WindowBits
The base two logarithm of the window size.
static int GetMemoryLevelMax(void)
EFlags
Compression/decompression flags.
virtual ~CZipCloudflareDecompressor(void)
Destructor.
CZipCloudflareDecompressor * GetDecompressor(void) const
Return a pointer to decompressor.
EStatus
Type of the result of all basic functions.
Definition: compress.hpp:467
static int GetStrategyMin(void)
static int GetStrategyMax(void)
EMode m_Mode
I/O mode (read/write).
CZipCloudflareCompressionFile(const CZipCloudflareCompressionFile &)
Private copy constructor to prohibit copy.
virtual EStatus Flush(char *out_buf, size_t out_size, size_t *out_avail)
Flush compressed/decompressed data from the output buffer.
virtual bool DecompressFile(const string &src_file, const string &dst_file, size_t file_io_bufsize=kCompressionDefaultBufSize, size_t decompression_in_bufsize=kCompressionDefaultBufSize, size_t decompression_out_bufsize=kCompressionDefaultBufSize)
Decompress file.
static bool Initialize(void)
Initialize compression library (for API compatibility, zlib don't need it).
size_t m_SkipInput
Number of bytes to skip from input stream.
static int GetStrategyDefault(void)
void SetMemoryLevel(int mem_level)
Memory level.
void g_GZip_Cloudflare_ScanForChunks(CNcbiIstream &is, IChunkHandler &handler)
Get list of positions of separate gzip files in the concatenated gzip file.
virtual ELevel GetDefaultLevel(void) const
Returns default compression level for a compression algorithm.
virtual EStatus Process(const char *in_buf, size_t in_len, char *out_buf, size_t out_size, size_t *in_avail, size_t *out_avail)
Compress/decompress as much data as possible, and stops when the input buffer becomes empty or the ou...
virtual EStatus Init(void)
Initialize the internal stream state for compression/decompression.
CZipCloudflareStreamCompressor(CZipCloudflareCompression::ELevel level, CZipCloudflareCompression::TZipFlags flags=0)
Conventional constructor.
string m_Cache
Buffer to cache small pieces of data.
ESupportFeature
Supported features.
Definition: compress.hpp:191
virtual EStatus End(int abandon=0)
Free all dynamically allocated data structures.
EMode
File open mode.
Definition: compress.hpp:371
virtual EStatus Process(const char *in_buf, size_t in_len, char *out_buf, size_t out_size, size_t *in_avail, size_t *out_avail)
Compress/decompress as much data as possible, and stops when the input buffer becomes empty or the ou...
CCompressionIOStream * m_Stream
[De]comression stream.
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
Definition: ncbistre.hpp:146
IO_PREFIX::fstream CNcbiFstream
Portable alias for fstream.
Definition: ncbistre.hpp:538
ENcbiOwnership
Ownership relations between objects.
Definition: ncbi_types.h:134
strategy
Block allocation strategies.
Definition: bmconst.h:146
char * buf
int len
static MDB_envinfo info
Definition: mdb_load.c:37
mdb_mode_t mode
Definition: lmdb++.h:38
NCBI_XUTIL_EXPORT
Parameter to control printing diagnostic message about conversion of static array data from a differe...
Definition: static_set.hpp:72
Structure to get information about recommended buffer sizes for file/stream I/O to tune up a (de)comp...
Definition: compress.hpp:299
Structure to keep compressed file information.
ZLib Compression API.
Modified on Mon Jun 24 05:21:25 2024 by modify_doxy.py rev. 669887