NCBI C++ ToolKit
bdb_ext_blob.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef BDB___EXT_BLOB_HPP__
2 #define BDB___EXT_BLOB_HPP__
3 
4 /* $Id: bdb_ext_blob.hpp 91315 2020-10-08 18:28:17Z grichenk $
5  * ===========================================================================
6  *
7  * PUBLIC DOMAIN NOTICE
8  * National Center for Biotechnology Information
9  *
10  * This software/database is a "United States Government Work" under the
11  * terms of the United States Copyright Act. It was written as part of
12  * the author's official duties as a United States Government employee and
13  * thus cannot be copyrighted. This software/database is freely available
14  * to the public for use. The National Library of Medicine and the U.S.
15  * Government have not placed any restriction on its use or reproduction.
16  *
17  * Although all reasonable efforts have been taken to ensure the accuracy
18  * and reliability of the software and data, the NLM and the U.S.
19  * Government do not and cannot warrant the performance or results that
20  * may be obtained by using this software or data. The NLM and the U.S.
21  * Government disclaim all warranties, express or implied, including
22  * warranties of performance, merchantability or fitness for any particular
23  * purpose.
24  *
25  * Please cite the author in any work or product based on this material.
26  *
27  * ===========================================================================
28  *
29  * Author: Anatoliy Kuznetsov
30  *
31  * File Description: BDB library external archival BLOB store.
32  *
33  */
34 /// @file bdb_ext_blob.hpp
35 /// BDB library external archival BLOB store
36 
37 #include <corelib/ncbistd.hpp>
38 #include <corelib/ncbimtx.hpp>
39 #include <corelib/ncbistre.hpp>
40 #include <corelib/ncbistr.hpp>
41 #include <corelib/ncbifile.hpp>
42 
45 #include <util/bitset/bmserial.h>
46 
47 #include <db/bdb/bdb_blob.hpp>
48 #include <db/bdb/bdb_cursor.hpp>
49 
50 #include <db/error_codes.hpp>
51 
52 
54 
55 /** @addtogroup BDB_BLOB
56  *
57  * @{
58  */
59 
60 
61 
62 /// BLOB map, encapsulates collection of BLOB ids and BLOB
63 /// locations. BLOB location encoded as a BLOB location table
64 /// (list of chunks and sizes).
65 /// This map is easily serializable so it can be stored in BDB file.
66 ///
67 /// It is intentional that this file supports multi-chunk BLOBs even
68 /// if neat term use is just single chunk.
69 ///
71 {
72 public:
73  /// BLOB chunk location: offset in file + chunk size
74  ///
76  {
77  Uint8 offset; ///< chunk offset
78  Uint8 size; ///< chunk size
79 
80  SBlobChunkLoc() { offset = size = 0; }
81 
83  : offset(off), size(s)
84  {}
85 
86  };
87 
88  /// BLOB location table (list of chunks and sizes)
89  typedef vector<SBlobChunkLoc> TBlobChunkVec;
90 
91  /// Blob id + blob location table (list of chunks and sizes)
92  ///
93  struct SBlobLoc
94  {
97 
98  SBlobLoc() : blob_id(0) {}
99 
100  /// Construct one-chunk blob locator
102  : blob_id(id), blob_location_table(1)
103  {
104  blob_location_table[0].offset = offset;
105  blob_location_table[0].size = size;
106  }
107  };
108 
109  /// Collection of BLOBs (id + allocation table)
110  typedef vector<SBlobLoc> TBlobMap;
111 
112 
113  CBDB_ExtBlobMap();
114 
115 
116  /// Add BLOB. BLOB consists of one single chunk.
117  void Add(Uint4 blob_id, Uint8 offset, Uint8 size);
118 
119  /// Returns TRUE if blob exists in the map
120  bool HasBlob(Uint4 blob_id) const;
121 
122  /// Get BLOB location
123  bool GetBlobLoc(Uint4 blob_id, Uint8* offset, Uint8* size) const;
124 
125  /// Number of BLOBs registered in the map
126  size_t Size() const { return m_BlobMap.size(); }
127 
128  /// Get BLOB id min and max range (closed interval)
129  void GetBlobIdRange(Uint4* min_id, Uint4* max_id) const;
130 
131  /// @name Serialization
132  /// @{
133 
134  /// Serialize map for storage
135  /// @param buf
136  /// destination buffer
137  /// @param buf_offset
138  /// start offset in the destination buffer
139  ///
141  Uint8 buf_offset = 0) const;
142 
143  /// DeSerialize map
145  Uint8 buf_offset = 0);
146 
147  /// Compute maximum serialization size
148  size_t ComputeSerializationSize() const;
149 
150  ///@}
151 private:
152  /// Compute serialization size and effective number of bits used
153  /// for offset/size storage (16, 32, 64)
154  /// @param bits_used
155  /// Number of bits used to represent offses & sizes
156  /// Valid values: 16, 32, 64
157  /// @param is_single_chunk
158  /// returned TRUE if there is no blob fragmentation
159  size_t x_ComputeSerializationSize(unsigned* bits_used,
160  bool* is_single_chunk) const;
161 private:
163 };
164 
165 /// Container of BLOB attributes
166 /// Encapsulates:
167 /// BLOB maps of several BLOBs (offsets there point in super BLOB)
168 /// Super BLOB location table (offsets and sizes in external file)
169 ///
170 /// Super BLOB can be put to external file in chunks and then reassembled
171 /// BLOB map itself allows to read and reassemble BLOBs in the super-blob
172 ///
174 {
175 public:
177 
178  const CBDB_ExtBlobMap& GetBlobMap() const { return m_BlobMap; }
179  CBDB_ExtBlobMap& SetBlobMap() { return m_BlobMap; }
180 
181 
182  /// @name Interface for BLOB container location table access
183  ///
184  /// @{
185 
186  /// Set container location (one chunk)
187  ///
188  void SetLoc(Uint8 offset, Uint8 size);
189 
190  /// Get container location (throws an exception if more than one chunk)
191  ///
192  void GetLoc(Uint8* offset, Uint8* size);
193 
194  /// Get location table of a super BLOB
195  /// Location table is used to reassemble BLOB from chunks
196  ///
198  GetSuperLoc() const { return m_Loc; }
199 
200  /// Get Edit access to location table
201  ///
203 
204  /// @}
205 
206 
207 
208  /// @name Serialization
209  /// @{
210 
212  Uint8 buf_offset = 0) const;
213 
215  Uint8 buf_offset = 0);
216 
217  /// Compute maximum serialization size
218  size_t ComputeSerializationSize() const;
219 
220  /// @}
221 
222 private:
223  /// Super BLOB location vector
225  /// Blob attributes (super BLOB content)
227 
228 };
229 
230 
231 
232 /// Dictionary file, storing references on external BLOB file
233 /// (super BLOB structure).
234 ///
235 /// All BLOB meta attributes here are stored in
236 /// this table and can be transactionally protected and caches.
237 /// External file in this case works as a raw partition stores
238 /// only BLOB bytes.
239 ///
240 /// An important implication is that we almost immediately know
241 /// if target range of ids contains our BLOB without reading
242 /// the super BLOB (could be slow). Search through overlapping
243 /// ranges can relatively fast.
244 ///
245 ///
246 /// File encodes id range -> super BLOB description
247 ///
248 /// The general design idea is that we take
249 /// certain number of BLOBs (each is has a unique id)
250 /// BLOBs are packed together into one large super BLOB, compressed
251 /// and stored in an external file. Super BLOB description is stored
252 /// in a CExtBlobLocDB.
253 ///
254 /// Getting a BLOB is a multi-step procedure:
255 /// - first we find the right id range
256 /// - read super BLOB description and deserialize it
257 /// - before reading the super BLOB
258 /// - reassemble super BLOB from chunks (one chunk in most cases)
259 /// the most efficient read would probably be mmap
260 /// - decompress super-BLOB (optional) (may be coupled with mmap)
261 /// - find blob id and location in the meta-information
262 /// - read BLOB from super BLOB (reassemble using location table)
263 ///
265 {
267 
268  CBDB_FieldUint4 id_from; ///< Id range from
269  CBDB_FieldUint4 id_to; ///< Id range to
270 
271  CBlobMetaDB();
272 
273  /// Find the meta container storing our target blob_id
274  /// Function is doing the cursor range scan sequentially reading
275  /// range-matching BLOB descriptions
276  ///
277  /// @param blob_id
278  /// BLOB id to search for
279  /// @param meta_container
280  /// Output: Container with BLOBs meta information
281  /// @param id_from
282  /// Output: Range from where BLOB has been found
283  /// @param id_to
284  /// Output: Range to where BLOB has been found
285  ///
286  EBDB_ErrCode FetchMeta(Uint4 blob_id,
287  CBDB_BlobMetaContainer* meta_container,
288  Uint4* id_from = 0,
289  Uint4* id_to = 0);
290 
291  /// Insert new super BLOB metainfo. Range (id_from, id_to)
292  /// is determined automatically
293  ///
294  EBDB_ErrCode UpdateInsert(const CBDB_BlobMetaContainer& meta_container);
295 
296 private:
299 };
300 
301 /// External BLOB store. BLOBs are stored in chunks in an external file.
302 ///
303 /// BLOB attributes are in a BDB attribute dictionary.
304 ///
305 template<class TBV>
307 {
308 public:
309  typedef TBV TBitVector;
311 public:
314 
315  /// External store location
316  void SetStoreDataDir(const string& dir_name);
317 
318  /// Store attributes DB location
319  /// (by default it is the same as SetStoreDataDir).
320  /// Works in the absense of BDB environment
321  ///
322  /// @sa SetEnv
323  ///
324  void SetStoreAttrDir(const string& dir_name);
325 
326 
327  void SetEnv(CBDB_Env& env) { m_Env = &env; }
328  CBDB_Env* GetEnv(void) const { return m_Env; }
329 
330  /// Set compressor for external BLOB
331  void SetCompressor(ICompression* compressor,
332  EOwnership own = eTakeOwnership);
333 
334  /// Open external store
335  void Open(const string& storage_name,
336  CBDB_RawFile::EOpenMode open_mode);
337 
338  /// Close store
339  void Close();
340 
341  /// Save all changes (flush buffers, store attributes, etc.)
342  void Save();
343 
344  /// Set maximum size of BLOB container.
345  /// All BLOBs smaller than container are getting packed together
346  ///
347  void SetContainerMaxSize(unsigned max_size) { m_ContainerMax = max_size; }
348 
349  /// Get container max size
350  unsigned GetContainerMaxSize() const { return m_ContainerMax; }
351 
352  /// Add blob to external store.
353  ///
354  /// BLOB is not written to stor immediately, but accumulated in the buffer
355  ///
356  void StoreBlob(unsigned blob_id, const CBDB_RawFile::TBuffer& buf);
357 
358  /// Flush current container to disk
359  void Flush();
360 
361  /// Read blob from external store
362  EBDB_ErrCode ReadBlob(unsigned blob_id, CBDB_RawFile::TBuffer& buf);
363 
364 private:
367 private:
368 
369  /// Last operation type.
370  ///
371  enum ELastOp {
373  eWrite
374  };
375 
376  /// Try to read BLOB from the recently loaded container
377  bool x_ReadCache(unsigned blob_id, CBDB_RawFile::TBuffer& buf);
378 
379 protected:
380  TBitVector m_BlobIds; ///< List of BLOB ids stored
381  /// temp block for bitvector serialization
383 
388 
389  AutoPtr<ICompression> m_Compressor; ///< Record compressor
391 
394 
395  unsigned m_ContainerMax; ///< Max size of a BLOB container
397  CBDB_BlobMetaContainer* m_AttrContainer; ///< Blob attributes container
398  ELastOp m_LastOp; ///< Last operation status
399  unsigned m_LastFromBlobId; ///< Recently read id interval
400  unsigned m_LastToBlobId; ///< Recently read id interval
401 };
402 
403 
404 /* @} */
405 
406 
407 /////////////////////////////////////////////////////////////////////////////
408 // IMPLEMENTATION of INLINE functions
409 /////////////////////////////////////////////////////////////////////////////
410 
411 
412 template<class TBV>
414 : m_OpenMode(CBDB_RawFile::eReadWriteCreate),
415  m_Env(0),
416  m_BlobAttrDB(0),
417  m_ExtStore(0),
418  m_Compressor(0, eTakeOwnership),
419  m_ContainerMax(64 * 1024),
420  m_AttrContainer(0),
421  m_LastOp(eRead)
422 {
426 }
427 
428 template<class TBV>
430 {
431  if (m_BlobAttrDB) { // save if store is still open
432  try {
433  if (m_OpenMode != CBDB_RawFile::eReadOnly) {
434  Save();
435  }
436  Close();
437 
438  }
439  catch (std::exception& ex)
440  {
441  ERR_POST_XX(Db_Bdb_Blob, 1, Error <<
442  "Exception in ~CBDB_ExtBlobStore " << ex.what());
443  }
444  }
445  delete m_AttrContainer;
446  if (m_STmpBlock) {
447  bm::aligned_free(m_STmpBlock);
448  }
449 
450 }
451 
452 template<class TBV>
454  EOwnership own)
455 {
456  m_Compressor.reset(compressor, own);
457 }
458 
459 template<class TBV>
460 void CBDB_ExtBlobStore<TBV>::SetStoreDataDir(const string& dir_name)
461 {
462  m_StoreDataDir = CDirEntry::AddTrailingPathSeparator(dir_name);
463 }
464 
465 template<class TBV>
466 void CBDB_ExtBlobStore<TBV>::SetStoreAttrDir(const string& dir_name)
467 {
468  m_StoreAttrDir = CDirEntry::AddTrailingPathSeparator(dir_name);
469 }
470 
471 template<class TBV>
473 {
474  delete m_BlobAttrDB; m_BlobAttrDB = 0;
475  delete m_ExtStore; m_ExtStore = 0;
476 }
477 
478 template<class TBV>
480 {
481  Flush();
482 
483  typename TBitVector::statistics st1;
484  m_BlobIds.optimize(0, TBV::opt_compress, &st1);
485  m_CompressBuffer.resize_mem(st1.max_serialize_mem);
486 
487  size_t size = bm::serialize(m_BlobIds,
488  (unsigned char*)&m_CompressBuffer[0],
489  m_STmpBlock,
491  m_CompressBuffer.resize(size);
492 
493  // create a magic record 0-0 storing the sum bit-vector
494 
495  m_BlobAttrDB->id_from = 0;
496  m_BlobAttrDB->id_to = 0;
497 
498  EBDB_ErrCode ret = m_BlobAttrDB->CBDB_BLobFile::UpdateInsert(m_CompressBuffer);
499  if (ret != eBDB_Ok) {
500  BDB_THROW(eInvalidOperation, "Cannot save ext. blob summary");
501  }
502 
503 }
504 
505 template<class TBV>
506 void CBDB_ExtBlobStore<TBV>::Open(const string& storage_name,
507  CBDB_RawFile::EOpenMode open_mode)
508 {
509  Close();
510 
511  m_OpenMode = open_mode;
512 
513  // Make sure dir exists
514  if (!m_StoreAttrDir.empty() && m_Env == 0) {
515  CDir dir(m_StoreAttrDir);
516  if ( !dir.Exists() ) {
517  dir.Create();
518  }
519  }
520  if (!m_StoreDataDir.empty()) {
521  CDir dir(m_StoreDataDir);
522  if ( !dir.Exists() ) {
523  dir.Create();
524  }
525  }
526 
527  // Open attributes database
528  {{
529  string attr_fname;
530  const string attr_fname_postf = "_ext.db";
531  m_BlobAttrDB = new CBlobMetaDB;
532  if (m_Env) {
533  m_BlobAttrDB->SetEnv(*m_Env);
534  attr_fname = storage_name + attr_fname_postf;
535  } else {
536  if (!m_StoreAttrDir.empty()) {
537  attr_fname = m_StoreAttrDir + storage_name + attr_fname_postf;
538  }
539  else {
540  attr_fname = storage_name + attr_fname_postf;
541  }
542  }
543  m_BlobAttrDB->Open(attr_fname, open_mode);
544  }}
545 
546  // Open external store
547  {{
548  string ext_fname;
549  const string ext_fname_postf = "_store.blob";
550  if (!m_StoreDataDir.empty()) {
551  ext_fname = m_StoreDataDir + storage_name + ext_fname_postf;
552  }
553  else {
554  ext_fname = storage_name + ext_fname_postf;
555  }
556 
557  IOS_BASE::openmode om = IOS_BASE::binary;
558  switch (open_mode) {
562  break;
564  om |= IOS_BASE::in;
565  break;
568  break;
569  default:
570  _ASSERT(0);
571  } // switch
572 
573  m_ExtStore = new CNcbiFstream(ext_fname.c_str(), om);
574  if (!m_ExtStore->is_open() || m_ExtStore->bad()) {
575  delete m_ExtStore; m_ExtStore = 0;
576  BDB_THROW(eFileIO, "Cannot open file " + ext_fname);
577  }
578  }}
579 
580  m_BlobAttrDB->id_from = 0;
581  m_BlobAttrDB->id_to = 0;
582 
583  EBDB_ErrCode ret = m_BlobAttrDB->ReadRealloc(m_CompressBuffer);
584  if (ret == eBDB_Ok) {
586  od.deserialize(m_BlobIds, m_CompressBuffer.data(), m_STmpBlock,
588 
589  } else {
590  m_BlobIds.clear();
591  }
592 }
593 
594 template<class TBV>
595 void CBDB_ExtBlobStore<TBV>::StoreBlob(unsigned blob_id,
596  const CBDB_RawFile::TBuffer& buf)
597 {
598  _ASSERT(blob_id);
599 
600  if (m_BlobIds[blob_id]) {
601  string msg =
602  "External store already has BLOB id="
603  + NStr::UIntToString(blob_id);
604  BDB_THROW(eInvalidValue, msg);
605  }
606 
607  unsigned offset;
608  if (m_LastOp == eRead && m_AttrContainer) {
609  delete m_AttrContainer; m_AttrContainer = 0;
610  }
611 
612  m_LastOp = eWrite;
613 
614  if (!m_AttrContainer) {
615  m_AttrContainer = new CBDB_BlobMetaContainer;
616  m_BlobContainer.resize_mem(buf.size());
617  offset = 0;
618  if (buf.size()) {
619  ::memcpy(m_BlobContainer.data(), buf.data(), buf.size());
620  }
621 
622  CBDB_ExtBlobMap& ext_attr = m_AttrContainer->SetBlobMap();
623  ext_attr.Add(blob_id, offset, buf.size());
624 
625  if (m_BlobContainer.size() > m_ContainerMax) {
626  Flush();
627  }
628  m_BlobIds.set(blob_id);
629  return;
630  }
631 
632  if (buf.size() + m_BlobContainer.size() > m_ContainerMax) {
633  Flush();
634  this->StoreBlob(blob_id, buf);
635  return;
636  }
637 
638  offset = (unsigned)m_BlobContainer.size();
639  m_BlobContainer.resize(m_BlobContainer.size() + buf.size());
640  if (buf.size()) {
641  ::memcpy(m_BlobContainer.data() + offset, buf.data(), buf.size());
642  }
643  CBDB_ExtBlobMap& ext_attr = m_AttrContainer->SetBlobMap();
644  ext_attr.Add(blob_id, offset, buf.size());
645  m_BlobIds.set(blob_id);
646 }
647 
648 template<class TBV>
650 {
651  if (!m_AttrContainer) {
652  return; // nothing to do
653  }
654 
655  _ASSERT(m_ExtStore);
656  _ASSERT(m_BlobAttrDB);
657 
658  CNcbiStreampos pos = m_ExtStore->tellg();
659  Int8 stream_offset = NcbiStreamposToInt8(pos);
660 
661 
662  // compress and write blob container
663  //
664  if (m_Compressor.get()) {
665  m_CompressBuffer.resize_mem(m_BlobContainer.size() * 2);
666 
667  size_t compressed_len;
668  bool compressed =
669  m_Compressor->CompressBuffer(m_BlobContainer.data(),
670  m_BlobContainer.size(),
671  m_CompressBuffer.data(),
672  m_CompressBuffer.size(),
673  &compressed_len);
674  if (!compressed) {
675  BDB_THROW(eInvalidOperation, "Cannot compress BLOB");
676  }
677 
678  m_ExtStore->write((char*)m_CompressBuffer.data(),
680  m_AttrContainer->SetLoc(stream_offset, compressed_len);
681  } else {
682  m_ExtStore->write((char*)m_BlobContainer.data(),
683  m_BlobContainer.size());
684  m_AttrContainer->SetLoc(stream_offset, m_BlobContainer.size());
685  }
686 
687  if (m_ExtStore->bad()) {
688  BDB_THROW(eFileIO, "Cannot write to external store file ");
689  }
690 
691  EBDB_ErrCode ret = m_BlobAttrDB->UpdateInsert(*m_AttrContainer);
692  delete m_AttrContainer; m_AttrContainer = 0;
693  if (ret != eBDB_Ok) {
694  BDB_THROW(eInvalidOperation, "Cannot store BLOB metainfo");
695  }
696 
697 }
698 
699 template<class TBV>
700 bool
703 {
704  if (m_AttrContainer) {
705  // check if this call can be resolved out of the same BLOB
706  // container
707 
708  if (blob_id >= m_LastFromBlobId && blob_id <= m_LastToBlobId) {
709  const CBDB_ExtBlobMap& ext_attr = m_AttrContainer->GetBlobMap();
710  Uint8 offset, size;
711  bool has_blob = ext_attr.GetBlobLoc(blob_id, &offset, &size);
712  if (has_blob) {
713  // empty BLOB
714  if (!size) {
715  buf.resize_mem((size_t)size);
716  return true;
717  }
718  buf.resize_mem((size_t)size);
719  if (m_Compressor.get()) {
720  // check logicall correctness of the decompressed container
721  _DEBUG_ARG(Uint8 sz = m_CompressBuffer.size();)
722  _ASSERT(offset < sz);
723  _ASSERT(offset + size <= sz);
724  ::memcpy(buf.data(),
725  m_CompressBuffer.data() + (size_t)offset,
726  (size_t)size);
727  } else {
728  ::memcpy(buf.data(),
729  m_BlobContainer.data() + (size_t)offset,
730  (size_t)size);
731  }
732  return true;
733  }
734  }
735  }
736 
737  return false;
738 }
739 
740 template<class TBV>
743 {
744  // Current implementation uses attribute container for both read and write
745  // the read-after-write mechanism is underdeveloped, so you have always to
746  // call Flush to turn between read and write. Something to improve in the
747  // future.
748  //
749  if (m_LastOp == eWrite && m_AttrContainer) {
750  BDB_THROW(eInvalidOperation, "Cannot read on unflushed data. ");
751  }
752  m_LastOp = eRead;
753 
754  if (!m_BlobIds[blob_id]) {
755  return eBDB_NotFound;
756  }
757 
758  // check if BLOB can be restored from the current container
759  if (m_AttrContainer) {
760  if (x_ReadCache(blob_id, buf)) {
761  return eBDB_Ok;
762  }
763  delete m_AttrContainer; m_AttrContainer = 0;
764  }
765 
766  // open new container
767  m_AttrContainer = new CBDB_BlobMetaContainer;
768 
769  EBDB_ErrCode ret;
770  ret = m_BlobAttrDB->FetchMeta(blob_id,
771  m_AttrContainer,
772  &m_LastFromBlobId,
773  &m_LastToBlobId);
774  if (ret != eBDB_Ok) {
775  delete m_AttrContainer; m_AttrContainer = 0;
776  return ret;
777  }
778 
779  // read the container from the external file
780 
781  {{
782  Uint8 offset, size;
783  m_AttrContainer->GetLoc(&offset, &size);
784 
786  m_ExtStore->seekg(pos, IOS_BASE::beg);
787  if (m_ExtStore->bad()) {
788  BDB_THROW(eFileIO, "Cannot read from external store file ");
789  }
790  m_BlobContainer.resize_mem((size_t)size);
791  m_ExtStore->read((char*)m_BlobContainer.data(), (size_t)size);
792  if (m_ExtStore->bad()) {
793  BDB_THROW(eFileIO, "Cannot read from external store file ");
794  }
795 
796  if (m_Compressor.get()) {
797  m_CompressBuffer.resize_mem((size_t)m_ContainerMax * 10);
798  size_t dst_len;
799  bool ok = m_Compressor->DecompressBuffer(m_BlobContainer.data(),
800  m_BlobContainer.size(),
801  m_CompressBuffer.data(),
802  m_CompressBuffer.size(),
803  &dst_len);
804  if (!ok) {
805  BDB_THROW(eInvalidOperation, "Cannot decompress BLOB");
806  }
807  m_CompressBuffer.resize(dst_len);
808  }
809 
810  }}
811 
812  if (x_ReadCache(blob_id, buf)) {
813  return eBDB_Ok;
814  }
815  delete m_AttrContainer; m_AttrContainer = 0;
816  return eBDB_NotFound;
817 }
818 
819 
820 
822 
823 
824 #endif
BDB library BLOB support.
Berkeley BDB file cursor.
Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.
Berkeley DB BLOB File class.
Definition: bdb_blob.hpp:59
Container of BLOB attributes Encapsulates: BLOB maps of several BLOBs (offsets there point in super B...
BDB environment object a collection including support for some or all of caching, locking,...
Definition: bdb_env.hpp:61
BLOB map, encapsulates collection of BLOB ids and BLOB locations.
External BLOB store.
Uint4 field type.
Definition: bdb_types.hpp:1267
Raw file class wraps up basic Berkeley DB operations.
Definition: bdb_file.hpp:73
CDir –.
Definition: ncbifile.hpp:1695
Reallocable memory buffer (no memory copy overhead) Mimics vector<>, without the overhead of explicit...
void reserve(size_type new_size)
Deserializer, performs logical operations between bit-vector and serialized bit-vector.
Definition: bmserial.h:927
size_type deserialize(bvector_type &bv, const unsigned char *buf, set_operation op, bool exit_on_one=false)
Deserialize bvector using buffer as set operation argument.
Definition: bmserial.h:6581
Include a standard set of the NCBI C++ Toolkit most basic headers.
The NCBI C++ standard methods for dealing with std::string.
static ulg compressed_len
std::ofstream out("events_result.xml")
main entry point for tests
static int trunc
Definition: array_out.c:8
static HENV env
Definition: transaction2.c:38
int offset
Definition: replacements.h:160
@ eTakeOwnership
An object can take ownership of another.
Definition: ncbi_types.h:136
CBDB_RawFile::TBuffer m_BlobContainer
Blob container.
void SetEnv(CBDB_Env &env)
CBDB_RawFile::EOpenMode m_OpenMode
SBlobLoc(Uint4 id, Uint8 offset, Uint8 size)
Construct one-chunk blob locator.
unsigned m_ContainerMax
Max size of a BLOB container.
bm::word_t * m_STmpBlock
temp block for bitvector serialization
vector< SBlobLoc > TBlobMap
Collection of BLOBs (id + allocation table)
CBDB_ExtBlobMap::TBlobChunkVec m_Loc
Super BLOB location vector.
CBDB_FieldUint4 id_from
Id range from.
CBDB_ExtBlobStore & operator=(const CBDB_ExtBlobStore &)
CBDB_ExtBlobStore(const CBDB_ExtBlobStore &)
const CBDB_ExtBlobMap::TBlobChunkVec & GetSuperLoc() const
Get location table of a super BLOB Location table is used to reassemble BLOB from chunks.
CBlobMetaDB(const CBlobMetaDB &)
CBlobMetaDB * m_BlobAttrDB
unsigned m_LastFromBlobId
Recently read id interval.
void SetStoreAttrDir(const string &dir_name)
Store attributes DB location (by default it is the same as SetStoreDataDir).
ELastOp m_LastOp
Last operation status.
CBDB_ExtBlobMap m_BlobMap
Blob attributes (super BLOB content)
void Flush()
Flush current container to disk.
void Add(Uint4 blob_id, Uint8 offset, Uint8 size)
Add BLOB. BLOB consists of one single chunk.
vector< SBlobChunkLoc > TBlobChunkVec
BLOB location table (list of chunks and sizes)
EBDB_ErrCode ReadBlob(unsigned blob_id, CBDB_RawFile::TBuffer &buf)
Read blob from external store.
bool GetBlobLoc(Uint4 blob_id, Uint8 *offset, Uint8 *size) const
Get BLOB location.
TBitVector m_BlobIds
List of BLOB ids stored.
TBlobChunkVec blob_location_table
CBDB_Env * GetEnv(void) const
CBlobMetaDB & operator=(const CBlobMetaDB &)
AutoPtr< ICompression > m_Compressor
Record compressor.
CBDB_ExtBlobMap::TBlobChunkVec & SetSuperLoc()
Get Edit access to location table.
void Save()
Save all changes (flush buffers, store attributes, etc.)
unsigned m_LastToBlobId
Recently read id interval.
CNcbiFstream * m_ExtStore
void Close()
Close store.
size_t Size() const
Number of BLOBs registered in the map.
void SetCompressor(ICompression *compressor, EOwnership own=eTakeOwnership)
Set compressor for external BLOB.
const CBDB_ExtBlobMap & GetBlobMap() const
void SetStoreDataDir(const string &dir_name)
External store location.
void Open(const string &storage_name, CBDB_RawFile::EOpenMode open_mode)
Open external store.
ELastOp
Last operation type.
unsigned GetContainerMaxSize() const
Get container max size.
CBDB_FieldUint4 id_to
Id range to.
void StoreBlob(unsigned blob_id, const CBDB_RawFile::TBuffer &buf)
Add blob to external store.
SBlobChunkLoc(Uint8 off, Uint8 s)
CBDB_BLobFile TParent
bool x_ReadCache(unsigned blob_id, CBDB_RawFile::TBuffer &buf)
Try to read BLOB from the recently loaded container.
CBDB_RawFile::TBuffer m_CompressBuffer
void SetContainerMaxSize(unsigned max_size)
Set maximum size of BLOB container.
CBDB_BlobMetaContainer * m_AttrContainer
Blob attributes container.
CBDB_RawFile::TBuffer TBuffer
CBDB_ExtBlobMap & SetBlobMap()
EOpenMode
BDB file open mode.
Definition: bdb_file.hpp:78
void SetEnv(CBDB_Env &env)
Associate file with environment.
Definition: bdb_file.cpp:203
EBDB_ErrCode
BDB Return codes.
Definition: bdb_file.hpp:57
@ eReadWriteCreate
read-write, create if it doesn't exist
Definition: bdb_file.hpp:82
@ eCreate
implies 'eReadWrite' too
Definition: bdb_file.hpp:81
@ eBDB_Ok
Definition: bdb_file.hpp:58
@ eBDB_NotFound
Definition: bdb_file.hpp:59
#define BDB_THROW(errcode, message)
Definition: bdb_expt.hpp:178
#define _DEBUG_ARG(arg)
Definition: ncbidbg.hpp:134
#define ERR_POST_XX(error_name, err_subcode, message)
Error posting with error code having given name and with given error subcode.
Definition: ncbidiag.hpp:564
void Error(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1197
static string AddTrailingPathSeparator(const string &path)
Add trailing path separator, if needed.
Definition: ncbifile.cpp:455
virtual bool Exists(void) const
Check if directory "dirname" exists.
Definition: ncbifile.hpp:4065
bool Create(TCreateFlags flags=fCreate_Default) const
Create the directory using "dirname" passed in the constructor.
Definition: ncbifile.cpp:4071
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
int64_t Int8
8-byte (64-bit) signed integer
Definition: ncbitype.h:104
uint64_t Uint8
8-byte (64-bit) unsigned integer
Definition: ncbitype.h:105
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
Int8 NcbiStreamposToInt8(NCBI_NS_STD::char_traits< char >::pos_type stream_pos)
Convert stream position to 64-bit int.
Definition: ncbistre.hpp:771
NCBI_NS_STD::char_traits< char >::pos_type NcbiInt8ToStreampos(Int8 pos)
Convert plain numeric stream position (offset) into stream position usable with STL stream library.
Definition: ncbistre.hpp:782
IO_PREFIX::streampos CNcbiStreampos
Portable alias for streampos.
Definition: ncbistre.hpp:134
IO_PREFIX::fstream CNcbiFstream
Portable alias for fstream.
Definition: ncbistre.hpp:538
static string UIntToString(unsigned int value, TNumToStringFlags flags=0, int base=10)
Convert UInt to string.
Definition: ncbistr.hpp:5109
enum ENcbiOwnership EOwnership
Ownership relations between objects.
#define NCBI_BDB_EXPORT
Definition: ncbi_export.h:272
@ set_ASSIGN
Definition: bmconst.h:173
size_t serialize(const BV &bv, unsigned char *buf, bm::word_t *temp_block=0, unsigned serialization_flags=0)
Saves bitvector into memory.
Definition: bmserial.h:3071
@ BM_NO_BYTE_ORDER
save no byte-order info (save some space)
Definition: bmserial.h:3024
Definition of all error codes used in bdb library (bdb.lib and ncbi_xcache_bdb.lib).
char * buf
unsigned int word_t
Definition: bmconst.h:39
void aligned_free(void *ptr) BMNOEXCEPT
Aligned free.
Definition: bmalloc.h:464
void * aligned_new_malloc(size_t size)
Aligned malloc (unlike classic malloc it throws bad_alloc exception)
Definition: bmalloc.h:436
const unsigned set_block_alloc_size
Definition: bmconst.h:61
const struct ncbi::grid::netcache::search::fields::SIZE size
Compressed bitset (entry point to bm.h)
Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...
Multi-threading – mutexes; rw-locks; semaphore.
NCBI C++ stream class wrappers for triggering between "new" and "old" C++ stream libraries.
std::istream & in(std::istream &in_, double &x_)
@ eRead
Definition: ns_types.hpp:56
int GetLoc(const string &acc, const string &pat, CSeq_loc &loc, CScope &scope)
CRef< objects::CObjectManager > om
BLOB chunk location: offset in file + chunk size.
Blob id + blob location table (list of chunks and sizes)
Dictionary file, storing references on external BLOB file (super BLOB structure).
#define _ASSERT
void Serialize(CNcbiOstream &, const CRawScoreVector< Key, Score > &)
Generics These throw an exception; we must implement serialization for each type.
void Deserialize(CNcbiIstream &istr, CRawScoreVector< Key, Score > &)
Modified on Sun May 19 04:42:01 2024 by modify_doxy.py rev. 669887