NCBI C++ ToolKit
storage_types.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef NETCACHE__STORAGE_TYPES__HPP
2 #define NETCACHE__STORAGE_TYPES__HPP
3 /* $Id: storage_types.hpp 93908 2021-06-03 14:08:09Z ucko $
4  * ===========================================================================
5  *
6  * PUBLIC DOMAIN NOTICE
7  * National Center for Biotechnology Information
8  *
9  * This software/database is a "United States Government Work" under the
10  * terms of the United States Copyright Act. It was written as part of
11  * the author's official duties as a United States Government employee and
12  * thus cannot be copyrighted. This software/database is freely available
13  * to the public for use. The National Library of Medicine and the U.S.
14  * Government have not placed any restriction on its use or reproduction.
15  *
16  * Although all reasonable efforts have been taken to ensure the accuracy
17  * and reliability of the software and data, the NLM and the U.S.
18  * Government do not and cannot warrant the performance or results that
19  * may be obtained by using this software or data. The NLM and the U.S.
20  * Government disclaim all warranties, express or implied, including
21  * warranties of performance, merchantability or fitness for any particular
22  * purpose.
23  *
24  * Please cite the author in any work or product based on this material.
25  *
26  * ===========================================================================
27  *
28  * Authors: Pavel Ivanov
29  */
30 
31 
32 #include "nc_utils.hpp"
33 #include "nc_db_info.hpp"
34 
35 
36 namespace intr = boost::intrusive;
37 
38 
40 
41 
42 struct SFileRecHeader;
43 struct SFileMetaRec;
44 class SNCCacheData;
45 struct SCacheDeadCompare;
46 struct SCacheKeyCompare;
47 
48 
55 };
56 
58 {
66 };
67 
68 // Meta-type records (kMetaSignature)
70 {
73  Uint2 map_size; // max number of down_coords in map record - see SFileChunkMapRec
75  Uint8 size; // blob size
76  Uint8 create_time; // time of completion of blob storage request
77  Uint8 create_server; // unique server id
78  Uint4 create_id; // blob id (unique for each server id)
79  Int4 dead_time; // time when this blob should be deleted from db
80  Int4 ttl; // time to live since last read, or creation
81  Int4 expire; // blob expiration time (blob ceases to exist for client)
82  Int4 blob_ver; // blob version (client supplied)
83  Int4 ver_ttl; // time to live for this version
84  Int4 ver_expire; // blob version expiration time
85  char key_data[1]; // key + MD5 of password, if any
86 };
87 
88 // Map-type records (kMapsSignature)
90 {
91  Uint2 map_idx; // index of this map in higher level map, if that exists
92  Uint1 map_depth; // map depth; it can be tree of maps
93  SNCDataCoord down_coords[1] ATTR_ALIGNED_8; // coords of lower levels
94 };
95 
96 // Data-type records (kDataSignature)
98 {
99  Uint8 chunk_num; // chunk index in blob
100  Uint2 chunk_idx; // chunk index in map
101  Uint1 chunk_data[1]; // chunk data, see kNCMaxBlobChunkSize
102 };
103 
104 
106 {
112 
115  }
116 };
117 
118 
120 {
121  string key;
128  int expire;
130 
132  : size(0), create_time(0), create_server(0),
133  create_id(0), dead_time(0), expire(0), ver_expire(0) {
134  }
135  SNCTempBlobInfo(const SNCCacheData& cache_info)
136  : key(cache_info.key),
137  size(cache_info.size),
138  create_time(cache_info.create_time),
139  create_server(cache_info.create_server),
140  coord(cache_info.coord),
141  create_id(cache_info.create_id),
142  dead_time(cache_info.dead_time),
143  expire(cache_info.expire),
144  ver_expire(cache_info.ver_expire)
145  {
146  }
147 };
148 
149 
152 
153 
154 /*
155  on startup:
156  loads blobs metadata into memory,
157  creates initial DB files
158  checks DB consistency
159 
160  begin: x_StartCaching
161  -> x_StartCaching: get first DB file, goto x_PreCacheRecNums
162  -> x_PreCacheRecNums:
163  for each DB file, collect valid record numbers
164  goto x_StartCreateFiles
165  -> x_StartCreateFiles
166  goto x_CreateInitialFile
167  -> x_CreateInitialFile
168  create 3 'current' files and 3 'next' files
169  if file creation failed, goto x_DelFileAndRetryCreate
170  when complete, goto x_StartCacheBlobs
171  -> x_DelFileAndRetryCreate
172  if cannot delete any files, request shutdown, goto x_CancelCaching
173  try to delete existing file (prefers data files)
174  goto x_CreateInitialFile
175  -> x_StartCacheBlobs: get first file, goto x_CacheNextFile
176  -> x_CacheNextFile
177  if file type is eDBFileMeta, goto x_CacheNextRecord
178  after that, goto x_CleanOrphanRecs
179  -> x_CacheNextRecord
180  for each record, check record validity, remember
181  goto x_CacheNextFile
182  -> x_CleanOrphanRecs
183  when all files are processed, delete abandoned records
184  goto x_FinishCaching
185  -> x_FinishCaching
186  notify NC that caching complete,
187  start DiskFlusher, RecNoSaver, SpaceShrinker, ExpiredCleaner
188 */
189 class CBlobCacher : public CSrvStatesTask<CBlobCacher>
190 {
191 public:
192  CBlobCacher(void);
193  virtual ~CBlobCacher(void);
194 
195 private:
196  State x_StartCaching(void);
197  State x_PreCacheRecNums(void);
198  State x_CancelCaching(void);
199  State x_StartCreateFiles(void);
200  State x_CreateInitialFile(void);
201  State x_DelFileAndRetryCreate(void);
202  State x_StartCacheBlobs(void);
203  State x_CacheNextFile(void);
204  State x_CacheNextRecord(void);
205  State x_CleanOrphanRecs(void);
206  State x_FinishCaching(void);
207 
208  bool x_CacheMetaRec(SNCDBFileInfo* file_info,
209  SFileIndexRec* ind_rec,
210  SNCDataCoord coord);
211  bool x_CacheMapRecs(SNCDataCoord map_coord,
212  Uint1 map_depth,
213  SNCDataCoord up_coord,
214  Uint2 up_index,
215  SNCCacheData* cache_data,
216  Uint8 cnt_chunks,
217  Uint8& chunk_num,
218  map<Uint4, Uint4>& sizes_map);
219  void x_DeleteIndexes(SNCDataCoord map_coord, Uint1 map_depth);
220 
221 
229 };
230 
231 
232 class CNewFileCreator : public CSrvTask
233 {
234 public:
235  CNewFileCreator(void);
236  virtual ~CNewFileCreator(void);
237 
238 private:
239  virtual void ExecuteSlice(TSrvThreadNum thr_num);
240 };
241 
242 
243 /*
244  flush all DB files
245 */
246 class CDiskFlusher : public CSrvStatesTask<CDiskFlusher>
247 {
248 public:
249  CDiskFlusher(void);
250  virtual ~CDiskFlusher(void);
251 
252 private:
253  State x_CheckFlushTime(void);
254  State x_FlushNextFile(void);
255 
256 
258 };
259 
260 
261 class CRecNoSaver : public CSrvTask
262 {
263 public:
264  CRecNoSaver(void);
265  virtual ~CRecNoSaver(void);
266 
267 private:
268  virtual void ExecuteSlice(TSrvThreadNum thr_num);
269 };
270 
271 
272 // move blob chunks, or data from old files (which are 'almost empty')
273 // into new ones, then deletes old files
274 /*
275  begin: x_PrepareToShrink
276  -> x_PrepareToShrink: analyze what to move and what to delete;
277  -> x_DeleteNextFile: has smth to delete ? delete : x_StartMoves
278  -> x_StartMoves: has smth to move ? x_MoveNextRecord : x_FinishSession
279  -> x_MoveNextRecord:
280  find what to move;
281  if not found, goto x_FinishMoves;
282  if VerMgr for this key exists, goto x_CheckCurVersion;
283  else x_MoveRecord;
284  -> x_CheckCurVersion
285  if this record already deleted (IndexDeleted), goto x_FinishMoveRecord;
286  if VerMgr has no current version (this is either prev version, or one
287  that was not written yet), goto x_FinishMoveRecord;
288  if record to move does not belong to current version of blob, goto x_FinishMoveRecord;
289  (this means this version will be deleted soon, probably)
290  if this record stores blob metadata, check that another thread BlobVerData does not update it;
291  if this record will be deeted soon, goto x_FinishMoveRecord;
292  goto x_MoveRecord
293  -> x_MoveRecord: move, goto x_FinishMoveRecord
294  -> x_FinishMoveRecord: release used resources;
295  if move failed ? x_FinishMoves : x_MoveNextRecord
296  -> x_FinishMoves: if the file from which we moved records is empty now, delete it;
297  save some statistics
298  goto x_FinishSession
299  -> x_FinishSession: run again in a second, or sooner
300 */
301 
302 
303 class CSpaceShrinker : public CSrvStatesTask<CSpaceShrinker>,
304  public CSrvTransConsumer
305 {
306 public:
307  CSpaceShrinker(void);
308  virtual ~CSpaceShrinker(void);
309 
310 private:
311  State x_PrepareToShrink(void);
312  State x_DeleteNextFile(void);
313  State x_StartMoves(void);
314  State x_MoveNextRecord(void);
315  State x_CheckCurVersion(void);
316  State x_MoveRecord(void);
317  State x_FinishMoveRecord(void);
318  State x_FinishMoves(void);
319  State x_FinishSession(void);
320 
321  SNCDataCoord x_FindMetaCoord(SNCDataCoord coord, Uint1 max_map_depth);
322 
323 
324  typedef vector<CSrvRef<SNCDBFileInfo> > TFilesList;
325 
327  TFilesList::iterator m_CurDelFile;
340  bool m_Failed;
343 };
344 
345 
346 /*
347 delete expired blobs:
348  by dead_time
349  or, when DB is too big, also delete oldest blobs
350 
351 slot is made of Buckets (see ini file: cnt_slot_buckets)
352 
353  begin: x_StartSession
354  -> x_StartSession
355  analyze DB size (do we need to reduce it?)
356  goto x_CleanNextBucket
357  -> x_CleanNextBucket
358  if all buckets clean, goto x_FinishSession;
359  select bucket, goto x_DeleteNextData
360  -> x_DeleteNextData
361  if nothing to delete, goto x_CleanNextBucket
362  delete blob, goto x_DeleteNextData
363  -> x_FinishSession
364  goto x_StartSession
365 */
366 
367 class CExpiredCleaner : public CSrvStatesTask<CExpiredCleaner>
368 {
369 public:
370  CExpiredCleaner(void);
371  virtual ~CExpiredCleaner(void);
372 
373  static void x_DeleteData(SNCCacheData* cache_data);
374 
375 private:
376  State x_StartSession(void);
377  State x_CleanNextBucket(void);
378  State x_DeleteNextData(void);
379  State x_FinishSession(void);
380 
381 
386  size_t m_CurDelData;
389  vector<SNCCacheData*> m_CacheDatas;
390 };
391 
392 /*
393  delete multiple blobs created earlier than a certain time
394  by filter (usually cache|key|* )
395 
396  begin: x_StartSession
397  -> x_StartSession
398  get first filter
399  if found then goto x_FindNext
400  -> x_FindNext
401  for each bucket: get full blob key
402  if found then goto x_RequestBlobAccess
403  if not found then goto x_FinishSession
404  -> x_RequestBlobAccess
405  create CNCBlobAccessor, request blob meta info
406  goto x_RemoveBlob
407  -> x_RemoveBlob
408  once the info is ready, check it once again and remove
409  goto x_Finalize
410  -> x_Finalize
411  finalize accessor, goto x_FindNext
412  -> x_FinishSession
413  erase filter
414  goto x_StartSession
415 */
416 class CBulkCleaner : public CSrvStatesTask<CBulkCleaner>
417 {
418 public:
419  CBulkCleaner(void);
420  virtual ~CBulkCleaner(void);
421 
422 private:
423  State x_StartSession(void);
424  State x_FindNext(void);
425  State x_RequestBlobAccess(void);
426  State x_RemoveBlob(void);
427  State x_Finalize(void);
428  State x_FinishSession(void);
429 
432  string m_Filter, m_Key;
434 };
435 
436 
438 {
439 public:
440  CMovedRecDeleter(SNCDBFileInfo* file_info, SFileIndexRec* ind_rec);
441  virtual ~CMovedRecDeleter(void);
442 
443 private:
444  virtual void ExecuteRCU(void);
445 
446 
449 };
450 
452 
453 #endif /* NETCACHE__STORAGE_TYPES__HPP */
TRecNumsSet * m_CurRecsSet
bool x_CacheMetaRec(SNCDBFileInfo *file_info, SFileIndexRec *ind_rec, SNCDataCoord coord)
State x_PreCacheRecNums(void)
State x_StartCreateFiles(void)
size_t m_CurCreatePass
State x_FinishCaching(void)
size_t m_CurCreateFile
CBlobCacher(void)
TFileRecsMap m_RecsMap
State x_StartCacheBlobs(void)
TRecNumsSet::iterator m_CurRecIt
virtual ~CBlobCacher(void)
TRecNumsSet m_NewFileIds
State x_CacheNextRecord(void)
void x_DeleteIndexes(SNCDataCoord map_coord, Uint1 map_depth)
State x_CancelCaching(void)
TNCDBFilesMap::const_iterator m_CurFile
State x_CleanOrphanRecs(void)
bool x_CacheMapRecs(SNCDataCoord map_coord, Uint1 map_depth, SNCDataCoord up_coord, Uint2 up_index, SNCCacheData *cache_data, Uint8 cnt_chunks, Uint8 &chunk_num, map< Uint4, Uint4 > &sizes_map)
State x_CacheNextFile(void)
State x_DelFileAndRetryCreate(void)
State x_StartCaching(void)
State x_CreateInitialFile(void)
State x_RemoveBlob(void)
virtual ~CBulkCleaner(void)
State x_RequestBlobAccess(void)
State x_Finalize(void)
State x_StartSession(void)
State x_FindNext(void)
State x_FinishSession(void)
CNCBlobAccessor * m_BlobAccess
CDiskFlusher(void)
virtual ~CDiskFlusher(void)
State x_FlushNextFile(void)
State x_CheckFlushTime(void)
static void x_DeleteData(SNCCacheData *cache_data)
vector< SNCCacheData * > m_CacheDatas
virtual ~CExpiredCleaner(void)
State x_CleanNextBucket(void)
State x_DeleteNextData(void)
State x_FinishSession(void)
State x_StartSession(void)
virtual ~CMovedRecDeleter(void)
SFileIndexRec * m_IndRec
CMovedRecDeleter(SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
virtual void ExecuteRCU(void)
Method implementing RCU job that was scheduled earlier by CallRCU().
CSrvRef< SNCDBFileInfo > m_FileInfo
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
virtual ~CNewFileCreator(void)
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
virtual ~CRecNoSaver(void)
CRecNoSaver(void)
State x_CheckCurVersion(void)
State x_FinishMoveRecord(void)
SNCCacheData * m_CacheData
vector< CSrvRef< SNCDBFileInfo > > TFilesList
CSrvRef< SNCDBFileInfo > m_MaxFile
TFilesList::iterator m_CurDelFile
State x_MoveNextRecord(void)
State x_StartMoves(void)
State x_DeleteNextFile(void)
CNCBlobVerManager * m_VerMgr
CSrvRef< SNCBlobVerData > m_CurVer
SFileIndexRec * m_IndRec
State x_PrepareToShrink(void)
TFilesList m_FilesToDel
TFileRecsMap m_RecsMap
State x_FinishSession(void)
virtual ~CSpaceShrinker(void)
State x_FinishMoves(void)
SNCDataCoord x_FindMetaCoord(SNCDataCoord coord, Uint1 max_map_depth)
State x_MoveRecord(void)
Class to derive from to use RCU mechanism.
Definition: task_server.hpp:75
Special task which executes as finite state machine.
Definition: srv_tasks.hpp:283
Main working entity in TaskServer.
Definition: srv_tasks.hpp:88
Consumer of notification about transition completeness in CSrvTransitionTask.
Definition: srv_tasks.hpp:424
container_type::const_iterator const_iterator
Definition: map.hpp:53
parent_type::iterator iterator
Definition: set.hpp:80
uint8_t Uint1
1-byte (8-bit) unsigned integer
Definition: ncbitype.h:99
int32_t Int4
4-byte (32-bit) signed integer
Definition: ncbitype.h:102
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
uint16_t Uint2
2-byte (16-bit) unsigned integer
Definition: ncbitype.h:101
uint64_t Uint8
8-byte (64-bit) unsigned integer
Definition: ncbitype.h:105
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
#define ATTR_ALIGNED_8
Definition: nc_utils.hpp:48
#define ATTR_PACKED
Definition: nc_utils.hpp:39
map< Uint4, TRecNumsSet > TFileRecsMap
EFileRecType
@ eFileRecChunkData
@ eFileRecChunkMap
@ eFileRecMeta
@ eFileRecNone
@ eFileRecAny
set< Uint4 > TRecNumsSet
SNCDataCoord chain_coord
SNCCacheData * cache_data
Information about database part in NetCache storage.
Definition: nc_db_info.hpp:325
SNCDataCoord coord
SNCTempBlobInfo(const SNCCacheData &cache_info)
CSrvRef< SNCDBFileInfo > next_file
CSrvRef< SNCDBFileInfo > cur_file
Uint2 TSrvThreadNum
Type for thread number in TaskServer.
Definition: task_server.hpp:42
Modified on Tue Apr 16 20:08:14 2024 by modify_doxy.py rev. 669887