NCBI C++ ToolKit
nc_storage_blob.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef NETCACHE__NC_STORAGE_BLOB__HPP
2 #define NETCACHE__NC_STORAGE_BLOB__HPP
3 /* $Id: nc_storage_blob.hpp 77868 2017-05-11 13:15:22Z gouriano $
4  * ===========================================================================
5  *
6  * PUBLIC DOMAIN NOTICE
7  * National Center for Biotechnology Information
8  *
9  * This software/database is a "United States Government Work" under the
10  * terms of the United States Copyright Act. It was written as part of
11  * the author's official duties as a United States Government employee and
12  * thus cannot be copyrighted. This software/database is freely available
13  * to the public for use. The National Library of Medicine and the U.S.
14  * Government have not placed any restriction on its use or reproduction.
15  *
16  * Although all reasonable efforts have been taken to ensure the accuracy
17  * and reliability of the software and data, the NLM and the U.S.
18  * Government do not and cannot warrant the performance or results that
19  * may be obtained by using this software or data. The NLM and the U.S.
20  * Government disclaim all warranties, express or implied, including
21  * warranties of performance, merchantability or fitness for any particular
22  * purpose.
23  *
24  * Please cite the author in any work or product based on this material.
25  *
26  * ===========================================================================
27  *
28  * Authors: Pavel Ivanov
29  */
30 
31 
32 #include "nc_db_info.hpp"
33 
34 
36 
37 
38 class CNCBlobStorage;
39 class CNCBlobAccessor;
40 class SNCCacheData;
41 class CCurVerReader;
42 struct SNCStateStat;
43 
44 
45 class CNCBlobVerManager : public CObject, public CSrvTask
46 {
47 public:
48  static CNCBlobVerManager* Get(Uint2 time_bucket,
49  const string& key,
50  SNCCacheData* cache_data,
51  bool for_new_version);
52  void ObtainReference(void);
53  void Release(void);
54 
55  void RequestCurVersion(CSrvTransConsumer* consumer);
58 
59  void FinalizeWriting(SNCBlobVerData* ver_data);
60  void DeadTimeChanged(SNCBlobVerData* ver_data);
61  void DeleteVersion(const SNCBlobVerData* ver_data);
62  void DeleteDeadVersion(int cut_time);
63 
65 
66 public:
67  void RevokeDataWrite();
68  void DataWritten(void);
69  void RequestMemRelease(void);
70  const string& GetKey(void);
71 
72 private:
76 
77  CNCBlobVerManager(Uint2 time_bucket,
78  const string& key,
79  SNCCacheData* cache_data);
80  virtual ~CNCBlobVerManager(void);
81 
82  virtual void DeleteThis(void);
83  virtual void ExecuteSlice(TSrvThreadNum thr_num);
84 
85  void x_DeleteCurVersion(void);
86  void x_ReleaseMgr(void);
87 
88 
89  friend class CCurVerReader;
90 
91 
97  string m_Key;
99 };
100 
101 
103 {
104 public:
105  void RequestMetaInfo(CSrvTask* owner);
106  bool IsMetaInfoReady(void);
107  /// Check if blob exists.
108  /// Method can be called only after lock is acquired.
109  bool IsBlobExists(void) const;
110  /// Check if password provided for accessing the blob was correct
111  bool IsAuthorized(void) const;
112  bool HasError(void) const;
113  /// Get key of the blob.
114  /// Method can be called only after lock is acquired.
115  const string& GetBlobKey (void) const;
116  Uint2 GetTimeBucket (void) const;
117  /// Get size of the blob.
118  /// Method can be called only after lock is acquired.
119  /// If blob doesn't exist then value is undefined.
120  Uint8 GetCurBlobSize (void) const;
121  Uint8 GetNewBlobSize (void) const;
122  Uint8 GetSizeRead (void) const;
123  /// Check if blob is already expired but not yet deleted by GC.
124  /// Method can be called only after lock is acquired.
125  /// If blob doesn't exist then value is undefined.
126  bool IsCurBlobExpired (void) const;
127  bool IsCurVerExpired (void) const;
128  bool IsCurBlobDead (void) const;
129  /// Get type of access this holder was created for
130  ENCAccessType GetAccessType (void) const;
131 
132  /// Initially set current position in the blob to start reading from
133  void SetPosition(Uint8 pos);
134  Uint8 GetPosition(void);
135  Uint4 GetReadMemSize(void);
136  const void* GetReadMemPtr(void);
137  void MoveReadPos(Uint4 move_size);
138  unsigned int GetCurBlobTTL(void) const;
139  unsigned int GetNewBlobTTL(void) const;
140  /// Set blob's timeout after last access before it will be deleted.
141  /// Method can be called only after lock is acquired, if blob exists and
142  /// lock is acquired for eNCDelete or eNCCreate access.
143  void SetBlobTTL(unsigned int ttl);
144  unsigned int GetCurVersionTTL(void) const;
145  unsigned int GetNewVersionTTL(void) const;
146  void SetVersionTTL(int ttl);
147  int GetCurBlobVersion(void) const;
148  void SetBlobVersion(int ver);
149  Uint8 GetCurBlobCreateTime(void) const;
150  Uint8 GetNewBlobCreateTime(void) const;
151  void SetBlobCreateTime(Uint8 create_time);
152  int GetCurBlobDeadTime(void) const;
153  int GetCurBlobExpire(void) const;
154  int GetNewBlobExpire(void) const;
155  int GetCurVerExpire(void) const;
156  void SetCurBlobExpire(int expire, int dead_time = 0);
157  void SetNewBlobExpire(int expire, int dead_time = 0);
158  void SetCurVerExpire(int dead_time);
159  void SetNewVerExpire(int dead_time);
160  Uint8 GetCurCreateServer(void) const;
161  Uint8 GetNewCreateServer(void) const;
162  Uint4 GetCurCreateId(void) const;
163  void SetCreateServer(Uint8 create_server, Uint4 create_id);
164  void UpdateMetaInfo(Uint8 upd_server, Uint8 upd_time);
165  bool IsValid(void) const;
166  Uint8 GetValidServer(void) const;
167  string GetCurPassword(void) const;
168  void SetPassword(CTempString password);
170 
171  size_t GetWriteMemSize(void);
172  void* GetWriteMemPtr(void);
173  void MoveWritePos(Uint4 move_size);
174  void Finalize(void);
175 
176  /// Release blob lock.
177  /// No other method can be called after call to this one.
178  void Release(void);
179 
180 public:
181  // For internal use only
182 
183  /// Create holder of blob lock bound to the given NetCache storage.
184  /// Lock holder is yet not usable after construction until Prepare()
185  /// and InitializeLock() are called.
186  CNCBlobAccessor(void);
187  virtual ~CNCBlobAccessor(void);
188 
189  bool IsPurged(const CNCBlobKeyLight& nc_key) const;
190  static Uint8 GetPurgeCount();
191  static bool Purge(const CNCBlobKeyLight& nc_key, Uint8 when);
192  static string GetPurgeData(char separator='\n');
193  static bool UpdatePurgeData(const string& data, char separator='\n');
194 
195  static void SetFailedWriteCount(int failed_write);
196  static int GetFailedWriteCount(void);
197  static void PutFailed(const string& blob_key);
198  static void PutSucceeded(const string& blob_key);
199  static bool HasPutSucceeded(const string& blob_key);
200 
201  /// Prepare lock for the blob identified by key, subkey and version.
202  /// Method only initializes necessary variables, actual acquiring of the
203  /// lock happens in InitializeLock() method.
204  /// If access is eNCRead and storage's IsChangeTimeOnRead() returns TRUE
205  /// then holder will change blob's access time when lock is released. If
206  /// access is eNCCreate then blob will be automatically deleted
207  /// when lock is released if blob wasn't properly finalized.
208  ///
209  /// @param key
210  /// Key of the blob
211  /// @param access
212  /// Required access to the blob
213  void Prepare(const string& key,
214  const string& password,
215  Uint2 time_bucket,
216  ENCAccessType access_type);
217  /// Initialize and acquire the lock.
218  /// Should be called only after Prepare().
219  void Initialize(SNCCacheData* cache_data);
220  void Deinitialize(void);
221 
222 private:
225 
226  virtual void ExecuteSlice(TSrvThreadNum thr_num);
227 
228  void x_CreateNewData(void);
229  void x_DelCorruptedVersion(void);
230 
231 
232  /// Type of access requested for the blob
234  string m_BlobKey;
235  /// Password that was used for access to the blob
236  string m_Password;
246  /// Current position of reading/writing inside blob's chunk
250  char* m_Buffer;
252 };
253 
254 
256 {
257 public:
259  virtual ~CCurVerReader(void);
260 
261 private:
262  virtual void ExecuteSlice(TSrvThreadNum thr_num);
263 
264 
266 };
267 
268 
270 {
271 public:
272  static void Initialize(void);
273  static void ReadState(SNCStateStat& state);
274 
275  // statistics
276  static void AnotherServerMain(void);
277  static void StartSyncBlob(Uint8 create_time);
278  static void RecordNotifyUpdateBlob(Uint8 update_received);
279  static void ResetStatCounters(void);
280 
281 private:
282  CWriteBackControl(void);
283  virtual ~CWriteBackControl(void);
284 
285  virtual void ExecuteSlice(TSrvThreadNum thr_num);
286 };
287 
288 
291 int GetWBWriteTimeout(void);
292 int GetWBFailedWriteDelay(void);
293 
294 void SetWBSoftSizeLimit(Uint8 limit);
295 void SetWBHardSizeLimit(Uint8 limit);
296 void SetWBWriteTimeout(int timeout1, int timeout2);
297 void SetWBFailedWriteDelay(int delay);
298 void SetWBInitialSyncComplete(void);
299 
300 
302 {
303 public:
304  CWBMemDeleter(char* mem, Uint4 mem_size);
305  virtual ~CWBMemDeleter(void);
306 
307 private:
308  virtual void ExecuteRCU(void);
309 
310 
311  char* m_Mem;
313 };
314 
315 
316 
317 //////////////////////////////////////////////////////////////////////////
318 // Inline methods
319 //////////////////////////////////////////////////////////////////////////
320 
321 inline SNCCacheData*
323 {
324  return m_CacheData;
325 }
326 
327 inline const string&
329 {
330  return m_Key;
331 }
332 
333 inline void
335 {
337 }
338 
339 
340 inline bool
342 {
343  return m_MetaInfoReady;
344 }
345 
346 inline bool
348 {
349  return m_CurData.NotNull();
350 }
351 
352 inline bool
354 {
355  return !IsBlobExists() || IsCurBlobExpired()
357 }
358 
359 inline bool
361 {
362  return m_HasError;
363 }
364 
365 inline const string&
367 {
368  return m_BlobKey;
369 }
370 
371 inline Uint2
373 {
374  return m_TimeBucket;
375 }
376 
377 inline Uint8
379 {
380  return m_CurData->size;
381 }
382 
383 inline Uint8
385 {
386  return m_NewData? m_NewData->size: 0;
387 }
388 
389 inline Uint8
391 {
392  return m_SizeRead;
393 }
394 
395 inline int
397 {
398  return m_CurData->blob_ver;
399 }
400 
401 inline unsigned int
403 {
404  return m_CurData->ttl;
405 }
406 
407 inline unsigned int
409 {
410  return m_NewData->ttl;
411 }
412 
413 inline bool
415 {
416  return m_CurData->expire <= CSrvTime::CurSecs();
417 }
418 
419 inline bool
421 {
423 }
424 inline bool
426 {
428 }
429 
430 inline void
432 {
433  x_CreateNewData();
435  if (create_time < prev) {
436 // new data cannot be created earlier than the current one.
437 // due to time differences between servers, or who knows what else..
438 // anyway, check here
439  create_time = prev + 1;
440  }
441  m_NewData->create_time = create_time;
442 }
443 
444 inline Uint8
446 {
447  return m_CurData.NotNull()? m_CurData->create_time: 0;
448 }
449 
450 inline Uint8
452 {
453  return m_NewData->create_time;
454 }
455 
456 inline int
458 {
459  return m_CurData->dead_time;
460 }
461 
462 inline int
464 {
465  return m_CurData->expire;
466 }
467 
468 inline int
470 {
471  return m_NewData->expire;
472 }
473 
474 inline int
476 {
477  return m_CurData->ver_expire;
478 }
479 
480 inline void
481 CNCBlobAccessor::SetNewBlobExpire(int expire, int dead_time /* = 0 */)
482 {
483  x_CreateNewData();
484  m_NewData->expire = expire;
485  int d_time = max(dead_time, expire + 40);
486  if (dead_time == 0 && m_CurData) {
487  d_time = max(m_CurData->dead_time, d_time);
488  }
489  m_NewData->dead_time = d_time;
490 }
491 
492 inline unsigned int
494 {
495  return m_CurData->ver_ttl;
496 }
497 
498 inline unsigned int
500 {
501  return m_NewData->ver_ttl;
502 }
503 
504 inline void
506 {
507  x_CreateNewData();
508  m_NewData->ver_ttl = ttl;
509 }
510 
511 inline void
513 {
514  x_CreateNewData();
515  m_NewData->ttl = ttl;
516 }
517 
518 inline void
520 {
521  x_CreateNewData();
522  m_NewData->blob_ver = ver;
523 }
524 
525 inline void
526 CNCBlobAccessor::SetCurBlobExpire(int expire, int dead_time /* = 0 */)
527 {
528  m_CurData->expire = expire;
529  m_CurData->dead_time = max(expire + 40, max(m_CurData->dead_time, dead_time));
531 }
532 
533 inline void
535 {
536  m_CurData->ver_expire = expire;
538 }
539 
540 inline void
542 {
543  x_CreateNewData();
544  m_NewData->ver_expire = expire;
545 }
546 
547 inline void
549  Uint4 create_id)
550 {
551  x_CreateNewData();
552  m_NewData->create_server = create_server;
553  m_NewData->create_id = create_id;
554 }
555 inline void
557 {
558  if (m_CurData->create_time < upd_time) {
559  m_CurData->updated_on_server = upd_server;
560  m_CurData->updated_at_time = upd_time;
562  }
563 }
564 
565 inline bool
567 {
568  return m_CurData.NotNull()? (m_CurData->updated_on_server == 0) : true;
569 }
570 
571 inline Uint8
573 {
575 }
576 
577 
578 inline Uint8
580 {
582 }
583 
584 inline Uint8
586 {
587  return m_NewData->create_server;
588 }
589 
590 inline Uint4
592 {
593  return m_CurData.NotNull()? m_CurData->create_id: 0;
594 }
595 
596 inline void
598 {
599  x_CreateNewData();
600  m_NewData->password = password;
601 }
602 
603 inline ENCAccessType
605 {
606  return m_AccessType;
607 }
608 
609 inline void
611 {
613  m_ChunkPos = size_t(pos % m_CurData->chunk_size);
614 }
615 
616 inline Uint8
618 {
620 }
621 
622 inline const void*
624 {
625  return m_Buffer + m_ChunkPos;
626 }
627 
628 inline void*
630 {
631  return m_Buffer + m_ChunkPos;
632 }
633 
634 inline void
636 {
637  m_ChunkPos += move_size;
638  m_NewData->size += move_size;
639 }
640 
642 
643 #endif /* NETCACHE__NC_STORAGE_BLOB__HPP */
virtual ~CCurVerReader(void)
CNCBlobVerManager * m_VerMgr
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
CCurVerReader(CNCBlobVerManager *mgr)
Uint8 GetNewBlobSize(void) const
static string GetPurgeData(char separator='\n')
ENCAccessType m_AccessType
Type of access requested for the blob.
void * GetWriteMemPtr(void)
Uint8 GetCurBlobSize(void) const
Get size of the blob.
ENCAccessType GetAccessType(void) const
Get type of access this holder was created for.
virtual ~CNCBlobAccessor(void)
void RequestMetaInfo(CSrvTask *owner)
Uint2 GetTimeBucket(void) const
void SetNewVerExpire(int dead_time)
const string & GetBlobKey(void) const
Get key of the blob.
static bool Purge(const CNCBlobKeyLight &nc_key, Uint8 when)
void MoveReadPos(Uint4 move_size)
bool IsMetaInfoReady(void)
static bool HasPutSucceeded(const string &blob_key)
static void SetFailedWriteCount(int failed_write)
size_t GetWriteMemSize(void)
void SetCurBlobExpire(int expire, int dead_time=0)
static int GetFailedWriteCount(void)
unsigned int GetCurBlobTTL(void) const
void SetCurVerExpire(int dead_time)
unsigned int GetCurVersionTTL(void) const
void SetPassword(CTempString password)
static void PutFailed(const string &blob_key)
bool IsCurBlobDead(void) const
CNCBlobVerManager * m_VerManager
void SetNewBlobExpire(int expire, int dead_time=0)
int GetCurBlobDeadTime(void) const
bool IsPurged(const CNCBlobKeyLight &nc_key) const
bool ReplaceBlobInfo(const SNCBlobVerData &new_info)
CSrvRef< SNCBlobVerData > m_CurData
void SetVersionTTL(int ttl)
void x_CreateNewData(void)
static bool UpdatePurgeData(const string &data, char separator='\n')
Uint4 GetCurCreateId(void) const
void SetBlobTTL(unsigned int ttl)
Set blob's timeout after last access before it will be deleted.
unsigned int GetNewVersionTTL(void) const
string m_Password
Password that was used for access to the blob.
CSrvRef< SNCBlobVerData > m_NewData
CNCBlobAccessor(const CNCBlobAccessor &)
SNCChunkMaps * m_ChunkMaps
void Prepare(const string &key, const string &password, Uint2 time_bucket, ENCAccessType access_type)
Prepare lock for the blob identified by key, subkey and version.
Uint8 GetCurBlobCreateTime(void) const
unsigned int GetNewBlobTTL(void) const
Uint8 GetCurCreateServer(void) const
void x_DelCorruptedVersion(void)
bool IsAuthorized(void) const
Check if password provided for accessing the blob was correct.
void SetCreateServer(Uint8 create_server, Uint4 create_id)
void MoveWritePos(Uint4 move_size)
int GetCurBlobVersion(void) const
Uint4 GetReadMemSize(void)
bool IsValid(void) const
static Uint8 GetPurgeCount()
bool IsCurVerExpired(void) const
Uint8 GetNewCreateServer(void) const
Uint8 GetPosition(void)
Uint4 m_ChunkPos
Current position of reading/writing inside blob's chunk.
int GetCurBlobExpire(void) const
string GetCurPassword(void) const
void SetPosition(Uint8 pos)
Initially set current position in the blob to start reading from.
int GetCurVerExpire(void) const
bool HasError(void) const
static void PutSucceeded(const string &blob_key)
void SetBlobVersion(int ver)
Uint8 GetValidServer(void) const
CNCBlobAccessor(void)
Create holder of blob lock bound to the given NetCache storage.
Uint8 GetSizeRead(void) const
void UpdateMetaInfo(Uint8 upd_server, Uint8 upd_time)
void SetBlobCreateTime(Uint8 create_time)
const void * GetReadMemPtr(void)
bool IsBlobExists(void) const
Check if blob exists.
CNCBlobAccessor & operator=(const CNCBlobAccessor &)
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
Uint8 GetNewBlobCreateTime(void) const
bool IsCurBlobExpired(void) const
Check if blob is already expired but not yet deleted by GC.
void Release(void)
Release blob lock.
void Initialize(SNCCacheData *cache_data)
Initialize and acquire the lock.
int GetNewBlobExpire(void) const
Blob storage for NetCache.
Definition: nc_storage.hpp:97
virtual ~CNCBlobVerManager(void)
CNCBlobVerManager(const CNCBlobVerManager &)
void x_DeleteCurVersion(void)
const string & GetKey(void)
SNCCacheData * GetCacheData(void)
void ObtainReference(void)
void RequestCurVersion(CSrvTransConsumer *consumer)
void DeleteVersion(const SNCBlobVerData *ver_data)
CSrvRef< SNCBlobVerData > CreateNewVersion(void)
static CNCBlobVerManager * Get(Uint2 time_bucket, const string &key, SNCCacheData *cache_data, bool for_new_version)
CCurVerReader * m_CurVerReader
void DeadTimeChanged(SNCBlobVerData *ver_data)
SNCCacheData * m_CacheData
void RequestMemRelease(void)
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
void DeleteDeadVersion(int cut_time)
CSrvRef< SNCBlobVerData > GetCurVersion(void)
void FinalizeWriting(SNCBlobVerData *ver_data)
CNCBlobVerManager & operator=(const CNCBlobVerManager &)
virtual void DeleteThis(void)
Virtual method "deleting" this object.
CSrvRef< SNCBlobVerData > m_CurVersion
CObject –.
Definition: ncbiobj.hpp:180
Class to derive from to use RCU mechanism.
Definition: task_server.hpp:75
Main working entity in TaskServer.
Definition: srv_tasks.hpp:88
Class incorporating convenient methods to work with struct timespec.
Definition: srv_time.hpp:61
static int CurSecs(void)
Current time in seconds since epoch (time_t).
static CSrvTime Current(void)
Exact current time with precision up to nanoseconds.
Uint8 AsUSec(void) const
Converts object's value to microseconds since epoch.
Consumer of notification about transition completeness in CSrvTransitionTask.
Definition: srv_tasks.hpp:424
Special task that has some information to give to its consumers but first this information should be ...
Definition: srv_tasks.hpp:380
void RequestTransition(CSrvTransConsumer *consumer)
Requests task's state transition with the provided consumer to be notified when transition is finishe...
Definition: srv_tasks.cpp:167
CTempString implements a light-weight string on top of a storage buffer whose lifetime management is ...
Definition: tempstr.hpp:65
CWBMemDeleter(char *mem, Uint4 mem_size)
virtual void ExecuteRCU(void)
Method implementing RCU job that was scheduled earlier by CallRCU().
virtual ~CWBMemDeleter(void)
static void Initialize(void)
virtual void ExecuteSlice(TSrvThreadNum thr_num)
This is the main method to do all work this task should do.
static void RecordNotifyUpdateBlob(Uint8 update_received)
static void StartSyncBlob(Uint8 create_time)
static void ReadState(SNCStateStat &state)
static void ResetStatCounters(void)
virtual ~CWriteBackControl(void)
static void AnotherServerMain(void)
static DLIST_TYPE *DLIST_NAME() prev(DLIST_LIST_TYPE *list, DLIST_TYPE *item)
Definition: dlist.tmpl.h:61
bool NotNull(void) const THROWS_NONE
Check if pointer is not null – same effect as NotEmpty().
Definition: ncbiobj.hpp:744
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
uint16_t Uint2
2-byte (16-bit) unsigned integer
Definition: ncbitype.h:101
uint64_t Uint8
8-byte (64-bit) unsigned integer
Definition: ncbitype.h:105
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
const struct ncbi::grid::netcache::search::fields::KEY key
void SetWBHardSizeLimit(Uint8 limit)
void SetWBSoftSizeLimit(Uint8 limit)
void SetWBWriteTimeout(int timeout1, int timeout2)
Uint8 GetWBSoftSizeLimit(void)
int GetWBWriteTimeout(void)
Uint8 GetWBHardSizeLimit(void)
void SetWBInitialSyncComplete(void)
int GetWBFailedWriteDelay(void)
void SetWBFailedWriteDelay(int delay)
ENCAccessType
Type of access to NetCache blob.
Definition: nc_utils.hpp:85
T max(T x_, T y_)
static int new_info(pcre *re, pcre_extra *study, int option, void *ptr)
Definition: pcretest.c:2421
Full information about NetCache blob (excluding key)
Definition: nc_db_info.hpp:96
Uint8 updated_on_server
Definition: nc_db_info.hpp:105
Uint8 updated_at_time
Definition: nc_db_info.hpp:106
Uint8 update_received
Definition: nc_db_info.hpp:107
unsigned int ttl
Definition: nc_db_info.hpp:108
Uint2 TSrvThreadNum
Type for thread number in TaskServer.
Definition: task_server.hpp:42
Modified on Mon Feb 26 04:03:15 2024 by modify_doxy.py rev. 669887