NCBI C++ ToolKit
Classes | Macros | Typedefs | Enumerations | Functions | Variables
nc_storage.cpp File Reference
#include "nc_pch.hpp"
#include <corelib/ncbireg.hpp>
#include <corelib/ncbifile.hpp>
#include <corelib/request_ctx.hpp>
#include <corelib/ncbi_process.hpp>
#include "netcached.hpp"
#include "nc_storage.hpp"
#include "storage_types.hpp"
#include "nc_db_files.hpp"
#include "distribution_conf.hpp"
#include "nc_storage_blob.hpp"
#include "sync_log.hpp"
#include "nc_stat.hpp"
#include "logging.hpp"
#include "peer_control.hpp"
+ Include dependency graph for nc_storage.cpp:

Go to the source code of this file.

Go to the SVN repository for this file.

Classes

struct  SCacheDeadCompare
 
struct  SCacheKeyCompare
 
struct  SBucketCache
 
struct  STimeTable
 

Macros

#define __NC_CACHEDATA_ALL_MONITOR   0
 
#define __NC_CACHEDATA_INTR_SET   1
 
#define DB_CORRUPTED(msg)
 

Typedefs

typedef intr::rbtree< SNCCacheData, intr::base_hook< TTimeTableHook >, intr::constant_time_size< false >, intr::compare< SCacheDeadCompare > > TTimeTableMap
 
typedef intr::rbtree< SNCCacheData, intr::base_hook< TKeyMapHook >, intr::constant_time_size< true >, intr::compare< SCacheKeyCompare > > TKeyMap
 
typedef map< Uint2, SBucketCache * > TBucketCacheMap
 
typedef map< Uint2, STimeTable * > TTimeBuckets
 

Enumerations

enum  EStopCause {
  eNoStop , eStopWarning , eStopDBSize , eStopDiskSpace ,
  eStopDiskCritical
}
 

Functions

static char * s_MapFile (TFileHandle fd, size_t file_size)
 
static void s_UnmapFile (char *mem_ptr, size_t file_size)
 
static void s_LockFileMem (const void *mem_ptr, size_t mem_size)
 
static bool s_EnsureDirExist (const string &dir_name)
 
static bool s_ReadVariableParams (const CNcbiRegistry &reg)
 Read from registry only those parameters that can be changed on the fly, without re-starting the application. More...
 
static bool s_ReadStorageParams (void)
 Read all storage parameters from registry. More...
 
static string s_GetIndexFileName (void)
 Make name of the index file in the storage. More...
 
static string s_GetFileName (Uint4 file_id, ENCDBFileType file_type)
 Make name of file with meta-information in given database part. More...
 
static bool s_LockInstanceGuard (void)
 Make sure that current storage database is used only with one instance of NetCache. More...
 
static void s_UnlockInstanceGuard (void)
 Unlock and delete specially named file used to ensure only one instance working with database. More...
 
static bool s_OpenIndexDB (void)
 Open and read index database file. More...
 
static void s_CleanDatabase (void)
 Reinitialize database cleaning all data from it. More...
 
static bool s_ReinitializeStorage (void)
 Check if database need re-initialization depending on different parameters and state of the guard file protecting from several instances running on the same database. More...
 
static bool s_IsIndexDeleted (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static SFileIndexRecs_GetIndOrDeleted (SNCDBFileInfo *file_info, Uint4 rec_num)
 
static SFileIndexRecs_GetIndexRec (SNCDBFileInfo *file_info, Uint4 rec_num)
 
static SFileIndexRecs_GetIndexRecTry (SNCDBFileInfo *file_info, Uint4 rec_num)
 
static void s_DeleteIndexRecNoLock (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static void s_DeleteIndexRec (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static void s_MoveRecToGarbageNoLock (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static void s_MoveRecToGarbage (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static CSrvRef< SNCDBFileInfos_GetDBFileNoLock (Uint4 file_id)
 
static CSrvRef< SNCDBFileInfos_GetDBFile (Uint4 file_id)
 
static CSrvRef< SNCDBFileInfos_GetDBFileTry (Uint4 file_id)
 
static Uint1 s_CalcMapDepthImpl (Uint8 size, Uint4 chunk_size, Uint2 map_size)
 
static Uint1 s_CalcMapDepth (Uint8 size, Uint4 chunk_size, Uint2 map_size)
 
static Uint2 s_CalcCntMapDowns (Uint4 rec_size)
 
static Uint4 s_CalcChunkDataSize (Uint4 rec_size)
 
static Uint4 s_CalcMetaRecSize (Uint2 key_size)
 
static Uint4 s_CalcMapRecSize (Uint2 cnt_downs)
 
static Uint4 s_CalcChunkRecSize (size_t data_size)
 
static char * s_CalcRecordAddress (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static SFileMetaRecs_CalcMetaAddress (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static SFileChunkMapRecs_CalcMapAddress (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static SFileChunkDataRecs_CalcChunkAddress (SNCDBFileInfo *file_info, SFileIndexRec *ind_rec)
 
static bool s_CreateNewFile (size_t type_idx, bool need_lock)
 Do set of procedures creating and initializing new database part and switching storage to using new database part as current one. More...
 
static void s_DeleteDBFile (const CSrvRef< SNCDBFileInfo > &file_info, bool need_lock)
 
static SBucketCaches_GetBucketCache (Uint2 bucket)
 
static SNCCacheDatas_GetKeyCacheData (Uint2 time_bucket, const string &key, bool need_create)
 
static void s_InitializeAccessor (CNCBlobAccessor *acessor)
 
static void s_SwitchToNextFile (SWritingInfo &w_info)
 
static bool s_GetNextWriteCoord (EDBFileIndex file_index, Uint4 rec_size, SNCDataCoord &coord, CSrvRef< SNCDBFileInfo > &file_info, SFileIndexRec *&ind_rec)
 
static bool s_UpdateUpCoords (SFileChunkMapRec *map_rec, SFileIndexRec *map_ind, SNCDataCoord coord)
 
static bool s_SaveOneMapImpl (SNCChunkMapInfo *save_map, SNCChunkMapInfo *up_map, Uint2 cnt_downs, Uint2 map_size, Uint1 map_depth, SNCCacheData *cache_data)
 
static bool s_SaveChunkMap (SNCBlobVerData *ver_data, SNCCacheData *cache_data, SNCChunkMapInfo **maps, Uint2 cnt_downs, bool save_all_deps)
 
static void s_MoveDataToGarbage (SNCDataCoord coord, Uint1 map_depth, SNCDataCoord up_coord, bool need_lock)
 
static bool s_ReadMapInfo (SNCDataCoord map_coord, SNCChunkMapInfo *map_info, Uint1 map_depth)
 

Variables

static const char * kNCStorage_DBFileExt = ".db"
 
static const char * kNCStorage_MetaFileSuffix = ""
 
static const char * kNCStorage_DataFileSuffix = ""
 
static const char * kNCStorage_MapsFileSuffix = ""
 
static const char * kNCStorage_IndexFileSuffix = ".index"
 
static const char * kNCStorage_StartedFileName = "__ncbi_netcache_started__"
 
static const char * kNCStorage_RegSection = "storage"
 
static const char * kNCStorage_PathParam = "path"
 
static const char * kNCStorage_FilePrefixParam = "prefix"
 
static const char * kNCStorage_GuardNameParam = "guard_file_name"
 
static const char * kNCStorage_FileSizeParam = "each_file_size"
 
static const char * kNCStorage_GarbagePctParam = "max_garbage_pct"
 
static const char * kNCStorage_MinDBSizeParam = "min_storage_size"
 
static const char * kNCStorage_MoveLifeParam = "min_lifetime_to_move"
 
static const char * kNCStorage_FailedMoveParam = "failed_move_delay"
 
static const char * kNCStorage_GCBatchParam = "gc_batch_size"
 
static const char * kNCStorage_FlushTimeParam = "sync_time_period"
 
static const char * kNCStorage_ExtraGCOnParam = "db_limit_del_old_on"
 
static const char * kNCStorage_ExtraGCOffParam = "db_limit_del_old_off"
 
static const char * kNCStorage_StopWriteOnParam = "db_limit_stop_write_on"
 
static const char * kNCStorage_StopWriteOffParam = "db_limit_stop_write_off"
 
static const char * kNCStorage_MinFreeDiskParam = "disk_free_limit"
 
static const char * kNCStorage_DiskCriticalParam = "critical_disk_free_limit"
 
static const char * kNCStorage_MinRecNoSaveParam = "min_rec_no_save_period"
 
static const char * kNCStorage_FailedWriteSize = "failed_write_blob_key_count"
 
static const char * kNCStorage_MaxBlobSizeStore = "max_blob_size_store"
 
static const char * kNCStorage_WbMemRelease = "task_priority_wb_memrelease"
 
static const Uint8 kMetaSignature = NCBI_CONST_UINT8(0xeed5be66cdafbfa3)
 
static const Uint8 kDataSignature = NCBI_CONST_UINT8(0xaf9bedf24cfa05ed)
 
static const Uint8 kMapsSignature = NCBI_CONST_UINT8(0xba6efd7b61fdff6c)
 
static const Uint1 kSignatureSize = sizeof(kMetaSignature)
 
static const size_t kMemPageSize = 4 * 1024
 Size of memory page that is a granularity of all allocations from OS. More...
 
static const size_t kMemPageAlignMask = ~(kMemPageSize - 1)
 Mask that can move pointer address or memory size to the memory page boundary. More...
 
static string s_Path
 Directory for all database files of the storage. More...
 
static string s_Prefix
 Name of the storage. More...
 
static int s_GCBatchSize = 0
 Number of blobs treated by GC and by caching mechanism in one batch. More...
 
static int s_FlushTimePeriod = 0
 
static int s_MaxGarbagePct = 0
 
static int s_MinMoveLife = 0
 
static int s_FailedMoveDelay = 0
 
static Int8 s_MinDBSize = 0
 
static string s_GuardName
 Name of guard file excluding several instances to run on the same database. More...
 
static unique_ptr< CFileLocks_GuardLock
 Lock for guard file representing this instance running. More...
 
static CMiniMutex s_IndexLock
 manages access to s_IndexDB More...
 
static unique_ptr< CNCDBIndexFiles_IndexDB
 Index database file. More...
 
static CMiniMutex s_DBFilesLock
 Read-write lock to work with s_DBFiles. More...
 
static TNCDBFilesMaps_DBFiles = nullptr
 List of all database parts in the storage. More...
 
static Uint4 s_LastFileId = 0
 
static CMiniMutex s_NextWriteLock
 manages access to s_AllWritings More...
 
static ENCDBFileType const s_AllFileTypes []
 
static size_t const s_CntAllFiles = sizeof(s_AllFileTypes) / sizeof(s_AllFileTypes[0])
 
static SWritingInfo s_AllWritings [s_CntAllFiles]
 
static Uint4 s_NewFileSize = 0
 
static TTimeBuckets s_TimeTables
 
static Int8 s_CurBlobsCnt = 0
 
static Int8 s_CurKeysCnt = 0
 
static bool s_Draining = false
 
static bool s_AbandonDB = false
 
static volatile Int8 s_CurDBSize = 0
 Current size of storage database. Kept here for printing statistics. More...
 
static volatile Int8 s_GarbageSize = 0
 
static int s_LastFlushTime = 0
 
static TBucketCacheMap s_BucketsCache
 Internal cache of blobs identification information sorted to be able to search by key, subkey and version. More...
 
static EStopCause s_IsStopWrite = eNoStop
 
static bool s_CleanStart = false
 
static bool s_NeedSaveLogRecNo = false
 
static bool s_NeedSavePurgeData = false
 
static int s_WarnLimitOnPct = 0
 
static int s_WarnLimitOffPct = 0
 
static int s_MinRecNoSavePeriod = 0
 
static int s_LastRecNoSaveTime = 0
 
static CAtomicCounter s_BlobCounter
 
static Int8 s_ExtraGCOnSize = 0
 
static Int8 s_ExtraGCOffSize = 0
 
static Int8 s_StopWriteOnSize = 0
 
static Int8 s_StopWriteOffSize = 0
 
static Int8 s_DiskFreeLimit = 0
 
static Int8 s_DiskCritical = 0
 
static Uint8 s_MaxBlobSizeStore = 0
 
static CNewFileCreators_NewFileCreator = nullptr
 
static CDiskFlushers_DiskFlusher = nullptr
 
static CRecNoSavers_RecNoSaver = nullptr
 
static CSpaceShrinkers_SpaceShrinker = nullptr
 
static CExpiredCleaners_ExpiredCleaner = nullptr
 
Uint4 s_TaskPriorityWbMemRelease = 10
 

Macro Definition Documentation

◆ __NC_CACHEDATA_ALL_MONITOR

#define __NC_CACHEDATA_ALL_MONITOR   0

Definition at line 55 of file nc_storage.cpp.

◆ __NC_CACHEDATA_INTR_SET

#define __NC_CACHEDATA_INTR_SET   1

Definition at line 57 of file nc_storage.cpp.

◆ DB_CORRUPTED

#define DB_CORRUPTED (   msg)
Value:
do { \
SRV_LOG(Fatal, "Database is corrupted. " << msg \
<< " Bug, faulty disk or somebody writes to database?"); \
abort(); \
} while (0) \
void Fatal(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1209

Definition at line 297 of file nc_storage.cpp.

Typedef Documentation

◆ TBucketCacheMap

Definition at line 178 of file nc_storage.cpp.

◆ TKeyMap

typedef intr::rbtree<SNCCacheData, intr::base_hook<TKeyMapHook>, intr::constant_time_size<true>, intr::compare<SCacheKeyCompare> > TKeyMap

Definition at line 151 of file nc_storage.cpp.

◆ TTimeBuckets

Definition at line 185 of file nc_storage.cpp.

◆ TTimeTableMap

typedef intr::rbtree<SNCCacheData, intr::base_hook<TTimeTableHook>, intr::constant_time_size<false>, intr::compare<SCacheDeadCompare> > TTimeTableMap

Definition at line 147 of file nc_storage.cpp.

Enumeration Type Documentation

◆ EStopCause

enum EStopCause
Enumerator
eNoStop 
eStopWarning 
eStopDBSize 
eStopDiskSpace 
eStopDiskCritical 

Definition at line 110 of file nc_storage.cpp.

Function Documentation

◆ s_CalcChunkAddress()

static SFileChunkDataRec* s_CalcChunkAddress ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_CalcChunkDataSize()

static Uint4 s_CalcChunkDataSize ( Uint4  rec_size)
inlinestatic

◆ s_CalcChunkRecSize()

static Uint4 s_CalcChunkRecSize ( size_t  data_size)
inlinestatic

Definition at line 1044 of file nc_storage.cpp.

References SFileChunkDataRec::chunk_data.

Referenced by CNCBlobStorage::WriteChunkData().

◆ s_CalcCntMapDowns()

static Uint2 s_CalcCntMapDowns ( Uint4  rec_size)
inlinestatic

◆ s_CalcMapAddress()

static SFileChunkMapRec* s_CalcMapAddress ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_CalcMapDepth()

static Uint1 s_CalcMapDepth ( Uint8  size,
Uint4  chunk_size,
Uint2  map_size 
)
static

◆ s_CalcMapDepthImpl()

static Uint1 s_CalcMapDepthImpl ( Uint8  size,
Uint4  chunk_size,
Uint2  map_size 
)
inlinestatic

◆ s_CalcMapRecSize()

static Uint4 s_CalcMapRecSize ( Uint2  cnt_downs)
inlinestatic

Definition at line 1037 of file nc_storage.cpp.

References SFileChunkMapRec::down_coords.

Referenced by s_SaveOneMapImpl().

◆ s_CalcMetaAddress()

static SFileMetaRec* s_CalcMetaAddress ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_CalcMetaRecSize()

static Uint4 s_CalcMetaRecSize ( Uint2  key_size)
inlinestatic

Definition at line 1030 of file nc_storage.cpp.

References SFileMetaRec::key_data.

Referenced by CNCBlobStorage::WriteBlobInfo().

◆ s_CalcRecordAddress()

static char* s_CalcRecordAddress ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_CleanDatabase()

static void s_CleanDatabase ( void  )
static

Reinitialize database cleaning all data from it.

Only database is cleaned, internal cache is left intact.

Definition at line 809 of file nc_storage.cpp.

References map_checker< Container >::clear(), CNCAlerts::eStorageReinit, INFO, CNCAlerts::Register(), s_DBFiles, s_IndexDB, s_Path, and s_Prefix.

Referenced by s_ReinitializeStorage().

◆ s_CreateNewFile()

static bool s_CreateNewFile ( size_t  type_idx,
bool  need_lock 
)
static

◆ s_DeleteDBFile()

static void s_DeleteDBFile ( const CSrvRef< SNCDBFileInfo > &  file_info,
bool  need_lock 
)
static

◆ s_DeleteIndexRec()

static void s_DeleteIndexRec ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
inlinestatic

◆ s_DeleteIndexRecNoLock()

static void s_DeleteIndexRecNoLock ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_EnsureDirExist()

static bool s_EnsureDirExist ( const string dir_name)
static

Definition at line 520 of file nc_storage.cpp.

References CDir::Create(), and CDir::Exists().

Referenced by s_ReadStorageParams().

◆ s_GetBucketCache()

static SBucketCache* s_GetBucketCache ( Uint2  bucket)
static

◆ s_GetDBFile()

static CSrvRef<SNCDBFileInfo> s_GetDBFile ( Uint4  file_id)
static

◆ s_GetDBFileNoLock()

static CSrvRef<SNCDBFileInfo> s_GetDBFileNoLock ( Uint4  file_id)
static

◆ s_GetDBFileTry()

static CSrvRef<SNCDBFileInfo> s_GetDBFileTry ( Uint4  file_id)
static

◆ s_GetFileName()

static string s_GetFileName ( Uint4  file_id,
ENCDBFileType  file_type 
)
static

◆ s_GetIndexFileName()

static string s_GetIndexFileName ( void  )
static

◆ s_GetIndexRec()

static SFileIndexRec* s_GetIndexRec ( SNCDBFileInfo file_info,
Uint4  rec_num 
)
static

◆ s_GetIndexRecTry()

static SFileIndexRec* s_GetIndexRecTry ( SNCDBFileInfo file_info,
Uint4  rec_num 
)
static

◆ s_GetIndOrDeleted()

static SFileIndexRec* s_GetIndOrDeleted ( SNCDBFileInfo file_info,
Uint4  rec_num 
)
static

◆ s_GetKeyCacheData()

static SNCCacheData* s_GetKeyCacheData ( Uint2  time_bucket,
const string key,
bool  need_create 
)
static

◆ s_GetNextWriteCoord()

static bool s_GetNextWriteCoord ( EDBFileIndex  file_index,
Uint4  rec_size,
SNCDataCoord coord,
CSrvRef< SNCDBFileInfo > &  file_info,
SFileIndexRec *&  ind_rec 
)
static

◆ s_InitializeAccessor()

static void s_InitializeAccessor ( CNCBlobAccessor acessor)
static

◆ s_IsIndexDeleted()

static bool s_IsIndexDeleted ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
inlinestatic

◆ s_LockFileMem()

static void s_LockFileMem ( const void *  mem_ptr,
size_t  mem_size 
)
inlinestatic

Definition at line 505 of file nc_storage.cpp.

References Critical(), and SRV_LOG.

Referenced by s_GetNextWriteCoord(), and CBlobCacher::x_PreCacheRecNums().

◆ s_LockInstanceGuard()

static bool s_LockInstanceGuard ( void  )
static

Make sure that current storage database is used only with one instance of NetCache.

Lock specially named file exclusively and hold the lock for all time while process is working. This file also is used for determining if previous instance of NetCache was closed gracefully. If another instance of NetCache using the database method will throw an exception.

Returns
TRUE if guard file existed and was unlocked meaning that previous instance of NetCache was terminated inappropriately. FALSE if file didn't exist so that this storage instance is a clean start.

Definition at line 677 of file nc_storage.cpp.

References Critical(), CFileIO_Base::eBegin, CFileLock::eExclusive, CFileIO_Base::eOpenAlways, CNCAlerts::eStartAfterCrash, CFile::Exists(), CFileLock::fDefault, CDirEntry::fRead, GetLength(), CDirEntry::GetMode(), CCurrentProcess::GetPid(), INFO, CNCAlerts::Register(), s_CleanStart, s_GuardLock, s_GuardName, CFileIO::SetFileHandle(), CFileIO::SetFileSize(), CDirEntry::SetMode(), SRV_LOG, NStr::UInt8ToString(), and CFileIO::Write().

Referenced by CNCBlobStorage::Initialize().

◆ s_MapFile()

static char* s_MapFile ( TFileHandle  fd,
size_t  file_size 
)
inlinestatic

Definition at line 480 of file nc_storage.cpp.

References Critical(), kMemPageAlignMask, kMemPageSize, NULL, and SRV_LOG.

Referenced by s_CreateNewFile(), and s_OpenIndexDB().

◆ s_MoveDataToGarbage()

static void s_MoveDataToGarbage ( SNCDataCoord  coord,
Uint1  map_depth,
SNCDataCoord  up_coord,
bool  need_lock 
)
static

◆ s_MoveRecToGarbage()

static void s_MoveRecToGarbage ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_MoveRecToGarbageNoLock()

static void s_MoveRecToGarbageNoLock ( SNCDBFileInfo file_info,
SFileIndexRec ind_rec 
)
static

◆ s_OpenIndexDB()

static bool s_OpenIndexDB ( void  )
static

◆ s_ReadMapInfo()

static bool s_ReadMapInfo ( SNCDataCoord  map_coord,
SNCChunkMapInfo map_info,
Uint1  map_depth 
)
static

◆ s_ReadStorageParams()

static bool s_ReadStorageParams ( void  )
static

◆ s_ReadVariableParams()

static bool s_ReadVariableParams ( const CNcbiRegistry reg)
static

◆ s_ReinitializeStorage()

static bool s_ReinitializeStorage ( void  )
static

Check if database need re-initialization depending on different parameters and state of the guard file protecting from several instances running on the same database.

Definition at line 822 of file nc_storage.cpp.

References Error(), CDirEntry::Remove(), s_CleanDatabase(), s_GetIndexFileName(), s_IndexDB, s_OpenIndexDB(), and SRV_LOG.

Referenced by CNCBlobStorage::Initialize().

◆ s_SaveChunkMap()

static bool s_SaveChunkMap ( SNCBlobVerData ver_data,
SNCCacheData cache_data,
SNCChunkMapInfo **  maps,
Uint2  cnt_downs,
bool  save_all_deps 
)
static

◆ s_SaveOneMapImpl()

static bool s_SaveOneMapImpl ( SNCChunkMapInfo save_map,
SNCChunkMapInfo up_map,
Uint2  cnt_downs,
Uint2  map_size,
Uint1  map_depth,
SNCCacheData cache_data 
)
static

◆ s_SwitchToNextFile()

static void s_SwitchToNextFile ( SWritingInfo w_info)
inlinestatic

◆ s_UnlockInstanceGuard()

static void s_UnlockInstanceGuard ( void  )
static

Unlock and delete specially named file used to ensure only one instance working with database.

Small race exists here now which cannot be resolved for Windows without re-writing of the mechanism using low-level functions. Race can appear like this: if another instance will start right when this instance stops then other instance can be thinking that this instance terminated incorrectly.

Definition at line 726 of file nc_storage.cpp.

References CDirEntry::Remove(), s_AbandonDB, s_GuardLock, and s_GuardName.

Referenced by CNCBlobStorage::Finalize().

◆ s_UnmapFile()

static void s_UnmapFile ( char *  mem_ptr,
size_t  file_size 
)
inlinestatic

Definition at line 496 of file nc_storage.cpp.

References kMemPageAlignMask, and kMemPageSize.

Referenced by SNCDBFileInfo::~SNCDBFileInfo().

◆ s_UpdateUpCoords()

static bool s_UpdateUpCoords ( SFileChunkMapRec map_rec,
SFileIndexRec map_ind,
SNCDataCoord  coord 
)
static

Variable Documentation

◆ kDataSignature

const Uint8 kDataSignature = NCBI_CONST_UINT8(0xaf9bedf24cfa05ed)
static

Definition at line 98 of file nc_storage.cpp.

Referenced by s_CreateNewFile(), and s_OpenIndexDB().

◆ kMapsSignature

const Uint8 kMapsSignature = NCBI_CONST_UINT8(0xba6efd7b61fdff6c)
static

Definition at line 99 of file nc_storage.cpp.

Referenced by s_CreateNewFile(), and s_OpenIndexDB().

◆ kMemPageAlignMask

const size_t kMemPageAlignMask = ~(kMemPageSize - 1)
static

Mask that can move pointer address or memory size to the memory page boundary.

Definition at line 107 of file nc_storage.cpp.

Referenced by s_MapFile(), and s_UnmapFile().

◆ kMemPageSize

const size_t kMemPageSize = 4 * 1024
static

Size of memory page that is a granularity of all allocations from OS.

Definition at line 104 of file nc_storage.cpp.

Referenced by s_MapFile(), and s_UnmapFile().

◆ kMetaSignature

const Uint8 kMetaSignature = NCBI_CONST_UINT8(0xeed5be66cdafbfa3)
static

Definition at line 97 of file nc_storage.cpp.

Referenced by s_CreateNewFile(), and s_OpenIndexDB().

◆ kNCStorage_DataFileSuffix

const char* kNCStorage_DataFileSuffix = ""
static

Definition at line 68 of file nc_storage.cpp.

Referenced by s_GetFileName().

◆ kNCStorage_DBFileExt

const char* kNCStorage_DBFileExt = ".db"
static

Definition at line 66 of file nc_storage.cpp.

Referenced by s_GetFileName(), and s_GetIndexFileName().

◆ kNCStorage_DiskCriticalParam

const char* kNCStorage_DiskCriticalParam = "critical_disk_free_limit"
static

Definition at line 89 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_ExtraGCOffParam

const char* kNCStorage_ExtraGCOffParam = "db_limit_del_old_off"
static

Definition at line 85 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_ExtraGCOnParam

const char* kNCStorage_ExtraGCOnParam = "db_limit_del_old_on"
static

Definition at line 84 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_FailedMoveParam

const char* kNCStorage_FailedMoveParam = "failed_move_delay"
static

Definition at line 81 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_FailedWriteSize

const char* kNCStorage_FailedWriteSize = "failed_write_blob_key_count"
static

Definition at line 91 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_FilePrefixParam

const char* kNCStorage_FilePrefixParam = "prefix"
static

Definition at line 75 of file nc_storage.cpp.

Referenced by s_ReadStorageParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_FileSizeParam

const char* kNCStorage_FileSizeParam = "each_file_size"
static

Definition at line 77 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_FlushTimeParam

const char* kNCStorage_FlushTimeParam = "sync_time_period"
static

Definition at line 83 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_GarbagePctParam

const char* kNCStorage_GarbagePctParam = "max_garbage_pct"
static

Definition at line 78 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_GCBatchParam

const char* kNCStorage_GCBatchParam = "gc_batch_size"
static

Definition at line 82 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_GuardNameParam

const char* kNCStorage_GuardNameParam = "guard_file_name"
static

◆ kNCStorage_IndexFileSuffix

const char* kNCStorage_IndexFileSuffix = ".index"
static

Definition at line 70 of file nc_storage.cpp.

Referenced by s_GetIndexFileName().

◆ kNCStorage_MapsFileSuffix

const char* kNCStorage_MapsFileSuffix = ""
static

Definition at line 69 of file nc_storage.cpp.

Referenced by s_GetFileName().

◆ kNCStorage_MaxBlobSizeStore

const char* kNCStorage_MaxBlobSizeStore = "max_blob_size_store"
static

Definition at line 92 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_MetaFileSuffix

const char* kNCStorage_MetaFileSuffix = ""
static

Definition at line 67 of file nc_storage.cpp.

Referenced by s_GetFileName().

◆ kNCStorage_MinDBSizeParam

const char* kNCStorage_MinDBSizeParam = "min_storage_size"
static

Definition at line 79 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_MinFreeDiskParam

const char* kNCStorage_MinFreeDiskParam = "disk_free_limit"
static

Definition at line 88 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_MinRecNoSaveParam

const char* kNCStorage_MinRecNoSaveParam = "min_rec_no_save_period"
static

Definition at line 90 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_MoveLifeParam

const char* kNCStorage_MoveLifeParam = "min_lifetime_to_move"
static

Definition at line 80 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_PathParam

const char* kNCStorage_PathParam = "path"
static

Definition at line 74 of file nc_storage.cpp.

Referenced by s_ReadStorageParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_RegSection

const char* kNCStorage_RegSection = "storage"
static

Definition at line 73 of file nc_storage.cpp.

Referenced by s_ReadStorageParams(), and s_ReadVariableParams().

◆ kNCStorage_StartedFileName

const char* kNCStorage_StartedFileName = "__ncbi_netcache_started__"
static

Definition at line 71 of file nc_storage.cpp.

Referenced by s_ReadStorageParams().

◆ kNCStorage_StopWriteOffParam

const char* kNCStorage_StopWriteOffParam = "db_limit_stop_write_off"
static

Definition at line 87 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_StopWriteOnParam

const char* kNCStorage_StopWriteOnParam = "db_limit_stop_write_on"
static

Definition at line 86 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kNCStorage_WbMemRelease

const char* kNCStorage_WbMemRelease = "task_priority_wb_memrelease"
static

Definition at line 93 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), and CNCBlobStorage::WriteSetup().

◆ kSignatureSize

const Uint1 kSignatureSize = sizeof(kMetaSignature)
static

◆ s_AbandonDB

bool s_AbandonDB = false
static

◆ s_AllFileTypes

ENCDBFileType const s_AllFileTypes[]
static
Initial value:
}
@ eDBFileMeta
Definition: nc_db_info.hpp:294
@ eDBFileMaps
Definition: nc_db_info.hpp:296
@ eDBFileData
Definition: nc_db_info.hpp:295

Definition at line 248 of file nc_storage.cpp.

Referenced by s_CreateNewFile().

◆ s_AllWritings

SWritingInfo s_AllWritings[s_CntAllFiles]
static

◆ s_BlobCounter

CAtomicCounter s_BlobCounter
static

Definition at line 280 of file nc_storage.cpp.

Referenced by CNCBlobStorage::GetNewBlobId(), and CNCBlobStorage::Initialize().

◆ s_BucketsCache

TBucketCacheMap s_BucketsCache
static

◆ s_CleanStart

bool s_CleanStart = false
static

◆ s_CntAllFiles

size_t const s_CntAllFiles = sizeof(s_AllFileTypes) / sizeof(s_AllFileTypes[0])
static

◆ s_CurBlobsCnt

Int8 s_CurBlobsCnt = 0
static

◆ s_CurDBSize

volatile Int8 s_CurDBSize = 0
static

◆ s_CurKeysCnt

Int8 s_CurKeysCnt = 0
static

◆ s_DBFiles

TNCDBFilesMap* s_DBFiles = nullptr
static

◆ s_DBFilesLock

CMiniMutex s_DBFilesLock
static

◆ s_DiskCritical

Int8 s_DiskCritical = 0
static

◆ s_DiskFlusher

CDiskFlusher* s_DiskFlusher = nullptr
static

Definition at line 289 of file nc_storage.cpp.

Referenced by CNCBlobStorage::Initialize(), and CBlobCacher::x_FinishCaching().

◆ s_DiskFreeLimit

Int8 s_DiskFreeLimit = 0
static

◆ s_Draining

bool s_Draining = false
static

Definition at line 258 of file nc_storage.cpp.

Referenced by CNCBlobStorage::IsDraining(), and CNCBlobStorage::SetDraining().

◆ s_ExpiredCleaner

CExpiredCleaner* s_ExpiredCleaner = nullptr
static

Definition at line 292 of file nc_storage.cpp.

Referenced by CNCBlobStorage::Initialize(), and CBlobCacher::x_FinishCaching().

◆ s_ExtraGCOffSize

Int8 s_ExtraGCOffSize = 0
static

◆ s_ExtraGCOnSize

Int8 s_ExtraGCOnSize = 0
static

◆ s_FailedMoveDelay

int s_FailedMoveDelay = 0
static

◆ s_FlushTimePeriod

int s_FlushTimePeriod = 0
static

◆ s_GarbageSize

volatile Int8 s_GarbageSize = 0
static

◆ s_GCBatchSize

int s_GCBatchSize = 0
static

Number of blobs treated by GC and by caching mechanism in one batch.

Definition at line 219 of file nc_storage.cpp.

Referenced by s_ReadVariableParams(), CNCBlobStorage::WriteSetup(), CExpiredCleaner::x_CleanNextBucket(), and CExpiredCleaner::x_DeleteNextData().

◆ s_GuardLock

unique_ptr<CFileLock> s_GuardLock
static

Lock for guard file representing this instance running.

It will hold exclusive lock all the time while NetCache is working, so that other instance of NetCache on the same database will be unable to start.

Definition at line 232 of file nc_storage.cpp.

Referenced by s_LockInstanceGuard(), and s_UnlockInstanceGuard().

◆ s_GuardName

string s_GuardName
static

Name of guard file excluding several instances to run on the same database.

Definition at line 227 of file nc_storage.cpp.

Referenced by s_LockInstanceGuard(), s_ReadStorageParams(), s_UnlockInstanceGuard(), CNCBlobStorage::WriteEnvInfo(), and CNCBlobStorage::WriteSetup().

◆ s_IndexDB

unique_ptr<CNCDBIndexFile> s_IndexDB
static

◆ s_IndexLock

CMiniMutex s_IndexLock
static

◆ s_IsStopWrite

EStopCause s_IsStopWrite = eNoStop
static

◆ s_LastFileId

Uint4 s_LastFileId = 0
static

Definition at line 244 of file nc_storage.cpp.

Referenced by CNCBlobStorage::Initialize(), and s_CreateNewFile().

◆ s_LastFlushTime

int s_LastFlushTime = 0
static

◆ s_LastRecNoSaveTime

int s_LastRecNoSaveTime = 0
static

Definition at line 279 of file nc_storage.cpp.

Referenced by CRecNoSaver::ExecuteSlice().

◆ s_MaxBlobSizeStore

Uint8 s_MaxBlobSizeStore = 0
static

◆ s_MaxGarbagePct

int s_MaxGarbagePct = 0
static

◆ s_MinDBSize

Int8 s_MinDBSize = 0
static

◆ s_MinMoveLife

int s_MinMoveLife = 0
static

◆ s_MinRecNoSavePeriod

int s_MinRecNoSavePeriod = 0
static

◆ s_NeedSaveLogRecNo

bool s_NeedSaveLogRecNo = false
static

◆ s_NeedSavePurgeData

bool s_NeedSavePurgeData = false
static

Definition at line 275 of file nc_storage.cpp.

Referenced by CRecNoSaver::ExecuteSlice(), and CNCBlobStorage::SavePurgeData().

◆ s_NewFileCreator

CNewFileCreator* s_NewFileCreator = nullptr
static

Definition at line 288 of file nc_storage.cpp.

Referenced by CNCBlobStorage::Initialize(), and s_GetNextWriteCoord().

◆ s_NewFileSize

Uint4 s_NewFileSize = 0
static

◆ s_NextWriteLock

CMiniMutex s_NextWriteLock
static

◆ s_Path

string s_Path
static

◆ s_Prefix

string s_Prefix
static

Name of the storage.

Definition at line 217 of file nc_storage.cpp.

Referenced by s_CleanDatabase(), s_GetFileName(), s_GetIndexFileName(), s_ReadStorageParams(), and CNCBlobStorage::WriteSetup().

◆ s_RecNoSaver

CRecNoSaver* s_RecNoSaver = nullptr
static

◆ s_SpaceShrinker

CSpaceShrinker* s_SpaceShrinker = nullptr
static

Definition at line 291 of file nc_storage.cpp.

Referenced by CNCBlobStorage::Initialize(), and CBlobCacher::x_FinishCaching().

◆ s_StopWriteOffSize

Int8 s_StopWriteOffSize = 0
static

◆ s_StopWriteOnSize

Int8 s_StopWriteOnSize = 0
static

◆ s_TaskPriorityWbMemRelease

Uint4 s_TaskPriorityWbMemRelease = 10

◆ s_TimeTables

TTimeBuckets s_TimeTables
static

◆ s_WarnLimitOffPct

int s_WarnLimitOffPct = 0
static

◆ s_WarnLimitOnPct

int s_WarnLimitOnPct = 0
static
Modified on Sat Jul 20 11:11:33 2024 by modify_doxy.py rev. 669887