46 #define NCBI_USE_ERRCODE_X Objtools_Reader
57 # define TRACE_SET(m) \
58 LOG_POST(Info<<SetPostFlags(eDPF_DateTime|eDPF_TID)<<m)
63 *
this << name <<
'(' <<
conn <<
"): ";
69 *
this << name <<
": ";
80 #define DEFAULT_PREOPEN false
81 #define DEFAULT_RETRY_COUNT 5
82 #define DEFAULT_WAIT_TIME_ERRORS 2
83 #define DEFAULT_WAIT_TIME 1
84 #define DEFAULT_WAIT_TIME_MULTIPLIER 1.5
85 #define DEFAULT_WAIT_TIME_INCREMENT 1
86 #define DEFAULT_WAIT_TIME_MAX 30
132 const string& driver_name,
133 int default_max_conn)
141 bool open_initial_connection =
154 int max_connections = conf.
GetInt(driver_name,
158 if ( max_connections == -1 ) {
159 max_connections = conf.
GetInt(driver_name,
171 for (
int attempt = 1; ; ++attempt ) {
185 "cannot open initial connection: "<<exc.
what());
189 "cannot open initial connection");
195 "cannot open initial connection: "<<exc.
what());
199 "cannot open initial connection");
222 max = default_max_conn;
243 catch ( exception& exc ) {
245 "CReader: cannot add connection: "<<exc.what());
276 double wait_seconds =
278 if ( wait_seconds > 0 ) {
280 wait_seconds<<
"s before new connection");
281 _TRACE(
"CReader: waiting "<<wait_seconds<<
282 "s before new connection");
293 if ( wait_seconds > 0 ) {
295 wait_seconds<<
"s before new connection");
296 _TRACE(
"CReader: waiting "<<wait_seconds<<
297 "s before new connection");
332 catch ( exception& ) {
386 wait_seconds<<
"s before next command");
387 _TRACE(
"CReader: waiting "<<wait_seconds<<
388 "s before next connection");
427 catch ( exception& exc ) {
429 "cannot reuse connection: "<<exc.what());
435 catch ( exception& ) {
449 " GenBank connection failed: reconnecting...");
453 " GenBank connection too old: reconnecting...");
498 if ( !ids_lock.IsLoaded() ) {
500 if ( !ids_lock.IsLoaded() ) {
517 if ( !ids_lock.IsLoaded() ) {
519 if ( !ids_lock.IsLoaded() ) {
536 if ( !ids_lock.IsLoaded() ) {
538 if ( !ids_lock.IsLoaded() ) {
550 if (
result.IsLoadedTaxId(seq_id) ) {
570 taxid = seq->GetTaxId();
582 if ( !
result.IsLoadedHash(seq_id) ) {
585 if ( !ids.IsLoaded() ) {
599 if (
result.IsLoadedLength(seq_id) ) {
619 length = seq->GetInst().GetLength();
631 if (
result.IsLoadedType(seq_id) ) {
651 type.
type = seq->GetInst().GetMol();
652 type.sequence_found =
true;
664 size_t count = ids.size();
665 for (
size_t i = 0;
i <
count; ++
i ) {
670 if ( !lock.IsLoaded() ) {
673 if ( lock.IsLoaded() ) {
675 if (
data.IsFound() ) {
688 size_t count = ids.size();
689 for (
size_t i = 0;
i <
count; ++
i ) {
712 size_t count = ids.size();
713 for (
size_t i = 0;
i <
count; ++
i ) {
736 size_t count = ids.size();
737 for (
size_t i = 0;
i <
count; ++
i ) {
757 size_t count = ids.size();
758 for (
size_t i = 0;
i <
count; ++
i ) {
779 size_t count = ids.size();
780 for (
size_t i = 0;
i <
count; ++
i ) {
790 if (
hash.sequence_found ) {
793 known[
i] =
hash.hash_known;
804 size_t count = ids.size();
805 for (
size_t i = 0;
i <
count; ++
i ) {
825 size_t count = ids.size();
826 for (
size_t i = 0;
i <
count; ++
i ) {
849 size_t count = ids.size();
850 for (
size_t i = 0;
i <
count; ++
i ) {
855 if ( !lock.IsLoaded() ) {
858 if ( lock.IsLoaded() ) {
868 if ( !state_lock.IsLoaded() ) {
871 if ( state_lock.IsLoaded() &&
894 if ( !src_ids.IsLoaded() ) {
909 if ( !ids.IsLoaded() ) {
913 if ( !ids.IsLoaded() ) {
927 int loaded_count = 0;
938 if (
info.IsSetAnnotInfo() ) {
939 if (
info.GetAnnotInfo()->GetAnnotInfo().empty() ) {
941 auto&
names =
info.GetAnnotInfo()->GetNamedAnnotNames();
942 if (
names.size() == 1 ) {
962 return loaded_count > 0;
1009 for (
auto& blob_chunks : chunk_ids ) {
1031 for (
auto&
info : blob_infos ) {
1038 if (
info.IsSetAnnotInfo() ) {
1039 if (
info.GetAnnotInfo()->GetAnnotInfo().empty() ) {
1041 auto&
names =
info.GetAnnotInfo()->GetNamedAnnotNames();
1042 if (
names.size() == 1 ) {
1073 result.SetNoBlob(blob_id, blob_state);
1101 if ( !
result.SetLoadedBlobState(blob_id, blob_state) ) {
1105 writer->SaveBlobState(
result, blob_id, blob_state);
1131 if ( !
result.SetLoadedSeqIds(seq_id, seq_ids) ) {
1134 TRACE_SET(
"SetAndSaveSeq_idSeq_ids("<<seq_id<<
") set");
1136 writer->SaveSeq_idSeq_ids(
result, seq_id);
1145 if ( !
result.SetLoadedSeqIds(seq_id, seq_ids) ) {
1149 writer->SaveSeq_idSeq_ids(
result, seq_id);
1158 if ( !
result.SetLoadedAccFromSeqIds(seq_id, seq_ids) ) {
1162 writer->SaveSeq_idAccVer(
result, seq_id);
1171 if ( !
result.SetLoadedGiFromSeqIds(seq_id, seq_ids) ) {
1175 writer->SaveSeq_idGi(
result, seq_id);
1184 if ( !
result.SetLoadedLabelFromSeqIds(seq_id, seq_ids) ) {
1188 writer->SaveSeq_idLabel(
result, seq_id);
1197 if ( !
result.SetLoadedSeqIdsFromZeroGi(seq_id, gi_lock) ) {
1201 writer->SaveSeq_idSeq_ids(
result, seq_id);
1211 TRACE_SET(
"SetAndSaveNoSeq_idBlob_ids("<<seq_id<<
") from gi");
1212 if ( !
result.SetLoadedBlobIdsFromZeroGi(seq_id, sel, gi_lock) ) {
1215 TRACE_SET(
"SetAndSaveNoSeq_idBlob_ids("<<seq_id<<
") from gi set");
1217 writer->SaveSeq_idBlob_ids(
result, seq_id, sel);
1237 if ( !
result.SetLoadedGi(seq_id, gi) ) {
1241 writer->SaveSeq_idGi(
result, seq_id);
1250 if ( !
result.SetLoadedAcc(seq_id, acc_id) ) {
1254 writer->SaveSeq_idAccVer(
result, seq_id);
1261 const string&
label)
const
1267 writer->SaveSeq_idLabel(
result, seq_id);
1276 if ( !
result.SetLoadedTaxId(seq_id, taxid) ) {
1280 writer->SaveSeq_idTaxId(
result, seq_id);
1293 writer->SaveSequenceHash(
result, seq_id);
1302 if ( !
result.SetLoadedLength(seq_id, length) ) {
1306 writer->SaveSequenceLength(
result, seq_id);
1319 writer->SaveSequenceType(
result, seq_id);
1330 TRACE_SET(
"SetAndSaveSeq_idBlob_ids("<<seq_id<<
")");
1334 TRACE_SET(
"SetAndSaveSeq_idBlob_ids("<<seq_id<<
") set");
1336 writer->SaveSeq_idBlob_ids(
result, seq_id, sel);
1347 TRACE_SET(
"SetAndSaveSeq_idBlob_ids("<<seq_id<<
")");
1351 TRACE_SET(
"SetAndSaveSeq_idBlob_ids("<<seq_id<<
") set");
1353 writer->SaveSeq_idBlob_ids(
result, seq_id, sel);
1367 TRACE_SET(
"SetAndSaveNoSeq_idBlob_ids("<<seq_id<<
")");
1371 TRACE_SET(
"SetAndSaveNoSeq_idBlob_ids("<<seq_id<<
") set");
1373 writer->SaveSeq_idBlob_ids(
result, seq_id, sel);
1381 stream.read(
reinterpret_cast<char*
>(&
value),
sizeof(
value));
1382 if ( stream.gcount() !=
sizeof(
value) ) {
1384 "cannot read value");
1404 "HUP is supported only by pubseqos or pubseqos2 readers");
1438 : m_Result(0), m_Reader(0), m_Conn(0), m_Restart(
false)
1445 result.ReleaseNotLoadedBlobs();
1449 result.ClearRetryDelay();
1450 result.m_AllocatedConnection =
this;
1452 else if ( pconn->
m_Reader == reader ) {
1460 result.m_AllocatedConnection =
this;
1464 "Only one reader can allocate connection for a result");
const CConstRef< CBlob_id > & GetBlob_id(void) const
bool IsSetAnnotInfo(void) const
TState GetState(void) const
double GetTime(int step) const
void Init(CConfig &conf, const string &driver_name, const SAllParams ¶ms)
static bool IsFound(const TData &data)
TData GetAccVer(void) const
static const CSeq_id_Handle & GetAcc(const TData &data)
bool IsLoadedAccVer(void) const
bool SetNoBlob_ids(const CFixedBlob_ids::TState &state, TExpirationTime expiration_time)
TData GetBlob_ids(void) const
bool SetLoadedBlob_ids(const TData &data, TExpirationTime expiration_time)
TData GetBlobState(void) const
bool IsLoadedBlob(void) const
CTSE_LoadLock & GetTSE_LoadLock(void)
bool IsLoadedGi(void) const
static bool IsFound(const TData &data)
static TGi GetGi(const TData &data)
bool IsLoadedHash(void) const
static int GetHash(const TData &data)
bool IsLoadedLabel(void) const
TData GetLabel(void) const
TData GetLength(void) const
bool IsLoadedLength(void) const
TData GetSeq_ids(void) const
void AllowIncompleteLoading()
CTSE_LoadLock & GetTSE_LoadLock(void)
bool IsLoadedTaxId(void) const
TData GetTaxId(void) const
static bool IsFound(const TData &data)
static CSeq_inst::TMol GetType(const TData &data)
bool IsLoadedType(void) const
Data loader exceptions, used by GenBank loader.
CNcbiOstrstreamToString class helps convert CNcbiOstrstream to a string Sample usage:
Base class for all object manager exceptions.
static void LoadBlob(CReaderRequestResult &result, const CBlob_Info &blob_info)
void LoadSeq_idLabel(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadSeq_idGi(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
static bool SetBlobState(size_t i, CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TStates &ret)
void LoadSequenceType(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadSequenceHash(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadBlobState(CReaderRequestResult &result, const TBlobId &blob_id)
void LoadSeq_idTaxId(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadSeq_idSeq_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadBlobs(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, TContentsMask mask, const SAnnotSelector *sel)
void LoadSeq_idAccVer(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadSequenceLength(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void LoadSeq_idBlob_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const SAnnotSelector *sel)
void LoadBlob(CReaderRequestResult &result, const CBlob_id &blob_id)
static bool CannotProcess(const CSeq_id_Handle &sih)
~CReaderAllocatedConnection(void)
CReaderRequestResult * m_Result
CReaderAllocatedConnection(CReaderRequestResult &result, CReader *reader)
virtual ~CReaderCacheManager(void)
CReaderCacheManager(void)
CReaderAllocatedConnection * m_AllocatedConnection
double GetRetryDelay(void) const
void ReleaseNotLoadedBlobs(void)
CDebugPrinter(TConn conn, const char *name)
CSemaphore m_NumFreeConnections
void SetPreopenConnection(bool preopen=true)
vector< CSeq_id_Handle > TSeqIds
virtual void x_DisconnectAtSlot(TConn conn, bool failed)
virtual int GetRetryCount(void) const
virtual bool LoadBlobSet(CReaderRequestResult &result, const TSeqIds &seq_ids)
virtual bool LoadHashes(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, THashes &ret, TKnown &known)
CReadDispatcher * m_Dispatcher
void SetAndSaveSeq_idAccVer(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const TSequenceAcc &acc_id) const
virtual bool LoadStates(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TStates &ret)
virtual bool LoadSequenceHash(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
virtual bool LoadSeq_idTaxId(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void x_ReleaseClosedConnection(TConn conn)
void SetAndSaveNoSeq_idBlob_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const SAnnotSelector *sel, const CLoadLockGi &gi_lock) const
void x_RemoveConnection(void)
void OpenInitialConnection(bool force)
virtual bool LoadLabels(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TLabels &ret)
void SetAndSaveSequenceType(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const TSequenceType &type) const
virtual void x_ConnectAtSlot(TConn conn)=0
virtual void InitializeCache(CReaderCacheManager &cache_manager, const TPluginManagerParamTree *params)
void SetAndSaveSeq_idLabelFromSeqIds(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const CLoadLockSeqIds &seq_ids) const
virtual void ConnectSucceeds(TConn conn)
virtual bool LoadTypes(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TTypes &ret)
void SetAndSaveSeq_idGi(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const TSequenceGi &gi) const
void SetAndSaveBlobVersion(CReaderRequestResult &result, const TBlobId &blob_id, TBlobVersion version) const
virtual bool MayBeSkippedOnErrors(void) const
vector< CBlob_Info > TBlobIds
TFreeConnections m_FreeConnections
void SetAndSaveBlobState(CReaderRequestResult &result, const TBlobId &blob_id, TBlobState blob_state) const
virtual bool LoadTaxIds(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TTaxIds &ret)
virtual void ConnectFailed(TConn conn)
void SetAndSaveNoBlob(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, TBlobState blob_state)
static int ReadInt(CNcbiIstream &stream)
virtual bool LoadSequenceType(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
CIncreasingTime m_WaitTime
TConn x_AllocConnection(bool oldest=false)
void SetAndSaveSeq_idSeq_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const CFixedSeq_ids &seq_ids) const
virtual bool LoadBlob(CReaderRequestResult &result, const CBlob_id &blob_id)=0
atomic< int > m_ConnectFailCount
void SetAndSaveSeq_idGiFromSeqIds(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const CLoadLockSeqIds &seq_ids) const
virtual bool LoadBulkIds(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TBulkIds &ret)
virtual bool LoadSequenceLength(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
virtual bool LoadChunks(CReaderRequestResult &result, const TBlobId &blob_id, const TChunkIds &chunk_ids)
void SetAndSaveSequenceLength(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, TSeqPos length) const
void SetAndSaveSeq_idTaxId(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, TTaxId taxid) const
void x_ReleaseConnection(TConn conn, double retry_delay=0)
virtual void ResetCache(void)
TConn m_NextNewConnection
vector< TSeqPos > TLengths
vector< pair< TBlobId, TChunkIds > > TBlobChunkIds
virtual bool LoadGis(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TGis &ret)
void x_AbortConnection(TConn conn, bool failed)
void SetAndSaveSeq_idLabel(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const string &label) const
virtual bool LoadAccVers(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TIds &ret)
void x_AddConnection(void)
vector< CSeq_inst::EMol > TTypes
virtual void SetIncludeHUP(bool include_hup=true, const string &web_cookie=NcbiEmptyString)
virtual int GetMaximumConnectionsLimit(void) const
virtual bool LoadSeq_idGi(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void SetMaximumRetryCount(int retry_count)
vector< CSeq_id_Handle > TIds
void x_ReportDisconnect(const char *reader, const char *server, TConn conn, bool failed) const
virtual bool LoadSeq_idAccVer(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
virtual void OpenConnection(TConn conn)
void SetAndSaveSeq_idBlob_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const SAnnotSelector *sel, CLoadLockBlobIds &lock, const CLoadLockBlobIds &blob_ids) const
int GetMaximumConnections(void) const
virtual void WaitBeforeNewConnection(TConn conn)
virtual void x_AddConnectionSlot(TConn conn)=0
void SetAndSaveSequenceHash(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const TSequenceHash &hash) const
virtual bool LoadChunk(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id)
virtual int GetConst(const string &const_name) const
void InitParams(CConfig &conf, const string &driver_name, int default_max_conn)
vector< TChunkId > TChunkIds
virtual bool LoadLengths(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TLengths &ret)
virtual void SetParams(const CReaderParams ¶ms)
int SetMaximumConnections(int max)
CMutex m_ConnectionsMutex
virtual void SetNewConnectionDelayMicroSec(unsigned long micro_sec)
virtual bool LoadSeq_idBlob_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const SAnnotSelector *sel)
All LoadXxx() methods should return false if there is no requested data in the reader.
bool GetPreopenConnection(void) const
void SetAndSaveSeq_idAccFromSeqIds(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const CLoadLockSeqIds &seq_ids) const
virtual bool LoadBlobs(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, TContentsMask mask, const SAnnotSelector *sel)
virtual void x_RemoveConnectionSlot(TConn conn)=0
void SetAndSaveNoSeq_idSeq_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, TState state) const
virtual bool LoadSeq_idLabel(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
CConstRef< CBioseq_Info > FindMatchingBioseq(const CSeq_id_Handle &id) const
void SetName(const CAnnotName &name)
definition of a Culling tree
BLOB cache read/write/maintenance interface.
static CS_CONNECTION * conn
static const struct name_t names[]
unsigned int TSeqPos
Type for sequence locations and lengths.
#define ITERATE(Type, Var, Cont)
ITERATE macro to sequence through container elements.
SStrictId_Tax::TId TTaxId
Taxon id type.
const TSeqPos kInvalidSeqPos
Define special value for invalid sequence position.
#define LOG_POST_X(err_subcode, message)
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
@ eDPF_DateTime
Include date and time.
TErrCode GetErrCode(void) const
Get error code.
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
void Warning(CExceptionArgs_Base &args)
virtual const char * what(void) const noexcept
Standard report (includes full backlog).
#define NCBI_RETHROW(prev_exception, exception_class, err_code, message)
Generic macro to re-throw an exception.
void Info(CExceptionArgs_Base &args)
int GetInt(const string &driver_name, const string ¶m_name, EErrAction on_error, int default_value, const list< string > *synonyms=NULL)
Utility function to get an integer element of parameter tree Throws an exception when mandatory param...
bool GetBool(const string &driver_name, const string ¶m_name, EErrAction on_error, bool default_value, const list< string > *synonyms=NULL)
Utility function to get an integer element of parameter tree Throws an exception when mandatory param...
@ eErr_NoThrow
Return default value on error.
bool IsIncludedAnyNamedAnnotAccession(void) const
check if any named annot accession is included in the search
#define END_NCBI_SCOPE
End previously defined NCBI scope.
#define END_SCOPE(ns)
End the previously defined scope.
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
#define BEGIN_SCOPE(ns)
Define a new scope.
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
void Wait(void)
Wait on semaphore.
void Post(unsigned int count=1)
Increment the semaphore by "count".
CTime & AddTimeSpan(const CTimeSpan ×pan)
Add specified time span.
CTime & Clear(void)
Make the time "empty",.
bool IsEmpty(void) const
Is time object empty (date and time)?
double DiffNanoSecond(const CTime &t) const
Difference in nanoseconds from specified time.
@ eCurrent
Use current time. See also CCurrentTime.
static const char label[]
const string version
version string
const GenericPointer< typename T::ValueType > T2 value
void SleepMicroSec(unsigned long mc_sec, EInterruptOnSignal onsignal=eRestartOnSignal)
#define DEFAULT_WAIT_TIME_MAX
static CIncreasingTime::SAllParams s_WaitTimeParams
#define DEFAULT_RETRY_COUNT
NCBI_DEFINE_ERR_SUBCODE_X(9)
#define DEFAULT_WAIT_TIME_MULTIPLIER
#define DEFAULT_WAIT_TIME_ERRORS
#define DEFAULT_WAIT_TIME_INCREMENT
#define DEFAULT_WAIT_TIME
#define NCBI_GBLOADER_READER_PARAM_WAIT_TIME_MAX
#define NCBI_GBLOADER_READER_PARAM_WAIT_TIME_ERRORS
#define NCBI_GBLOADER_READER_PARAM_WAIT_TIME_MULTIPLIER
#define NCBI_GBLOADER_READER_PARAM_WAIT_TIME
#define NCBI_GBLOADER_READER_PARAM_RETRY_COUNT
#define NCBI_GBLOADER_READER_PARAM2_NUM_CONN
#define NCBI_GBLOADER_READER_PARAM_PREOPEN
#define NCBI_GBLOADER_READER_PARAM_NUM_CONN
#define NCBI_GBLOADER_READER_PARAM_WAIT_TIME_INCREMENT
Better replacement of GetAccVer(), this method should be defined in data loaders, GetAccVer() is left...
Better replacement of GetGi(), this method should be defined in data loaders, GetGi() is left for com...
Better replacement of GetSequenceHash(), this method should be defined in data loaders,...
Better replacement of GetSequenceType(), this method should be defined in data loaders,...
SReaderCacheInfo(ICache &cache, ECacheType cache_type)