NCBI C++ ToolKit
reader_cache.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: reader_cache.cpp 99954 2023-05-24 12:59:04Z vasilche $
2  * ===========================================================================
3  * PUBLIC DOMAIN NOTICE
4  * National Center for Biotechnology Information
5  *
6  * This software/database is a "United States Government Work" under the
7  * terms of the United States Copyright Act. It was written as part of
8  * the author's official duties as a United States Government employee and
9  * thus cannot be copyrighted. This software/database is freely available
10  * to the public for use. The National Library of Medicine and the U.S.
11  * Government have not placed any restriction on its use or reproduction.
12  *
13  * Although all reasonable efforts have been taken to ensure the accuracy
14  * and reliability of the software and data, the NLM and the U.S.
15  * Government do not and cannot warrant the performance or results that
16  * may be obtained by using this software or data. The NLM and the U.S.
17  * Government disclaim all warranties, express or implied, including
18  * warranties of performance, merchantability or fitness for any particular
19  * purpose.
20  *
21  * Please cite the author in any work or product based on this material.
22  *
23  * ===========================================================================
24  *
25  * Author: Eugene Vasilchenko, Anatoliy Kuznetsov
26  *
27  * File Description: Cached extension of data reader from ID1
28  *
29  */
30 #include <ncbi_pch.hpp>
34 #include <objtools/data_loaders/genbank/readers.hpp> // for entry point
39 
40 #include <corelib/ncbitime.hpp>
41 #include <corelib/rwstream.hpp>
43 
44 #include <util/cache/icache.hpp>
46 
51 
52 #include <serial/objistrasnb.hpp> // for reading Seq-ids
53 #include <serial/serial.hpp> // for reading Seq-ids
54 #include <objects/seqloc/Seq_id.hpp> // for reading Seq-ids
55 
56 #define FIX_BAD_ID2S_REPLY_DATA 1
57 
60 
61 NCBI_PARAM_DECL(int, GENBANK, CACHE_DEBUG);
62 
63 NCBI_PARAM_DEF_EX(int, GENBANK, CACHE_DEBUG, 0,
64  eParam_NoThread, GENBANK_CACHE_DEBUG);
65 
67 {
68  static CSafeStatic<NCBI_PARAM_TYPE(GENBANK, CACHE_DEBUG)> s_Value;
69  return s_Value->Get();
70 }
71 
72 const int SCacheInfo::BLOB_IDS_MAGIC = 0x32fd0108;
73 const char* const SCacheInfo::BLOB_IDS_SUBKEY = "Blobs8";
74 
75 string SCacheInfo::GetBlobKey(const CBlob_id& blob_id)
76 {
77  CNcbiOstrstream oss;
78  oss << blob_id.GetSat();
79  if ( blob_id.GetSubSat() != 0 ) {
80  oss << '.' << blob_id.GetSubSat();
81  }
82  oss << '-' << blob_id.GetSatKey();
83  return CNcbiOstrstreamToString(oss);
84 }
85 
86 
88 {
89  return NStr::NumericToString(gi);
90 }
91 
92 
93 string SCacheInfo::GetIdKey(const CSeq_id& id)
94 {
95  return id.IsGi()? GetIdKey(id.GetGi()): id.AsFastaString();
96 }
97 
98 
100 {
101  return id.IsGi()? GetIdKey(id.GetGi()): id.AsString();
102 }
103 
104 
105 static const size_t kHashLimit = 100;
106 
107 
109  string& subkey,
110  string& true_subkey)
111 {
112  if ( !sel ) {
114  return;
115  }
118  if ( accs.empty() ) {
120  return;
121  }
122 
124  str << BLOB_IDS_SUBKEY;
125  size_t total_size = 0;
127  total_size += 1+it->first.size();
128  }
129  bool add_hash = total_size > kHashLimit;
130  if ( add_hash ) {
131  size_t hash = 5381;
133  hash = hash*17 + it->first.size();
134  ITERATE ( string, i, it->first ) {
135  hash = hash*17 + (*i & 0xff);
136  }
137  }
138  str << ";#" << hex << hash << dec;
139  }
141  sel->GetNamedAnnotAccessions() ) {
142  str << ';';
143  str << it->first;
144  }
145  if ( add_hash ) {
146  true_subkey = CNcbiOstrstreamToString(str);
147  subkey = true_subkey.substr(0, kHashLimit);
148  }
149  else {
151  }
152 }
153 
154 
155 const char* SCacheInfo::GetGiSubkey(void)
156 {
157  return "Gi";
158 }
159 
160 
162 {
163  return "Accver";
164 }
165 
166 
167 const char* SCacheInfo::GetLabelSubkey(void)
168 {
169  return "Label";
170 }
171 
172 
173 const char* SCacheInfo::GetTaxIdSubkey(void)
174 {
175  return "Taxid";
176 }
177 
178 
179 const char* SCacheInfo::GetHashSubkey(void)
180 {
181  return "Hash";
182 }
183 
184 
186 {
187  return "Length";
188 }
189 
190 
191 const char* SCacheInfo::GetTypeSubkey(void)
192 {
193  return "Type";
194 }
195 
196 
198 {
199  return "Ids";
200 }
201 
202 
204 {
205  return "State";
206 }
207 
208 
210 {
211  return "Ver";
212 }
213 
214 
215 string SCacheInfo::GetBlobSubkey(CLoadLockBlob& blob, int chunk_id)
216 {
217  if ( chunk_id == kMain_ChunkId )
218  return string();
219  else if ( chunk_id == kDelayedMain_ChunkId )
220  return "ext";
221  else {
222  CNcbiOstrstream oss;
223  oss << chunk_id << '-' << blob.GetSplitInfo().GetSplitVersion();
224  return CNcbiOstrstreamToString(oss);;
225  }
226 }
227 
228 
229 string SCacheInfo::GetBlobSubkey(int split_version, int chunk_id)
230 {
231  if ( chunk_id == kMain_ChunkId )
232  return string();
233  else if ( chunk_id == kDelayedMain_ChunkId )
234  return "ext";
235  else {
236  CNcbiOstrstream oss;
237  oss << chunk_id << '-' << split_version;
238  return CNcbiOstrstreamToString(oss);;
239  }
240 }
241 
242 
243 /////////////////////////////////////////////////////////////////////////////
244 // CCacheHolder
245 /////////////////////////////////////////////////////////////////////////////
246 
248  : m_BlobCache(0), m_IdCache(0)
249 {
250 }
251 
252 
254 {
255  SetIdCache(0);
256  SetBlobCache(0);
257 }
258 
259 
261 {
262  m_IdCache = id_cache;
263 }
264 
265 
267 {
268  m_BlobCache = blob_cache;
269 }
270 
271 
272 /////////////////////////////////////////////////////////////////////////
273 
274 
276  : m_JoinedBlobVersion(eDefault)
277 {
279 }
280 
281 
283  const string& driver_name)
284  : m_JoinedBlobVersion(eDefault)
285 {
286  CConfig conf(params);
287  bool joined_blob_version = conf.GetBool(
288  driver_name,
291  true);
292  m_JoinedBlobVersion = joined_blob_version? eDefault: eOff;
294 }
295 
296 
298 {
299 }
300 
301 
303 {
304  SetIdCache(0);
305  SetBlobCache(0);
306 }
307 
308 
309 void CCacheReader::x_DisconnectAtSlot(TConn /*conn*/, bool /*failed*/)
310 {
311 }
312 
313 
315 {
316 }
317 
318 
320 {
321  return 2;
322 }
323 
324 
326 {
327  return true;
328 }
329 
330 
332 {
333  return 1;
334 }
335 
336 
337 //////////////////////////////////////////////////////////////////
338 
339 
340 namespace {
341  class CParseBuffer : public IReader {
342  public:
343  typedef CReaderRequestResult::TExpirationTime TExpirationTime;
344 
345  CParseBuffer(CReaderRequestResult& result, ICache* cache,
346  const string& key, const string& subkey);
347  CParseBuffer(CReaderRequestResult& result, ICache* cache,
348  const string& key, const string& subkey,
349  int version);
350  CParseBuffer(CReaderRequestResult& result, ICache* cache,
351  const string& key, const string& subkey,
352  int* get_current_version);
353 
354  bool Found(void) const
355  {
356  return m_Descr.blob_found;
357  }
358  bool FoundSome(void) const
359  {
360  return m_Descr.actual_age != unsigned(-1);
361  }
362  TExpirationTime GetExpirationTime(void) const
363  {
364  return m_ExpirationTime;
365  }
366  bool GotCurrentVersion(void) const
367  {
368  return m_Descr.return_current_version_supported;
369  }
370  bool CurrentVersionExpired(void) const
371  {
372  return GetExpirationTime() == TExpirationTime(-1);
373  }
374  bool Done(void) const
375  {
376  if ( m_Ptr ) {
377  return m_Size == 0;
378  }
379  else {
380  return x_Eof();
381  }
382  }
383 
384  Uint1 ParseUint1(void)
385  {
386  return *x_NextBytes(1);
387  }
388  bool ParseBool(void)
389  {
390  return ParseUint1();
391  }
392  Uint4 ParseUint4(void)
393  {
394  const char* ptr = x_NextBytes(4);
395  return ((ptr[0]&0xff)<<24)|((ptr[1]&0xff)<<16)|
396  ((ptr[2]&0xff)<<8)|(ptr[3]&0xff);
397  }
398  Int4 ParseInt4(void)
399  {
400  return ParseUint4();
401  }
402  Int8 ParseInt8(void)
403  {
404  Int8 n = Int8(ParseInt4()) << 32;
405  n |= ParseUint4();
406  return n;
407  }
408  string ParseString(void);
409  string FullString(void);
410 
411  IReader* GetReader(void);
412 
413  virtual ERW_Result Read(void* buf,
414  size_t count,
415  size_t* bytes_read = 0)
416  {
417  if ( !m_Size ) {
418  if ( bytes_read ) {
419  *bytes_read = 0;
420  }
421  return eRW_Eof;
422  }
423  count = min(count, m_Size);
424  memcpy(buf, m_Ptr, count);
425  if ( bytes_read ) {
426  *bytes_read = count;
427  }
428  m_Ptr += count;
429  m_Size -= count;
430  return eRW_Success;
431  }
432  virtual ERW_Result PendingCount(size_t* count)
433  {
434  *count = m_Size;
435  return eRW_Success;
436  }
437 
438  protected:
439  void x_Init(CReaderRequestResult& result,
440  ICache* cache,
441  const string& key,
442  const string& subkey,
443  int version,
444  int* get_current_version,
445  bool set_expiration_time);
446  bool x_Eof(void) const;
447  const char* x_NextBytes(size_t size);
448 
449  private:
450  CParseBuffer(const CParseBuffer&);
451  void operator=(const CParseBuffer&);
452 
453  char m_Buffer[4096];
454  ICache::SBlobAccessDescr m_Descr;
455  TExpirationTime m_ExpirationTime;
456  const char* m_Ptr;
457  size_t m_Size;
458  };
459 
460  void CParseBuffer::x_Init(CReaderRequestResult& result,
461  ICache* cache,
462  const string& key,
463  const string& subkey,
464  int version,
465  int* get_current_version,
466  bool set_expiration_time)
467  {
468  if ( set_expiration_time ) {
469  m_Descr.maximum_age = result.GetIdExpirationTimeout(GBL::eExpire_normal);
470  }
471  if ( get_current_version ) {
472  // Decrease blob version timeout for easier debugging
473  // m_Descr.maximum_age /= 10;
474  m_Descr.return_current_version = true;
475  }
476  cache->GetBlobAccess(key, version, subkey, &m_Descr);
477 
478  if ( SCacheInfo::GetDebugLevel() > 0 ) {
479  CReader::CDebugPrinter s("CCacheReader");
480  s << "Read";
481  if ( get_current_version ) {
482  s << "V";
483  }
484  s << ": "<<key<<","<<subkey;
485  if ( !get_current_version ) {
486  s << ","<<version;
487  }
488  if ( !Found() ) {
489  s << " not found";
490  if ( get_current_version && GotCurrentVersion() ) {
491  s << ", ver="<<m_Descr.current_version;
492  }
493  }
494  else {
495  s << " found";
496  if ( get_current_version && GotCurrentVersion() ) {
497  s << ", ver="<<m_Descr.current_version;
498  }
499  }
500  s << ", age="<<int(m_Descr.actual_age);
501  }
502  m_ExpirationTime = result.GetNewIdExpirationTime(GBL::eExpire_normal);
503  if ( FoundSome() ) {
504  if ( m_Descr.actual_age > m_ExpirationTime ) {
505  m_ExpirationTime = TExpirationTime(-1);
506  }
507  else {
508  m_ExpirationTime -= m_Descr.actual_age;
509  }
510  }
511  if ( get_current_version ) {
512  if ( GotCurrentVersion() ) {
513  *get_current_version = m_Descr.current_version;
514  }
515  else {
516  m_ExpirationTime = TExpirationTime(-1);
517  *get_current_version = 0;
518  }
519  }
520  if ( Found() && !m_Descr.reader.get() ) {
521  m_Ptr = m_Descr.buf;
522  m_Size = m_Descr.blob_size;
523  }
524  }
525 
526  CParseBuffer::CParseBuffer(CReaderRequestResult& result,
527  ICache* cache,
528  const string& key,
529  const string& subkey)
530  : m_Descr(m_Buffer, sizeof(m_Buffer)), m_Ptr(0), m_Size(0)
531  {
532  x_Init(result, cache, key, subkey, 0, 0, true);
533  }
534 
535  CParseBuffer::CParseBuffer(CReaderRequestResult& result,
536  ICache* cache,
537  const string& key,
538  const string& subkey,
539  int version)
540  : m_Descr(m_Buffer, sizeof(m_Buffer)), m_Ptr(0), m_Size(0)
541  {
542  x_Init(result, cache, key, subkey, version, 0, false);
543  }
544 
545  CParseBuffer::CParseBuffer(CReaderRequestResult& result,
546  ICache* cache,
547  const string& key,
548  const string& subkey,
549  int* get_current_version)
550  : m_Descr(m_Buffer, sizeof(m_Buffer)), m_Ptr(0), m_Size(0)
551  {
552  x_Init(result, cache, key, subkey, -1, get_current_version, true);
553  }
554 
555  string CParseBuffer::ParseString(void)
556  {
557  _ASSERT(Found());
558  string ret;
559  size_t size = ParseUint4();
560  if ( m_Ptr ) {
561  ret.assign(x_NextBytes(size), size);
562  }
563  else {
564  ret.reserve(size);
565  while ( size ) {
566  size_t count = min(size, sizeof(m_Buffer));
567  ret.assign(x_NextBytes(count), count);
568  size -= count;
569  }
570  }
571  return ret;
572  }
573 
574  string CParseBuffer::FullString(void)
575  {
576  _ASSERT(Found());
577  string ret;
578  if ( m_Ptr ) {
579  ret.assign(m_Ptr, m_Size);
580  m_Ptr += m_Size;
581  m_Size = 0;
582  }
583  else {
584  size_t count = 0;
585  while ( m_Descr.reader->Read(m_Buffer, sizeof(m_Buffer), &count) == eRW_Success ) {
586  ret.append(m_Buffer, count);
587  }
588  }
589  return ret;
590  }
591 
592  const char* CParseBuffer::x_NextBytes(size_t size)
593  {
594  _ASSERT(Found());
595  const char* ret = m_Ptr;
596  if ( ret ) {
597  if ( m_Size >= size ) {
598  m_Ptr = ret + size;
599  m_Size -= size;
600  return ret;
601  }
602  }
603  else if ( size <= sizeof(m_Buffer) ) {
604  char* buf = m_Buffer;
605  while ( size ) {
606  size_t count = 0;
607  if ( m_Descr.reader->Read(buf, size, &count) != eRW_Success ) {
608  break;
609  }
610  buf += count;
611  size -= count;
612  }
613  if ( size == 0 ) {
614  return m_Buffer;
615  }
616  }
617  NCBI_THROW(CLoaderException, eLoaderFailed,
618  "parse buffer overflow");
619  }
620 
621  bool CParseBuffer::x_Eof(void) const
622  {
623  _ASSERT(Found());
624  char buffer[1];
625  size_t count;
626  return m_Descr.reader->Read(buffer, 1, &count) == eRW_Eof;
627  }
628 
629  IReader* CParseBuffer::GetReader(void)
630  {
631  _ASSERT(Found());
632  if ( !m_Descr.reader.get() ) {
633  return this;
634  }
635  return m_Descr.reader.get();
636  }
637 }
638 
639 
640 
642  const string& key,
643  CLoadLockSeqIds& lock)
644 {
645  if ( !m_IdCache ) {
646  return false;
647  }
648 
649  if ( lock.IsLoaded() ) {
650  return true;
651  }
652 
653  CConn conn(result, this);
654  CParseBuffer str(result, m_IdCache, key, GetSeq_idsSubkey());
655  if ( !str.Found() ) {
656  conn.Release();
657  return false;
658  }
659  CRStream r_stream(str.GetReader());
660  CObjectIStreamAsnBinary obj_stream(r_stream);
661  size_t count = static_cast<CObjectIStream&>(obj_stream).ReadUint4();
662  TSeqIds seq_ids;
663  for ( size_t i = 0; i < count; ++i ) {
664  CSeq_id id;
665  obj_stream >> id;
666  seq_ids.push_back(CSeq_id_Handle::GetHandle(id));
667  }
668  conn.Release();
670  str.GetExpirationTime());
671  return true;
672 }
673 
674 
676  const CSeq_id_Handle& seq_id)
677 {
678  if ( !m_IdCache ) {
679  return false;
680  }
681 
682  CLoadLockSeqIds lock(result, seq_id);
683  return lock.IsLoaded() || ReadSeq_ids(result, GetIdKey(seq_id), lock);
684 }
685 
686 
688  const CSeq_id_Handle& seq_id)
689 {
690  if ( !m_IdCache ) {
691  return false;
692  }
693 
694  CLoadLockGi lock(result, seq_id);
695  if ( lock.IsLoadedGi() ) {
696  return true;
697  }
698 
700 
701  CConn conn(result, this);
702  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetGiSubkey());
703  if ( str.Found() ) {
704  Int8 gi_num = str.ParseInt8();
705 #ifdef NCBI_INT8_GI
706  TGi gi = GI_FROM(TIntId, gi_num);
707 #else
708  if ( gi_num != Int4(gi_num) ) {
709  NCBI_THROW(CLoaderException, eLoaderFailed,
710  "64-bit gi overflow");
711  }
712  TGi gi = Int4(gi_num);
713 #endif
714  if ( str.Done() ) {
715  conn.Release();
717  data.gi = gi;
718  data.sequence_found = true;
719  lock.SetLoadedGi(data, str.GetExpirationTime());
720  return true;
721  }
722  }
723  conn.Release();
724 
725  CLoadLockSeqIds ids_lock(result, seq_id);
726  LoadSeq_idSeq_ids(result, seq_id);
727  if ( ids_lock.IsLoaded() ) {
728  result.SetLoadedGiFromSeqIds(seq_id, ids_lock);
729  return true;
730  }
731 
732  return false;
733 }
734 
735 
737  const CSeq_id_Handle& seq_id)
738 {
739  if ( !m_IdCache ) {
740  return false;
741  }
742 
743  CLoadLockAcc lock(result, seq_id);
744  if ( lock.IsLoadedAccVer() ) {
745  return true;
746  }
747 
749 
750  CConn conn(result, this);
751  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetAccVerSubkey());
752  if ( str.Found() ) {
753  string data = str.FullString();
754  conn.Release();
755  TSequenceAcc acc;
756  if ( !data.empty() ) {
758  }
759  acc.sequence_found = true;
760  lock.SetLoadedAccVer(acc, str.GetExpirationTime());
761  return true;
762  }
763  conn.Release();
764 
765  CLoadLockSeqIds ids_lock(result, seq_id);
766  LoadSeq_idSeq_ids(result, seq_id);
767  if ( ids_lock.IsLoaded() ) {
768  result.SetLoadedAccFromSeqIds(seq_id, ids_lock);
769  return true;
770  }
771  /*
772  if ( !seq_id.IsGi() ) {
773  CLoadLockGi gi_lock(result, seq_id);
774  LoadSeq_idGi(result, seq_id);
775  if ( gi_lock.IsLoadedGi() ) {
776  TSequenceGi gi = gi_lock.GetGi();
777  if ( gi.second && gi.first ) {
778  CSeq_id_Handle gi_id =
779  CSeq_id_Handle::GetGiHandle(gi.first);
780  CLoadLockAcc gi_acc(result, gi_id);
781  LoadSeq_idAccVer(result, gi_id);
782  if ( gi_acc.IsLoadedAccVer() ) {
783  lock.SetLoadedAccVer(gi_acc.GetAccVer(),
784  gi_acc.GetExpirationTime());
785  return true;
786  }
787  }
788  }
789  }
790  */
791  return false;
792 }
793 
794 
796  const CSeq_id_Handle& seq_id)
797 {
798  if ( !m_IdCache ) {
799  return false;
800  }
801 
802  CLoadLockLabel lock(result, seq_id);
803  if ( lock.IsLoadedLabel() ) {
804  return true;
805  }
806 
807  CConn conn(result, this);
808  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetLabelSubkey());
809  if ( str.Found() ) {
810  string data = str.FullString();
811  conn.Release();
812  lock.SetLoadedLabel(data, str.GetExpirationTime());
813  return true;
814  }
815  conn.Release();
816 
817  CLoadLockSeqIds ids_lock(result, seq_id);
818  LoadSeq_idSeq_ids(result, seq_id);
819  if ( ids_lock.IsLoaded() ) {
820  lock.SetLoadedLabel(ids_lock.GetSeq_ids().FindLabel(),
821  ids_lock.GetExpirationTime());
822  return true;
823  }
824  return false;
825 }
826 
827 
829  const CSeq_id_Handle& seq_id)
830 {
831  if ( !m_IdCache ) {
832  return false;
833  }
834 
835  CLoadLockTaxId lock(result, seq_id);
836  if ( lock.IsLoadedTaxId() ) {
837  return true;
838  }
839 
840  CConn conn(result, this);
841  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetTaxIdSubkey());
842  if ( !str.Found() ) {
843  conn.Release();
844  return false;
845  }
846  TTaxId taxid = TAX_ID_FROM(Int4, str.ParseInt4());
847  if ( !str.Done() ) {
848  conn.Release();
849  return false;
850  }
851  conn.Release();
852  lock.SetLoadedTaxId(taxid, str.GetExpirationTime());
853  return true;
854 }
855 
856 
858  const CSeq_id_Handle& seq_id)
859 {
860  if ( !m_IdCache ) {
861  return false;
862  }
863 
864  CLoadLockHash lock(result, seq_id);
865  if ( lock.IsLoadedHash() ) {
866  return true;
867  }
868 
869  CConn conn(result, this);
870  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetHashSubkey());
871  if ( !str.Found() ) {
872  if ( !seq_id.IsGi() ) {
873  CLoadLockGi gi_lock(result, seq_id);
874  LoadSeq_idGi(result, seq_id);
875  if ( gi_lock.IsLoadedGi() ) {
876  TSequenceGi gi = gi_lock.GetGi();
877  if ( gi_lock.GetGi(gi) != ZERO_GI ) {
878  CSeq_id_Handle gi_id =
879  CSeq_id_Handle::GetGiHandle(gi_lock.GetGi(gi));
880  CLoadLockHash gi_hash(result, gi_id);
881  LoadSequenceHash(result, gi_id);
882  if ( gi_hash.IsLoadedHash() ) {
883  lock.SetLoadedHash(gi_hash.GetHash(),
884  gi_hash.GetExpirationTime());
885  return true;
886  }
887  }
888  }
889  }
890  conn.Release();
891  return false;
892  }
894  hash.hash = str.ParseInt4();
895  hash.sequence_found = str.ParseBool();
896  hash.hash_known = str.ParseBool();
897  if ( !str.Done() ) {
898  conn.Release();
899  return false;
900  }
901  conn.Release();
902  lock.SetLoadedHash(hash, str.GetExpirationTime());
903  return true;
904 }
905 
906 
908  const CSeq_id_Handle& seq_id)
909 {
910  if ( !m_IdCache ) {
911  return false;
912  }
913 
914  CLoadLockLength lock(result, seq_id);
915  if ( lock.IsLoadedLength() ) {
916  return true;
917  }
918 
919  CConn conn(result, this);
920  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetLengthSubkey());
921  if ( !str.Found() ) {
922  conn.Release();
923  return false;
924  }
925  TSeqPos length = str.ParseUint4();
926  if ( !str.Done() ) {
927  conn.Release();
928  return false;
929  }
930  conn.Release();
931  lock.SetLoadedLength(length, str.GetExpirationTime());
932  return true;
933 }
934 
935 
937  const CSeq_id_Handle& seq_id)
938 {
939  if ( !m_IdCache ) {
940  return false;
941  }
942 
943  CLoadLockType lock(result, seq_id);
944  if ( lock.IsLoadedType() ) {
945  return true;
946  }
947 
948  CConn conn(result, this);
949  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), GetTypeSubkey());
950  if ( !str.Found() ) {
951  conn.Release();
952  return false;
953  }
954  int type = str.ParseInt4();
955  if ( !str.Done() ) {
956  conn.Release();
957  return false;
958  }
959  conn.Release();
961  data.type = CSeq_inst::EMol(type);
962  data.sequence_found = true;
963  lock.SetLoadedType(data, str.GetExpirationTime());
964  return true;
965 }
966 
967 
969  const TIds& ids, TLoaded& loaded, TIds& ret)
970 {
971  if ( !m_IdCache ) {
972  return false;
973  }
974  size_t count = ids.size();
975  for ( size_t i = 0; i < count; ++i ) {
976  if ( loaded[i] || CReadDispatcher::CannotProcess(ids[i]) ) {
977  continue;
978  }
979  CLoadLockAcc lock(result, ids[i]);
980  if ( !lock.IsLoadedAccVer() ) {
981  LoadSeq_idAccVer(result, ids[i]);
982  }
983  if ( lock.IsLoadedAccVer() ) {
984  TSequenceAcc data = lock.GetAccVer();
985  if ( lock.IsFound(data) ) {
986  ret[i] = lock.GetAcc(data);
987  loaded[i] = true;
988  }
989  }
990  }
991  return false;
992 }
993 
994 
996  const TIds& ids, TLoaded& loaded, TGis& ret)
997 {
998  if ( !m_IdCache ) {
999  return false;
1000  }
1001  size_t count = ids.size();
1002  for ( size_t i = 0; i < count; ++i ) {
1003  if ( loaded[i] || CReadDispatcher::CannotProcess(ids[i]) ) {
1004  continue;
1005  }
1006  CLoadLockGi lock(result, ids[i]);
1007  if ( !lock.IsLoadedGi() ) {
1008  LoadSeq_idGi(result, ids[i]);
1009  }
1010  if ( lock.IsLoadedGi() ) {
1011  TSequenceGi data = lock.GetGi();
1012  if ( lock.IsFound(data) ) {
1013  ret[i] = lock.GetGi(data);
1014  loaded[i] = true;
1015  }
1016  }
1017  }
1018  return false;
1019 }
1020 
1021 
1023  const TIds& ids, TLoaded& loaded, TLabels& ret)
1024 {
1025  if ( !m_IdCache ) {
1026  return false;
1027  }
1028  size_t count = ids.size();
1029  for ( size_t i = 0; i < count; ++i ) {
1030  if ( loaded[i] || CReadDispatcher::CannotProcess(ids[i]) ) {
1031  continue;
1032  }
1033  CLoadLockLabel lock(result, ids[i]);
1034  if ( !lock.IsLoadedLabel() ) {
1035  LoadSeq_idLabel(result, ids[i]);
1036  }
1037  if ( lock.IsLoadedLabel() ) {
1038  ret[i] = lock.GetLabel();
1039  loaded[i] = true;
1040  continue;
1041  }
1042  }
1043  return false;
1044 }
1045 
1046 
1048  const TIds& ids, TLoaded& loaded, TTaxIds& ret)
1049 {
1050  if ( !m_IdCache ) {
1051  return false;
1052  }
1053  size_t count = ids.size();
1054  for ( size_t i = 0; i < count; ++i ) {
1055  if ( loaded[i] || CReadDispatcher::CannotProcess(ids[i]) ) {
1056  continue;
1057  }
1058  CLoadLockTaxId lock(result, ids[i]);
1059  if ( !lock.IsLoadedTaxId() ) {
1060  LoadSeq_idTaxId(result, ids[i]);
1061  }
1062  if ( lock.IsLoadedTaxId() ) {
1063  ret[i] = lock.GetTaxId();
1064  loaded[i] = true;
1065  continue;
1066  }
1067  }
1068  return false;
1069 }
1070 
1071 
1073  const CSeq_id_Handle& seq_id,
1074  const SAnnotSelector* sel)
1075 {
1076  if ( !m_IdCache ) {
1077  return false;
1078  }
1079 
1080  CLoadLockBlobIds ids(result, seq_id, sel);
1081  if( ids.IsLoaded() ) {
1082  return true;
1083  }
1084 
1085  CConn conn(result, this);
1086  string subkey, true_subkey;
1087  GetBlob_idsSubkey(sel, subkey, true_subkey);
1088  CParseBuffer str(result, m_IdCache, GetIdKey(seq_id), subkey);
1089  if ( !str.Found() ) {
1090  conn.Release();
1091  return false;
1092  }
1093  if ( str.ParseInt4() != BLOB_IDS_MAGIC ) {
1094  conn.Release();
1095  return false;
1096  }
1097 
1098  int state = str.ParseUint4();
1099  TBlobIds blob_ids;
1100  size_t blob_count = str.ParseUint4();
1101  blob_ids.reserve(blob_count);
1102 
1103  for ( size_t i = 0; i < blob_count; ++i ) {
1104  CRef<CBlob_id> id(new CBlob_id);
1105  id->SetSat(str.ParseInt4());
1106  id->SetSubSat(str.ParseInt4());
1107  id->SetSatKey(str.ParseInt4());
1108  CBlob_Info info(id, str.ParseUint4());
1109  CRef<CBlob_Annot_Info> annots_info;
1110  size_t name_count = str.ParseUint4();
1111  if ( name_count ) {
1112  annots_info = new CBlob_Annot_Info;
1113  for ( size_t j = 0; j < name_count; ++j ) {
1114  annots_info->AddNamedAnnotName(str.ParseString());
1115  }
1116  }
1117  string s = str.ParseString();
1118  if ( !s.empty() ) {
1119  if ( !annots_info ) {
1120  annots_info = new CBlob_Annot_Info;
1121  }
1122  CObjectIStreamAsnBinary stream(s.data(), s.size());
1123  CRef<CID2S_Seq_annot_Info> annot_info;
1124  while ( stream.HaveMoreData() ) {
1125  annot_info.Reset(new CID2S_Seq_annot_Info);
1126  stream >> *annot_info;
1127  annots_info->AddAnnotInfo(*annot_info);
1128  }
1129  }
1130  if ( annots_info ) {
1131  info.SetAnnotInfo(annots_info);
1132  }
1133  blob_ids.push_back(info);
1134  }
1135  if ( !true_subkey.empty() && str.ParseString() != true_subkey ) {
1136  conn.Release();
1137  return false;
1138  }
1139  if ( !str.Done() ) {
1140  conn.Release();
1141  return false;
1142  }
1143  conn.Release();
1145  str.GetExpirationTime());
1146  return true;
1147 }
1148 
1149 
1151  atomic<Uint8> load_count, save_count;
1152 
1153  void GoingToLoad() {
1154  ++load_count;
1155  }
1156  bool NoNeedToSave() {
1157  if ( save_count >= load_count ) {
1158  return true;
1159  }
1160  ++save_count;
1161  return false;
1162  }
1163 };
1164 
1166 
1167 
1169 {
1170  if ( type < eCacheEntry_Count ) {
1172  }
1173 }
1174 
1175 
1177 {
1178  if ( type < eCacheEntry_Count ) {
1180  }
1181  return false;
1182 }
1183 
1184 
1186  const TBlobId& blob_id)
1187 {
1188  if ( !m_IdCache ) {
1189  return false;
1190  }
1191 
1192  CLoadLockBlobState lock(result, blob_id);
1193  if( lock.IsLoadedBlobState() ) {
1194  return true;
1195  }
1196 
1198 
1199  CConn conn(result, this);
1200  CParseBuffer str(result, m_IdCache,
1201  GetBlobKey(blob_id), GetBlobStateSubkey());
1202  if ( !str.Found() ) {
1203  conn.Release();
1204  return false;
1205  }
1206  int state = str.ParseInt4();
1207  if ( !str.Done() ) {
1208  conn.Release();
1209  return false;
1210  }
1211  conn.Release();
1212  SetAndSaveBlobState(result, blob_id, state);
1213  return true;
1214 }
1215 
1216 
1218  const TBlobId& blob_id)
1219 {
1220  if ( !m_IdCache ) {
1221  return false;
1222  }
1223 
1224  CLoadLockBlobVersion lock(result, blob_id);
1225  if( lock.IsLoadedBlobVersion() ) {
1226  return true;
1227  }
1228 
1230 
1231  CConn conn(result, this);
1232  CParseBuffer str(result, m_IdCache,
1233  GetBlobKey(blob_id), GetBlobVersionSubkey());
1234  if ( !str.Found() ) {
1235  conn.Release();
1236  return false;
1237  }
1238  int version = str.ParseInt4();
1239  if ( !str.Done() ) {
1240  conn.Release();
1241  return false;
1242  }
1243  conn.Release();
1245  return true;
1246 }
1247 
1248 
1250  const TBlobId& blob_id)
1251 {
1252  return LoadChunk(result, blob_id, kMain_ChunkId);
1253 }
1254 
1255 
1257  const string& key,
1258  const string& subkey,
1260 {
1261  // current blob version is still valid
1262  if ( GetDebugLevel() > 0 ) {
1263  CDebugPrinter s("CCacheReader");
1264  s << "SetBlobVersionAsCurrent("<<key<<", "<<subkey<<", "<<version<<")";
1265  }
1266  CConn conn(result, this);
1268  conn.Release();
1269 }
1270 
1271 
1273  const TBlobId& blob_id,
1274  TChunkId chunk_id)
1275 {
1276  if ( !m_BlobCache ) {
1277  return false;
1278  }
1279 
1280  CLoadLockBlob blob(result, blob_id, chunk_id);
1281  if ( blob.IsLoadedChunk() ) {
1282  return true;
1283  }
1284 
1285  string key = GetBlobKey(blob_id);
1286  string subkey = GetBlobSubkey(blob, chunk_id);
1287  TBlobVersion cache_version = -1;
1289  if ( chunk_id == kMain_ChunkId && CProcessor_ExtAnnot::IsExtAnnot(blob_id) ) {
1290  version = 0;
1291  }
1292  if ( version < 0 ) {
1293  CLoadLockBlobVersion lock(result, blob_id, eAlreadyLoaded);
1294  if ( lock ) {
1295  version = lock.GetBlobVersion();
1296  _ASSERT(version >= 0);
1297  }
1298  }
1299  if ( version < 0 ) {
1300  CConn conn(result, this);
1301  bool know_has_blobs = false;
1302  if ( m_JoinedBlobVersion != eOff ) {
1303  do { // artificial one-time cycle to allow breaking
1304  CParseBuffer str(result, m_BlobCache, key, subkey, &version);
1305  if ( !str.GotCurrentVersion() ) {
1306  // joined blob version is not supported by ICache
1307  if ( m_JoinedBlobVersion != eOff ) {
1308  if ( m_JoinedBlobVersion == eOn ) {
1309  ERR_POST("CCacheReader: "
1310  "stopped to get current blob version");
1311  }
1313  }
1314  break;
1315  }
1316  else {
1317  cache_version = version;
1318  // joined blob version is supported by ICache
1319  if ( m_JoinedBlobVersion == eDefault ) {
1321  }
1322  }
1323  if ( !str.Found() ) {
1324  know_has_blobs = str.FoundSome();
1325  break;
1326  }
1327  else if ( str.CurrentVersionExpired() ) {
1328  // check if blob version is still the same
1329 
1330  // read the blob to allow next ICache command
1332  {{
1333  CRStream stream(str.GetReader());
1334  data << stream.rdbuf();
1335  }}
1336  conn.Release();
1337 
1338  // get blob version from other readers (e.g. ID2)
1339  CLoadLockBlobVersion lock(result, blob_id);
1340  m_Dispatcher->LoadBlobVersion(result, blob_id, this);
1341  version = lock.GetBlobVersion();
1342  if ( version < 0 ) {
1343  // Cannot determine the blob version ->
1344  // pass the request to the next reader.
1345  return false;
1346  }
1347  if ( blob.GetKnownBlobVersion() >= 0 &&
1348  blob.GetKnownBlobVersion() != version ) {
1349  // Cached blob version is outdated ->
1350  // pass the request to the next reader.
1351  return false;
1352  }
1354  x_ProcessBlob(result, blob_id, chunk_id, data);
1355  return true;
1356  }
1357  else {
1358  // current blob version is valid
1359  result.SetAndSaveBlobVersion(blob_id, version);
1360  {{
1361  CRStream stream(str.GetReader());
1362  x_ProcessBlob(result, blob_id, chunk_id, stream);
1363  }}
1364  conn.Release();
1365  return true;
1366  }
1367  } while ( false );
1368  // failed, continue with old-style version
1369  }
1370 
1371  if ( !know_has_blobs ) {
1372  bool has_blobs = m_BlobCache->HasBlobs(key, subkey);
1373  if ( !has_blobs ) {
1374  // shurely there are no versions
1375  conn.Release();
1376  return false;
1377  }
1378  }
1379  conn.Release();
1380 
1381  version = blob.GetKnownBlobVersion();
1382  if ( version < 0 ) {
1383  // get blob version from other readers (e.g. ID2)
1384  CLoadLockBlobVersion lock(result, blob_id);
1385  if ( m_JoinedBlobVersion != eOff ) {
1386  m_Dispatcher->LoadBlobVersion(result, blob_id, this);
1387  }
1388  else {
1389  m_Dispatcher->LoadBlobVersion(result, blob_id);
1390  }
1391  version = lock.GetBlobVersion();
1392  if ( version < 0 ) {
1393  // Cannot determine the blob version ->
1394  // pass the request to the next reader.
1395  return false;
1396  }
1397  }
1398 
1399  _ASSERT(version >= 0);
1400  if ( m_JoinedBlobVersion != eOff && version == cache_version) {
1402  }
1403  }
1404  if ( cache_version != -1 && version != cache_version ) {
1405  // cache version is surely different
1406  return false;
1407  }
1408 
1409  CConn conn(result, this);
1410  CParseBuffer buffer(result, m_BlobCache, key, subkey, version);
1411  if ( !buffer.Found() ) {
1412  conn.Release();
1413  return false;
1414  }
1415 
1416  CRStream stream(buffer.GetReader());
1417  x_ProcessBlob(result, blob_id, chunk_id, stream);
1418  conn.Release();
1419  return true;
1420 }
1421 
1422 
1424  const TBlobId& blob_id,
1425  TChunkId chunk_id,
1426  CNcbiIstream& stream)
1427 {
1428  int processor_type = ReadInt(stream);
1429  const CProcessor& processor =
1430  m_Dispatcher->GetProcessor(CProcessor::EType(processor_type));
1431  if ( processor_type != processor.GetType() ) {
1432  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1433  "CCacheReader::LoadChunk: "
1434  "invalid processor type: "<<processor_type);
1435  }
1436  int processor_magic = ReadInt(stream);
1437  if ( processor_magic != int(processor.GetMagic()) ) {
1438  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1439  "CCacheReader::LoadChunk: "
1440  "invalid processor magic number: "<<processor_magic);
1441  }
1442  processor.ProcessStream(result, blob_id, chunk_id, stream);
1443 }
1444 
1445 
1447 {
1449 
1450  struct SDefaultValue {
1451  const char* name;
1452  const char* value;
1453  };
1454 
1455 
1456  static
1457  TParams* FindSubNode(TParams* params, const string& name)
1458  {
1459  return params? params->FindSubNode(name): 0;
1460  }
1461 
1462 
1463  static
1464  const TParams* FindSubNode(const TParams* params,
1465  const string& name)
1466  {
1467  return params? params->FindSubNode(name): 0;
1468  }
1469 
1470 
1471  static
1473  const string& name,
1474  const char* default_value = "")
1475  {
1476  _ASSERT(!name.empty());
1477  TParams* node = FindSubNode(params, name);
1478  if ( !node ) {
1479  node = params->AddNode(TParams::TValueType(name, default_value));
1480  }
1481  return node;
1482  }
1483 
1484 
1485  static
1486  TParams* SetSubSection(TParams* params, const string& name)
1487  {
1488  return SetSubNode(params, name, "");
1489  }
1490 
1491 
1492  static
1493  const string& SetDefaultValue(TParams* params,
1494  const string& name,
1495  const char* value)
1496  {
1497  return SetSubNode(params, name, value)->GetValue().value;
1498  }
1499 
1500 
1501  static
1502  const string& SetDefaultValue(TParams* params, const SDefaultValue& value)
1503  {
1504  return SetDefaultValue(params, value.name, value.value);
1505  }
1506 
1507 
1508  static
1509  void SetDefaultValues(TParams* params, const SDefaultValue* values)
1510  {
1511  for ( ; values->name; ++values ) {
1512  SetDefaultValue(params, *values);
1513  }
1514  }
1515 
1516 };
1517 
1518 
1519 static
1521 {
1522  const SCacheInfo::TParams* driver =
1525  if ( driver ) {
1526  if ( driver->GetValue().value.empty() ) {
1527  // driver is set empty, it means no cache
1528  return true;
1529  }
1530  }
1531  return false;
1532 }
1533 
1534 
1535 static
1537 {
1538  const string& driver_name =
1541  "bdb");
1542  return SPluginParams::SetSubSection(params, driver_name);
1543 }
1544 
1545 
1546 static
1549  const char* section_name)
1550 {
1551  const SCacheInfo::TParams* src_section =
1552  SPluginParams::FindSubNode(src_params, section_name);
1553  if ( IsDisabledCache(src_section) ) {
1554  // no cache
1555  return 0;
1556  }
1557  if ( src_section ) {
1558  // make a copy of params
1559  return new SCacheInfo::TParams(*src_section);
1560  }
1561  else {
1562  // new param tree, if section is absent
1563  return new SCacheInfo::TParams();
1564  }
1565 }
1566 
1568  // common:
1569  { "keep_versions", "all" },
1570  { "write_sync", "no" },
1571  // bdb:
1572  { "path", ".genbank_cache" },
1573  { "mem_size", "20M" },
1574  { "log_file_max", "20M" },
1575  { "purge_batch_sleep", "500" }, // .5 sec
1576  { "purge_thread_delay", "3600" }, // 1 hour
1577  { "purge_clean_log", "16" },
1578  // netcache:
1579  { "connection_max_retries", "0" },
1580  { "connection_timeout", "0.3" },
1581  { "communication_timeout", "0.1" },
1582  { "max_connection_pool_size", "30" },
1583  { "max_find_lbname_retries", "2" },
1584  { "retry_delay", "0.001" },
1585 
1586  { 0, 0 }
1587 };
1589  // common:
1590  { "name", "ids" },
1591  { "timeout", "172800" }, // 2 days
1592  { "timestamp", "subkey check_expiration" /* purge_on_startup"*/ },
1593  // bdb:
1594  { "page_size", "small" },
1595 
1596  { 0, 0 }
1597 };
1599  // common:
1600  { "name", "blobs" },
1601  { "timeout", "432000" }, // 5 days
1602  { "timestamp", "onread expire_not_used" /* purge_on_startup"*/ },
1603 
1604  { 0, 0 }
1605 };
1607  // bdb:
1608  { "purge_thread", "yes" },
1609 
1610  { 0, 0 }
1611 };
1613  // bdb:
1614  { "purge_thread", "no" },
1615 
1616  { 0, 0 }
1617 };
1618 
1621  SCacheInfo::EReaderOrWriter reader_or_writer,
1622  SCacheInfo::EIdOrBlob id_or_blob)
1623 {
1624  const char* section_name;
1625  if ( id_or_blob == SCacheInfo::eIdCache ) {
1627  }
1628  else {
1630  }
1631  unique_ptr<SCacheInfo::TParams> section
1632  (GetCacheParamsCopy(src_params, section_name));
1633  if ( !section.get() ) {
1634  // disabled
1635  return 0;
1636  }
1637  // fill driver section with default values
1638  SCacheInfo::TParams* driver_params = GetDriverParams(section.get());
1640  if ( id_or_blob == SCacheInfo::eIdCache ) {
1642  }
1643  else {
1645  }
1646  if ( reader_or_writer == SCacheInfo::eCacheReader ) {
1648  }
1649  else {
1651  }
1652  return section.release();
1653 }
1654 
1655 
1657  EReaderOrWriter reader_or_writer,
1658  EIdOrBlob id_or_blob)
1659 {
1660  unique_ptr<TParams> cache_params
1661  (GetCacheParams(params, reader_or_writer, id_or_blob));
1662  if ( !cache_params.get() ) {
1663  return 0;
1664  }
1665  typedef CPluginManager<ICache> TCacheManager;
1667  _ASSERT(manager);
1668  return manager->CreateInstanceFromKey
1669  (cache_params.get(), NCBI_GBLOADER_READER_CACHE_PARAM_DRIVER);
1670 }
1671 
1672 
1674  const TPluginManagerParamTree* params)
1675 {
1676  const TPluginManagerParamTree* reader_params = params ?
1678  ICache* id_cache = 0;
1679  ICache* blob_cache = 0;
1680  unique_ptr<TParams> id_params
1681  (GetCacheParams(reader_params, eCacheReader, eIdCache));
1682  unique_ptr<TParams> blob_params
1683  (GetCacheParams(reader_params, eCacheReader, eBlobCache));
1684  _ASSERT(id_params.get());
1685  _ASSERT(blob_params.get());
1686  const TParams* share_id_param =
1688  bool share_id = !share_id_param ||
1689  NStr::StringToBool(share_id_param->GetValue().value);
1690  const TParams* share_blob_param =
1692  bool share_blob = !share_blob_param ||
1693  NStr::StringToBool(share_blob_param->GetValue().value);
1694  if (share_id || share_blob) {
1695  if ( share_id ) {
1696  ICache* cache = cache_manager.
1698  id_params.get());
1699  if ( cache ) {
1700  _ASSERT(!id_cache);
1701  id_cache = cache;
1702  }
1703  }
1704  if ( share_blob ) {
1705  ICache* cache = cache_manager.
1707  blob_params.get());
1708  if ( cache ) {
1709  _ASSERT(!blob_cache);
1710  blob_cache = cache;
1711  }
1712  }
1713  }
1714  if ( !id_cache ) {
1715  id_cache = CreateCache(reader_params, eCacheReader, eIdCache);
1716  if ( id_cache ) {
1717  cache_manager.RegisterCache(*id_cache,
1719  }
1720  }
1721  if ( !blob_cache ) {
1722  blob_cache = CreateCache(reader_params, eCacheReader, eBlobCache);
1723  if ( blob_cache ) {
1724  cache_manager.RegisterCache(*blob_cache,
1726  }
1727  }
1728  SetIdCache(id_cache);
1729  SetBlobCache(blob_cache);
1730 }
1731 
1732 
1734 {
1735  SetIdCache(0);
1736  SetBlobCache(0);
1737 }
1738 
1739 
1741 
1742 
1743 /// Class factory for Cache reader
1744 ///
1745 /// @internal
1746 ///
1747 using namespace objects;
1748 
1750  public CSimpleClassFactoryImpl<CReader, CCacheReader>
1751 {
1753 public:
1756  {
1757  }
1759  {
1760  }
1761 
1762 
1763  CReader*
1764  CreateInstance(const string& driver = kEmptyStr,
1766  const TPluginManagerParamTree* params = 0) const
1767  {
1768  if ( !driver.empty() && driver != m_DriverName ) {
1769  return 0;
1770  }
1771  if ( !version.Match(NCBI_INTERFACE_VERSION(CReader)) ) {
1772  return 0;
1773  }
1774  return new CCacheReader(params, driver);
1775  }
1776 };
1777 
1778 
1782 {
1784  method);
1785 }
1786 
1787 
1791 {
1792  NCBI_EntryPoint_CacheReader(info_list, method);
1793 }
1794 
1795 
1797 {
1798  RegisterEntryPoint<CReader>(NCBI_EntryPoint_CacheReader);
1799 }
1800 
1801 
TSat GetSat() const
Definition: blob_id.hpp:56
TSatKey GetSatKey() const
Definition: blob_id.hpp:64
TSubSat GetSubSat() const
Definition: blob_id.hpp:60
void SetIdCache(ICache *id_cache)
ICache * m_BlobCache
void SetBlobCache(ICache *blob_cache)
ICache * m_IdCache
CSimpleClassFactoryImpl< CReader, CCacheReader > TParent
CReader * CreateInstance(const string &driver=kEmptyStr, CVersionInfo version=NCBI_INTERFACE_VERSION(CReader), const TPluginManagerParamTree *params=0) const
Create instance of TDriver.
void x_ProcessBlob(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CNcbiIstream &stream)
ESwitch m_JoinedBlobVersion
bool LoadSeq_idSeq_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
bool LoadBlob(CReaderRequestResult &result, const TBlobId &blob_id)
bool LoadGis(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TGis &ret)
bool LoadBlobState(CReaderRequestResult &result, const TBlobId &blob_id)
bool LoadSeq_idGi(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
bool LoadBlobVersion(CReaderRequestResult &result, const TBlobId &blob_id)
bool LoadSeq_idTaxId(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
int GetRetryCount(void) const
bool LoadTaxIds(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TTaxIds &ret)
bool LoadSequenceLength(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void x_SetBlobVersionAsCurrent(CReaderRequestResult &result, const string &key, const string &subkey, TBlobVersion version)
virtual void ResetCache(void)
int GetMaximumConnectionsLimit(void) const
bool MayBeSkippedOnErrors(void) const
bool LoadAccVers(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TIds &ret)
bool LoadSeq_idLabel(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
bool LoadLabels(CReaderRequestResult &result, const TIds &ids, TLoaded &loaded, TLabels &ret)
bool LoadSeq_idAccVer(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
static bool NoNeedToSave(ECacheEntryType type)
void x_AddConnectionSlot(TConn conn)
static void GoingToLoad(ECacheEntryType type)
bool LoadSequenceType(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
bool LoadSeq_idBlob_ids(CReaderRequestResult &result, const CSeq_id_Handle &seq_id, const SAnnotSelector *sel)
All LoadXxx() methods should return false if there is no requested data in the reader.
bool LoadChunk(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id)
bool ReadSeq_ids(CReaderRequestResult &result, const string &key, CLoadLockSeqIds &ids)
void x_ConnectAtSlot(TConn conn)
virtual void InitializeCache(CReaderCacheManager &cache_manager, const TPluginManagerParamTree *params)
void x_DisconnectAtSlot(TConn conn, bool failed)
bool LoadSequenceHash(CReaderRequestResult &result, const CSeq_id_Handle &seq_id)
void x_RemoveConnectionSlot(TConn conn)
In-memory stream (a la strstream or stringstream)
CID2S_Seq_annot_Info –.
static bool IsFound(const TData &data)
TData GetAccVer(void) const
static const CSeq_id_Handle & GetAcc(const TData &data)
bool IsLoadedAccVer(void) const
bool SetLoadedAccVer(const TData &data)
bool SetLoadedBlob_ids(const TData &data, TExpirationTime expiration_time)
bool IsLoadedBlobState(void) const
bool IsLoadedBlobVersion(void) const
TData GetBlobVersion(void) const
bool IsLoadedChunk(void) const
const CTSE_Split_Info & GetSplitInfo(void) const
TBlobVersion GetKnownBlobVersion(void) const
bool SetLoadedGi(const TData &data)
bool IsLoadedGi(void) const
static bool IsFound(const TData &data)
static TGi GetGi(const TData &data)
bool SetLoadedHash(const TData &data)
bool IsLoadedHash(void) const
static int GetHash(const TData &data)
bool IsLoadedLabel(void) const
bool SetLoadedLabel(const TData &data)
TData GetLabel(void) const
bool IsLoadedLength(void) const
bool SetLoadedLength(const TData &data)
TData GetSeq_ids(void) const
bool SetLoadedSeq_ids(const TData &data)
bool IsLoadedTaxId(void) const
bool SetLoadedTaxId(const TData &data)
TData GetTaxId(void) const
bool SetLoadedType(const TData &data)
bool IsLoadedType(void) const
Data loader exceptions, used by GenBank loader.
CNcbiOstrstreamToString class helps convert CNcbiOstrstream to a string Sample usage:
Definition: ncbistre.hpp:802
CObjectIStreamAsnBinary –.
Definition: objistrasnb.hpp:59
CObjectIStream –.
Definition: objistr.hpp:93
CPluginManager<> –.
static bool IsExtAnnot(const TBlobId &blob_id)
virtual void ProcessStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CNcbiIstream &stream) const
Definition: processors.cpp:237
virtual EType GetType(void) const =0
virtual TMagic GetMagic(void) const =0
Note about the "buf_size" parameter for streams in this API.
Definition: rwstream.hpp:122
void LoadBlobVersion(CReaderRequestResult &result, const TBlobId &blob_id, const CReader *asking_reader=0)
const CProcessor & GetProcessor(CProcessor::EType type) const
Definition: dispatcher.cpp:219
static bool CannotProcess(const CSeq_id_Handle &sih)
Definition: dispatcher.cpp:261
virtual void RegisterCache(ICache &cache, ECacheType cache_type)=0
GBL::CInfo_Base::TExpirationTime TExpirationTime
vector< CSeq_id_Handle > TSeqIds
Definition: reader.hpp:96
CReadDispatcher * m_Dispatcher
Definition: reader.hpp:353
void SetAndSaveBlobVersion(CReaderRequestResult &result, const TBlobId &blob_id, TBlobVersion version) const
Definition: reader.cpp:1110
int TBlobVersion
Definition: reader.hpp:91
vector< CBlob_Info > TBlobIds
Definition: reader.hpp:97
void SetAndSaveBlobState(CReaderRequestResult &result, const TBlobId &blob_id, TBlobState blob_state) const
Definition: reader.cpp:1097
static int ReadInt(CNcbiIstream &stream)
Definition: reader.cpp:1378
vector< string > TLabels
Definition: reader.hpp:127
vector< TGi > TGis
Definition: reader.hpp:126
unsigned TConn
Definition: reader.hpp:87
vector< CSeq_id_Handle > TIds
Definition: reader.hpp:123
int TChunkId
Definition: reader.hpp:93
vector< bool > TLoaded
Definition: reader.hpp:124
int SetMaximumConnections(int max)
Definition: reader.cpp:228
vector< TTaxId > TTaxIds
Definition: reader.hpp:128
CRef –.
Definition: ncbiobj.hpp:618
CSafeStatic<>::
Template class helps to implement one driver class factory.
TSplitVersion GetSplitVersion(void) const
definition of a Culling tree
Definition: ncbi_tree.hpp:100
CVersionInfo –.
BLOB cache read/write/maintenance interface.
Definition: icache.hpp:64
virtual bool HasBlobs(const string &key, const string &subkey)=0
Check if any BLOB exists (any version)
virtual void GetBlobAccess(const string &key, TBlobVersion version, const string &subkey, SBlobAccessDescr *blob_descr)=0
Get BLOB access using BlobAccessDescr.
virtual void SetBlobVersionAsCurrent(const string &key, const string &subkey, TBlobVersion version)=0
Set current valid version for a BLOB.
A very basic data-read interface.
bool empty() const
Definition: map.hpp:149
#define Done
Definition: fastme.h:82
static CS_CONNECTION * conn
Definition: ct_dynamic.c:25
static int type
Definition: getdata.c:31
static const char * str(char *buf, int n)
Definition: stats.c:84
char data[12]
Definition: iconv.c:80
#define GI_FROM(T, value)
Definition: ncbimisc.hpp:1086
unsigned int TSeqPos
Type for sequence locations and lengths.
Definition: ncbimisc.hpp:875
#define ITERATE(Type, Var, Cont)
ITERATE macro to sequence through container elements.
Definition: ncbimisc.hpp:815
Int8 TIntId
Definition: ncbimisc.hpp:999
SStrictId_Tax::TId TTaxId
Taxon id type.
Definition: ncbimisc.hpp:1048
#define TAX_ID_FROM(T, value)
Definition: ncbimisc.hpp:1111
#define ZERO_GI
Definition: ncbimisc.hpp:1088
@ eDefault
Definition: ncbi_types.h:112
@ eOn
Definition: ncbi_types.h:111
@ eTakeOwnership
An object can take ownership of another.
Definition: ncbi_types.h:136
string
Definition: cgiapp.hpp:687
#define ERR_POST(message)
Error posting with file, line number information but without error codes.
Definition: ncbidiag.hpp:186
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
Definition: ncbiexpt.hpp:704
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
void Read(CObjectIStream &in, TObjectPtr object, const CTypeRef &type)
Definition: serial.cpp:60
bool GetBool(const string &driver_name, const string &param_name, EErrAction on_error, bool default_value, const list< string > *synonyms=NULL)
Utility function to get an integer element of parameter tree Throws an exception when mandatory param...
@ eErr_NoThrow
Return default value on error.
static CSeq_id_Handle GetGiHandle(TGi gi)
Faster way to create a handle for a gi.
bool IsGi(void) const
static CSeq_id_Handle GetHandle(const CSeq_id &id)
Normal way of getting a handle, works for any seq-id.
bool HaveMoreData(void)
const TNamedAnnotAccessions & GetNamedAnnotAccessions(void) const
void Reset(void)
Reset reference object.
Definition: ncbiobj.hpp:773
#define NCBI_PARAM_TYPE(section, name)
Generate typename for a parameter from its {section, name} attributes.
Definition: ncbi_param.hpp:149
@ eParam_NoThread
Do not use per-thread values.
Definition: ncbi_param.hpp:418
static void NCBI_EntryPointImpl(TDriverInfoList &info_list, EEntryPointRequest method)
Entry point implementation.
#define NCBI_INTERFACE_VERSION(iface)
Macro to construct CVersionInfo class using interface name (relies on CInterfaceVersion class)
list< SDriverInfo > TDriverInfoList
List of driver information.
EEntryPointRequest
Actions performed by the entry point.
uint8_t Uint1
1-byte (8-bit) unsigned integer
Definition: ncbitype.h:99
int32_t Int4
4-byte (32-bit) signed integer
Definition: ncbitype.h:102
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
int64_t Int8
8-byte (64-bit) signed integer
Definition: ncbitype.h:104
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define END_SCOPE(ns)
End the previously defined scope.
Definition: ncbistl.hpp:75
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
#define BEGIN_SCOPE(ns)
Define a new scope.
Definition: ncbistl.hpp:72
ERW_Result
Result codes for I/O operations.
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
Definition: ncbistre.hpp:146
@ eRW_Eof
End of data, should be considered permanent.
@ eRW_Success
Everything is okay, I/O completed.
static bool StringToBool(const CTempString str)
Convert string to bool.
Definition: ncbistr.cpp:2821
#define kEmptyStr
Definition: ncbistr.hpp:123
static enable_if< is_arithmetic< TNumeric >::value||is_convertible< TNumeric, Int8 >::value, string >::type NumericToString(TNumeric value, TNumToStringFlags flags=0, int base=10)
Convert numeric value to string.
Definition: ncbistr.hpp:673
const TTreeType * FindNode(const TKeyType &key, TNodeSearchMode sflag=eImmediateAndTop) const
Search for node.
Definition: ncbi_tree.hpp:970
void AddNode(TTreeType *subnode)
Add new subnode.
Definition: ncbi_tree.hpp:743
const TTreeType * FindSubNode(const TKeyType &key) const
Non recursive linear scan of all subnodes, with key comparison.
Definition: ncbi_tree.hpp:940
const TValue & GetValue(void) const
Return node's value.
Definition: ncbi_tree.hpp:184
TValue TValueType
Definition: ncbi_tree.hpp:102
EMol
molecule class in living organism
Definition: Seq_inst_.hpp:108
unsigned int
A callback function used to compare two keys in a database.
Definition: types.hpp:1210
Interfaces for a local cache of versioned binary large objects (BLOBS).
@ eExpire_normal
Definition: info_cache.hpp:79
char * buf
int i
yy_size_t n
static void hex(unsigned char c)
Definition: mdb_dump.c:56
static MDB_envinfo info
Definition: mdb_load.c:37
static int version
Definition: mdb_load.c:29
const struct ncbi::grid::netcache::search::fields::SIZE size
const struct ncbi::grid::netcache::search::fields::KEY key
const struct ncbi::grid::netcache::search::fields::SUBKEY subkey
string s_Value(TValue value)
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
Defines: CTimeFormat - storage class for time format.
T min(T x_, T y_)
@ kMain_ChunkId
Definition: blob_id.hpp:147
@ kDelayedMain_ChunkId
Definition: blob_id.hpp:149
static pcre_uint8 * buffer
Definition: pcretest.c:1051
void NCBI_EntryPoint_xreader_cache(CPluginManager< CReader >::TDriverInfoList &info_list, CPluginManager< CReader >::EEntryPointRequest method)
static const SPluginParams::SDefaultValue s_DefaultBlobParams[]
static const SPluginParams::SDefaultValue s_DefaultIdParams[]
void GenBankReaders_Register_Cache(void)
static const SPluginParams::SDefaultValue s_DefaultReaderParams[]
static SCacheInfo::TParams * GetCacheParamsCopy(const SCacheInfo::TParams *src_params, const char *section_name)
static SCacheEntryAccessCount s_CacheEntryAccessCounts[CCacheReader::eCacheEntry_Count]
SCacheInfo::TParams * GetCacheParams(const SCacheInfo::TParams *src_params, SCacheInfo::EReaderOrWriter reader_or_writer, SCacheInfo::EIdOrBlob id_or_blob)
NCBI_PARAM_DECL(int, GENBANK, CACHE_DEBUG)
void NCBI_EntryPoint_CacheReader(CPluginManager< CReader >::TDriverInfoList &info_list, CPluginManager< CReader >::EEntryPointRequest method)
static const SPluginParams::SDefaultValue s_DefaultParams[]
NCBI_PARAM_DEF_EX(int, GENBANK, CACHE_DEBUG, 0, eParam_NoThread, GENBANK_CACHE_DEBUG)
static bool IsDisabledCache(const SCacheInfo::TParams *params)
static SCacheInfo::TParams * GetDriverParams(SCacheInfo::TParams *params)
static const SPluginParams::SDefaultValue s_DefaultWriterParams[]
static const size_t kHashLimit
#define NCBI_GBLOADER_READER_CACHE_PARAM_ID_SECTION
#define NCBI_GBLOADER_WRITER_CACHE_PARAM_SHARE
#define NCBI_GBLOADER_READER_CACHE_PARAM_DRIVER
#define NCBI_GBLOADER_READER_CACHE_PARAM_JOINED_BLOB_VERSION
#define NCBI_GBLOADER_READER_CACHE_DRIVER_NAME
#define NCBI_GBLOADER_READER_CACHE_PARAM_BLOB_SECTION
@ eAlreadyLoaded
Reader-writer based streams.
static int GetDebugLevel()
Definition: snpptis.cpp:81
Better replacement of GetAccVer(), this method should be defined in data loaders, GetAccVer() is left...
Better replacement of GetGi(), this method should be defined in data loaders, GetGi() is left for com...
Better replacement of GetSequenceHash(), this method should be defined in data loaders,...
Better replacement of GetSequenceType(), this method should be defined in data loaders,...
BLOB access descriptor.
Definition: icache.hpp:332
unsigned maximum_age
Set to a non-zero value to return a version not older than the specified value.
Definition: icache.hpp:355
SAnnotSelector –.
atomic< Uint8 > save_count
atomic< Uint8 > load_count
structure for common cache reader&writer implementation
static void GetBlob_idsSubkey(const SAnnotSelector *sel, string &subkey, string &true_subkey)
Id cache subkeys:
CConfig::TParamTree TParams
static ICache * CreateCache(const TParams *params, EReaderOrWriter reader_or_writer, EIdOrBlob id_or_blob)
static string GetBlobSubkey(CLoadLockBlob &blob, int chunk_id=kMain_ChunkId)
BLOB cache subkeys:
static string GetIdKey(const CSeq_id &id)
Return Id cache key string based on CSeq_id of gi.
static const char * GetAccVerSubkey(void)
static const char * GetTypeSubkey(void)
static const char * GetBlobStateSubkey(void)
static const char * GetGiSubkey(void)
static int GetDebugLevel(void)
static const char * GetHashSubkey(void)
static const char * GetLabelSubkey(void)
static const char *const BLOB_IDS_SUBKEY
static const char * GetLengthSubkey(void)
static const char * GetTaxIdSubkey(void)
static const char * GetSeq_idsSubkey(void)
static const char * GetBlobVersionSubkey(void)
static string GetBlobKey(const CBlob_id &blob_id)
Return BLOB cache key string based on Sat() and SatKey()
static const int BLOB_IDS_MAGIC
static const string & SetDefaultValue(TParams *params, const string &name, const char *value)
static const TParams * FindSubNode(const TParams *params, const string &name)
static const string & SetDefaultValue(TParams *params, const SDefaultValue &value)
static TParams * SetSubNode(TParams *params, const string &name, const char *default_value="")
static TParams * FindSubNode(TParams *params, const string &name)
SCacheInfo::TParams TParams
static TParams * SetSubSection(TParams *params, const string &name)
static void SetDefaultValues(TParams *params, const SDefaultValue *values)
Definition: _hash_fun.h:40
Definition: type.c:6
#define _ASSERT
else result
Definition: token2.c:20
Modified on Thu May 30 12:27:16 2024 by modify_doxy.py rev. 669887