NCBI C++ ToolKit
vdbread.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: vdbread.cpp 102235 2024-04-10 14:30:48Z vasilche $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Authors: Eugene Vasilchenko
27  *
28  * File Description:
29  * Access to SRA files
30  *
31  */
32 
33 #include <ncbi_pch.hpp>
35 #include <common/ncbi_source_ver.h>
37 
38 #include <klib/rc.h>
39 #include <klib/log.h>
40 #include <klib/text.h>
41 #include <klib/sra-release-version.h>
42 #include <kfg/config.h>
43 #include <kdb/manager.h>
44 #include <kdb/database.h>
45 #include <kdb/kdb-priv.h>
46 #include <kns/manager.h>
47 #include <kns/http.h>
48 #include <kns/tls.h>
49 
50 #include <vfs/manager-priv.h>
51 #include <vfs/manager.h>
52 #include <vfs/path.h>
53 #include <vfs/resolver.h>
54 
55 #include <sra/sradb-priv.h>
56 
57 #include <vdb/vdb-priv.h>
58 #include <vdb/manager.h>
59 #include <vdb/database.h>
60 #include <vdb/schema.h>
61 #include <vdb/table.h>
62 #include <vdb/cursor.h>
63 
64 
65 #include <corelib/ncbimtx.hpp>
66 #include <corelib/ncbifile.hpp>
67 #include <corelib/ncbi_param.hpp>
68 #include <corelib/request_ctx.hpp>
71 #include <objects/seq/seq__.hpp>
74 #include <sra/error_codes.hpp>
75 #include <util/random_gen.hpp>
76 
77 #include <cstring>
78 #include <algorithm>
79 #include <thread>
80 
82 
83 #define NCBI_USE_ERRCODE_X VDBReader
85 
87 
88 class CSeq_entry;
89 
90 
91 NCBI_PARAM_DECL(int, VDB, DIAG_HANDLER);
92 NCBI_PARAM_DEF(int, VDB, DIAG_HANDLER, 1);
93 
94 
95 static int s_GetDiagHandler(void)
96 {
97  static CSafeStatic<NCBI_PARAM_TYPE(VDB, DIAG_HANDLER)> s_Value;
98  return s_Value->Get();
99 }
100 
101 
102 
104 NCBI_PARAM_DEF_EX(int, VDB, DEBUG, 0, eParam_NoThread, VDB_DEBUG);
105 
106 
107 static int s_GetDebugLevel(void)
108 {
109  static int value = NCBI_PARAM_TYPE(VDB, DEBUG)::GetDefault();
110  return value;
111 }
112 
113 
114 NCBI_PARAM_DECL(bool, VDB, DISABLE_PAGEMAP_THREAD);
115 NCBI_PARAM_DEF(bool, VDB, DISABLE_PAGEMAP_THREAD, false);
116 
117 
118 
119 DEFINE_SRA_REF_TRAITS(VDBManager, const);
120 DEFINE_SRA_REF_TRAITS(VDatabase, const);
121 DEFINE_SRA_REF_TRAITS(VTable, const);
122 DEFINE_SRA_REF_TRAITS(VCursor, const);
123 DEFINE_SRA_REF_TRAITS(KIndex, const);
125 DEFINE_SRA_REF_TRAITS(KDBManager, const);
126 DEFINE_SRA_REF_TRAITS(KNSManager, );
127 DEFINE_SRA_REF_TRAITS(VFSManager, );
128 DEFINE_SRA_REF_TRAITS(VPath, const);
130 
131 //#define VDB_RANDOM_FAILS 1
132 
133 #define FAILS_VAR(name, suffix) NCBI_NAME3(VDB_, name, suffix)
134 
135 #define VDB_RESOLVE_RANDOM_FAILS 1
136 #define VDB_OPEN_RANDOM_FAILS 1
137 #define VDB_SCHEMA_RANDOM_FAILS 1
138 #define VDB_READ_RANDOM_FAILS 1
139 
140 #define VDB_RESOLVE_RANDOM_FAILS_FREQUENCY 20
141 #define VDB_RESOLVE_RANDOM_FAILS_RECOVER 10
142 #define VDB_OPEN_RANDOM_FAILS_FREQUENCY 10
143 #define VDB_OPEN_RANDOM_FAILS_RECOVER 5
144 #define VDB_SCHEMA_RANDOM_FAILS_FREQUENCY 50
145 #define VDB_SCHEMA_RANDOM_FAILS_RECOVER 100
146 #define VDB_READ_RANDOM_FAILS_FREQUENCY 20000
147 #define VDB_READ_RANDOM_FAILS_RECOVER 1000
148 
149 #if defined(VDB_RANDOM_FAILS)
150 static bool s_Fails(unsigned frequency)
151 {
152  static CSafeStatic<CRandom> s_Random([]()->CRandom*{CRandom* r=new CRandom();r->Randomize();return r;}, nullptr);
153  return s_Random->GetRandIndex(frequency) == 0;
154 }
155 # define SIMULATE_ERROR(name) \
156  if ( !(FAILS_VAR(name, _RANDOM_FAILS)) ) { \
157  } \
158  else { \
159  static thread_local unsigned recover_counter = 0; \
160  if ( recover_counter > 0 ) { \
161  --recover_counter; \
162  } \
163  else { \
164  if ( s_Fails(FAILS_VAR(name, _RANDOM_FAILS_FREQUENCY)) ) { \
165  recover_counter = (FAILS_VAR(name, _RANDOM_FAILS_RECOVER)); \
166  ERR_POST("VDB: Simulated " #name " failure: "<<CStackTrace()); \
167  NCBI_THROW(CSraException, eOtherError, \
168  "Simulated " #name " failure"); \
169  } \
170  } \
171  }
172 #else
173 # define SIMULATE_ERROR(name) ((void)0)
174 #endif
175 
176 #define SIMULATE_RESOLVE_ERROR() SIMULATE_ERROR(RESOLVE)
177 #define SIMULATE_OPEN_ERROR() SIMULATE_ERROR(OPEN)
178 #define SIMULATE_SCHEMA_ERROR() SIMULATE_ERROR(SCHEMA)
179 #define SIMULATE_READ_ERROR() SIMULATE_ERROR(READ)
180 
181 
182 /////////////////////////////////////////////////////////////////////////////
183 // CKConfig
184 
186 {
187  KConfig* cfg;
188  if ( rc_t rc = KConfigMake(&cfg, 0) ) {
189  *x_InitPtr() = 0;
190  NCBI_THROW2(CSraException, eInitFailed,
191  "Cannot create KConfig", rc);
192  }
193  *x_InitPtr() = cfg;
194 }
195 
196 
198 {
199  *x_InitPtr() = const_cast<KConfig*>(VFSManagerGetConfig(CVFSManager(mgr)));
200  if ( rc_t rc = KConfigAddRef(*this) ) {
201  *x_InitPtr() = 0;
202  NCBI_THROW2(CSraException, eInitFailed,
203  "Cannot get reference to KConfig", rc);
204  }
205 }
206 
207 
209 {
210  if ( rc_t rc = KConfigMake(x_InitPtr(), NULL) ) {
211  NCBI_THROW2(CSraException, eInitFailed,
212  "Cannot create KConfig singleton", rc);
213  }
214 }
215 
216 
217 void CKConfig::Commit() const
218 {
219  if ( rc_t rc = KConfigCommit(const_cast<KConfig*>(GetPointer())) ) {
220  NCBI_THROW2(CSraException, eOtherError,
221  "CKConfig: Cannot commit config changes", rc);
222  }
223 }
224 
225 
226 /////////////////////////////////////////////////////////////////////////////
227 // CVFSManager
228 
230 {
231  x_InitNew();
232 }
233 
234 
236 {
237  x_InitNew();
238 }
239 
240 
242 {
243  if ( rc_t rc = VFSManagerMake(x_InitPtr()) ) {
244  *x_InitPtr() = 0;
245  NCBI_THROW2(CSraException, eInitFailed,
246  "Cannot create VFSManager", rc);
247  }
248 }
249 
250 
252 {
253  if ( rc_t rc = KDBManagerGetVFSManager(CKDBManager(mgr), x_InitPtr()) ) {
254  *x_InitPtr() = 0;
255  NCBI_THROW2(CSraException, eInitFailed,
256  "Cannot get VFSManager", rc);
257  }
258 }
259 
260 
261 /////////////////////////////////////////////////////////////////////////////
262 // CKDBManager
263 
264 
266 {
267  if ( rc_t rc = VDBManagerGetKDBManagerRead(mgr, x_InitPtr()) ) {
268  *x_InitPtr() = 0;
269  NCBI_THROW2(CSraException, eInitFailed,
270  "Cannot get KDBManager", rc);
271  }
272 }
273 
274 
275 /////////////////////////////////////////////////////////////////////////////
276 // CKNSManager
277 
278 
280 {
281  if ( rc_t rc = KNSManagerMake(x_InitPtr()) ) {
282  *x_InitPtr() = 0;
283  NCBI_THROW2(CSraException, eInitFailed,
284  "Cannot make KNSManager", rc);
285  }
286 }
287 
288 
290 {
291  if ( rc_t rc = VFSManagerGetKNSMgr(mgr, x_InitPtr()) ) {
292  *x_InitPtr() = 0;
293  NCBI_THROW2(CSraException, eInitFailed,
294  "Cannot get KNSManager", rc);
295  }
296 }
297 
298 
299 /////////////////////////////////////////////////////////////////////////////
300 // CVPath
301 
302 CVPath::CVPath(const CVFSManager& mgr, const string& path, EType type)
303 {
304  x_Init(mgr, path, type);
305 }
306 
307 
308 CVPath::CVPath(const string& path, EType type)
309 {
311 }
312 
313 
314 void CVPath::x_Init(const CVFSManager& mgr, const string& path, EType type)
315 {
316  VPath* vpath = 0;
317  if ( type == eSys ) {
318  if ( rc_t rc = VFSManagerMakeSysPath(mgr, &vpath, path.c_str()) ) {
319  *x_InitPtr() = 0;
320  NCBI_THROW2_FMT(CSraException, eInitFailed,
321  "Cannot create sys VPath: "<<path, rc);
322  }
323  }
324  else if ( type == eAcc ) {
325  if ( rc_t rc = VFSManagerMakeAccPath(mgr, &vpath, path.c_str()) ) {
326  *x_InitPtr() = 0;
327  NCBI_THROW2_FMT(CSraException, eInitFailed,
328  "Cannot create acc VPath: "<<path, rc);
329  }
330  }
331  else {
332  if ( rc_t rc = VFSManagerMakePath(mgr, &vpath, path.c_str()) ) {
333  *x_InitPtr() = 0;
334  NCBI_THROW2_FMT(CSraException, eInitFailed,
335  "Cannot create VPath: "<<path, rc);
336  }
337  }
338  *x_InitPtr() = vpath;
339 }
340 
341 
343 {
344  char buffer[32];
345  size_t size;
346  if ( VPathReadScheme(*this, buffer, sizeof(buffer), &size) != 0 ) {
347  return false;
348  }
349  if ( size == 4 && memcmp(buffer, "file", 4) == 0 ) {
350  return true;
351  }
352  if ( size == 9 && memcmp(buffer, "ncbi-file", 9) == 0 ) {
353  return true;
354  }
355  return false;
356 }
357 
358 
360 {
361  const String* str = 0;
362  if (type == eSys && IsLocalFile()) {
363  if (rc_t rc = VPathMakeSysPath(*this, &str)) {
364  NCBI_THROW2(CSraException, eOtherError,
365  "Cannot get path from VPath", rc);
366  }
367  }
368  else {
369  if (rc_t rc = VPathMakeString(*this, &str)) {
370  NCBI_THROW2(CSraException, eOtherError,
371  "Cannot get path from VPath", rc);
372  }
373  }
374  string ret(str->addr, str->size);
375  StringWhack(str);
376  return ret;
377 }
378 
379 
380 #ifdef NCBI_OS_MSWIN
381 static inline
382 bool s_HasWindowsDriveLetter(const string& s)
383 {
384  // first symbol is letter, and second symbol is colon (':')
385  return s.length() >= 2 && isalpha(s[0]&0xff) && s[1] == ':';
386 }
387 #endif
388 
389 
390 bool CVPath::IsPlainAccession(const string& acc_or_path)
391 {
392 #ifdef NCBI_OS_MSWIN
393  return !s_HasWindowsDriveLetter(acc_or_path) &&
394  acc_or_path.find_first_of("/\\") == NPOS;
395 #else
396  return acc_or_path.find('/') == NPOS;
397 #endif
398 }
399 
400 
401 #ifdef NCBI_OS_MSWIN
402 string CVPath::ConvertSysPathToPOSIX(const string& sys_path)
403 {
404  // Convert Windows path with drive letter
405  // C:\Users\Public -> /C/Users/Public
406  if ( sys_path[0] == 'h' &&
407  (NStr::StartsWith(sys_path, "http://") ||
408  NStr::StartsWith(sys_path, "https://")) ) {
409  return sys_path;
410  }
411  try {
412  string path = CDirEntry::CreateAbsolutePath(sys_path);
413  replace(path.begin(), path.end(), '\\', '/');
414  if (s_HasWindowsDriveLetter(path)) {
415  // move drive letter from first symbol to second (in place of ':')
416  path[1] = toupper(path[0] & 0xff);
417  // add leading slash
418  path[0] = '/';
419  }
420  return path;
421  }
422  catch (exception&) {
423  // CDirEntry::CreateAbsolutePath() can fail on URL for remote access
424  return sys_path;
425  }
426 }
427 
428 
429 string CVPath::ConvertAccOrSysPathToPOSIX(const string& acc_or_path)
430 {
431  return IsPlainAccession(acc_or_path) ?
432  acc_or_path :
433  ConvertSysPathToPOSIX(acc_or_path);
434 }
435 #endif
436 
437 
438 /////////////////////////////////////////////////////////////////////////////
439 // CVResolver
440 
442  : m_Mgr(mgr)
443 {
444  if ( rc_t rc = VFSManagerGetResolver(mgr, x_InitPtr()) ) {
445  *x_InitPtr() = 0;
446  NCBI_THROW2(CSraException, eInitFailed,
447  "Cannot get VResolver", rc);
448  }
449 }
450 
451 
453  : m_Mgr(mgr)
454 {
455  if ( rc_t rc = VFSManagerMakeResolver(mgr, x_InitPtr(), cfg) ) {
456  *x_InitPtr() = 0;
457  NCBI_THROW2(CSraException, eInitFailed,
458  "Cannot create VResolver", rc);
459  }
460 }
461 
462 
463 string CVResolver::Resolve(const string& acc_or_path) const
464 {
465  if ( !CVPath::IsPlainAccession(acc_or_path) ) {
466  // already a path
467  return acc_or_path;
468  }
470  CVPath acc(m_Mgr, acc_or_path, CVPath::eAcc);
471  const VPath* path;
472  rc_t rc = VResolverLocal(*this, acc, &path);
473  if ( rc ) {
474  if ( GetRCObject(rc) == rcName &&
475  GetRCState(rc) == rcNotFound &&
476  GetRCContext(rc) == rcResolving ) {
477  // continue with other ways to resolve
478  }
479  else {
480  NCBI_THROW2_FMT(CSraException, eOtherError,
481  "VResolverLocal("<<acc_or_path<<") failed", rc);
482  }
483  rc = VResolverRemote(*this, eProtocolNone, acc, &path);
484  }
485  if ( rc ) {
486  CHECK_VDB_TIMEOUT_FMT("Cannot find acc path: "<<acc_or_path, rc);
487  if ( GetRCObject(rc) == rcName &&
488  GetRCState(rc) == rcNotFound &&
489  GetRCContext(rc) == rcResolving ) {
490  // continue with other ways to resolve
491  }
492  else {
493  NCBI_THROW2_FMT(CSraException, eOtherError,
494  "VResolverRemote("<<acc_or_path<<") failed", rc);
495  }
496  if ( CDirEntry(acc_or_path).Exists() ) {
497  // local file
498  return acc_or_path;
499  }
500  // couldnt resolve with any way
501  NCBI_THROW2_FMT(CSraException, eNotFoundDb,
502  "Cannot find acc path: "<<acc_or_path, rc);
503  }
504  return CVPath(path).ToString();
505 }
506 
507 
508 /////////////////////////////////////////////////////////////////////////////
509 // CVDBMgr
510 
512  : m_Resolver(null)
513 {
514  x_Init();
515 }
516 
517 
518 string CVDBMgr::FindAccPath(const string& acc) const
519 {
520  if ( !m_Resolver ) {
522  }
523  return m_Resolver.Resolve(acc);
524 }
525 
526 
527 string CVDBMgr::FindDereferencedAccPath(const string& acc_or_path) const
528 {
529  string path;
530  if ( CVPath::IsPlainAccession(acc_or_path) ) {
531  // resolve VDB accessions
532  path = FindAccPath(acc_or_path);
533  if ( s_GetDebugLevel() >= 2 ) {
534  LOG_POST_X(4, "CVDBMgr: resolved accession: " << acc_or_path << " -> " << path);
535  }
536  }
537  else {
538  // real path, http:, etc.
539  path = acc_or_path;
540  }
541 
542  // resolve symbolic links for correct timestamp and longer-living reference
543  CDirEntry de(path);
544  if ( de.Exists() ) {
545  de.DereferencePath();
546  if ( de.GetPath() != path ) {
547  path = de.GetPath();
548  if ( s_GetDebugLevel() >= 2 ) {
549  LOG_POST_X(5, "CVDBMgr: resolved file link: " << acc_or_path << " -> " << path);
550  }
551  }
552  }
553  return path;
554 }
555 
556 
557 CTime CVDBMgr::GetTimestamp(const string& path) const
558 {
559  CTime timestamp;
560  if ( path[0] == 'h' &&
561  (NStr::StartsWith(path, "http://") ||
562  NStr::StartsWith(path, "https://")) ) {
563  // try http:
564  timestamp = GetURLTimestamp(path);
565  }
566  else {
567  // try direct file access
568  if ( !CDirEntry(path).GetTime(&timestamp) ) {
569  NCBI_THROW_FMT(CSraException, eInitFailed,
570  "Cannot get timestamp of local path: " << path);
571  }
572  timestamp.ToUniversalTime();
573  }
574  _ASSERT(!timestamp.IsEmpty());
575  _ASSERT(timestamp.IsUniversalTime());
576  if ( s_GetDebugLevel() >= 2 ) {
577  LOG_POST_X(3, "CVDBMgr: timestamp of " << path << ": " << CTime(timestamp).ToLocalTime());
578  }
579  return timestamp;
580 }
581 
582 
583 DECLARE_SRA_REF_TRAITS(KClientHttpRequest, );
584 DECLARE_SRA_REF_TRAITS(KClientHttpResult, );
585 
586 DEFINE_SRA_REF_TRAITS(KClientHttpRequest, );
587 DEFINE_SRA_REF_TRAITS(KClientHttpResult, );
588 
590  : public CSraRef<KClientHttpRequest>
591 {
592 public:
593  CClientHttpRequest(const CKNSManager& mgr, const string& path)
594  {
595  const ver_t kHTTP_1_1 = 0x01010000;
596  if ( rc_t rc = KNSManagerMakeClientRequest(mgr, x_InitPtr(),
597  kHTTP_1_1, nullptr,
598  "%s", path.c_str()) ) {
599  *x_InitPtr() = 0;
600  CHECK_VDB_TIMEOUT("Cannot create http request", rc);
601  NCBI_THROW2(CSraException, eInitFailed,
602  "Cannot create http client request", rc);
603  }
604  }
605 
606 private:
607 };
608 
609 
611  : public CSraRef<KClientHttpResult>
612 {
613 public:
614  enum EHead {
615  eHead
616  };
617 
619  {
620  if ( rc_t rc = KClientHttpRequestHEAD(request, x_InitPtr()) ) {
621  *x_InitPtr() = 0;
622  CHECK_VDB_TIMEOUT("Cannot get http HEAD", rc);
623  NCBI_THROW2(CSraException, eInitFailed,
624  "Cannot get http HEAD", rc);
625  }
626  }
627 
628 private:
629 };
630 
631 
632 CTime CVDBMgr::GetURLTimestamp(const string& path) const
633 {
634  CVFSManager vfs(*this);
635  CKNSManager kns(vfs);
636  CClientHttpRequest request(kns, path);
638  char buffer[99];
639  size_t size;
640  if ( rc_t rc = KClientHttpResultGetHeader(result, "Last-Modified",
641  buffer, sizeof(buffer), &size) ) {
642  CHECK_VDB_TIMEOUT("No Last-Modified header in HEAD response", rc);
643  NCBI_THROW2(CSraException, eNotFoundValue,
644  "No Last-Modified header in HEAD response", rc);
645  }
647  CTimeFormat fmt("w, d b Y h:m:s Z"); // standard Last-Modified HTTP header format
648  return CTime(str, fmt);
649 }
650 
651 
652 //#define GUARD_SDK
653 #ifdef NCBI_COMPILER_MSVC
654 //# define GUARD_SDK_GET
655 #endif
656 
657 #ifdef GUARD_SDK
658 # define DECLARE_SDK_GUARD() CFastMutexGuard guard(sx_SDKMutex)
659 #else
660 # define DECLARE_SDK_GUARD()
661 #endif
662 
663 #ifdef GUARD_SDK_GET
664 # define DECLARE_SDK_GET_GUARD() CFastMutexGuard guard(sx_SDKMutex)
665 #else
666 # define DECLARE_SDK_GET_GUARD()
667 #endif
668 
669 
670 /////////////////////////////////////////////////////////////////////////////
671 // VDB library initialization code
672 // similar code is located in bamread.cpp
673 /////////////////////////////////////////////////////////////////////////////
674 
676 
677 static char s_VDBVersion[32]; // enough for 255.255.65535-dev4000000000
678 
679 static
681 {
682  if ( !s_VDBVersion[0] ) {
683  ostringstream s;
684  {{ // format VDB version string
685  SraReleaseVersion release_version;
686  SraReleaseVersionGet(&release_version);
687  s << (release_version.version>>24) << '.'
688  << ((release_version.version>>16)&0xff) << '.'
689  << (release_version.version&0xffff);
690  if ( release_version.revision != 0 ||
691  release_version.type != SraReleaseVersion::eSraReleaseVersionTypeFinal ) {
692  const char* type = "";
693  switch ( release_version.type ) {
694  case SraReleaseVersion::eSraReleaseVersionTypeDev: type = "dev"; break;
695  case SraReleaseVersion::eSraReleaseVersionTypeAlpha: type = "a"; break;
696  case SraReleaseVersion::eSraReleaseVersionTypeBeta: type = "b"; break;
697  case SraReleaseVersion::eSraReleaseVersionTypeRC: type = "RC"; break;
698  default: type = ""; break;
699  }
700  s << '-' << type << release_version.revision;
701  }
702  }}
703  string v = s.str();
704  if ( !v.empty() ) {
705  if ( v.size() >= sizeof(s_VDBVersion) ) {
706  v.resize(sizeof(s_VDBVersion)-1);
707  }
708  copy(v.begin()+1, v.end(), s_VDBVersion+1);
709  s_VDBVersion[0] = v[0];
710  }
711  }
712 }
713 
714 struct SVDBSeverityTag {
715  const char* tag;
717 };
718 static const SVDBSeverityTag kSeverityTags[] = {
719  { "err:", Error },
720  { "int:", Error },
721  { "sys:", Error },
722  { "info:", Info },
723  { "warn:", Warning },
724  { "debug:", Trace },
725  { "fatal:", Fatal },
726 };
728 {
729  if ( !token.empty() && token[token.size()-1] == ':' ) {
730  for ( auto& tag : kSeverityTags ) {
731  if ( token == tag.tag ) {
732  return &tag;
733  }
734  }
735  }
736  return 0;
737 }
738 
739 #ifndef NCBI_THREADS
740 static thread::id s_DiagCheckThreadID;
741 #endif
742 
743 static inline void s_InitDiagCheck()
744 {
745 #ifndef NCBI_THREADS
746  s_DiagCheckThreadID = this_thread::get_id();
747 #endif
748 }
749 
750 static inline bool s_DiagIsSafe()
751 {
752 #ifndef NCBI_THREADS
753  return s_DiagCheckThreadID == this_thread::get_id();
754 #else
755  return true;
756 #endif
757 }
758 
759 
760 static
761 rc_t VDBLogWriter(void* /*data*/, const char* buffer, size_t size, size_t* written)
762 {
763  CTempString msg(buffer, size);
765  CNcbiDiag::FManip sev_manip = Error;
766 
767  for ( SIZE_TYPE token_pos = 0, token_end; token_pos < msg.size(); token_pos = token_end + 1 ) {
768  token_end = msg.find(' ', token_pos);
769  if ( token_end == NPOS ) {
770  token_end = msg.size();
771  }
772  if ( auto tag = s_GetVDBSeverityTag(CTempString(msg, token_pos, token_end-token_pos)) ) {
773  sev_manip = tag->manip;
774  break;
775  }
776  }
777  if ( sev_manip == Trace ) {
778  if ( s_DiagIsSafe() ) {
779  _TRACE("VDB "<<s_VDBVersion<<": "<<msg);
780  }
781  }
782  else {
783  if ( s_DiagIsSafe() ) {
784  ERR_POST_X(2, sev_manip<<"VDB "<<s_VDBVersion<<": "<<msg);
785  }
786  }
787  *written = size;
788  return 0;
789 }
790 
791 
793 {
794  CKConfig config(null);
796  string host = app->GetConfig().GetString("CONN", "HTTP_PROXY_HOST", kEmptyStr);
797  int port = app->GetConfig().GetInt("CONN", "HTTP_PROXY_PORT", 0);
798  if ( !host.empty() && port != 0 ) {
800  string path = host + ':' + NStr::IntToString(port);
801  if ( rc_t rc = KConfigWriteString(config,
802  "/http/proxy/path", path.c_str()) ) {
803  NCBI_THROW2(CSraException, eInitFailed,
804  "Cannot set KConfig proxy path", rc);
805  }
806  if ( rc_t rc = KConfigWriteBool(config,
807  "/http/proxy/enabled", true) ) {
808  NCBI_THROW2(CSraException, eInitFailed,
809  "Cannot set KConfig proxy enabled", rc);
810  }
811  }
812  }
813  return config;
814 }
815 
816 
817 static DECLARE_TLS_VAR(const CRequestContext*, s_LastRequestContext);
818 static DECLARE_TLS_VAR(CRequestContext::TVersion, s_LastRequestContextVersion);
819 
820 /*
821 // Performance collecting code
822 static struct SReport {
823  size_t count;
824  double time;
825  SReport() : count(0), time(0) {}
826  ~SReport() {
827  LOG_POST("GetRequestContext() called "<<count<<" times, spent "<<time<<"s");
828  }
829 } dummy;
830 */
831 
832 static void s_UpdateVDBRequestContext(void)
833 {
834  // Performance collecting code
835  // CStopWatch sw(CStopWatch::eStart);
837  // Performance collecting code
838  // dummy.count += 1; dummy.time += sw.Elapsed();
839  auto req_ctx_version = req_ctx.GetVersion();
840  if ( &req_ctx == s_LastRequestContext && req_ctx_version == s_LastRequestContextVersion ) {
841  return;
842  }
843  _TRACE("CVDBMgr: Updating request context with version: "<<req_ctx_version);
844  s_LastRequestContext = &req_ctx;
845  s_LastRequestContextVersion = req_ctx_version;
847  if ( req_ctx.IsSetSessionID() ) {
848  _TRACE("CVDBMgr: Updating session ID: "<<req_ctx.GetSessionID());
849  KNSManagerSetSessionID(kns_mgr, req_ctx.GetSessionID().c_str());
850  }
851  if ( req_ctx.IsSetClientIP() ) {
852  _TRACE("CVDBMgr: Updating client IP: "<<req_ctx.GetClientIP());
853  KNSManagerSetClientIP(kns_mgr, req_ctx.GetClientIP().c_str());
854  }
855  if ( req_ctx.IsSetHitID() ) {
856  _TRACE("CVDBMgr: Updating hit ID: "<<req_ctx.GetHitID());
857  KNSManagerSetPageHitID(kns_mgr, req_ctx.GetHitID().c_str());
858  }
859 }
860 
861 
862 static void s_InitAllKNS(KNSManager* kns_mgr)
863 {
865  if ( app && app->GetConfig().GetBool("VDB", "ALLOW_ALL_CERTS", false) ) {
866  if ( rc_t rc = KNSManagerSetAllowAllCerts(kns_mgr, true) ) {
867  NCBI_THROW2(CSraException, eInitFailed,
868  "Cannot enable all HTTPS certificates in KNSManager", rc);
869  }
870  }
871  {{ // set user agent
873  if ( app ) {
874  str << app->GetAppName() << ": " << app->GetVersion().Print() << "; ";
875  }
876 #if NCBI_PACKAGE
877  str << "Package: " << NCBI_PACKAGE_NAME << ' ' <<
878  NCBI_PACKAGE_VERSION << "; ";
879 #endif
880  str << "C++ ";
881 #ifdef NCBI_PRODUCTION_VER
882  str << NCBI_PRODUCTION_VER << "/";
883 #endif
884 #ifdef NCBI_DEVELOPMENT_VER
886 #endif
888  KNSManagerSetUserAgent(kns_mgr, "%s; VDB %s",
889  prefix.c_str(),
890  s_VDBVersion);
891  }}
892 }
893 
894 
895 static void s_InitStaticKNS(KNSManager* kns_mgr)
896 {
897  s_InitAllKNS(kns_mgr);
898 }
899 
900 
901 static void s_InitLocalKNS(KNSManager* kns_mgr)
902 {
903  s_InitAllKNS(kns_mgr);
904 }
905 
906 
907 static void s_VDBInit()
908 {
909  CFastMutexGuard guard(sx_SDKMutex);
910  static bool initialized = false;
911  if ( !initialized ) {
913  // redirect VDB log to C++ Toolkit
914  if ( s_GetDiagHandler() ) {
915  KLogInit();
916  KLogLevel ask_level;
917 #ifdef _DEBUG
918  ask_level = klogDebug;
919 #else
920  ask_level = klogInfo;
921 #endif
922  s_InitDiagCheck();
923  KLogLevelSet(ask_level);
924  KLogHandlerSet(VDBLogWriter, 0);
925  KLogLibHandlerSet(VDBLogWriter, 0);
926  if ( s_GetDebugLevel() >= 2 ) {
927  const char* msg = "info: VDB initialized";
928  size_t written;
929  VDBLogWriter(0, msg, strlen(msg), &written);
930  }
931  }
934  s_InitStaticKNS(kns_mgr);
935  initialized = true;
936  }
937 }
938 
939 /////////////////////////////////////////////////////////////////////////////
940 // end of VDB library initialization code
941 /////////////////////////////////////////////////////////////////////////////
942 
943 
944 //#define UPDATE_REQUEST_CONTEXT_FINAL
945 
946 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
947 static DECLARE_TLS_VAR(size_t, s_RequestContextUpdaterRecursion);
948 #endif
949 
951 public:
955 };
956 
957 
959 {
960 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
961  size_t r = s_RequestContextUpdaterRecursion;
962  s_RequestContextUpdaterRecursion = r+1;
963  if ( r != 0 ) {
964  return;
965  }
968 #endif
969 }
970 
971 
973 {
974 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
975  size_t r = s_RequestContextUpdaterRecursion;
976  s_RequestContextUpdaterRecursion = r+1;
977  if ( r != 0 ) {
978  return;
979  }
980 #endif
983 }
984 
985 
987 {
988 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
989  size_t r = s_RequestContextUpdaterRecursion;
990  s_RequestContextUpdaterRecursion = r-1;
991 #endif
992 }
993 
994 
996 {
997 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
998  size_t r = s_RequestContextUpdaterRecursion;
999  s_RequestContextUpdaterRecursion = r-1;
1000 #endif
1001 }
1002 
1003 
1005 {
1006  s_VDBInit();
1007  if ( rc_t rc = VDBManagerMakeRead(x_InitPtr(), 0) ) {
1008  *x_InitPtr() = 0;
1009  NCBI_THROW2(CSraException, eInitFailed,
1010  "Cannot open VDBManager", rc);
1011  }
1012  CVFSManager vfs_mgr(*this);
1013  VFSManagerLogNamesServiceErrors(vfs_mgr, false);
1014  s_InitLocalKNS(CKNSManager(vfs_mgr));
1015  if ( NCBI_PARAM_TYPE(VDB, DISABLE_PAGEMAP_THREAD)().Get() ) {
1016  VDBManagerDisablePagemapThread(*this);
1017  }
1018 }
1019 
1020 
1022 {
1023  const VPath* ret;
1024  if ( rc_t rc = VDBManagerGetCacheRoot(*this, &ret) ) {
1025  if ( GetRCObject(rc) == RCObject(rcPath) &&
1026  GetRCState(rc) == rcNotFound ) {
1027  return kEmptyStr;
1028  }
1029  NCBI_THROW2(CSraException, eOtherError,
1030  "CVDBMgr: Cannot get cache root", rc);
1031  }
1032  return CVPath(ret).ToString(CVPath::eSys);
1033 }
1034 
1035 
1036 void CVDBMgr::SetCacheRoot(const string& path)
1037 {
1038  CVPath vpath(CVFSManager(*this), path, CVPath::eSys);
1039  if ( rc_t rc = VDBManagerSetCacheRoot(*this, vpath) ) {
1040  NCBI_THROW2(CSraException, eOtherError,
1041  "CVDBMgr: Cannot set cache root", rc);
1042  }
1043 }
1044 
1045 
1047 {
1048  if ( rc_t rc = VDBManagerDeleteCacheOlderThan(*this, days) ) {
1049  NCBI_THROW2(CSraException, eOtherError,
1050  "CVDBMgr: Cannot delete old cache files", rc);
1051  }
1052 }
1053 
1054 
1056 {
1057  CKConfig(*this).Commit();
1058 }
1059 
1060 
1061 /////////////////////////////////////////////////////////////////////////////
1062 // CVDB
1063 
1064 CVDB::CVDB(const CVDBMgr& mgr, const string& acc_or_path)
1065  : m_Name(acc_or_path)
1066 {
1068  CFinalRequestContextUpdater ctx_updater;
1069  string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
1071  if ( rc_t rc = VDBManagerOpenDBRead(mgr, x_InitPtr(), 0, "%.*s",
1072  int(path.size()), path.data()) ) {
1073  *x_InitPtr() = 0;
1074  string msg = "Cannot open VDB: "+acc_or_path;
1075  CHECK_VDB_TIMEOUT(msg, rc);
1076  if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
1077  GetRCObject(rc) == RCObject(rcPath) ||
1078  GetRCObject(rc) == RCObject(rcFile)) &&
1079  GetRCState(rc) == rcNotFound ) {
1080  // no VDB accession
1081  NCBI_THROW2(CSraException, eNotFoundDb, msg, rc);
1082  }
1083  else if ( GetRCObject(rc) == rcName &&
1084  GetRCState(rc) == rcNotFound &&
1085  GetRCContext(rc) == rcResolving ) {
1086  // invalid VDB database
1087  NCBI_THROW2(CSraException, eNotFoundDb, msg, rc);
1088  }
1089  else if ( GetRCObject(rc) == RCObject(rcFile) &&
1090  GetRCState(rc) == rcUnauthorized ) {
1091  // no permission to use VDB database
1092  NCBI_THROW2(CSraException, eProtectedDb, msg, rc);
1093  }
1094  else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
1095  GetRCState(rc) == rcIncorrect ) {
1096  // invalid VDB database
1097  NCBI_THROW2(CSraException, eDataError, msg, rc);
1098  }
1099  else {
1100  // other errors
1101  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1102  }
1103  }
1104  if ( 0 ) {
1105  const KDatabase* kdb = 0;
1106  if ( rc_t rc = VDatabaseOpenKDatabaseRead(*this, &kdb) ) {
1107  NCBI_THROW2(CSraException, eOtherError, "VDatabaseOpenKDatabaseRead() failed", rc);
1108  }
1109  const char* kdb_path = 0;
1110  if ( rc_t rc = KDatabaseGetPath(kdb, &kdb_path) ) {
1111  NCBI_THROW2(CSraException, eOtherError, "KDatabaseGetPath() failed", rc);
1112  }
1113  LOG_POST("CVDB("<<acc_or_path<<") -> "<<path<<" -> "<<kdb_path);
1114  if ( rc_t rc = KDatabaseRelease(kdb) ) {
1115  NCBI_THROW2(CSraException, eOtherError, "KDatabaseRelease() failed", rc);
1116  }
1117  }
1118 }
1119 
1120 
1122 {
1123  return out << GetName();
1124 }
1125 
1126 
1127 /////////////////////////////////////////////////////////////////////////////
1128 // CVDBTable
1129 
1130 static inline
1132 {
1133  return obj.PrintFullName(out);
1134 }
1135 
1136 
1138  const char* table_name,
1139  EMissing missing)
1140  : m_Db(db),
1141  m_Name(table_name)
1142 {
1144  CFinalRequestContextUpdater ctx_updater;
1146  if ( rc_t rc = VDatabaseOpenTableRead(db, x_InitPtr(), table_name) ) {
1147  *x_InitPtr() = 0;
1148  string msg = "Cannot open VDB: "+GetFullName();
1149  CHECK_VDB_TIMEOUT(msg, rc);
1150  RCState rc_state = GetRCState(rc);
1151  int rc_object = GetRCObject(rc);
1152  if ( rc_state == rcNotFound &&
1153  (rc_object == rcParam ||
1154  rc_object == rcPath) ) {
1155  // missing table in the DB
1156  if ( missing != eMissing_Throw ) {
1157  return;
1158  }
1159  NCBI_THROW2(CSraException, eNotFoundTable, msg, rc);
1160  }
1161  else {
1162  // other errors
1163  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1164  }
1165  }
1166 }
1167 
1168 
1170  const string& acc_or_path,
1171  EMissing missing)
1172  : m_Name(acc_or_path)
1173 {
1174  *x_InitPtr() = 0;
1176  CFinalRequestContextUpdater ctx_updater;
1177  string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
1179  if ( rc_t rc = VDBManagerOpenTableRead(mgr, x_InitPtr(), 0, "%.*s",
1180  int(path.size()), path.data()) ) {
1181  *x_InitPtr() = 0;
1182  string msg = "Cannot open VDB: "+acc_or_path;
1183  CHECK_VDB_TIMEOUT(msg, rc);
1184  if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
1185  GetRCObject(rc) == RCObject(rcPath)) &&
1186  GetRCState(rc) == rcNotFound ) {
1187  // no SRA accession
1188  if ( missing != eMissing_Throw ) {
1189  return;
1190  }
1191  NCBI_THROW2(CSraException, eNotFoundTable, msg, rc);
1192  }
1193  else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
1194  GetRCState(rc) == rcIncorrect ) {
1195  // invalid SRA database
1196  NCBI_THROW2(CSraException, eDataError, msg, rc);
1197  }
1198  else {
1199  // other errors
1200  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1201  }
1202  }
1203 }
1204 
1205 
1206 string CVDBTable::GetFullName(void) const
1207 {
1208  string ret;
1209  if ( GetDb() ) {
1210  ret = GetDb().GetFullName();
1211  ret += '.';
1212  }
1213  ret += GetName();
1214  return ret;
1215 }
1216 
1217 
1219 {
1220  if ( GetDb() ) {
1221  GetDb().PrintFullName(out) << '.';
1222  }
1223  return out << GetName();
1224 }
1225 
1226 
1227 /////////////////////////////////////////////////////////////////////////////
1228 // CVDBTableIndex
1229 
1230 static inline
1232 {
1233  return obj.PrintFullName(out);
1234 }
1235 
1236 
1238  const char* index_name,
1239  EMissing missing)
1240  : m_Table(table),
1241  m_Name(index_name)
1242 {
1243  CFinalRequestContextUpdater ctx_updater;
1245  if ( rc_t rc = VTableOpenIndexRead(table, x_InitPtr(), index_name) ) {
1246  *x_InitPtr() = 0;
1247  string msg = "Cannot open VDB table index: "+GetFullName();
1248  CHECK_VDB_TIMEOUT(msg, rc);
1249  if ( GetRCObject(rc) == RCObject(rcIndex) &&
1250  GetRCState(rc) == rcNotFound ) {
1251  // no such index
1252  if ( missing != eMissing_Throw ) {
1253  return;
1254  }
1255  NCBI_THROW2(CSraException, eNotFoundIndex, msg, rc);
1256  }
1257  else {
1258  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1259  }
1260  }
1261 }
1262 
1263 
1265 {
1266  return GetTable().GetFullName()+'.'+GetName();
1267 }
1268 
1269 
1271 {
1272  return GetTable().PrintFullName(out) << '.' << GetName();
1273 }
1274 
1275 
1277 {
1279  if ( rc_t rc = KIndexFindText(*this, value.c_str(),
1280  &range.first, &range.second, 0, 0) ) {
1281  CHECK_VDB_TIMEOUT_FMT("Cannot find value in index: "<<*this<<": "<<value, rc);
1282  if ( GetRCObject(rc) == RCObject(rcString) &&
1283  GetRCState(rc) == rcNotFound ) {
1284  // no such value
1285  range.first = range.second = 0;
1286  }
1287  else {
1288  NCBI_THROW2_FMT(CSraException, eOtherError,
1289  "Cannot find value in index: "<<*this<<": "<<value, rc);
1290  }
1291  }
1292  return range;
1293 }
1294 
1295 
1296 /////////////////////////////////////////////////////////////////////////////
1297 // CVDBCursor
1298 
1299 static inline
1301 {
1302  return out << obj.GetTable();
1303 }
1304 
1305 
1307 {
1308  CFinalRequestContextUpdater ctx_updater;
1309  if ( *this ) {
1310  NCBI_THROW2(CSraException, eInvalidState,
1311  "Cannot init VDB cursor again",
1312  RC(rcApp, rcCursor, rcConstructing, rcSelf, rcOpen));
1313  }
1314  if ( !table ) { // VTableCreateCursorRead lacks check for null table argument
1315  NCBI_THROW2(CSraException, eNullPtr,
1316  "Cannot init VDB cursor",
1317  RC(rcApp, rcCursor, rcConstructing, rcTable, rcNull));
1318  }
1319  if ( rc_t rc = VTableCreateCursorRead(table, x_InitPtr()) ) {
1320  *x_InitPtr() = 0;
1321  CHECK_VDB_TIMEOUT_FMT("Cannot create VDB cursor: "<<table, rc);
1322  NCBI_THROW2_FMT(CSraException, eInitFailed,
1323  "Cannot create VDB cursor: "<<table, rc);
1324  }
1325  if ( rc_t rc = VCursorPermitPostOpenAdd(*this) ) {
1326  NCBI_THROW2_FMT(CSraException, eInitFailed,
1327  "Cannot allow VDB cursor post open column add: "<<table, rc);
1328  }
1329  if ( rc_t rc = VCursorOpen(*this) ) {
1330  CHECK_VDB_TIMEOUT_FMT("Cannot open VDB cursor: "<<table, rc);
1331  NCBI_THROW2_FMT(CSraException, eInitFailed,
1332  "Cannot open VDB cursor: "<<table, rc);
1333  }
1334  m_Table = table;
1335 }
1336 
1337 
1339 {
1340  if ( !RowIsOpened() ) {
1341  return;
1342  }
1343  if ( rc_t rc = VCursorCloseRow(*this) ) {
1344  NCBI_THROW2(CSraException, eInitFailed,
1345  "Cannot close VDB cursor row", rc);
1346  }
1347  m_RowOpened = false;
1348 }
1349 
1350 
1352 {
1353  CloseRow();
1354  CFinalRequestContextUpdater ctx_updater;
1355  if ( rc_t rc = VCursorSetRowId(*this, row_id) ) {
1356  return rc;
1357  }
1358  if ( rc_t rc = VCursorOpenRow(*this) ) {
1359  return rc;
1360  }
1361  m_RowOpened = true;
1362  return 0;
1363 }
1364 
1365 
1367 {
1368  if ( rc_t rc = OpenRowRc(row_id) ) {
1369  CHECK_VDB_TIMEOUT_FMT("Cannot open VDB cursor row: "<<*this<<": "<<row_id, rc);
1370  NCBI_THROW2_FMT(CSraException, eInitFailed,
1371  "Cannot open VDB cursor row: "<<*this<<": "<<row_id, rc);
1372  }
1373 }
1374 
1375 
1377 {
1378  TVDBRowIdRange ret;
1379  if ( rc_t rc = VCursorIdRange(*this, column, &ret.first, &ret.second) ) {
1380  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB cursor row range: "<<*this<<": "<<column, rc);
1381  NCBI_THROW2_FMT(CSraException, eInitFailed,
1382  "Cannot get VDB cursor row range: "<<*this<<": "<<column, rc);
1383  }
1384  return ret;
1385 }
1386 
1387 
1389 {
1391  return range.first+range.second-1;
1392 }
1393 
1394 
1395 void CVDBCursor::SetParam(const char* name, const CTempString& value) const
1396 {
1397  CFinalRequestContextUpdater ctx_updater;
1398  if ( rc_t rc = VCursorParamsSet
1399  ((struct VCursorParams *)GetPointer(),
1400  name, "%.*s", value.size(), value.data()) ) {
1402  "Cannot set VDB cursor param: "<<*this<<": "<<name,
1403  rc);
1404  }
1405 }
1406 
1407 
1409  uint32_t elem_bits) const
1410 {
1412  CFinalRequestContextUpdater ctx_updater;
1413  uint32_t read_count, remaining_count;
1414  if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
1415  elem_bits, 0, 0, 0, 0,
1416  &read_count, &remaining_count) ) {
1417  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value array size: "<<*this<<column<<
1418  '['<<row<<']', rc);
1419  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1420  "Cannot read VDB value array size: "<<*this<<column<<
1421  '['<<row<<']', rc);
1422  }
1423  return remaining_count;
1424 }
1425 
1426 
1428  uint32_t elem_bits,
1429  uint32_t start, uint32_t count,
1430  void* buffer) const
1431 {
1433  CFinalRequestContextUpdater ctx_updater;
1434  uint32_t read_count, remaining_count;
1435  if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
1436  elem_bits, start, buffer, 0, count,
1437  &read_count, &remaining_count) ) {
1438  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value array: "<<*this<<column<<
1439  '['<<row<<"]["<<start<<".."<<(start+count-1)<<']', rc);
1440  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1441  "Cannot read VDB value array: "<<*this<<column<<
1442  '['<<row<<"]["<<start<<".."<<(start+count-1)<<']', rc);
1443  }
1444  if ( read_count != count ) {
1445  NCBI_THROW_FMT(CSraException, eNotFoundValue,
1446  "Cannot read VDB value array: "<<*this<<column<<
1447  '['<<row<<"]["<<start<<".."<<(start+count-1)<<
1448  "] only "<<read_count<<" elements are read");
1449  }
1450  if ( s_GetDebugLevel() >= 9 ) {
1451  LOG_POST(Info<<"VDB "<<*this<<column<<'['<<row<<"]: @"<<start<<" #"<<count);
1452  }
1453 }
1454 
1455 
1456 /////////////////////////////////////////////////////////////////////////////
1457 // CVDBObjectCache
1458 
1459 static const size_t kCacheSize = 7;
1460 
1461 
1463 {
1464  m_Objects.reserve(kCacheSize);
1465 }
1466 
1467 
1469 {
1470 }
1471 
1472 
1474 
1475 
1477 {
1478  CFastMutexGuard guard(sm_CacheMutex);
1479  m_Objects.clear();
1480 }
1481 
1482 
1484 {
1485  CFastMutexGuard guard(sm_CacheMutex);
1486  if ( m_Objects.empty() ) {
1487  return 0;
1488  }
1489  TObjects::iterator best_it;
1492  TVDBRowId slot_row = it->first;
1493  if ( slot_row >= row ) {
1494  TVDBRowId d = slot_row - row;
1495  if ( d <= best_d ) {
1496  best_d = d;
1497  best_it = it;
1498  }
1499  }
1500  else {
1501  TVDBRowId d = row - slot_row;
1502  if ( d < best_d ) {
1503  best_d = d;
1504  best_it = it;
1505  }
1506  }
1507  }
1508  CObject* obj = best_it->second.Release();
1509  *best_it = m_Objects.back();
1510  m_Objects.pop_back();
1511  _ASSERT(!obj->Referenced());
1512  return obj;
1513 }
1514 
1515 
1517 {
1518  if ( obj->Referenced() ) {
1519  return;
1520  }
1521  //row = 0;
1522  CFastMutexGuard guard(sm_CacheMutex);
1523  if ( m_Objects.size() < kCacheSize ) {
1524  m_Objects.push_back(TSlot());
1525  m_Objects.back().first = row;
1526  m_Objects.back().second = obj;
1527  }
1528  else {
1529  CRef<CObject> ref(obj); // delete the object
1530  }
1531 }
1532 
1533 
1534 /////////////////////////////////////////////////////////////////////////////
1535 // CVDBColumn
1536 
1537 static inline
1539 {
1540  return out << '.' << column.GetName();
1541 }
1542 
1543 
1544 void CVDBColumn::Init(const CVDBCursor& cursor,
1545  size_t element_bit_size,
1546  const char* name,
1547  const char* backup_name,
1548  EMissing missing)
1549 {
1551  CFinalRequestContextUpdater ctx_updater;
1552  if ( !name ) {
1553  if ( missing == eMissing_Throw ) {
1554  NCBI_THROW_FMT(CSraException, eInvalidArg,
1555  "NULL column name is not allowed"<<cursor<<*this);
1556  }
1558  return;
1559  }
1560  m_Name = name;
1561  //SIMULATE_SCHEMA_ERROR();
1562  if ( rc_t rc = VCursorAddColumn(cursor, &m_Index, name) ) {
1563  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB column: "<<cursor<<*this, rc);
1564  if ( backup_name &&
1565  (rc = VCursorAddColumn(cursor, &m_Index, backup_name)) == 0 ) {
1566  m_Name = backup_name;
1567  }
1568  else {
1569  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB column: "<<cursor<<*this, rc);
1571  if ( missing == eMissing_Throw ) {
1572  NCBI_THROW2_FMT(CSraException, eNotFoundColumn,
1573  "Cannot get VDB column: "<<cursor<<*this,rc);
1574  }
1575  else {
1576  return;
1577  }
1578  }
1579  }
1580  if ( element_bit_size ) {
1581  VTypedesc type;
1582  if ( rc_t rc = VCursorDatatype(cursor, m_Index, 0, &type) ) {
1583  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB column type: "<<cursor<<*this,rc);
1584  NCBI_THROW2_FMT(CSraException, eInvalidState,
1585  "Cannot get VDB column type: "<<cursor<<*this,rc);
1586  }
1587  size_t size = type.intrinsic_bits*type.intrinsic_dim;
1588  if ( size != element_bit_size ) {
1589  ERR_POST_X(1, "Wrong VDB column size "<<cursor<<*this<<
1590  " expected "<<element_bit_size<<" bits != "<<
1591  type.intrinsic_dim<<"*"<<type.intrinsic_bits<<" bits");
1592  NCBI_THROW2_FMT(CSraException, eInvalidState,
1593  "Wrong VDB column size: "<<cursor<<*this<<": "<<size,
1594  RC(rcApp, rcColumn, rcConstructing, rcSelf, rcIncorrect));
1595  }
1596  }
1597 }
1598 
1599 
1600 bool CVDBColumn::IsStatic(const CVDBCursor& cursor) const
1601 {
1602  bool static_value;
1603  if ( rc_t rc = VCursorIsStaticColumn(cursor, GetIndex(), &static_value) ) {
1604  NCBI_THROW2_FMT(CSraException, eInvalidState,
1605  "Cannot get static status of "<<cursor<<*this,rc);
1606  }
1607  return static_value;
1608 }
1609 
1610 
1612 {
1613  try {
1614  if ( IsStatic(cursor) && CVDBValue(cursor, 1, *this).empty() ) {
1615  // value is always empty
1616  Reset();
1617  }
1618  }
1619  catch ( CException& /*ignored*/ ) {
1620  }
1621 }
1622 
1623 
1624 /////////////////////////////////////////////////////////////////////////////
1625 // CVDBValue
1626 
1628 {
1629  if ( m_Table ) {
1631  }
1632  if ( m_Row ) {
1633  out << '[' << m_Row << ']';
1634  }
1635  if ( m_ColumnName ) {
1636  out << '.' << m_ColumnName;
1637  }
1638  return out;
1639 }
1640 
1641 
1642 static inline
1644 {
1645  return obj.PrintFullName(out);
1646 }
1647 
1648 
1649 void CVDBValue::x_Get(const CVDBCursor& cursor, const CVDBColumn& column)
1650 {
1652  CFinalRequestContextUpdater ctx_updater;
1653  uint32_t bit_offset, bit_length;
1654  if ( rc_t rc = VCursorCellData(cursor, column.GetIndex(),
1655  &bit_length, &m_Data, &bit_offset,
1656  &m_ElemCount) ) {
1657  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value: "<<cursor<<column, rc);
1658  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1659  "Cannot read VDB value: "<<cursor<<column, rc);
1660  }
1661  if ( bit_offset ) {
1662  NCBI_THROW2_FMT(CSraException, eInvalidState,
1663  "Cannot read VDB value with non-zero bit offset: "
1664  <<cursor<<column<<": "<<bit_offset,
1665  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1666  }
1667  if ( s_GetDebugLevel() >= 9 ) {
1668  CNcbiOstrstream s;
1669  if ( bit_length == 8 ) {
1670  s << '"' << NStr::PrintableString(CTempString((const char*)m_Data, m_ElemCount)) << '"';
1671  }
1672  else if ( bit_length == 16 ) {
1673  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1674  if ( i ) {
1675  s << ", ";
1676  }
1677  s << ((const int16_t*)m_Data)[i];
1678  }
1679  }
1680  else if ( bit_length == 32 ) {
1681  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1682  if ( i ) {
1683  s << ", ";
1684  }
1685  s << ((const int32_t*)m_Data)[i];
1686  }
1687  }
1688  else if ( bit_length == 64 ) {
1689  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1690  if ( i ) {
1691  s << ", ";
1692  }
1693  s << ((const int64_t*)m_Data)[i];
1694  }
1695  }
1696  else {
1697  s << "*** bad bit_length="<<bit_length;
1698  }
1699  LOG_POST(Info<<"VDB "<<cursor<<column<<": "<<CNcbiOstrstreamToString(s));
1700  }
1701  m_Ref.Set(cursor, 0, column);
1702 }
1703 
1704 
1705 void CVDBValue::x_Get(const CVDBCursor& cursor,
1706  TVDBRowId row,
1707  const CVDBColumn& column,
1708  EMissing missing)
1709 {
1711  CFinalRequestContextUpdater ctx_updater;
1712  uint32_t bit_offset, bit_length;
1713  if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1714  &bit_length, &m_Data, &bit_offset,
1715  &m_ElemCount) ) {
1716  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value: "<<cursor<<column<<'['<<row<<']', rc);
1717  if ( missing != eMissing_Throw ) {
1718  m_Data = 0;
1719  m_ElemCount = 0;
1720  return;
1721  }
1722  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1723  "Cannot read VDB value: "<<cursor<<column<<'['<<row<<']', rc);
1724  }
1725  if ( bit_offset ) {
1726  NCBI_THROW2_FMT(CSraException, eInvalidState,
1727  "Cannot read VDB value with non-zero bit offset: "<<
1728  cursor<<column<<'['<<row<<"]: "<<bit_offset,
1729  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1730  }
1731  if ( s_GetDebugLevel() >= 9 ) {
1732  CNcbiOstrstream s;
1733  if ( bit_length == 8 ) {
1734  s << '"' << NStr::PrintableString(CTempString((const char*)m_Data, m_ElemCount)) << '"';
1735  }
1736  else if ( bit_length == 16 ) {
1737  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1738  if ( i ) {
1739  s << ", ";
1740  }
1741  s << ((const int16_t*)m_Data)[i];
1742  }
1743  }
1744  else if ( bit_length == 32 ) {
1745  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1746  if ( i ) {
1747  s << ", ";
1748  }
1749  s << ((const int32_t*)m_Data)[i];
1750  }
1751  }
1752  else if ( bit_length == 64 ) {
1753  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1754  if ( i ) {
1755  s << ", ";
1756  }
1757  s << ((const int64_t*)m_Data)[i];
1758  }
1759  }
1760  else {
1761  s << "*** bad bit_length="<<bit_length;
1762  }
1763  LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
1764  }
1765  m_Ref.Set(cursor, row, column);
1766 }
1767 
1768 
1769 void CVDBValue::x_ReportIndexOutOfBounds(size_t index) const
1770 {
1771  if ( index >= size() ) {
1772  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1773  "Invalid index for VDB value array: "<<
1774  *this<<'['<<index<<']',
1775  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1776  }
1777 }
1778 
1779 
1781 {
1782  if ( size() != 1 ) {
1783  NCBI_THROW2_FMT(CSraException, eDataError,
1784  "VDB value array doen't have single value: "<<
1785  *this<<'['<<size()<<']',
1786  RC(rcApp, rcData, rcRetrieving, rcSize, rcIncorrect));
1787  }
1788 }
1789 
1790 
1791 void CVDBValue::x_CheckRange(size_t pos, size_t len) const
1792 {
1793  if ( pos > size() ) {
1794  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1795  "Invalid index for VDB value array: "<<
1796  *this<<'['<<pos<<']',
1797  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1798  }
1799  if ( pos+len < pos ) {
1800  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1801  "Invalid length for VDB value sub-array: "<<
1802  *this<<'['<<pos<<','<<len<<']',
1803  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1804  }
1805  if ( pos+len > size() ) {
1806  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1807  "Invalid end of VDB value sub-array: "<<
1808  *this<<'['<<pos<<','<<len<<']',
1809  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1810  }
1811 }
1812 
1813 
1814 static inline
1816 {
1817  return obj.PrintFullName(out);
1818 }
1819 
1820 
1822  TVDBRowId row,
1823  const CVDBColumn& column)
1824 {
1826  CFinalRequestContextUpdater ctx_updater;
1827  uint32_t bit_offset, bit_length, elem_count;
1828  const void* data;
1829  if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1830  &bit_length, &data, &bit_offset,
1831  &elem_count) ) {
1832  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB 4-bits value array: "<<
1833  cursor<<column<<'['<<row<<']', rc);
1834  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1835  "Cannot read VDB 4-bits value array: "<<
1836  cursor<<column<<'['<<row<<']', rc);
1837  }
1838  if ( bit_offset >= 8 || (bit_offset&3) ) {
1839  NCBI_THROW2_FMT(CSraException, eInvalidState,
1840  "Cannot read VDB 4-bits value array with odd bit offset"<<
1841  cursor<<column<<'['<<row<<"]: "<<bit_offset,
1842  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1843  }
1844  m_RawData = static_cast<const char*>(data);
1845  m_ElemOffset = bit_offset >> 2;
1846  m_ElemCount = elem_count;
1847  if ( s_GetDebugLevel() >= 9 ) {
1848  CNcbiOstrstream s;
1849  if ( bit_length == 4 ) {
1850  for ( uint32_t i = 0; i < elem_count; ++i ) {
1851  if ( i ) {
1852  s << ", ";
1853  }
1854  s << '?';
1855  }
1856  }
1857  else {
1858  s << "*** bad bit_length="<<bit_length;
1859  }
1860  LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
1861  }
1862  m_Ref.Set(cursor, row, column);
1863 }
1864 
1865 
1867 {
1868  if ( index >= size() ) {
1869  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1870  "Invalid index for VDB 4-bits value array: "<<
1871  *this<<'['<<index<<']',
1872  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1873  }
1874 }
1875 
1876 
1877 inline
1878 void CVDBValueFor4Bits::x_CheckRange(size_t pos, size_t len) const
1879 {
1880  if ( pos > size() ) {
1881  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1882  "Invalid index for VDB 4-bits value array: "<<
1883  *this<<'['<<pos<<']',
1884  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1885  }
1886  if ( pos+len < pos ) {
1887  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1888  "Invalid length for VDB 4-bits value sub-array: "<<
1889  *this<<'['<<pos<<','<<len<<']',
1890  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1891  }
1892  if ( pos+len > size() ) {
1893  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1894  "Invalid end of VDB 4-bits value sub-array: "<<
1895  *this<<'['<<pos<<','<<len<<']',
1896  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1897  }
1898 }
1899 
1900 
1902 {
1903  x_CheckRange(pos, len);
1904  size_t offset = m_ElemOffset + pos;
1905  const char* raw_data = m_RawData + offset/2;
1906  offset %= 2;
1907  // limits are checked above
1908  return CVDBValueFor4Bits(m_Ref, raw_data, uint32_t(offset), uint32_t(len));
1909 }
1910 
1911 
1912 static inline
1914 {
1915  return obj.PrintFullName(out);
1916 }
1917 
1918 
1920  TVDBRowId row,
1921  const CVDBColumn& column)
1922 {
1924  CFinalRequestContextUpdater ctx_updater;
1925  uint32_t bit_offset, bit_length, elem_count;
1926  const void* data;
1927  if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1928  &bit_length, &data, &bit_offset,
1929  &elem_count) ) {
1930  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB 2-bits value array: "<<
1931  cursor<<column<<'['<<row<<']', rc);
1932  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1933  "Cannot read VDB 2-bits value array: "<<
1934  cursor<<column<<'['<<row<<']', rc);
1935  }
1936  if ( bit_offset >= 8 || (bit_offset&1) ) {
1937  NCBI_THROW2_FMT(CSraException, eInvalidState,
1938  "Cannot read VDB 2-bits value array with odd bit offset"<<
1939  cursor<<column<<'['<<row<<"]: "<<bit_offset,
1940  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1941  }
1942  m_RawData = static_cast<const char*>(data);
1943  m_ElemOffset = bit_offset >> 1;
1944  m_ElemCount = elem_count;
1945  if ( s_GetDebugLevel() >= 9 ) {
1946  CNcbiOstrstream s;
1947  if ( bit_length == 2 ) {
1948  for ( uint32_t i = 0; i < elem_count; ++i ) {
1949  if ( i ) {
1950  s << ", ";
1951  }
1952  s << '?';
1953  }
1954  }
1955  else {
1956  s << "*** bad bit_length="<<bit_length;
1957  }
1958  LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
1959  }
1960  m_Ref.Set(cursor, row, column);
1961 }
1962 
1963 
1965 {
1966  if ( index >= size() ) {
1967  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1968  "Invalid index for VDB 2-bits value array: "<<
1969  *this<<'['<<index<<']',
1970  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1971  }
1972 }
1973 
1974 
1975 inline
1976 void CVDBValueFor2Bits::x_CheckRange(size_t pos, size_t len) const
1977 {
1978  if ( pos > size() ) {
1979  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1980  "Invalid index for VDB 2-bits value array: "<<
1981  *this<<'['<<pos<<']',
1982  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1983  }
1984  if ( pos+len < pos ) {
1985  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1986  "Invalid length for VDB 2-bits value sub-array: "<<
1987  *this<<'['<<pos<<','<<len<<']',
1988  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1989  }
1990  if ( pos+len > size() ) {
1991  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1992  "Invalid end of VDB 2-bits value sub-array: "<<
1993  *this<<'['<<pos<<','<<len<<']',
1994  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1995  }
1996 }
1997 
1998 
2000 {
2001  x_CheckRange(pos, len);
2002  size_t offset = m_ElemOffset + pos;
2003  const char* raw_data = m_RawData + offset/4;
2004  offset %= 4;
2005  // limits are checked above
2006  return CVDBValueFor2Bits(m_Ref, raw_data, uint32_t(offset), uint32_t(len));
2007 }
2008 
2009 
uint32_t rc_t
CClientHttpRequest(const CKNSManager &mgr, const string &path)
Definition: vdbread.cpp:593
CClientHttpResult(const CClientHttpRequest &request, EHead)
Definition: vdbread.cpp:618
CDirEntry –.
Definition: ncbifile.hpp:262
CFinalRequestContextUpdater(const CFinalRequestContextUpdater &)=delete
void Commit() const
Definition: vdbread.cpp:217
CKConfig(void)
Definition: vdbread.cpp:185
CKDBManager(const CVDBMgr &mgr)
Definition: vdbread.cpp:265
CKNSManager(const CVFSManager &mgr)
Definition: vdbread.cpp:289
CNcbiOstrstreamToString class helps convert CNcbiOstrstream to a string Sample usage:
Definition: ncbistre.hpp:802
CObject –.
Definition: ncbiobj.hpp:180
CRandom::
Definition: random_gen.hpp:66
CSafeStatic<>::
Definition: Seq_entry.hpp:56
Definition: sdk.hpp:85
TObject ** x_InitPtr(void)
Definition: sdk.hpp:175
TObject * GetPointer(void) const
Definition: sdk.hpp:146
CTempString implements a light-weight string on top of a storage buffer whose lifetime management is ...
Definition: tempstr.hpp:65
CTimeFormat –.
Definition: ncbitime.hpp:131
CTime –.
Definition: ncbitime.hpp:296
@ kInvalidIndex
Definition: vdbread.hpp:478
@ eMissing_Throw
Definition: vdbread.hpp:473
TVDBColumnIdx GetIndex(void) const
Definition: vdbread.hpp:523
const char * m_Name
Definition: vdbread.hpp:542
TVDBColumnIdx m_Index
Definition: vdbread.hpp:543
void ResetIfAlwaysEmpty(const CVDBCursor &cursor)
Definition: vdbread.cpp:1611
bool IsStatic(const CVDBCursor &cursor) const
Definition: vdbread.cpp:1600
void Init(const CVDBCursor &cursor, size_t element_bit_size, const char *name, const char *backup_name, EMissing missing)
Definition: vdbread.cpp:1544
void Reset(void)
Definition: vdbread.hpp:510
rc_t OpenRowRc(TVDBRowId row_id)
Definition: vdbread.cpp:1351
void OpenRow(TVDBRowId row_id)
Definition: vdbread.cpp:1366
bool RowIsOpened(void) const
Definition: vdbread.hpp:391
void CloseRow(void)
Definition: vdbread.cpp:1338
uint32_t GetElementCount(TVDBRowId row, const CVDBColumn &column, uint32_t elem_bits) const
Definition: vdbread.cpp:1408
const CVDBTable & GetTable(void) const
Definition: vdbread.hpp:386
void ReadElements(TVDBRowId row, const CVDBColumn &column, uint32_t elem_bits, uint32_t start, uint32_t count, void *buffer) const
Definition: vdbread.cpp:1427
bool m_RowOpened
Definition: vdbread.hpp:426
void SetParam(const char *name, const CTempString &value) const
Definition: vdbread.cpp:1395
CVDBTable m_Table
Definition: vdbread.hpp:425
void Init(const CVDBTable &table)
Definition: vdbread.cpp:1306
TVDBRowId GetMaxRowId(void) const
Definition: vdbread.cpp:1388
TVDBRowIdRange GetRowIdRange(TVDBColumnIdx column=0) const
Definition: vdbread.cpp:1376
CVDBMgr(void)
Definition: vdbread.cpp:511
void DeleteCacheOlderThan(Uint4 days)
Definition: vdbread.cpp:1046
CVResolver m_Resolver
Definition: vdbread.hpp:261
string GetCacheRoot() const
Definition: vdbread.cpp:1021
string FindDereferencedAccPath(const string &acc_or_path) const
Definition: vdbread.cpp:527
CTime GetTimestamp(const string &path) const
Definition: vdbread.cpp:557
CTime GetURLTimestamp(const string &url) const
Definition: vdbread.cpp:632
string FindAccPath(const string &acc) const
Definition: vdbread.cpp:518
void SetCacheRoot(const string &path)
Definition: vdbread.cpp:1036
void CommitConfig() const
Definition: vdbread.cpp:1055
void x_Init(void)
Definition: vdbread.cpp:1004
CObject * Get(TVDBRowId row)
Definition: vdbread.cpp:1483
TObjects m_Objects
Definition: vdbread.hpp:445
void Put(CObject *curs, TVDBRowId row)
Definition: vdbread.cpp:1516
pair< TVDBRowId, CRef< CObject > > TSlot
Definition: vdbread.hpp:443
~CVDBObjectCacheBase(void)
Definition: vdbread.cpp:1468
vector< TSlot > TObjects
Definition: vdbread.hpp:444
void Clear(void)
Definition: vdbread.cpp:1476
string GetFullName(void) const
Definition: vdbread.cpp:1264
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1270
const char * GetName(void) const
Definition: vdbread.hpp:353
const CVDBTable & GetTable(void) const
Definition: vdbread.hpp:349
TVDBRowIdRange Find(const string &value) const
Definition: vdbread.cpp:1276
CVDBTableIndex(void)
Definition: vdbread.hpp:342
string GetFullName(void) const
Definition: vdbread.cpp:1206
@ eMissing_Throw
Definition: vdbread.hpp:298
const CVDB & GetDb(void) const
Definition: vdbread.hpp:312
CVDBTable(void)
Definition: vdbread.hpp:302
const string & GetName(void) const
Definition: vdbread.hpp:316
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1218
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.hpp:885
void x_Get(const CVDBCursor &cursor, TVDBRowId row, const CVDBColumn &column)
Definition: vdbread.cpp:1919
void x_ReportIndexOutOfBounds(size_t index) const
Definition: vdbread.cpp:1964
CVDBValueFor2Bits substr(size_t pos, size_t len) const
Definition: vdbread.cpp:1999
void x_CheckRange(size_t pos, size_t len) const
Definition: vdbread.cpp:1976
void x_CheckRange(size_t pos, size_t len) const
Definition: vdbread.cpp:1878
CVDBValueFor4Bits substr(size_t pos, size_t len) const
Definition: vdbread.cpp:1901
void x_ReportIndexOutOfBounds(size_t index) const
Definition: vdbread.cpp:1866
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.hpp:786
void x_Get(const CVDBCursor &cursor, TVDBRowId row, const CVDBColumn &column)
Definition: vdbread.cpp:1821
void x_CheckRange(size_t pos, size_t len) const
Definition: vdbread.cpp:1791
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.hpp:695
const void * m_Data
Definition: vdbread.hpp:725
uint32_t m_ElemCount
Definition: vdbread.hpp:726
size_t size(void) const
Definition: vdbread.hpp:690
@ eMissing_Throw
Definition: vdbread.hpp:615
SSaveRef m_Ref
Definition: vdbread.hpp:724
void x_Get(const CVDBCursor &cursor, const CVDBColumn &column)
Definition: vdbread.cpp:1649
void x_ReportNotOneValue(void) const
Definition: vdbread.cpp:1780
void x_ReportIndexOutOfBounds(size_t index) const
Definition: vdbread.cpp:1769
const string & GetFullName(void) const
Definition: vdbread.hpp:278
CVDB(void)
Definition: vdbread.hpp:269
const string & GetName(void) const
Definition: vdbread.hpp:274
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1121
void x_InitNew(void)
Definition: vdbread.cpp:241
CVFSManager(void)
Definition: vdbread.cpp:229
@ eSys
Definition: vdbread.hpp:148
@ eAcc
Definition: vdbread.hpp:149
string ToString(EType type=eSys) const
Definition: vdbread.cpp:359
bool IsLocalFile() const
Definition: vdbread.cpp:342
static string ConvertAccOrSysPathToPOSIX(const string &acc_or_path)
Definition: vdbread.cpp:429
void x_Init(const CVFSManager &mgr, const string &path, EType type)
Definition: vdbread.cpp:314
static bool IsPlainAccession(const string &acc_or_path)
Definition: vdbread.cpp:390
static string ConvertSysPathToPOSIX(const string &sys_path)
Definition: vdbread.cpp:402
CVPath(void)
Definition: vdbread.hpp:136
CVFSManager m_Mgr
Definition: vdbread.hpp:211
CVResolver(const CVFSManager &mgr)
Definition: vdbread.cpp:441
string Resolve(const string &acc) const
Definition: vdbread.cpp:463
struct config config
std::ofstream out("events_result.xml")
main entry point for tests
static const char table_name[]
Definition: bcp.c:249
static int type
Definition: getdata.c:31
static const char * str(char *buf, int n)
Definition: stats.c:84
static const char * column
Definition: stats.c:23
int offset
Definition: replacements.h:160
char data[12]
Definition: iconv.c:80
Int4 int32_t
Int2 int16_t
Int8 int64_t
Uint4 uint32_t
static string GetAppName(EAppNameType name_type=eBaseName, int argc=0, const char *const *argv=NULL)
Definition: ncbiapp.cpp:1377
static CNcbiApplicationGuard InstanceGuard(void)
Singleton method.
Definition: ncbiapp.cpp:133
const CNcbiRegistry & GetConfig(void) const
Get the application's cached configuration parameters (read-only).
CVersionInfo GetVersion(void) const
Get the program version information.
Definition: ncbiapp.cpp:1184
#define NON_CONST_ITERATE(Type, Var, Cont)
Non constant version of ITERATE macro.
Definition: ncbimisc.hpp:822
@ null
Definition: ncbimisc.hpp:646
#define NULL
Definition: ncbistd.hpp:225
#define _TRACE(message)
Definition: ncbidbg.hpp:122
#define LOG_POST_X(err_subcode, message)
Definition: ncbidiag.hpp:553
const CNcbiDiag &(* FManip)(const CNcbiDiag &)
Diagnostic stream manipulator.
Definition: ncbidiag.hpp:954
string GetSessionID(void) const
Session ID.
CAtomicCounter::TValue TVersion
bool IsSetSessionID(void) const
static CRequestContext & GetRequestContext(void)
Shortcut to CDiagContextThreadData::GetThreadData().GetRequestContext()
Definition: ncbidiag.cpp:1901
string GetClientIP(void) const
Client IP/hostname.
bool IsSetHitID(EHitIDSource src=eHitID_Any) const
Check if there's an explicit hit id or the default one.
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
Definition: ncbidiag.hpp:550
bool IsSetClientIP(void) const
string GetHitID(void) const
Get explicit hit id or the default one (from HTTP_NCBI_PHID etc).
TVersion GetVersion(void) const
Return version increased on every context change (hit/subhit id, client ip, session id).
#define LOG_POST(message)
This macro is deprecated and it's strongly recomended to move in all projects (except tests) to macro...
Definition: ncbidiag.hpp:226
void Error(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1197
void Trace(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1179
void Warning(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1191
void Fatal(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1209
#define NCBI_THROW2(exception_class, err_code, message, extra)
Throw exception with extra parameter.
Definition: ncbiexpt.hpp:1754
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
void Info(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1185
static string CreateAbsolutePath(const string &path, ERelativeToWhat rtw=eRelativeToCwd)
Get an absolute path from some, possibly relative, path.
Definition: ncbifile.cpp:665
void DereferencePath(void)
Dereference a path.
Definition: ncbifile.cpp:2415
virtual bool Exists(void) const
Check the entry existence.
Definition: ncbifile.cpp:2325
const string & GetPath(void) const
Get entry path.
Definition: ncbifile.hpp:3910
bool Referenced(void) const THROWS_NONE
Check if object is referenced.
Definition: ncbiobj.hpp:468
#define NCBI_PARAM_TYPE(section, name)
Generate typename for a parameter from its {section, name} attributes.
Definition: ncbi_param.hpp:149
@ eParam_NoThread
Do not use per-thread values.
Definition: ncbi_param.hpp:418
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
virtual bool GetBool(const string &section, const string &name, bool default_value, TFlags flags=0, EErrAction err_action=eThrow) const
Get boolean value of specified parameter name.
Definition: ncbireg.cpp:391
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define END_SCOPE(ns)
End the previously defined scope.
Definition: ncbistl.hpp:75
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
#define BEGIN_SCOPE(ns)
Define a new scope.
Definition: ncbistl.hpp:72
IO_PREFIX::ostream CNcbiOstream
Portable alias for ostream.
Definition: ncbistre.hpp:149
NCBI_NS_STD::string::size_type SIZE_TYPE
Definition: ncbistr.hpp:132
static string PrintableString(const CTempString str, TPrintableMode mode=fNewLine_Quote|fNonAscii_Passthru)
Get a printable version of the specified string.
Definition: ncbistr.cpp:3953
#define kEmptyStr
Definition: ncbistr.hpp:123
#define NPOS
Definition: ncbistr.hpp:133
static void TruncateSpacesInPlace(string &str, ETrunc where=eTrunc_Both)
Truncate spaces in a string (in-place)
Definition: ncbistr.cpp:3201
static string IntToString(int value, TNumToStringFlags flags=0, int base=10)
Convert int to string.
Definition: ncbistr.hpp:5084
bool empty(void) const
Return true if the represented string is empty (i.e., the length is zero)
Definition: tempstr.hpp:334
static bool StartsWith(const CTempString str, const CTempString start, ECase use_case=eCase)
Check if a string starts with a specified prefix value.
Definition: ncbistr.hpp:5412
size_type find(const CTempString match, size_type pos=0) const
Find the first instance of the entire matching string within the current string, beginning at an opti...
Definition: tempstr.hpp:655
size_type size(void) const
Return the length of the represented array.
Definition: tempstr.hpp:327
bool IsUniversalTime(void) const
Is time universal (GMT/UTC/Z)?
Definition: ncbitime.hpp:2422
bool IsEmpty(void) const
Is time object empty (date and time)?
Definition: ncbitime.hpp:2378
CTime & ToUniversalTime(void)
Convert the time into universal (GMT/UTC) time.
Definition: ncbitime.hpp:2472
virtual string Print(void) const
Print version information.
Definition: version.cpp:120
#define NCBI_SRAREAD_EXPORT
Definition: ncbi_export.h:1227
#define NCBI_DEVELOPMENT_VER
#define DEBUG
Definition: config.h:32
Definition of all error codes used in SRA C++ support libraries.
<!DOCTYPE HTML >< html > n< header > n< title > PubSeq Gateway Help Page</title > n< style > n table
int i
int len
const TYPE & Get(const CNamedParameterList *param)
range(_Ty, _Ty) -> range< _Ty >
constexpr bool empty(list< Ts... >) noexcept
const struct ncbi::grid::netcache::search::fields::SIZE size
string s_Value(TValue value)
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
#define NCBI_PACKAGE_NAME
#define NCBI_PACKAGE_VERSION
const char * tag
int isalpha(Uchar c)
Definition: ncbictype.hpp:61
int toupper(Uchar c)
Definition: ncbictype.hpp:73
Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...
Multi-threading – mutexes; rw-locks; semaphore.
T max(T x_, T y_)
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
void copy(Njn::Matrix< S > *matrix_, const Njn::Matrix< T > &matrix0_)
Definition: njn_matrix.hpp:613
static const char * prefix[]
Definition: pcregrep.c:405
static pcre_uint8 * buffer
Definition: pcretest.c:1051
@ eNotFound
Not found.
Defines CRequestContext class for NCBI C++ diagnostic API.
#define CHECK_VDB_TIMEOUT_FMT(msg, rc)
Definition: exception.hpp:186
#define CHECK_VDB_TIMEOUT(msg, rc)
Definition: exception.hpp:181
#define NCBI_THROW2_FMT(exception_class, err_code, message, extra)
Definition: exception.hpp:176
#define uint32_t
Definition: config.h:42
#define row(bind, expected)
Definition: string_bind.c:73
void Set(const CVDBCursor &cursor, TVDBRowId row, const CVDBColumn &column)
Definition: vdbread.hpp:637
const CVDBTable * m_Table
Definition: vdbread.hpp:648
const char * m_ColumnName
Definition: vdbread.hpp:649
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1627
const char * tag
Definition: bamread.cpp:529
CNcbiDiag::FManip manip
Definition: bamread.cpp:530
Definition: type.c:6
#define _ASSERT
else result
Definition: token2.c:20
static const SVDBSeverityTag kSeverityTags[]
Definition: vdbread.cpp:718
static const size_t kCacheSize
Definition: vdbread.cpp:1459
static char s_VDBVersion[32]
Definition: vdbread.cpp:677
static bool s_HasWindowsDriveLetter(const string &s)
Definition: vdbread.cpp:382
static rc_t VDBLogWriter(void *, const char *buffer, size_t size, size_t *written)
Definition: vdbread.cpp:761
DEFINE_SRA_REF_TRAITS(VDBManager, const)
static void s_InitLocalKNS(KNSManager *kns_mgr)
Definition: vdbread.cpp:901
DECLARE_SRA_REF_TRAITS(KClientHttpRequest,)
static void s_InitAllKNS(KNSManager *kns_mgr)
Definition: vdbread.cpp:862
static int s_GetDiagHandler(void)
Definition: vdbread.cpp:95
static const SVDBSeverityTag * s_GetVDBSeverityTag(CTempString token)
Definition: vdbread.cpp:727
DEFINE_STATIC_FAST_MUTEX(sx_SDKMutex)
static void s_InitVDBVersion()
Definition: vdbread.cpp:680
#define SIMULATE_RESOLVE_ERROR()
Definition: vdbread.cpp:176
static CKConfig s_InitProxyConfig()
Definition: vdbread.cpp:792
#define DECLARE_SDK_GET_GUARD()
Definition: vdbread.cpp:666
static void s_InitStaticKNS(KNSManager *kns_mgr)
Definition: vdbread.cpp:895
#define SIMULATE_SCHEMA_ERROR()
Definition: vdbread.cpp:178
static void s_VDBInit()
Definition: vdbread.cpp:907
static void s_InitDiagCheck()
Definition: vdbread.cpp:743
#define DECLARE_SDK_GUARD()
Definition: vdbread.cpp:660
static bool s_DiagIsSafe()
Definition: vdbread.cpp:750
NCBI_DEFINE_ERR_SUBCODE_X(9)
NCBI_PARAM_DEF(int, VDB, DIAG_HANDLER, 1)
static int s_GetDebugLevel(void)
Definition: vdbread.cpp:107
static void s_UpdateVDBRequestContext(void)
Definition: vdbread.cpp:832
NCBI_PARAM_DECL(int, VDB, DIAG_HANDLER)
NCBI_PARAM_DEF_EX(int, VDB, DEBUG, 0, eParam_NoThread, VDB_DEBUG)
static DECLARE_TLS_VAR(const CRequestContext *, s_LastRequestContext)
static CNcbiOstream & operator<<(CNcbiOstream &out, const CVDBTable &obj)
Definition: vdbread.cpp:1131
#define SIMULATE_OPEN_ERROR()
Definition: vdbread.cpp:177
uint32_t TVDBColumnIdx
Definition: vdbread.hpp:84
pair< TVDBRowId, TVDBRowCount > TVDBRowIdRange
Definition: vdbread.hpp:83
int64_t TVDBRowId
Definition: vdbread.hpp:79
Modified on Fri Apr 12 17:19:50 2024 by modify_doxy.py rev. 669887