NCBI C++ ToolKit
vdbread.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: vdbread.cpp 102589 2024-06-06 17:12:19Z vasilche $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Authors: Eugene Vasilchenko
27  *
28  * File Description:
29  * Access to SRA files
30  *
31  */
32 
33 #include <ncbi_pch.hpp>
35 #include <common/ncbi_source_ver.h>
37 
38 #include <klib/rc.h>
39 #include <klib/log.h>
40 #include <klib/text.h>
41 #include <klib/sra-release-version.h>
42 #include <kfg/config.h>
43 #include <kdb/manager.h>
44 #include <kdb/database.h>
45 #include <kdb/kdb-priv.h>
46 #include <kns/manager.h>
47 #include <kns/http.h>
48 #include <kns/tls.h>
49 
50 #include <vfs/manager-priv.h>
51 #include <vfs/manager.h>
52 #include <vfs/path.h>
53 #include <vfs/resolver.h>
54 
55 #include <sra/sradb-priv.h>
56 
57 #include <vdb/vdb-priv.h>
58 #include <vdb/manager.h>
59 #include <vdb/database.h>
60 #include <vdb/schema.h>
61 #include <vdb/table.h>
62 #include <vdb/cursor.h>
63 
64 
65 #include <corelib/ncbimtx.hpp>
66 #include <corelib/ncbifile.hpp>
67 #include <corelib/ncbi_param.hpp>
68 #include <corelib/request_ctx.hpp>
71 #include <objects/seq/seq__.hpp>
74 #include <sra/error_codes.hpp>
75 #include <util/random_gen.hpp>
76 
77 #include <cstring>
78 #include <algorithm>
79 #include <thread>
80 
82 
83 #define NCBI_USE_ERRCODE_X VDBReader
85 
87 
88 class CSeq_entry;
89 
90 
91 NCBI_PARAM_DECL(int, VDB, DIAG_HANDLER);
92 NCBI_PARAM_DEF(int, VDB, DIAG_HANDLER, 1);
93 
94 
95 static int s_GetDiagHandler(void)
96 {
97  static CSafeStatic<NCBI_PARAM_TYPE(VDB, DIAG_HANDLER)> s_Value;
98  return s_Value->Get();
99 }
100 
101 
102 
104 NCBI_PARAM_DEF_EX(int, VDB, DEBUG, 0, eParam_NoThread, VDB_DEBUG);
105 
106 
107 static int& s_GetDebugLevelVar(void)
108 {
109  static int value = NCBI_PARAM_TYPE(VDB, DEBUG)::GetDefault();
110  return value;
111 }
112 
113 
114 inline
115 static int s_GetDebugLevel()
116 {
117  return s_GetDebugLevelVar();
118 }
119 
120 
121 inline
122 static int s_SetDebugLevel(int new_level)
123 {
124  auto& var = s_GetDebugLevelVar();
125  auto old_level = var;
126  var = new_level;
127  return old_level;
128 }
129 
130 
131 NCBI_PARAM_DECL(bool, VDB, DISABLE_PAGEMAP_THREAD);
132 NCBI_PARAM_DEF(bool, VDB, DISABLE_PAGEMAP_THREAD, false);
133 
134 
135 
136 DEFINE_SRA_REF_TRAITS(VDBManager, const);
137 DEFINE_SRA_REF_TRAITS(VDatabase, const);
138 DEFINE_SRA_REF_TRAITS(VTable, const);
139 DEFINE_SRA_REF_TRAITS(VCursor, const);
140 DEFINE_SRA_REF_TRAITS(KIndex, const);
142 DEFINE_SRA_REF_TRAITS(KDBManager, const);
143 DEFINE_SRA_REF_TRAITS(KNSManager, );
144 DEFINE_SRA_REF_TRAITS(VFSManager, );
145 DEFINE_SRA_REF_TRAITS(VPath, const);
147 
148 //#define VDB_RANDOM_FAILS 1
149 
150 #define FAILS_VAR(name, suffix) NCBI_NAME3(VDB_, name, suffix)
151 
152 #define VDB_RESOLVE_RANDOM_FAILS 1
153 #define VDB_OPEN_RANDOM_FAILS 1
154 #define VDB_SCHEMA_RANDOM_FAILS 1
155 #define VDB_READ_RANDOM_FAILS 1
156 
157 #define VDB_RESOLVE_RANDOM_FAILS_FREQUENCY 20
158 #define VDB_RESOLVE_RANDOM_FAILS_RECOVER 10
159 #define VDB_OPEN_RANDOM_FAILS_FREQUENCY 10
160 #define VDB_OPEN_RANDOM_FAILS_RECOVER 5
161 #define VDB_SCHEMA_RANDOM_FAILS_FREQUENCY 50
162 #define VDB_SCHEMA_RANDOM_FAILS_RECOVER 100
163 #define VDB_READ_RANDOM_FAILS_FREQUENCY 20000
164 #define VDB_READ_RANDOM_FAILS_RECOVER 1000
165 
166 #if defined(VDB_RANDOM_FAILS)
167 static bool s_Fails(unsigned frequency)
168 {
169  static CSafeStatic<CRandom> s_Random([]()->CRandom*{CRandom* r=new CRandom();r->Randomize();return r;}, nullptr);
170  return s_Random->GetRandIndex(frequency) == 0;
171 }
172 # define SIMULATE_ERROR(name) \
173  if ( !(FAILS_VAR(name, _RANDOM_FAILS)) ) { \
174  } \
175  else { \
176  static thread_local unsigned recover_counter = 0; \
177  if ( recover_counter > 0 ) { \
178  --recover_counter; \
179  } \
180  else { \
181  if ( s_Fails(FAILS_VAR(name, _RANDOM_FAILS_FREQUENCY)) ) { \
182  recover_counter = (FAILS_VAR(name, _RANDOM_FAILS_RECOVER)); \
183  ERR_POST("VDB: Simulated " #name " failure: "<<CStackTrace()); \
184  NCBI_THROW(CSraException, eOtherError, \
185  "Simulated " #name " failure"); \
186  } \
187  } \
188  }
189 #else
190 # define SIMULATE_ERROR(name) ((void)0)
191 #endif
192 
193 #define SIMULATE_RESOLVE_ERROR() SIMULATE_ERROR(RESOLVE)
194 #define SIMULATE_OPEN_ERROR() SIMULATE_ERROR(OPEN)
195 #define SIMULATE_SCHEMA_ERROR() SIMULATE_ERROR(SCHEMA)
196 #define SIMULATE_READ_ERROR() SIMULATE_ERROR(READ)
197 
198 
199 /////////////////////////////////////////////////////////////////////////////
200 // CKConfig
201 
203 {
204  KConfig* cfg;
205  if ( rc_t rc = KConfigMake(&cfg, 0) ) {
206  *x_InitPtr() = 0;
207  NCBI_THROW2(CSraException, eInitFailed,
208  "Cannot create KConfig", rc);
209  }
210  *x_InitPtr() = cfg;
211 }
212 
213 
215 {
216  *x_InitPtr() = const_cast<KConfig*>(VFSManagerGetConfig(CVFSManager(mgr)));
217  if ( rc_t rc = KConfigAddRef(*this) ) {
218  *x_InitPtr() = 0;
219  NCBI_THROW2(CSraException, eInitFailed,
220  "Cannot get reference to KConfig", rc);
221  }
222 }
223 
224 
226 {
227  if ( rc_t rc = KConfigMake(x_InitPtr(), NULL) ) {
228  NCBI_THROW2(CSraException, eInitFailed,
229  "Cannot create KConfig singleton", rc);
230  }
231 }
232 
233 
234 void CKConfig::Commit() const
235 {
236  if ( rc_t rc = KConfigCommit(const_cast<KConfig*>(GetPointer())) ) {
237  NCBI_THROW2(CSraException, eOtherError,
238  "CKConfig: Cannot commit config changes", rc);
239  }
240 }
241 
242 
243 /////////////////////////////////////////////////////////////////////////////
244 // CVFSManager
245 
247 {
248  x_InitNew();
249 }
250 
251 
253 {
254  x_InitNew();
255 }
256 
257 
259 {
260  if ( rc_t rc = VFSManagerMake(x_InitPtr()) ) {
261  *x_InitPtr() = 0;
262  NCBI_THROW2(CSraException, eInitFailed,
263  "Cannot create VFSManager", rc);
264  }
265 }
266 
267 
269 {
270  if ( rc_t rc = KDBManagerGetVFSManager(CKDBManager(mgr), x_InitPtr()) ) {
271  *x_InitPtr() = 0;
272  NCBI_THROW2(CSraException, eInitFailed,
273  "Cannot get VFSManager", rc);
274  }
275 }
276 
277 
278 /////////////////////////////////////////////////////////////////////////////
279 // CKDBManager
280 
281 
283 {
284  if ( rc_t rc = VDBManagerGetKDBManagerRead(mgr, x_InitPtr()) ) {
285  *x_InitPtr() = 0;
286  NCBI_THROW2(CSraException, eInitFailed,
287  "Cannot get KDBManager", rc);
288  }
289 }
290 
291 
292 /////////////////////////////////////////////////////////////////////////////
293 // CKNSManager
294 
295 
297 {
298  if ( rc_t rc = KNSManagerMake(x_InitPtr()) ) {
299  *x_InitPtr() = 0;
300  NCBI_THROW2(CSraException, eInitFailed,
301  "Cannot make KNSManager", rc);
302  }
303 }
304 
305 
307 {
308  if ( rc_t rc = VFSManagerGetKNSMgr(mgr, x_InitPtr()) ) {
309  *x_InitPtr() = 0;
310  NCBI_THROW2(CSraException, eInitFailed,
311  "Cannot get KNSManager", rc);
312  }
313 }
314 
315 
316 /////////////////////////////////////////////////////////////////////////////
317 // CVPath
318 
319 CVPath::CVPath(const CVFSManager& mgr, const string& path, EType type)
320 {
321  x_Init(mgr, path, type);
322 }
323 
324 
325 CVPath::CVPath(const string& path, EType type)
326 {
328 }
329 
330 
331 void CVPath::x_Init(const CVFSManager& mgr, const string& path, EType type)
332 {
333  VPath* vpath = 0;
334  if ( type == eSys ) {
335  if ( rc_t rc = VFSManagerMakeSysPath(mgr, &vpath, path.c_str()) ) {
336  *x_InitPtr() = 0;
337  NCBI_THROW2_FMT(CSraException, eInitFailed,
338  "Cannot create sys VPath: "<<path, rc);
339  }
340  }
341  else if ( type == eAcc ) {
342  if ( rc_t rc = VFSManagerMakeAccPath(mgr, &vpath, path.c_str()) ) {
343  *x_InitPtr() = 0;
344  NCBI_THROW2_FMT(CSraException, eInitFailed,
345  "Cannot create acc VPath: "<<path, rc);
346  }
347  }
348  else {
349  if ( rc_t rc = VFSManagerMakePath(mgr, &vpath, path.c_str()) ) {
350  *x_InitPtr() = 0;
351  NCBI_THROW2_FMT(CSraException, eInitFailed,
352  "Cannot create VPath: "<<path, rc);
353  }
354  }
355  *x_InitPtr() = vpath;
356 }
357 
358 
360 {
361  char buffer[32];
362  size_t size;
363  if ( VPathReadScheme(*this, buffer, sizeof(buffer), &size) != 0 ) {
364  return false;
365  }
366  if ( size == 4 && memcmp(buffer, "file", 4) == 0 ) {
367  return true;
368  }
369  if ( size == 9 && memcmp(buffer, "ncbi-file", 9) == 0 ) {
370  return true;
371  }
372  return false;
373 }
374 
375 
377 {
378  const String* str = 0;
379  if (type == eSys && IsLocalFile()) {
380  if (rc_t rc = VPathMakeSysPath(*this, &str)) {
381  NCBI_THROW2(CSraException, eOtherError,
382  "Cannot get path from VPath", rc);
383  }
384  }
385  else {
386  if (rc_t rc = VPathMakeString(*this, &str)) {
387  NCBI_THROW2(CSraException, eOtherError,
388  "Cannot get path from VPath", rc);
389  }
390  }
391  string ret(str->addr, str->size);
392  StringWhack(str);
393  return ret;
394 }
395 
396 
397 #ifdef NCBI_OS_MSWIN
398 static inline
399 bool s_HasWindowsDriveLetter(const string& s)
400 {
401  // first symbol is letter, and second symbol is colon (':')
402  return s.length() >= 2 && isalpha(s[0]&0xff) && s[1] == ':';
403 }
404 #endif
405 
406 
407 bool CVPath::IsPlainAccession(const string& acc_or_path)
408 {
409 #ifdef NCBI_OS_MSWIN
410  return !s_HasWindowsDriveLetter(acc_or_path) &&
411  acc_or_path.find_first_of("/\\") == NPOS;
412 #else
413  return acc_or_path.find('/') == NPOS;
414 #endif
415 }
416 
417 
418 #ifdef NCBI_OS_MSWIN
419 string CVPath::ConvertSysPathToPOSIX(const string& sys_path)
420 {
421  // Convert Windows path with drive letter
422  // C:\Users\Public -> /C/Users/Public
423  if ( sys_path[0] == 'h' &&
424  (NStr::StartsWith(sys_path, "http://") ||
425  NStr::StartsWith(sys_path, "https://")) ) {
426  return sys_path;
427  }
428  try {
429  string path = CDirEntry::CreateAbsolutePath(sys_path);
430  replace(path.begin(), path.end(), '\\', '/');
431  if (s_HasWindowsDriveLetter(path)) {
432  // move drive letter from first symbol to second (in place of ':')
433  path[1] = toupper(path[0] & 0xff);
434  // add leading slash
435  path[0] = '/';
436  }
437  return path;
438  }
439  catch (exception&) {
440  // CDirEntry::CreateAbsolutePath() can fail on URL for remote access
441  return sys_path;
442  }
443 }
444 
445 
446 string CVPath::ConvertAccOrSysPathToPOSIX(const string& acc_or_path)
447 {
448  return IsPlainAccession(acc_or_path) ?
449  acc_or_path :
450  ConvertSysPathToPOSIX(acc_or_path);
451 }
452 #endif
453 
454 
455 /////////////////////////////////////////////////////////////////////////////
456 // CVResolver
457 
459  : m_Mgr(mgr)
460 {
461  if ( rc_t rc = VFSManagerGetResolver(mgr, x_InitPtr()) ) {
462  *x_InitPtr() = 0;
463  NCBI_THROW2(CSraException, eInitFailed,
464  "Cannot get VResolver", rc);
465  }
466 }
467 
468 
470  : m_Mgr(mgr)
471 {
472  if ( rc_t rc = VFSManagerMakeResolver(mgr, x_InitPtr(), cfg) ) {
473  *x_InitPtr() = 0;
474  NCBI_THROW2(CSraException, eInitFailed,
475  "Cannot create VResolver", rc);
476  }
477 }
478 
479 
480 string CVResolver::Resolve(const string& acc_or_path) const
481 {
482  if ( !CVPath::IsPlainAccession(acc_or_path) ) {
483  // already a path
484  return acc_or_path;
485  }
487  CVPath acc(m_Mgr, acc_or_path, CVPath::eAcc);
488  const VPath* path;
489  rc_t rc = VResolverLocal(*this, acc, &path);
490  if ( rc ) {
491  if ( GetRCObject(rc) == rcName &&
492  GetRCState(rc) == rcNotFound &&
493  GetRCContext(rc) == rcResolving ) {
494  // continue with other ways to resolve
495  }
496  else {
497  NCBI_THROW2_FMT(CSraException, eOtherError,
498  "VResolverLocal("<<acc_or_path<<") failed", rc);
499  }
500  rc = VResolverRemote(*this, eProtocolNone, acc, &path);
501  }
502  if ( rc ) {
503  CHECK_VDB_TIMEOUT_FMT("Cannot find acc path: "<<acc_or_path, rc);
504  if ( GetRCObject(rc) == rcName &&
505  GetRCState(rc) == rcNotFound &&
506  GetRCContext(rc) == rcResolving ) {
507  // continue with other ways to resolve
508  }
509  else {
510  NCBI_THROW2_FMT(CSraException, eOtherError,
511  "VResolverRemote("<<acc_or_path<<") failed", rc);
512  }
513  if ( CDirEntry(acc_or_path).Exists() ) {
514  // local file
515  return acc_or_path;
516  }
517  // couldnt resolve with any way
518  NCBI_THROW2_FMT(CSraException, eNotFoundDb,
519  "Cannot find acc path: "<<acc_or_path, rc);
520  }
521  return CVPath(path).ToString();
522 }
523 
524 
525 /////////////////////////////////////////////////////////////////////////////
526 // CVDBMgr
527 
529  : m_Resolver(null)
530 {
531  x_Init();
532 }
533 
534 
536 {
537  return s_GetDebugLevel();
538 }
539 
540 
541 int CVDBMgr::SetDebugLevel(int new_level)
542 {
543  return s_SetDebugLevel(new_level);
544 }
545 
546 
547 string CVDBMgr::FindAccPath(const string& acc) const
548 {
549  if ( !m_Resolver ) {
551  }
552  return m_Resolver.Resolve(acc);
553 }
554 
555 
556 string CVDBMgr::FindDereferencedAccPath(const string& acc_or_path) const
557 {
558  string path;
559  if ( CVPath::IsPlainAccession(acc_or_path) ) {
560  // resolve VDB accessions
562  LOG_POST_X(6, "CVDBMgr: resolving VDB accession: " << acc_or_path);
563  }
564  path = FindAccPath(acc_or_path);
566  LOG_POST_X(4, "CVDBMgr: resolved VDB accession: " << acc_or_path << " -> " << path);
567  }
568  }
569  else {
570  // real path, http:, etc.
571  path = acc_or_path;
572  }
573 
574  // resolve symbolic links for correct timestamp and longer-living reference
576  LOG_POST_X(7, "CVDBMgr: checking file symbolic link: " << acc_or_path << " -> " << path);
577  }
578  CDirEntry de(path);
579  if ( de.Exists() ) {
580  de.DereferencePath();
581  if ( de.GetPath() != path ) {
582  path = de.GetPath();
584  LOG_POST_X(5, "CVDBMgr: resolved file symbolic link: " << acc_or_path << " -> " << path);
585  }
586  }
587  }
588  return path;
589 }
590 
591 
592 CTime CVDBMgr::GetTimestamp(const string& path) const
593 {
594  CTime timestamp;
595  if ( path[0] == 'h' &&
596  (NStr::StartsWith(path, "http://") ||
597  NStr::StartsWith(path, "https://")) ) {
598  // try http:
599  timestamp = GetURLTimestamp(path);
600  }
601  else {
602  // try direct file access
603  if ( !CDirEntry(path).GetTime(&timestamp) ) {
604  NCBI_THROW_FMT(CSraException, eInitFailed,
605  "Cannot get timestamp of local path: " << path);
606  }
607  timestamp.ToUniversalTime();
608  }
609  _ASSERT(!timestamp.IsEmpty());
610  _ASSERT(timestamp.IsUniversalTime());
612  LOG_POST_X(3, "CVDBMgr: timestamp of " << path << ": " << CTime(timestamp).ToLocalTime());
613  }
614  return timestamp;
615 }
616 
617 
618 DECLARE_SRA_REF_TRAITS(KClientHttpRequest, );
619 DECLARE_SRA_REF_TRAITS(KClientHttpResult, );
620 
621 DEFINE_SRA_REF_TRAITS(KClientHttpRequest, );
622 DEFINE_SRA_REF_TRAITS(KClientHttpResult, );
623 
625  : public CSraRef<KClientHttpRequest>
626 {
627 public:
628  CClientHttpRequest(const CKNSManager& mgr, const string& path)
629  {
630  const ver_t kHTTP_1_1 = 0x01010000;
631  if ( rc_t rc = KNSManagerMakeClientRequest(mgr, x_InitPtr(),
632  kHTTP_1_1, nullptr,
633  "%s", path.c_str()) ) {
634  *x_InitPtr() = 0;
635  CHECK_VDB_TIMEOUT("Cannot create http request", rc);
636  NCBI_THROW2(CSraException, eInitFailed,
637  "Cannot create http client request", rc);
638  }
639  }
640 
641 private:
642 };
643 
644 
646  : public CSraRef<KClientHttpResult>
647 {
648 public:
649  enum EHead {
650  eHead
651  };
652 
654  {
655  if ( rc_t rc = KClientHttpRequestHEAD(request, x_InitPtr()) ) {
656  *x_InitPtr() = 0;
657  CHECK_VDB_TIMEOUT("Cannot get http HEAD", rc);
658  NCBI_THROW2(CSraException, eInitFailed,
659  "Cannot get http HEAD", rc);
660  }
661  }
662 
663 private:
664 };
665 
666 
667 CTime CVDBMgr::GetURLTimestamp(const string& path) const
668 {
669  CVFSManager vfs(*this);
670  CKNSManager kns(vfs);
671  CClientHttpRequest request(kns, path);
673  char buffer[99];
674  size_t size;
675  if ( rc_t rc = KClientHttpResultGetHeader(result, "Last-Modified",
676  buffer, sizeof(buffer), &size) ) {
677  CHECK_VDB_TIMEOUT("No Last-Modified header in HEAD response", rc);
678  NCBI_THROW2(CSraException, eNotFoundValue,
679  "No Last-Modified header in HEAD response", rc);
680  }
682  CTimeFormat fmt("w, d b Y h:m:s Z"); // standard Last-Modified HTTP header format
683  return CTime(str, fmt);
684 }
685 
686 
687 //#define GUARD_SDK
688 #ifdef NCBI_COMPILER_MSVC
689 //# define GUARD_SDK_GET
690 #endif
691 
692 #ifdef GUARD_SDK
693 # define DECLARE_SDK_GUARD() CFastMutexGuard guard(sx_SDKMutex)
694 #else
695 # define DECLARE_SDK_GUARD()
696 #endif
697 
698 #ifdef GUARD_SDK_GET
699 # define DECLARE_SDK_GET_GUARD() CFastMutexGuard guard(sx_SDKMutex)
700 #else
701 # define DECLARE_SDK_GET_GUARD()
702 #endif
703 
704 
705 /////////////////////////////////////////////////////////////////////////////
706 // VDB library initialization code
707 // similar code is located in bamread.cpp
708 /////////////////////////////////////////////////////////////////////////////
709 
711 
712 static char s_VDBVersion[32]; // enough for 255.255.65535-dev4000000000
713 
714 static
716 {
717  if ( !s_VDBVersion[0] ) {
718  ostringstream s;
719  {{ // format VDB version string
720  SraReleaseVersion release_version;
721  SraReleaseVersionGet(&release_version);
722  s << (release_version.version>>24) << '.'
723  << ((release_version.version>>16)&0xff) << '.'
724  << (release_version.version&0xffff);
725  if ( release_version.revision != 0 ||
726  release_version.type != SraReleaseVersion::eSraReleaseVersionTypeFinal ) {
727  const char* type = "";
728  switch ( release_version.type ) {
729  case SraReleaseVersion::eSraReleaseVersionTypeDev: type = "dev"; break;
730  case SraReleaseVersion::eSraReleaseVersionTypeAlpha: type = "a"; break;
731  case SraReleaseVersion::eSraReleaseVersionTypeBeta: type = "b"; break;
732  case SraReleaseVersion::eSraReleaseVersionTypeRC: type = "RC"; break;
733  default: type = ""; break;
734  }
735  s << '-' << type << release_version.revision;
736  }
737  }}
738  string v = s.str();
739  if ( !v.empty() ) {
740  if ( v.size() >= sizeof(s_VDBVersion) ) {
741  v.resize(sizeof(s_VDBVersion)-1);
742  }
743  copy(v.begin()+1, v.end(), s_VDBVersion+1);
744  s_VDBVersion[0] = v[0];
745  }
746  }
747 }
748 
749 struct SVDBSeverityTag {
750  const char* tag;
752 };
753 static const SVDBSeverityTag kSeverityTags[] = {
754  { "err:", Error },
755  { "int:", Error },
756  { "sys:", Error },
757  { "info:", Info },
758  { "warn:", Warning },
759  { "debug:", Trace },
760  { "fatal:", Fatal },
761 };
763 {
764  if ( !token.empty() && token[token.size()-1] == ':' ) {
765  for ( auto& tag : kSeverityTags ) {
766  if ( token == tag.tag ) {
767  return &tag;
768  }
769  }
770  }
771  return 0;
772 }
773 
774 #ifndef NCBI_THREADS
775 static thread::id s_DiagCheckThreadID;
776 #endif
777 
778 static inline void s_InitDiagCheck()
779 {
780 #ifndef NCBI_THREADS
781  s_DiagCheckThreadID = this_thread::get_id();
782 #endif
783 }
784 
785 static inline bool s_DiagIsSafe()
786 {
787 #ifndef NCBI_THREADS
788  return s_DiagCheckThreadID == this_thread::get_id();
789 #else
790  return true;
791 #endif
792 }
793 
794 
795 static
796 rc_t VDBLogWriter(void* /*data*/, const char* buffer, size_t size, size_t* written)
797 {
800  CNcbiDiag::FManip sev_manip = Error;
801 
802  for ( SIZE_TYPE token_pos = 0, token_end; token_pos < msg.size(); token_pos = token_end + 1 ) {
803  token_end = msg.find(' ', token_pos);
804  if ( token_end == NPOS ) {
805  token_end = msg.size();
806  }
807  if ( auto tag = s_GetVDBSeverityTag(CTempString(msg, token_pos, token_end-token_pos)) ) {
808  sev_manip = tag->manip;
809  break;
810  }
811  }
812  if ( sev_manip == Trace ) {
813  if ( s_DiagIsSafe() ) {
814  _TRACE("VDB "<<s_VDBVersion<<": "<<msg);
815  }
816  }
817  else {
818  if ( s_DiagIsSafe() ) {
819  ERR_POST_X(2, sev_manip<<"VDB "<<s_VDBVersion<<": "<<msg);
820  }
821  }
822  *written = size;
823  return 0;
824 }
825 
826 
828 {
829  CKConfig config(null);
831  string host = app->GetConfig().GetString("CONN", "HTTP_PROXY_HOST", kEmptyStr);
832  int port = app->GetConfig().GetInt("CONN", "HTTP_PROXY_PORT", 0);
833  if ( !host.empty() && port != 0 ) {
835  string path = host + ':' + NStr::IntToString(port);
836  if ( rc_t rc = KConfigWriteString(config,
837  "/http/proxy/path", path.c_str()) ) {
838  NCBI_THROW2(CSraException, eInitFailed,
839  "Cannot set KConfig proxy path", rc);
840  }
841  if ( rc_t rc = KConfigWriteBool(config,
842  "/http/proxy/enabled", true) ) {
843  NCBI_THROW2(CSraException, eInitFailed,
844  "Cannot set KConfig proxy enabled", rc);
845  }
846  }
847  }
848  return config;
849 }
850 
851 
852 static DECLARE_TLS_VAR(const CRequestContext*, s_LastRequestContext);
853 static DECLARE_TLS_VAR(CRequestContext::TVersion, s_LastRequestContextVersion);
854 
855 /*
856 // Performance collecting code
857 static struct SReport {
858  size_t count;
859  double time;
860  SReport() : count(0), time(0) {}
861  ~SReport() {
862  LOG_POST("GetRequestContext() called "<<count<<" times, spent "<<time<<"s");
863  }
864 } dummy;
865 */
866 
867 static void s_UpdateVDBRequestContext(void)
868 {
869  // Performance collecting code
870  // CStopWatch sw(CStopWatch::eStart);
872  // Performance collecting code
873  // dummy.count += 1; dummy.time += sw.Elapsed();
874  auto req_ctx_version = req_ctx.GetVersion();
875  if ( &req_ctx == s_LastRequestContext && req_ctx_version == s_LastRequestContextVersion ) {
876  return;
877  }
878  _TRACE("CVDBMgr: Updating request context with version: "<<req_ctx_version);
879  s_LastRequestContext = &req_ctx;
880  s_LastRequestContextVersion = req_ctx_version;
882  if ( req_ctx.IsSetSessionID() ) {
883  _TRACE("CVDBMgr: Updating session ID: "<<req_ctx.GetSessionID());
884  KNSManagerSetSessionID(kns_mgr, req_ctx.GetSessionID().c_str());
885  }
886  if ( req_ctx.IsSetClientIP() ) {
887  _TRACE("CVDBMgr: Updating client IP: "<<req_ctx.GetClientIP());
888  KNSManagerSetClientIP(kns_mgr, req_ctx.GetClientIP().c_str());
889  }
890  if ( req_ctx.IsSetHitID() ) {
891  _TRACE("CVDBMgr: Updating hit ID: "<<req_ctx.GetHitID());
892  KNSManagerSetPageHitID(kns_mgr, req_ctx.GetHitID().c_str());
893  }
894 }
895 
896 
897 static void s_InitAllKNS(KNSManager* kns_mgr)
898 {
900  if ( app && app->GetConfig().GetBool("VDB", "ALLOW_ALL_CERTS", false) ) {
901  if ( rc_t rc = KNSManagerSetAllowAllCerts(kns_mgr, true) ) {
902  NCBI_THROW2(CSraException, eInitFailed,
903  "Cannot enable all HTTPS certificates in KNSManager", rc);
904  }
905  }
906  {{ // set user agent
908  if ( app ) {
909  str << app->GetAppName() << ": " << app->GetVersion().Print() << "; ";
910  }
911 #if NCBI_PACKAGE
912  str << "Package: " << NCBI_PACKAGE_NAME << ' ' <<
913  NCBI_PACKAGE_VERSION << "; ";
914 #endif
915  str << "C++ ";
916 #ifdef NCBI_PRODUCTION_VER
917  str << NCBI_PRODUCTION_VER << "/";
918 #endif
919 #ifdef NCBI_DEVELOPMENT_VER
921 #endif
922  string prefix = CNcbiOstrstreamToString(str);
923  KNSManagerSetUserAgent(kns_mgr, "%s; VDB %s",
924  prefix.c_str(),
925  s_VDBVersion);
926  }}
927 }
928 
929 
930 static void s_InitStaticKNS(KNSManager* kns_mgr)
931 {
932  s_InitAllKNS(kns_mgr);
933 }
934 
935 
936 static void s_InitLocalKNS(KNSManager* kns_mgr)
937 {
938  s_InitAllKNS(kns_mgr);
939 }
940 
941 
943 {
944  CFastMutexGuard guard(sx_SDKMutex);
945  static bool initialized = false;
946  if ( !initialized ) {
948  // redirect VDB log to C++ Toolkit
949  if ( s_GetDiagHandler() ) {
950  KLogInit();
951  KLogLevel ask_level;
952 #ifdef _DEBUG
953  ask_level = klogDebug;
954 #else
955  ask_level = klogInfo;
956 #endif
957  s_InitDiagCheck();
958  KLogLevelSet(ask_level);
959  KLogHandlerSet(VDBLogWriter, 0);
960  KLogLibHandlerSet(VDBLogWriter, 0);
962  const char* msg = "info: VDB initialized";
963  size_t written;
964  VDBLogWriter(0, msg, strlen(msg), &written);
965  }
966  }
969  s_InitStaticKNS(kns_mgr);
970  initialized = true;
971  }
972 }
973 
974 /////////////////////////////////////////////////////////////////////////////
975 // end of VDB library initialization code
976 /////////////////////////////////////////////////////////////////////////////
977 
978 
979 //#define UPDATE_REQUEST_CONTEXT_FINAL
980 
981 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
982 static DECLARE_TLS_VAR(size_t, s_RequestContextUpdaterRecursion);
983 #endif
984 
986 public:
990 };
991 
992 
994 {
995 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
996  size_t r = s_RequestContextUpdaterRecursion;
997  s_RequestContextUpdaterRecursion = r+1;
998  if ( r != 0 ) {
999  return;
1000  }
1003 #endif
1004 }
1005 
1006 
1008 {
1009 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
1010  size_t r = s_RequestContextUpdaterRecursion;
1011  s_RequestContextUpdaterRecursion = r+1;
1012  if ( r != 0 ) {
1013  return;
1014  }
1015 #endif
1018 }
1019 
1020 
1022 {
1023 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
1024  size_t r = s_RequestContextUpdaterRecursion;
1025  s_RequestContextUpdaterRecursion = r-1;
1026 #endif
1027 }
1028 
1029 
1031 {
1032 #ifndef UPDATE_REQUEST_CONTEXT_FINAL
1033  size_t r = s_RequestContextUpdaterRecursion;
1034  s_RequestContextUpdaterRecursion = r-1;
1035 #endif
1036 }
1037 
1038 
1040 {
1041  s_VDBInit();
1042  if ( rc_t rc = VDBManagerMakeRead(x_InitPtr(), 0) ) {
1043  *x_InitPtr() = 0;
1044  NCBI_THROW2(CSraException, eInitFailed,
1045  "Cannot open VDBManager", rc);
1046  }
1047  CVFSManager vfs_mgr(*this);
1048  VFSManagerLogNamesServiceErrors(vfs_mgr, false);
1049  s_InitLocalKNS(CKNSManager(vfs_mgr));
1050  if ( NCBI_PARAM_TYPE(VDB, DISABLE_PAGEMAP_THREAD)().Get() ) {
1051  VDBManagerDisablePagemapThread(*this);
1052  }
1053 }
1054 
1055 
1057 {
1058  const VPath* ret;
1059  if ( rc_t rc = VDBManagerGetCacheRoot(*this, &ret) ) {
1060  if ( GetRCObject(rc) == RCObject(rcPath) &&
1061  GetRCState(rc) == rcNotFound ) {
1062  return kEmptyStr;
1063  }
1064  NCBI_THROW2(CSraException, eOtherError,
1065  "CVDBMgr: Cannot get cache root", rc);
1066  }
1067  return CVPath(ret).ToString(CVPath::eSys);
1068 }
1069 
1070 
1071 void CVDBMgr::SetCacheRoot(const string& path)
1072 {
1073  CVPath vpath(CVFSManager(*this), path, CVPath::eSys);
1074  if ( rc_t rc = VDBManagerSetCacheRoot(*this, vpath) ) {
1075  NCBI_THROW2(CSraException, eOtherError,
1076  "CVDBMgr: Cannot set cache root", rc);
1077  }
1078 }
1079 
1080 
1082 {
1083  if ( rc_t rc = VDBManagerDeleteCacheOlderThan(*this, days) ) {
1084  NCBI_THROW2(CSraException, eOtherError,
1085  "CVDBMgr: Cannot delete old cache files", rc);
1086  }
1087 }
1088 
1089 
1091 {
1092  CKConfig(*this).Commit();
1093 }
1094 
1095 
1096 /////////////////////////////////////////////////////////////////////////////
1097 // CVDB
1098 
1099 CVDB::CVDB(const CVDBMgr& mgr, const string& acc_or_path)
1100  : m_Name(acc_or_path)
1101 {
1102  if ( s_GetDebugLevel() >= CVDBMgr::eDebugOpen ) {
1103  LOG_POST_X(8, "CVDBMgr: opening VDB: " << acc_or_path);
1104  }
1106  CFinalRequestContextUpdater ctx_updater;
1107  string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
1109  if ( rc_t rc = VDBManagerOpenDBRead(mgr, x_InitPtr(), 0, "%.*s",
1110  int(path.size()), path.data()) ) {
1111  *x_InitPtr() = 0;
1112  string msg = "Cannot open VDB: "+acc_or_path;
1113  CHECK_VDB_TIMEOUT(msg, rc);
1114  if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
1115  GetRCObject(rc) == RCObject(rcPath) ||
1116  GetRCObject(rc) == RCObject(rcFile)) &&
1117  GetRCState(rc) == rcNotFound ) {
1118  // no VDB accession
1119  NCBI_THROW2(CSraException, eNotFoundDb, msg, rc);
1120  }
1121  else if ( GetRCObject(rc) == rcName &&
1122  GetRCState(rc) == rcNotFound &&
1123  GetRCContext(rc) == rcResolving ) {
1124  // invalid VDB database
1125  NCBI_THROW2(CSraException, eNotFoundDb, msg, rc);
1126  }
1127  else if ( GetRCObject(rc) == RCObject(rcFile) &&
1128  GetRCState(rc) == rcUnauthorized ) {
1129  // no permission to use VDB database
1130  NCBI_THROW2(CSraException, eProtectedDb, msg, rc);
1131  }
1132  else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
1133  GetRCState(rc) == rcIncorrect ) {
1134  // invalid VDB database
1135  NCBI_THROW2(CSraException, eDataError, msg, rc);
1136  }
1137  else {
1138  // other errors
1139  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1140  }
1141  }
1142  if ( s_GetDebugLevel() >= CVDBMgr::eDebugOpen ) {
1143  LOG_POST_X(9, "CVDBMgr: opened VDB: " << acc_or_path << " (" << path << ")");
1144  }
1145  if ( 0 ) {
1146  const KDatabase* kdb = 0;
1147  if ( rc_t rc = VDatabaseOpenKDatabaseRead(*this, &kdb) ) {
1148  NCBI_THROW2(CSraException, eOtherError, "VDatabaseOpenKDatabaseRead() failed", rc);
1149  }
1150  const char* kdb_path = 0;
1151  if ( rc_t rc = KDatabaseGetPath(kdb, &kdb_path) ) {
1152  NCBI_THROW2(CSraException, eOtherError, "KDatabaseGetPath() failed", rc);
1153  }
1154  LOG_POST("CVDB("<<acc_or_path<<") -> "<<path<<" -> "<<kdb_path);
1155  if ( rc_t rc = KDatabaseRelease(kdb) ) {
1156  NCBI_THROW2(CSraException, eOtherError, "KDatabaseRelease() failed", rc);
1157  }
1158  }
1159 }
1160 
1161 
1163 {
1164  return out << GetName();
1165 }
1166 
1167 
1168 /////////////////////////////////////////////////////////////////////////////
1169 // CVDBTable
1170 
1171 static inline
1173 {
1174  return obj.PrintFullName(out);
1175 }
1176 
1177 
1179  const char* table_name,
1180  EMissing missing)
1181  : m_Db(db),
1182  m_Name(table_name)
1183 {
1185  CFinalRequestContextUpdater ctx_updater;
1187  if ( rc_t rc = VDatabaseOpenTableRead(db, x_InitPtr(), table_name) ) {
1188  *x_InitPtr() = 0;
1189  string msg = "Cannot open VDB: "+GetFullName();
1190  CHECK_VDB_TIMEOUT(msg, rc);
1191  RCState rc_state = GetRCState(rc);
1192  int rc_object = GetRCObject(rc);
1193  if ( rc_state == rcNotFound &&
1194  (rc_object == rcParam ||
1195  rc_object == rcPath) ) {
1196  // missing table in the DB
1197  if ( missing != eMissing_Throw ) {
1198  return;
1199  }
1200  NCBI_THROW2(CSraException, eNotFoundTable, msg, rc);
1201  }
1202  else {
1203  // other errors
1204  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1205  }
1206  }
1207 }
1208 
1209 
1211  const string& acc_or_path,
1212  EMissing missing)
1213  : m_Name(acc_or_path)
1214 {
1215  *x_InitPtr() = 0;
1217  CFinalRequestContextUpdater ctx_updater;
1218  string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
1220  if ( rc_t rc = VDBManagerOpenTableRead(mgr, x_InitPtr(), 0, "%.*s",
1221  int(path.size()), path.data()) ) {
1222  *x_InitPtr() = 0;
1223  string msg = "Cannot open VDB: "+acc_or_path;
1224  CHECK_VDB_TIMEOUT(msg, rc);
1225  if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
1226  GetRCObject(rc) == RCObject(rcPath)) &&
1227  GetRCState(rc) == rcNotFound ) {
1228  // no SRA accession
1229  if ( missing != eMissing_Throw ) {
1230  return;
1231  }
1232  NCBI_THROW2(CSraException, eNotFoundTable, msg, rc);
1233  }
1234  else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
1235  GetRCState(rc) == rcIncorrect ) {
1236  // invalid SRA database
1237  NCBI_THROW2(CSraException, eDataError, msg, rc);
1238  }
1239  else {
1240  // other errors
1241  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1242  }
1243  }
1244 }
1245 
1246 
1247 string CVDBTable::GetFullName(void) const
1248 {
1249  string ret;
1250  if ( GetDb() ) {
1251  ret = GetDb().GetFullName();
1252  ret += '.';
1253  }
1254  ret += GetName();
1255  return ret;
1256 }
1257 
1258 
1260 {
1261  if ( GetDb() ) {
1262  GetDb().PrintFullName(out) << '.';
1263  }
1264  return out << GetName();
1265 }
1266 
1267 
1268 /////////////////////////////////////////////////////////////////////////////
1269 // CVDBTableIndex
1270 
1271 static inline
1273 {
1274  return obj.PrintFullName(out);
1275 }
1276 
1277 
1279  const char* index_name,
1280  EMissing missing)
1281  : m_Table(table),
1282  m_Name(index_name)
1283 {
1284  CFinalRequestContextUpdater ctx_updater;
1286  if ( rc_t rc = VTableOpenIndexRead(table, x_InitPtr(), index_name) ) {
1287  *x_InitPtr() = 0;
1288  string msg = "Cannot open VDB table index: "+GetFullName();
1289  CHECK_VDB_TIMEOUT(msg, rc);
1290  if ( GetRCObject(rc) == RCObject(rcIndex) &&
1291  GetRCState(rc) == rcNotFound ) {
1292  // no such index
1293  if ( missing != eMissing_Throw ) {
1294  return;
1295  }
1296  NCBI_THROW2(CSraException, eNotFoundIndex, msg, rc);
1297  }
1298  else {
1299  NCBI_THROW2(CSraException, eOtherError, msg, rc);
1300  }
1301  }
1302 }
1303 
1304 
1306 {
1307  return GetTable().GetFullName()+'.'+GetName();
1308 }
1309 
1310 
1312 {
1313  return GetTable().PrintFullName(out) << '.' << GetName();
1314 }
1315 
1316 
1318 {
1320  if ( rc_t rc = KIndexFindText(*this, value.c_str(),
1321  &range.first, &range.second, 0, 0) ) {
1322  CHECK_VDB_TIMEOUT_FMT("Cannot find value in index: "<<*this<<": "<<value, rc);
1323  if ( GetRCObject(rc) == RCObject(rcString) &&
1324  GetRCState(rc) == rcNotFound ) {
1325  // no such value
1326  range.first = range.second = 0;
1327  }
1328  else {
1329  NCBI_THROW2_FMT(CSraException, eOtherError,
1330  "Cannot find value in index: "<<*this<<": "<<value, rc);
1331  }
1332  }
1333  return range;
1334 }
1335 
1336 
1337 /////////////////////////////////////////////////////////////////////////////
1338 // CVDBCursor
1339 
1340 static inline
1342 {
1343  return out << obj.GetTable();
1344 }
1345 
1346 
1348 {
1349  CFinalRequestContextUpdater ctx_updater;
1350  if ( *this ) {
1351  NCBI_THROW2(CSraException, eInvalidState,
1352  "Cannot init VDB cursor again",
1353  RC(rcApp, rcCursor, rcConstructing, rcSelf, rcOpen));
1354  }
1355  if ( !table ) { // VTableCreateCursorRead lacks check for null table argument
1356  NCBI_THROW2(CSraException, eNullPtr,
1357  "Cannot init VDB cursor",
1358  RC(rcApp, rcCursor, rcConstructing, rcTable, rcNull));
1359  }
1360  if ( rc_t rc = VTableCreateCursorRead(table, x_InitPtr()) ) {
1361  *x_InitPtr() = 0;
1362  CHECK_VDB_TIMEOUT_FMT("Cannot create VDB cursor: "<<table, rc);
1363  NCBI_THROW2_FMT(CSraException, eInitFailed,
1364  "Cannot create VDB cursor: "<<table, rc);
1365  }
1366  if ( rc_t rc = VCursorPermitPostOpenAdd(*this) ) {
1367  NCBI_THROW2_FMT(CSraException, eInitFailed,
1368  "Cannot allow VDB cursor post open column add: "<<table, rc);
1369  }
1370  if ( rc_t rc = VCursorOpen(*this) ) {
1371  CHECK_VDB_TIMEOUT_FMT("Cannot open VDB cursor: "<<table, rc);
1372  NCBI_THROW2_FMT(CSraException, eInitFailed,
1373  "Cannot open VDB cursor: "<<table, rc);
1374  }
1375  m_Table = table;
1376 }
1377 
1378 
1380 {
1381  if ( !RowIsOpened() ) {
1382  return;
1383  }
1384  if ( rc_t rc = VCursorCloseRow(*this) ) {
1385  NCBI_THROW2(CSraException, eInitFailed,
1386  "Cannot close VDB cursor row", rc);
1387  }
1388  m_RowOpened = false;
1389 }
1390 
1391 
1393 {
1394  CloseRow();
1395  CFinalRequestContextUpdater ctx_updater;
1396  if ( rc_t rc = VCursorSetRowId(*this, row_id) ) {
1397  return rc;
1398  }
1399  if ( rc_t rc = VCursorOpenRow(*this) ) {
1400  return rc;
1401  }
1402  m_RowOpened = true;
1403  return 0;
1404 }
1405 
1406 
1408 {
1409  if ( rc_t rc = OpenRowRc(row_id) ) {
1410  CHECK_VDB_TIMEOUT_FMT("Cannot open VDB cursor row: "<<*this<<": "<<row_id, rc);
1411  NCBI_THROW2_FMT(CSraException, eInitFailed,
1412  "Cannot open VDB cursor row: "<<*this<<": "<<row_id, rc);
1413  }
1414 }
1415 
1416 
1418 {
1419  TVDBRowIdRange ret;
1420  if ( rc_t rc = VCursorIdRange(*this, column, &ret.first, &ret.second) ) {
1421  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB cursor row range: "<<*this<<": "<<column, rc);
1422  NCBI_THROW2_FMT(CSraException, eInitFailed,
1423  "Cannot get VDB cursor row range: "<<*this<<": "<<column, rc);
1424  }
1425  return ret;
1426 }
1427 
1428 
1430 {
1432  return range.first+range.second-1;
1433 }
1434 
1435 
1436 void CVDBCursor::SetParam(const char* name, const CTempString& value) const
1437 {
1438  CFinalRequestContextUpdater ctx_updater;
1439  if ( rc_t rc = VCursorParamsSet
1440  ((struct VCursorParams *)GetPointer(),
1441  name, "%.*s", value.size(), value.data()) ) {
1443  "Cannot set VDB cursor param: "<<*this<<": "<<name,
1444  rc);
1445  }
1446 }
1447 
1448 
1450  uint32_t elem_bits) const
1451 {
1453  CFinalRequestContextUpdater ctx_updater;
1454  uint32_t read_count, remaining_count;
1455  if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
1456  elem_bits, 0, 0, 0, 0,
1457  &read_count, &remaining_count) ) {
1458  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value array size: "<<*this<<column<<
1459  '['<<row<<']', rc);
1460  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1461  "Cannot read VDB value array size: "<<*this<<column<<
1462  '['<<row<<']', rc);
1463  }
1464  return remaining_count;
1465 }
1466 
1467 
1469  uint32_t elem_bits,
1470  uint32_t start, uint32_t count,
1471  void* buffer) const
1472 {
1474  CFinalRequestContextUpdater ctx_updater;
1475  uint32_t read_count, remaining_count;
1476  if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
1477  elem_bits, start, buffer, 0, count,
1478  &read_count, &remaining_count) ) {
1479  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value array: "<<*this<<column<<
1480  '['<<row<<"]["<<start<<".."<<(start+count-1)<<']', rc);
1481  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1482  "Cannot read VDB value array: "<<*this<<column<<
1483  '['<<row<<"]["<<start<<".."<<(start+count-1)<<']', rc);
1484  }
1485  if ( read_count != count ) {
1486  NCBI_THROW_FMT(CSraException, eNotFoundValue,
1487  "Cannot read VDB value array: "<<*this<<column<<
1488  '['<<row<<"]["<<start<<".."<<(start+count-1)<<
1489  "] only "<<read_count<<" elements are read");
1490  }
1491  if ( s_GetDebugLevel() >= CVDBMgr::eDebugData ) {
1492  LOG_POST(Info<<"VDB "<<*this<<column<<'['<<row<<"]: @"<<start<<" #"<<count);
1493  }
1494 }
1495 
1496 
1497 /////////////////////////////////////////////////////////////////////////////
1498 // CVDBObjectCache
1499 
1500 static const size_t kCacheSize = 7;
1501 
1502 
1504 {
1505  m_Objects.reserve(kCacheSize);
1506 }
1507 
1508 
1510 {
1511 }
1512 
1513 
1515 
1516 
1518 {
1519  CFastMutexGuard guard(sm_CacheMutex);
1520  m_Objects.clear();
1521 }
1522 
1523 
1525 {
1526  CFastMutexGuard guard(sm_CacheMutex);
1527  if ( m_Objects.empty() ) {
1528  return 0;
1529  }
1530  TObjects::iterator best_it;
1533  TVDBRowId slot_row = it->first;
1534  if ( slot_row >= row ) {
1535  TVDBRowId d = slot_row - row;
1536  if ( d <= best_d ) {
1537  best_d = d;
1538  best_it = it;
1539  }
1540  }
1541  else {
1542  TVDBRowId d = row - slot_row;
1543  if ( d < best_d ) {
1544  best_d = d;
1545  best_it = it;
1546  }
1547  }
1548  }
1549  CObject* obj = best_it->second.Release();
1550  *best_it = m_Objects.back();
1551  m_Objects.pop_back();
1552  _ASSERT(!obj->Referenced());
1553  return obj;
1554 }
1555 
1556 
1558 {
1559  if ( obj->Referenced() ) {
1560  return;
1561  }
1562  //row = 0;
1563  CFastMutexGuard guard(sm_CacheMutex);
1564  if ( m_Objects.size() < kCacheSize ) {
1565  m_Objects.push_back(TSlot());
1566  m_Objects.back().first = row;
1567  m_Objects.back().second = obj;
1568  }
1569  else {
1570  CRef<CObject> ref(obj); // delete the object
1571  }
1572 }
1573 
1574 
1575 /////////////////////////////////////////////////////////////////////////////
1576 // CVDBColumn
1577 
1578 static inline
1580 {
1581  return out << '.' << column.GetName();
1582 }
1583 
1584 
1585 void CVDBColumn::Init(const CVDBCursor& cursor,
1586  size_t element_bit_size,
1587  const char* name,
1588  const char* backup_name,
1589  EMissing missing)
1590 {
1592  CFinalRequestContextUpdater ctx_updater;
1593  if ( !name ) {
1594  if ( missing == eMissing_Throw ) {
1595  NCBI_THROW_FMT(CSraException, eInvalidArg,
1596  "NULL column name is not allowed"<<cursor<<*this);
1597  }
1599  return;
1600  }
1601  m_Name = name;
1602  //SIMULATE_SCHEMA_ERROR();
1603  if ( rc_t rc = VCursorAddColumn(cursor, &m_Index, name) ) {
1604  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB column: "<<cursor<<*this, rc);
1605  if ( backup_name &&
1606  (rc = VCursorAddColumn(cursor, &m_Index, backup_name)) == 0 ) {
1607  m_Name = backup_name;
1608  }
1609  else {
1610  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB column: "<<cursor<<*this, rc);
1612  if ( missing == eMissing_Throw ) {
1613  NCBI_THROW2_FMT(CSraException, eNotFoundColumn,
1614  "Cannot get VDB column: "<<cursor<<*this,rc);
1615  }
1616  else {
1617  return;
1618  }
1619  }
1620  }
1621  if ( element_bit_size ) {
1622  VTypedesc type;
1623  if ( rc_t rc = VCursorDatatype(cursor, m_Index, 0, &type) ) {
1624  CHECK_VDB_TIMEOUT_FMT("Cannot get VDB column type: "<<cursor<<*this,rc);
1625  NCBI_THROW2_FMT(CSraException, eInvalidState,
1626  "Cannot get VDB column type: "<<cursor<<*this,rc);
1627  }
1628  size_t size = type.intrinsic_bits*type.intrinsic_dim;
1629  if ( size != element_bit_size ) {
1630  ERR_POST_X(1, "Wrong VDB column size "<<cursor<<*this<<
1631  " expected "<<element_bit_size<<" bits != "<<
1632  type.intrinsic_dim<<"*"<<type.intrinsic_bits<<" bits");
1633  NCBI_THROW2_FMT(CSraException, eInvalidState,
1634  "Wrong VDB column size: "<<cursor<<*this<<": "<<size,
1635  RC(rcApp, rcColumn, rcConstructing, rcSelf, rcIncorrect));
1636  }
1637  }
1638 }
1639 
1640 
1641 bool CVDBColumn::IsStatic(const CVDBCursor& cursor) const
1642 {
1643  bool static_value;
1644  if ( rc_t rc = VCursorIsStaticColumn(cursor, GetIndex(), &static_value) ) {
1645  NCBI_THROW2_FMT(CSraException, eInvalidState,
1646  "Cannot get static status of "<<cursor<<*this,rc);
1647  }
1648  return static_value;
1649 }
1650 
1651 
1653 {
1654  try {
1655  if ( IsStatic(cursor) && CVDBValue(cursor, 1, *this).empty() ) {
1656  // value is always empty
1657  Reset();
1658  }
1659  }
1660  catch ( CException& /*ignored*/ ) {
1661  }
1662 }
1663 
1664 
1665 /////////////////////////////////////////////////////////////////////////////
1666 // CVDBValue
1667 
1669 {
1670  if ( m_Table ) {
1672  }
1673  if ( m_Row ) {
1674  out << '[' << m_Row << ']';
1675  }
1676  if ( m_ColumnName ) {
1677  out << '.' << m_ColumnName;
1678  }
1679  return out;
1680 }
1681 
1682 
1683 static inline
1685 {
1686  return obj.PrintFullName(out);
1687 }
1688 
1689 
1690 void CVDBValue::x_Get(const CVDBCursor& cursor, const CVDBColumn& column)
1691 {
1693  CFinalRequestContextUpdater ctx_updater;
1694  uint32_t bit_offset, bit_length;
1695  if ( rc_t rc = VCursorCellData(cursor, column.GetIndex(),
1696  &bit_length, &m_Data, &bit_offset,
1697  &m_ElemCount) ) {
1698  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value: "<<cursor<<column, rc);
1699  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1700  "Cannot read VDB value: "<<cursor<<column, rc);
1701  }
1702  if ( bit_offset ) {
1703  NCBI_THROW2_FMT(CSraException, eInvalidState,
1704  "Cannot read VDB value with non-zero bit offset: "
1705  <<cursor<<column<<": "<<bit_offset,
1706  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1707  }
1708  if ( s_GetDebugLevel() >= CVDBMgr::eDebugData ) {
1709  CNcbiOstrstream s;
1710  if ( bit_length == 8 ) {
1711  s << '"' << NStr::PrintableString(CTempString((const char*)m_Data, m_ElemCount)) << '"';
1712  }
1713  else if ( bit_length == 16 ) {
1714  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1715  if ( i ) {
1716  s << ", ";
1717  }
1718  s << ((const int16_t*)m_Data)[i];
1719  }
1720  }
1721  else if ( bit_length == 32 ) {
1722  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1723  if ( i ) {
1724  s << ", ";
1725  }
1726  s << ((const int32_t*)m_Data)[i];
1727  }
1728  }
1729  else if ( bit_length == 64 ) {
1730  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1731  if ( i ) {
1732  s << ", ";
1733  }
1734  s << ((const int64_t*)m_Data)[i];
1735  }
1736  }
1737  else {
1738  s << "*** bad bit_length="<<bit_length;
1739  }
1740  LOG_POST(Info<<"VDB "<<cursor<<column<<": "<<CNcbiOstrstreamToString(s));
1741  }
1742  m_Ref.Set(cursor, 0, column);
1743 }
1744 
1745 
1746 void CVDBValue::x_Get(const CVDBCursor& cursor,
1747  TVDBRowId row,
1748  const CVDBColumn& column,
1749  EMissing missing)
1750 {
1752  CFinalRequestContextUpdater ctx_updater;
1753  uint32_t bit_offset, bit_length;
1754  if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1755  &bit_length, &m_Data, &bit_offset,
1756  &m_ElemCount) ) {
1757  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB value: "<<cursor<<column<<'['<<row<<']', rc);
1758  if ( missing != eMissing_Throw ) {
1759  m_Data = 0;
1760  m_ElemCount = 0;
1761  return;
1762  }
1763  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1764  "Cannot read VDB value: "<<cursor<<column<<'['<<row<<']', rc);
1765  }
1766  if ( bit_offset ) {
1767  NCBI_THROW2_FMT(CSraException, eInvalidState,
1768  "Cannot read VDB value with non-zero bit offset: "<<
1769  cursor<<column<<'['<<row<<"]: "<<bit_offset,
1770  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1771  }
1772  if ( s_GetDebugLevel() >= CVDBMgr::eDebugData ) {
1773  CNcbiOstrstream s;
1774  if ( bit_length == 8 ) {
1775  s << '"' << NStr::PrintableString(CTempString((const char*)m_Data, m_ElemCount)) << '"';
1776  }
1777  else if ( bit_length == 16 ) {
1778  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1779  if ( i ) {
1780  s << ", ";
1781  }
1782  s << ((const int16_t*)m_Data)[i];
1783  }
1784  }
1785  else if ( bit_length == 32 ) {
1786  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1787  if ( i ) {
1788  s << ", ";
1789  }
1790  s << ((const int32_t*)m_Data)[i];
1791  }
1792  }
1793  else if ( bit_length == 64 ) {
1794  for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1795  if ( i ) {
1796  s << ", ";
1797  }
1798  s << ((const int64_t*)m_Data)[i];
1799  }
1800  }
1801  else {
1802  s << "*** bad bit_length="<<bit_length;
1803  }
1804  LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
1805  }
1806  m_Ref.Set(cursor, row, column);
1807 }
1808 
1809 
1810 void CVDBValue::x_ReportIndexOutOfBounds(size_t index) const
1811 {
1812  if ( index >= size() ) {
1813  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1814  "Invalid index for VDB value array: "<<
1815  *this<<'['<<index<<']',
1816  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1817  }
1818 }
1819 
1820 
1822 {
1823  if ( size() != 1 ) {
1824  NCBI_THROW2_FMT(CSraException, eDataError,
1825  "VDB value array doen't have single value: "<<
1826  *this<<'['<<size()<<']',
1827  RC(rcApp, rcData, rcRetrieving, rcSize, rcIncorrect));
1828  }
1829 }
1830 
1831 
1832 void CVDBValue::x_CheckRange(size_t pos, size_t len) const
1833 {
1834  if ( pos > size() ) {
1835  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1836  "Invalid index for VDB value array: "<<
1837  *this<<'['<<pos<<']',
1838  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1839  }
1840  if ( pos+len < pos ) {
1841  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1842  "Invalid length for VDB value sub-array: "<<
1843  *this<<'['<<pos<<','<<len<<']',
1844  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1845  }
1846  if ( pos+len > size() ) {
1847  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1848  "Invalid end of VDB value sub-array: "<<
1849  *this<<'['<<pos<<','<<len<<']',
1850  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1851  }
1852 }
1853 
1854 
1855 static inline
1857 {
1858  return obj.PrintFullName(out);
1859 }
1860 
1861 
1863  TVDBRowId row,
1864  const CVDBColumn& column)
1865 {
1867  CFinalRequestContextUpdater ctx_updater;
1868  uint32_t bit_offset, bit_length, elem_count;
1869  const void* data;
1870  if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1871  &bit_length, &data, &bit_offset,
1872  &elem_count) ) {
1873  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB 4-bits value array: "<<
1874  cursor<<column<<'['<<row<<']', rc);
1875  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1876  "Cannot read VDB 4-bits value array: "<<
1877  cursor<<column<<'['<<row<<']', rc);
1878  }
1879  if ( bit_offset >= 8 || (bit_offset&3) ) {
1880  NCBI_THROW2_FMT(CSraException, eInvalidState,
1881  "Cannot read VDB 4-bits value array with odd bit offset"<<
1882  cursor<<column<<'['<<row<<"]: "<<bit_offset,
1883  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1884  }
1885  m_RawData = static_cast<const char*>(data);
1886  m_ElemOffset = bit_offset >> 2;
1887  m_ElemCount = elem_count;
1888  if ( s_GetDebugLevel() >= CVDBMgr::eDebugData ) {
1889  CNcbiOstrstream s;
1890  if ( bit_length == 4 ) {
1891  for ( uint32_t i = 0; i < elem_count; ++i ) {
1892  if ( i ) {
1893  s << ", ";
1894  }
1895  s << '?';
1896  }
1897  }
1898  else {
1899  s << "*** bad bit_length="<<bit_length;
1900  }
1901  LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
1902  }
1903  m_Ref.Set(cursor, row, column);
1904 }
1905 
1906 
1908 {
1909  if ( index >= size() ) {
1910  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1911  "Invalid index for VDB 4-bits value array: "<<
1912  *this<<'['<<index<<']',
1913  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1914  }
1915 }
1916 
1917 
1918 inline
1919 void CVDBValueFor4Bits::x_CheckRange(size_t pos, size_t len) const
1920 {
1921  if ( pos > size() ) {
1922  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1923  "Invalid index for VDB 4-bits value array: "<<
1924  *this<<'['<<pos<<']',
1925  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1926  }
1927  if ( pos+len < pos ) {
1928  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1929  "Invalid length for VDB 4-bits value sub-array: "<<
1930  *this<<'['<<pos<<','<<len<<']',
1931  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1932  }
1933  if ( pos+len > size() ) {
1934  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1935  "Invalid end of VDB 4-bits value sub-array: "<<
1936  *this<<'['<<pos<<','<<len<<']',
1937  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1938  }
1939 }
1940 
1941 
1943 {
1944  x_CheckRange(pos, len);
1945  size_t offset = m_ElemOffset + pos;
1946  const char* raw_data = m_RawData + offset/2;
1947  offset %= 2;
1948  // limits are checked above
1949  return CVDBValueFor4Bits(m_Ref, raw_data, uint32_t(offset), uint32_t(len));
1950 }
1951 
1952 
1953 static inline
1955 {
1956  return obj.PrintFullName(out);
1957 }
1958 
1959 
1961  TVDBRowId row,
1962  const CVDBColumn& column)
1963 {
1965  CFinalRequestContextUpdater ctx_updater;
1966  uint32_t bit_offset, bit_length, elem_count;
1967  const void* data;
1968  if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1969  &bit_length, &data, &bit_offset,
1970  &elem_count) ) {
1971  CHECK_VDB_TIMEOUT_FMT("Cannot read VDB 2-bits value array: "<<
1972  cursor<<column<<'['<<row<<']', rc);
1973  NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1974  "Cannot read VDB 2-bits value array: "<<
1975  cursor<<column<<'['<<row<<']', rc);
1976  }
1977  if ( bit_offset >= 8 || (bit_offset&1) ) {
1978  NCBI_THROW2_FMT(CSraException, eInvalidState,
1979  "Cannot read VDB 2-bits value array with odd bit offset"<<
1980  cursor<<column<<'['<<row<<"]: "<<bit_offset,
1981  RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1982  }
1983  m_RawData = static_cast<const char*>(data);
1984  m_ElemOffset = bit_offset >> 1;
1985  m_ElemCount = elem_count;
1986  if ( s_GetDebugLevel() >= CVDBMgr::eDebugData ) {
1987  CNcbiOstrstream s;
1988  if ( bit_length == 2 ) {
1989  for ( uint32_t i = 0; i < elem_count; ++i ) {
1990  if ( i ) {
1991  s << ", ";
1992  }
1993  s << '?';
1994  }
1995  }
1996  else {
1997  s << "*** bad bit_length="<<bit_length;
1998  }
1999  LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
2000  }
2001  m_Ref.Set(cursor, row, column);
2002 }
2003 
2004 
2006 {
2007  if ( index >= size() ) {
2008  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
2009  "Invalid index for VDB 2-bits value array: "<<
2010  *this<<'['<<index<<']',
2011  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
2012  }
2013 }
2014 
2015 
2016 inline
2017 void CVDBValueFor2Bits::x_CheckRange(size_t pos, size_t len) const
2018 {
2019  if ( pos > size() ) {
2020  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
2021  "Invalid index for VDB 2-bits value array: "<<
2022  *this<<'['<<pos<<']',
2023  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
2024  }
2025  if ( pos+len < pos ) {
2026  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
2027  "Invalid length for VDB 2-bits value sub-array: "<<
2028  *this<<'['<<pos<<','<<len<<']',
2029  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
2030  }
2031  if ( pos+len > size() ) {
2032  NCBI_THROW2_FMT(CSraException, eInvalidIndex,
2033  "Invalid end of VDB 2-bits value sub-array: "<<
2034  *this<<'['<<pos<<','<<len<<']',
2035  RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
2036  }
2037 }
2038 
2039 
2041 {
2042  x_CheckRange(pos, len);
2043  size_t offset = m_ElemOffset + pos;
2044  const char* raw_data = m_RawData + offset/4;
2045  offset %= 4;
2046  // limits are checked above
2047  return CVDBValueFor2Bits(m_Ref, raw_data, uint32_t(offset), uint32_t(len));
2048 }
2049 
2050 
uint32_t rc_t
CClientHttpRequest(const CKNSManager &mgr, const string &path)
Definition: vdbread.cpp:628
CClientHttpResult(const CClientHttpRequest &request, EHead)
Definition: vdbread.cpp:653
CDirEntry –.
Definition: ncbifile.hpp:262
CFinalRequestContextUpdater(const CFinalRequestContextUpdater &)=delete
void Commit() const
Definition: vdbread.cpp:234
CKConfig(void)
Definition: vdbread.cpp:202
CKDBManager(const CVDBMgr &mgr)
Definition: vdbread.cpp:282
CKNSManager(const CVFSManager &mgr)
Definition: vdbread.cpp:306
CNcbiOstrstreamToString class helps convert CNcbiOstrstream to a string Sample usage:
Definition: ncbistre.hpp:802
CObject –.
Definition: ncbiobj.hpp:180
CRandom::
Definition: random_gen.hpp:66
CSafeStatic<>::
Definition: Seq_entry.hpp:56
Definition: sdk.hpp:85
TObject ** x_InitPtr(void)
Definition: sdk.hpp:175
TObject * GetPointer(void) const
Definition: sdk.hpp:146
CTempString implements a light-weight string on top of a storage buffer whose lifetime management is ...
Definition: tempstr.hpp:65
CTimeFormat –.
Definition: ncbitime.hpp:131
CTime –.
Definition: ncbitime.hpp:296
@ eMissing_Throw
Definition: vdbread.hpp:490
TVDBColumnIdx GetIndex(void) const
Definition: vdbread.hpp:540
const char * m_Name
Definition: vdbread.hpp:559
TVDBColumnIdx m_Index
Definition: vdbread.hpp:560
void ResetIfAlwaysEmpty(const CVDBCursor &cursor)
Definition: vdbread.cpp:1652
bool IsStatic(const CVDBCursor &cursor) const
Definition: vdbread.cpp:1641
@ kInvalidIndex
Definition: vdbread.hpp:495
void Init(const CVDBCursor &cursor, size_t element_bit_size, const char *name, const char *backup_name, EMissing missing)
Definition: vdbread.cpp:1585
void Reset(void)
Definition: vdbread.hpp:527
rc_t OpenRowRc(TVDBRowId row_id)
Definition: vdbread.cpp:1392
void OpenRow(TVDBRowId row_id)
Definition: vdbread.cpp:1407
bool RowIsOpened(void) const
Definition: vdbread.hpp:408
void CloseRow(void)
Definition: vdbread.cpp:1379
uint32_t GetElementCount(TVDBRowId row, const CVDBColumn &column, uint32_t elem_bits) const
Definition: vdbread.cpp:1449
const CVDBTable & GetTable(void) const
Definition: vdbread.hpp:403
void ReadElements(TVDBRowId row, const CVDBColumn &column, uint32_t elem_bits, uint32_t start, uint32_t count, void *buffer) const
Definition: vdbread.cpp:1468
bool m_RowOpened
Definition: vdbread.hpp:443
void SetParam(const char *name, const CTempString &value) const
Definition: vdbread.cpp:1436
CVDBTable m_Table
Definition: vdbread.hpp:442
void Init(const CVDBTable &table)
Definition: vdbread.cpp:1347
TVDBRowId GetMaxRowId(void) const
Definition: vdbread.cpp:1429
TVDBRowIdRange GetRowIdRange(TVDBColumnIdx column=0) const
Definition: vdbread.cpp:1417
CVDBMgr(void)
Definition: vdbread.cpp:528
void DeleteCacheOlderThan(Uint4 days)
Definition: vdbread.cpp:1081
CVResolver m_Resolver
Definition: vdbread.hpp:278
static int SetDebugLevel(int new_level)
Definition: vdbread.cpp:541
string GetCacheRoot() const
Definition: vdbread.cpp:1056
string FindDereferencedAccPath(const string &acc_or_path) const
Definition: vdbread.cpp:556
@ eDebugInitialize
Definition: vdbread.hpp:226
@ eDebugResolve
Definition: vdbread.hpp:227
@ eDebugOpen
Definition: vdbread.hpp:225
@ eDebugData
Definition: vdbread.hpp:228
CTime GetTimestamp(const string &path) const
Definition: vdbread.cpp:592
static void s_VDBInit()
Definition: vdbread.cpp:942
CTime GetURLTimestamp(const string &url) const
Definition: vdbread.cpp:667
string FindAccPath(const string &acc) const
Definition: vdbread.cpp:547
void SetCacheRoot(const string &path)
Definition: vdbread.cpp:1071
static int GetDebugLevel()
Definition: vdbread.cpp:535
void CommitConfig() const
Definition: vdbread.cpp:1090
void x_Init(void)
Definition: vdbread.cpp:1039
CObject * Get(TVDBRowId row)
Definition: vdbread.cpp:1524
TObjects m_Objects
Definition: vdbread.hpp:462
void Put(CObject *curs, TVDBRowId row)
Definition: vdbread.cpp:1557
pair< TVDBRowId, CRef< CObject > > TSlot
Definition: vdbread.hpp:460
~CVDBObjectCacheBase(void)
Definition: vdbread.cpp:1509
vector< TSlot > TObjects
Definition: vdbread.hpp:461
void Clear(void)
Definition: vdbread.cpp:1517
string GetFullName(void) const
Definition: vdbread.cpp:1305
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1311
const char * GetName(void) const
Definition: vdbread.hpp:370
const CVDBTable & GetTable(void) const
Definition: vdbread.hpp:366
TVDBRowIdRange Find(const string &value) const
Definition: vdbread.cpp:1317
CVDBTableIndex(void)
Definition: vdbread.hpp:359
string GetFullName(void) const
Definition: vdbread.cpp:1247
@ eMissing_Throw
Definition: vdbread.hpp:315
const CVDB & GetDb(void) const
Definition: vdbread.hpp:329
CVDBTable(void)
Definition: vdbread.hpp:319
const string & GetName(void) const
Definition: vdbread.hpp:333
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1259
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.hpp:902
void x_Get(const CVDBCursor &cursor, TVDBRowId row, const CVDBColumn &column)
Definition: vdbread.cpp:1960
void x_ReportIndexOutOfBounds(size_t index) const
Definition: vdbread.cpp:2005
CVDBValueFor2Bits substr(size_t pos, size_t len) const
Definition: vdbread.cpp:2040
void x_CheckRange(size_t pos, size_t len) const
Definition: vdbread.cpp:2017
void x_CheckRange(size_t pos, size_t len) const
Definition: vdbread.cpp:1919
CVDBValueFor4Bits substr(size_t pos, size_t len) const
Definition: vdbread.cpp:1942
void x_ReportIndexOutOfBounds(size_t index) const
Definition: vdbread.cpp:1907
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.hpp:803
void x_Get(const CVDBCursor &cursor, TVDBRowId row, const CVDBColumn &column)
Definition: vdbread.cpp:1862
void x_CheckRange(size_t pos, size_t len) const
Definition: vdbread.cpp:1832
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.hpp:712
const void * m_Data
Definition: vdbread.hpp:742
uint32_t m_ElemCount
Definition: vdbread.hpp:743
size_t size(void) const
Definition: vdbread.hpp:707
@ eMissing_Throw
Definition: vdbread.hpp:632
SSaveRef m_Ref
Definition: vdbread.hpp:741
void x_Get(const CVDBCursor &cursor, const CVDBColumn &column)
Definition: vdbread.cpp:1690
void x_ReportNotOneValue(void) const
Definition: vdbread.cpp:1821
void x_ReportIndexOutOfBounds(size_t index) const
Definition: vdbread.cpp:1810
const string & GetFullName(void) const
Definition: vdbread.hpp:295
CVDB(void)
Definition: vdbread.hpp:286
const string & GetName(void) const
Definition: vdbread.hpp:291
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1162
void x_InitNew(void)
Definition: vdbread.cpp:258
CVFSManager(void)
Definition: vdbread.cpp:246
@ eSys
Definition: vdbread.hpp:149
@ eAcc
Definition: vdbread.hpp:150
string ToString(EType type=eSys) const
Definition: vdbread.cpp:376
bool IsLocalFile() const
Definition: vdbread.cpp:359
static string ConvertAccOrSysPathToPOSIX(const string &acc_or_path)
Definition: vdbread.cpp:446
void x_Init(const CVFSManager &mgr, const string &path, EType type)
Definition: vdbread.cpp:331
static bool IsPlainAccession(const string &acc_or_path)
Definition: vdbread.cpp:407
static string ConvertSysPathToPOSIX(const string &sys_path)
Definition: vdbread.cpp:419
CVPath(void)
Definition: vdbread.hpp:137
CVFSManager m_Mgr
Definition: vdbread.hpp:212
CVResolver(const CVFSManager &mgr)
Definition: vdbread.cpp:458
string Resolve(const string &acc) const
Definition: vdbread.cpp:480
struct config config
std::ofstream out("events_result.xml")
main entry point for tests
static const char table_name[]
Definition: bcp.c:249
static int type
Definition: getdata.c:31
static const char * str(char *buf, int n)
Definition: stats.c:84
static const char * column
Definition: stats.c:23
int offset
Definition: replacements.h:160
char data[12]
Definition: iconv.c:80
Int4 int32_t
Int2 int16_t
Int8 int64_t
Uint4 uint32_t
static string GetAppName(EAppNameType name_type=eBaseName, int argc=0, const char *const *argv=NULL)
Definition: ncbiapp.cpp:1390
static CNcbiApplicationGuard InstanceGuard(void)
Singleton method.
Definition: ncbiapp.cpp:133
const CNcbiRegistry & GetConfig(void) const
Get the application's cached configuration parameters (read-only).
CVersionInfo GetVersion(void) const
Get the program version information.
Definition: ncbiapp.cpp:1197
#define NON_CONST_ITERATE(Type, Var, Cont)
Non constant version of ITERATE macro.
Definition: ncbimisc.hpp:822
@ null
Definition: ncbimisc.hpp:646
#define NULL
Definition: ncbistd.hpp:225
#define _TRACE(message)
Definition: ncbidbg.hpp:122
#define LOG_POST_X(err_subcode, message)
Definition: ncbidiag.hpp:553
const CNcbiDiag &(* FManip)(const CNcbiDiag &)
Diagnostic stream manipulator.
Definition: ncbidiag.hpp:954
string GetSessionID(void) const
Session ID.
CAtomicCounter::TValue TVersion
bool IsSetSessionID(void) const
static CRequestContext & GetRequestContext(void)
Shortcut to CDiagContextThreadData::GetThreadData().GetRequestContext()
Definition: ncbidiag.cpp:1901
string GetClientIP(void) const
Client IP/hostname.
bool IsSetHitID(EHitIDSource src=eHitID_Any) const
Check if there's an explicit hit id or the default one.
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
Definition: ncbidiag.hpp:550
bool IsSetClientIP(void) const
string GetHitID(void) const
Get explicit hit id or the default one (from HTTP_NCBI_PHID etc).
TVersion GetVersion(void) const
Return version increased on every context change (hit/subhit id, client ip, session id).
#define LOG_POST(message)
This macro is deprecated and it's strongly recomended to move in all projects (except tests) to macro...
Definition: ncbidiag.hpp:226
void Error(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1197
void Trace(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1179
void Warning(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1191
void Fatal(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1209
#define NCBI_THROW2(exception_class, err_code, message, extra)
Throw exception with extra parameter.
Definition: ncbiexpt.hpp:1754
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
void Info(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1185
static string CreateAbsolutePath(const string &path, ERelativeToWhat rtw=eRelativeToCwd)
Get an absolute path from some, possibly relative, path.
Definition: ncbifile.cpp:665
void DereferencePath(void)
Dereference a path.
Definition: ncbifile.cpp:2415
virtual bool Exists(void) const
Check the entry existence.
Definition: ncbifile.cpp:2325
const string & GetPath(void) const
Get entry path.
Definition: ncbifile.hpp:3911
bool Referenced(void) const THROWS_NONE
Check if object is referenced.
Definition: ncbiobj.hpp:468
#define NCBI_PARAM_TYPE(section, name)
Generate typename for a parameter from its {section, name} attributes.
Definition: ncbi_param.hpp:149
@ eParam_NoThread
Do not use per-thread values.
Definition: ncbi_param.hpp:418
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
virtual bool GetBool(const string &section, const string &name, bool default_value, TFlags flags=0, EErrAction err_action=eThrow) const
Get boolean value of specified parameter name.
Definition: ncbireg.cpp:391
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define END_SCOPE(ns)
End the previously defined scope.
Definition: ncbistl.hpp:75
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
#define BEGIN_SCOPE(ns)
Define a new scope.
Definition: ncbistl.hpp:72
IO_PREFIX::ostream CNcbiOstream
Portable alias for ostream.
Definition: ncbistre.hpp:149
NCBI_NS_STD::string::size_type SIZE_TYPE
Definition: ncbistr.hpp:132
static string PrintableString(const CTempString str, TPrintableMode mode=fNewLine_Quote|fNonAscii_Passthru)
Get a printable version of the specified string.
Definition: ncbistr.cpp:3944
#define kEmptyStr
Definition: ncbistr.hpp:123
#define NPOS
Definition: ncbistr.hpp:133
static void TruncateSpacesInPlace(string &str, ETrunc where=eTrunc_Both)
Truncate whitespace in a string (in-place)
Definition: ncbistr.cpp:3192
static string IntToString(int value, TNumToStringFlags flags=0, int base=10)
Convert int to string.
Definition: ncbistr.hpp:5078
bool empty(void) const
Return true if the represented string is empty (i.e., the length is zero)
Definition: tempstr.hpp:334
static bool StartsWith(const CTempString str, const CTempString start, ECase use_case=eCase)
Check if a string starts with a specified prefix value.
Definition: ncbistr.hpp:5406
size_type size(void) const
Return the length of the represented array.
Definition: tempstr.hpp:327
bool IsUniversalTime(void) const
Is time universal (GMT/UTC/Z)?
Definition: ncbitime.hpp:2421
bool IsEmpty(void) const
Is time object empty (date and time)?
Definition: ncbitime.hpp:2377
CTime & ToUniversalTime(void)
Convert the time into universal (GMT/UTC) time.
Definition: ncbitime.hpp:2471
virtual string Print(void) const
Print version information.
Definition: version.cpp:120
#define NCBI_SRAREAD_EXPORT
Definition: ncbi_export.h:1227
#define NCBI_DEVELOPMENT_VER
#define DEBUG
Definition: config.h:32
Definition of all error codes used in SRA C++ support libraries.
<!DOCTYPE HTML >< html > n< header > n< title > PubSeq Gateway Help Page</title > n< style > n table
int i
int len
const TYPE & Get(const CNamedParameterList *param)
range(_Ty, _Ty) -> range< _Ty >
constexpr bool empty(list< Ts... >) noexcept
const struct ncbi::grid::netcache::search::fields::SIZE size
string s_Value(TValue value)
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
#define NCBI_PACKAGE_NAME
#define NCBI_PACKAGE_VERSION
const char * tag
int isalpha(Uchar c)
Definition: ncbictype.hpp:61
int toupper(Uchar c)
Definition: ncbictype.hpp:73
Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...
Multi-threading – mutexes; rw-locks; semaphore.
T max(T x_, T y_)
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
void copy(Njn::Matrix< S > *matrix_, const Njn::Matrix< T > &matrix0_)
Definition: njn_matrix.hpp:613
#define count
static uint8_t * buffer
Definition: pcre2test.c:1016
@ eNotFound
Not found.
Defines CRequestContext class for NCBI C++ diagnostic API.
#define RC(flags)
static SLJIT_INLINE sljit_ins msg(sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b)
#define CHECK_VDB_TIMEOUT_FMT(msg, rc)
Definition: exception.hpp:186
#define CHECK_VDB_TIMEOUT(msg, rc)
Definition: exception.hpp:181
#define NCBI_THROW2_FMT(exception_class, err_code, message, extra)
Definition: exception.hpp:176
#define uint32_t
Definition: config.h:42
#define row(bind, expected)
Definition: string_bind.c:73
void Set(const CVDBCursor &cursor, TVDBRowId row, const CVDBColumn &column)
Definition: vdbread.hpp:654
const CVDBTable * m_Table
Definition: vdbread.hpp:665
const char * m_ColumnName
Definition: vdbread.hpp:666
CNcbiOstream & PrintFullName(CNcbiOstream &out) const
Definition: vdbread.cpp:1668
const char * tag
Definition: bamread.cpp:529
CNcbiDiag::FManip manip
Definition: bamread.cpp:530
Definition: type.c:6
#define _ASSERT
else result
Definition: token2.c:20
static const SVDBSeverityTag kSeverityTags[]
Definition: vdbread.cpp:753
static int & s_GetDebugLevelVar(void)
Definition: vdbread.cpp:107
static const size_t kCacheSize
Definition: vdbread.cpp:1500
static char s_VDBVersion[32]
Definition: vdbread.cpp:712
static bool s_HasWindowsDriveLetter(const string &s)
Definition: vdbread.cpp:399
static rc_t VDBLogWriter(void *, const char *buffer, size_t size, size_t *written)
Definition: vdbread.cpp:796
DEFINE_SRA_REF_TRAITS(VDBManager, const)
static void s_InitLocalKNS(KNSManager *kns_mgr)
Definition: vdbread.cpp:936
DECLARE_SRA_REF_TRAITS(KClientHttpRequest,)
static void s_InitAllKNS(KNSManager *kns_mgr)
Definition: vdbread.cpp:897
static int s_GetDiagHandler(void)
Definition: vdbread.cpp:95
static const SVDBSeverityTag * s_GetVDBSeverityTag(CTempString token)
Definition: vdbread.cpp:762
DEFINE_STATIC_FAST_MUTEX(sx_SDKMutex)
static void s_InitVDBVersion()
Definition: vdbread.cpp:715
#define SIMULATE_RESOLVE_ERROR()
Definition: vdbread.cpp:193
static CKConfig s_InitProxyConfig()
Definition: vdbread.cpp:827
#define DECLARE_SDK_GET_GUARD()
Definition: vdbread.cpp:701
static void s_InitStaticKNS(KNSManager *kns_mgr)
Definition: vdbread.cpp:930
#define SIMULATE_SCHEMA_ERROR()
Definition: vdbread.cpp:195
static void s_InitDiagCheck()
Definition: vdbread.cpp:778
#define DECLARE_SDK_GUARD()
Definition: vdbread.cpp:695
static bool s_DiagIsSafe()
Definition: vdbread.cpp:785
NCBI_DEFINE_ERR_SUBCODE_X(9)
NCBI_PARAM_DEF(int, VDB, DIAG_HANDLER, 1)
static void s_UpdateVDBRequestContext(void)
Definition: vdbread.cpp:867
static int s_SetDebugLevel(int new_level)
Definition: vdbread.cpp:122
NCBI_PARAM_DECL(int, VDB, DIAG_HANDLER)
NCBI_PARAM_DEF_EX(int, VDB, DEBUG, 0, eParam_NoThread, VDB_DEBUG)
static DECLARE_TLS_VAR(const CRequestContext *, s_LastRequestContext)
static CNcbiOstream & operator<<(CNcbiOstream &out, const CVDBTable &obj)
Definition: vdbread.cpp:1172
static int s_GetDebugLevel()
Definition: vdbread.cpp:115
#define SIMULATE_OPEN_ERROR()
Definition: vdbread.cpp:194
uint32_t TVDBColumnIdx
Definition: vdbread.hpp:85
pair< TVDBRowId, TVDBRowCount > TVDBRowIdRange
Definition: vdbread.hpp:84
int64_t TVDBRowId
Definition: vdbread.hpp:80
Modified on Fri Sep 20 14:58:12 2024 by modify_doxy.py rev. 669887