NCBI C++ ToolKit
processors.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: processors.cpp 100805 2023-09-14 14:12:22Z vasilche $
2  * ===========================================================================
3  * PUBLIC DOMAIN NOTICE
4  * National Center for Biotechnology Information
5  *
6  * This software/database is a "United States Government Work" under the
7  * terms of the United States Copyright Act. It was written as part of
8  * the author's official duties as a United States Government employee and
9  * thus cannot be copyrighted. This software/database is freely available
10  * to the public for use. The National Library of Medicine and the U.S.
11  * Government have not placed any restriction on its use or reproduction.
12  *
13  * Although all reasonable efforts have been taken to ensure the accuracy
14  * and reliability of the software and data, the NLM and the U.S.
15  * Government do not and cannot warrant the performance or results that
16  * may be obtained by using this software or data. The NLM and the U.S.
17  * Government disclaim all warranties, express or implied, including
18  * warranties of performance, merchantability or fitness for any particular
19  * purpose.
20  *
21  * Please cite the author in any work or product based on this material.
22  * ===========================================================================
23  *
24  * Author: Eugene Vasilchenko
25  *
26  * File Description: blob stream processor interface
27  *
28  */
29 
30 #include <ncbi_pch.hpp>
31 
32 #include <corelib/rwstream.hpp>
33 
42 
44 #include <objtools/error_codes.hpp>
45 
50 
51 #include <objects/id1/id1__.hpp>
54 // for read hooks setup
59 
61 
62 #include <serial/objistr.hpp>
63 #include <serial/objostr.hpp>
64 #include <serial/objcopy.hpp>
65 #include <serial/objistrasnb.hpp>
66 #include <serial/objostrasnb.hpp>
67 #include <serial/delaybuf.hpp>
68 #include <serial/serial.hpp>
69 #include <serial/iterator.hpp>
70 
72 #include <util/compress/zlib.hpp>
73 
74 #include <serial/pack_string.hpp>
75 
76 
77 #define NCBI_USE_ERRCODE_X Objtools_Rd_Process
78 
81 
82 NCBI_PARAM_DEF_EX(bool, GENBANK, SNP_PACK_STRINGS, true,
83  eParam_NoThread, GENBANK_SNP_PACK_STRINGS);
84 NCBI_PARAM_DEF_EX(bool, GENBANK, SNP_SPLIT, true,
85  eParam_NoThread, GENBANK_SNP_SPLIT);
86 NCBI_PARAM_DEF_EX(bool, GENBANK, SNP_TABLE, true,
87  eParam_NoThread, GENBANK_SNP_TABLE);
88 NCBI_PARAM_DEF_EX(bool, GENBANK, USE_MEMORY_POOL, true,
89  eParam_NoThread, GENBANK_USE_MEMORY_POOL);
90 NCBI_PARAM_DEF_EX(int, GENBANK, READER_STATS, 0,
91  eParam_NoThread, GENBANK_READER_STATS);
92 NCBI_PARAM_DEF_EX(bool, GENBANK, CACHE_RECOMPRESS, true,
93  eParam_NoThread, GENBANK_CACHE_RECOMPRESS);
94 
95 
96 /////////////////////////////////////////////////////////////////////////////
97 // helper functions
98 /////////////////////////////////////////////////////////////////////////////
99 
100 namespace {
101  CProcessor::TMagic s_GetMagic(const char* s)
102  {
103  CProcessor::TMagic m = 0;
104  const char* p = s;
105  for ( size_t i = 0; i < sizeof(m); ++p, ++i ) {
106  if ( !*p ) {
107  p = s;
108  }
109  m = (m << 8) | (*p & 0xff);
110  }
111  return m;
112  }
113 
114 
115  class COSSReader : public IReader
116  {
117  public:
118  typedef vector<char> TOctetString;
119  typedef list<TOctetString*> TOctetStringSequence;
120 
121  COSSReader(const TOctetStringSequence& in)
122  : m_Input(in),
123  m_CurVec(in.begin())
124  {
125  x_SetVec();
126  }
127 
128  virtual ERW_Result Read(void* buffer,
129  size_t count,
130  size_t* bytes_read = 0)
131  {
132  size_t pending = x_Pending();
133  count = min(pending, count);
134  if ( bytes_read ) {
135  *bytes_read = count;
136  }
137  if ( pending == 0 ) {
138  return eRW_Eof;
139  }
140  if ( count ) {
141  memcpy(buffer, &(**m_CurVec)[m_CurPos], count);
142  m_CurPos += count;
143  }
144  return eRW_Success;
145  }
146 
147  virtual ERW_Result PendingCount(size_t* count)
148  {
149  size_t pending = x_Pending();
150  *count = pending;
151  return pending? eRW_Success: eRW_Eof;
152  }
153 
154  protected:
155  void x_SetVec(void)
156  {
157  m_CurPos = 0;
158  m_CurSize = m_CurVec == m_Input.end()? 0: (**m_CurVec).size();
159  }
160  size_t x_Pending(void)
161  {
162  size_t size;
163  while ( (size = m_CurSize - m_CurPos) == 0 &&
164  m_CurVec != m_Input.end() ) {
165  ++m_CurVec;
166  x_SetVec();
167  }
168  return size;
169  }
170  private:
171  const TOctetStringSequence& m_Input;
172  TOctetStringSequence::const_iterator m_CurVec;
173  size_t m_CurPos;
174  size_t m_CurSize;
175  };
176  class COSSWriter : public IWriter
177  {
178  public:
179  typedef vector<char> TOctetString;
180  typedef list<TOctetString*> TOctetStringSequence;
181 
183  : m_Output(out)
184  {
185  }
186 
187  virtual ERW_Result Write(const void* buffer,
188  size_t count,
189  size_t* written)
190  {
191  const char* data = static_cast<const char*>(buffer);
192  m_Output.push_back(new TOctetString(data, data+count));
193  if ( written ) {
194  *written = count;
195  }
196  return eRW_Success;
197  }
198  virtual ERW_Result Flush(void)
199  {
200  return eRW_Success;
201  }
202 
203  private:
205  };
206 }
207 
208 
209 /////////////////////////////////////////////////////////////////////////////
210 // CProcessor
211 /////////////////////////////////////////////////////////////////////////////
212 
213 
214 #define GB_STATS_STOP(action, stat, size) \
215  LogStat(action, blob_id, stat, r, size, result)
216 
217 
218 inline
220 {
221  static CSafeStatic<NCBI_PARAM_TYPE(GENBANK, READER_STATS)> s_Value;
222  return s_Value->Get();
223 }
224 
225 
227  : m_Dispatcher(&dispatcher)
228 {
229 }
230 
231 
233 {
234 }
235 
236 
238  const TBlobId& blob_id,
239  TChunkId chunk_id,
240  CNcbiIstream& stream) const
241 {
242  //CReaderRequestResult::CRecurse r(result);
243  CObjectIStreamAsnBinary obj_stream(stream);
244  ProcessObjStream(result, blob_id, chunk_id, obj_stream);
245  //LogStat(result, r, blob_id,
246  // CGBRequestStatistics::eStat_LoadBlob,
247  // "CProcessor: process stream",
248  // obj_stream.GetStreamPos());
249 }
250 
251 
253  const TBlobId& /*blob_id*/,
254  TChunkId /*chunk_id*/,
255  CObjectIStream& /*obj_stream*/) const
256 {
257  NCBI_THROW(CLoaderException, eLoaderFailed,
258  "CProcessor::ProcessObjStream() is not implemented");
259 }
260 
261 
263  const TBlobId& blob_id,
264  TChunkId chunk_id,
265  const CID2_Reply_Data& data) const
266 {
267  if ( !data.IsSetData() ) {
268  NCBI_THROW(CLoaderException, eLoaderFailed,
269  "CProcessor::ProcessBlobFromID2Data() no data");
270  }
271  if ( data.GetData_format() != data.eData_format_asn_binary ) {
272  NCBI_THROW(CLoaderException, eLoaderFailed,
273  "CProcessor::ProcessBlobFromID2Data() is not implemented");
274  }
275 
276  CRStream stream(new COSSReader(data.GetData()),
277  0, 0, CRWStreambuf::fOwnAll);
278  switch ( data.GetData_compression() ) {
280  {
281  ProcessStream(result, blob_id, chunk_id, stream);
282  break;
283  }
285  {
286  CCompressionIStream zip_stream(stream,
289  ProcessStream(result, blob_id, chunk_id, zip_stream);
290  break;
291  }
292  default:
293  NCBI_THROW(CLoaderException, eLoaderFailed,
294  "CProcessor::ProcessBlobFromID2Data() is not implemented");
295  }
296 }
297 
298 
300 {
311 }
312 
313 
315 {
316  typedef NCBI_PARAM_TYPE(GENBANK, SNP_PACK_STRINGS) TPackStrings;
317  if ( !TPackStrings::GetDefault() ) {
318  return false;
319  }
320  if ( !CPackString::TryStringPack() ) {
321  TPackStrings::SetDefault(false);
322  return false;
323  }
324  return true;
325 }
326 
327 
329 {
330  static CSafeStatic<NCBI_PARAM_TYPE(GENBANK, SNP_SPLIT)> s_Value;
331  return s_Value->Get();
332 }
333 
334 
336 {
337  static CSafeStatic<NCBI_PARAM_TYPE(GENBANK, SNP_TABLE)> s_Value;
338  return s_Value->Get();
339 }
340 
341 
342 static bool s_UseMemoryPool(void)
343 {
344  static CSafeStatic<NCBI_PARAM_TYPE(GENBANK, USE_MEMORY_POOL)> s_Value;
345  return s_Value->Get();
346 }
347 
348 
349 static bool s_CacheRecompress(void)
350 {
351  static CSafeStatic<NCBI_PARAM_TYPE(GENBANK, CACHE_RECOMPRESS)> s_Value;
352  return s_Value->Get();
353 }
354 
355 
357 {
358  if ( TryStringPack() ) {
360 
362  type.FindVariant("str").SetLocalReadHook(in, new CPackStringChoiceHook);
363 
365  type.FindMember("key").SetLocalReadHook(in,
366  new CPackStringClassHook(32, 128));
367 
369  type.FindMember("db").SetLocalReadHook(in, new CPackStringClassHook);
370 
371  type = CType<CGb_qual>();
372  type.FindMember("qual").SetLocalReadHook(in, new CPackStringClassHook);
373  }
374  if ( s_UseMemoryPool() ) {
375  in.UseMemoryPool();
376  }
377 }
378 
379 
381 {
382  if ( TryStringPack() ) {
384 
385  type = CType<CGb_qual>();
386  type.FindMember("qual").SetLocalReadHook(in, new CPackStringClassHook);
387  type.FindMember("val").SetLocalReadHook(in,
388  new CPackStringClassHook(4, 128));
389 
391  type.FindMember("key").SetLocalReadHook(in,
392  new CPackStringClassHook(32, 128));
393 
395  type.FindVariant("str").SetLocalReadHook(in, new CPackStringChoiceHook);
396 
398  type.FindMember("db").SetLocalReadHook(in, new CPackStringClassHook);
399 
401  type.FindMember("comment").SetLocalReadHook(in, new CPackStringClassHook);
402  }
403 }
404 
405 
406 inline
408 {
410 }
411 
412 
413 NCBI_PARAM_DEF_EX(Int8, GENBANK, GI_OFFSET, 0,
414  eParam_NoThread, GENBANK_GI_OFFSET);
415 
416 
418 {
419  static atomic<TIntId> gi_offset;
420  static atomic<bool> initialized;
421  if ( !initialized.load(memory_order_acquire) ) {
422  gi_offset.store(NCBI_PARAM_TYPE(GENBANK, GI_OFFSET)::GetDefault(), memory_order_relaxed);
423  initialized.store(true, memory_order_release);
424  }
425  return gi_offset.load(memory_order_relaxed);
426 }
427 
428 
429 bool CProcessor::OffsetId(CSeq_id& id, TIntId gi_offset)
430 {
431  if ( !gi_offset ) {
432  return false;
433  }
434  if ( id.IsGi() ) {
435  TGi gi = id.GetGi();
436  if ( gi != ZERO_GI ) {
437  id.SetGi(gi + GI_FROM(TIntId, gi_offset));
438  return true;
439  }
440  }
441  else if ( id.IsGeneral() ) {
442  CDbtag& dbtag = id.SetGeneral();
443  CObject_id& tag = dbtag.SetTag();
444  if ( tag.IsStr() &&
445  NStr::EqualNocase(dbtag.GetDb(), "NAnnot") ) {
446  // Named annotation virtual sequence id:
447  // gnl|NAnnot|12345:NA000012345.1
448  const string& s = tag.GetStr();
449  SIZE_TYPE sep = s.find(':');
450  Int8 id8;
451  if ( sep != NPOS &&
453  id8 != 0 ) {
454  // offset only non-zero GIs
455  tag.SetStr(NStr::NumericToString(id8 + gi_offset) +
456  s.substr(sep));
457  return true;
458  }
459  }
460  else if ( NStr::StartsWith(dbtag.GetDb(), "ANNOT:", NStr::eNocase) ) {
461  // External annotation virtual sequence id:
462  // gnl|Annot:SNP|12345
463  CObject_id::TId8 id8 = 0;
464  if ( tag.GetId8(id8) &&
465  id8 != 0 ) {
466  // offset only non-zero GIs
467  tag.SetId8(id8 + gi_offset);
468  return true;
469  }
470  }
471  }
472  return false;
473 }
474 
475 
477 {
478  if ( !gi_offset ) {
479  return false;
480  }
481  if ( id.IsGi() ) {
482  TGi gi = id.GetGi();
483  if ( gi != ZERO_GI ) {
484  id = CSeq_id_Handle::GetGiHandle(gi + GI_FROM(TIntId, gi_offset));
485  return true;
486  }
487  }
488  else if ( id.Which() == CSeq_id::e_General ) {
489  CRef<CSeq_id> seq_id(SerialClone(*id.GetSeqId()));
490  if ( OffsetId(*seq_id, gi_offset) ) {
491  id = CSeq_id_Handle::GetHandle(*seq_id);
492  return true;
493  }
494  }
495  return false;
496 }
497 
498 
500 {
501  if ( !gi_offset ) {
502  return;
503  }
504  for ( CTypeIterator<CSeq_id> it(obj); it; ++it ) {
505  OffsetId(*it, gi_offset);
506  }
507 
508  // ID2
509  for ( CTypeIterator<CID1server_request> it(obj); it; ++it ) {
510  CID1server_request& req = *it;
511  switch ( req.Which() ) {
513  OffsetGi(req.SetGetseqidsfromgi(), gi_offset);
514  break;
516  OffsetGi(req.SetGetgihist(), gi_offset);
517  break;
519  OffsetGi(req.SetGetgirev(), gi_offset);
520  break;
522  OffsetGi(req.SetGetgistate(), gi_offset);
523  break;
524  default:
525  break;
526  }
527  }
528  for ( CTypeIterator<CID1server_maxcomplex> it(obj); it; ++it ) {
529  OffsetGi(it->SetGi(), gi_offset);
530  }
531  for ( CTypeIterator<CID1server_back> it(obj); it; ++it ) {
532  CID1server_back& reply = *it;
533  if ( reply.IsGotgi() ) {
534  OffsetGi(reply.SetGotgi(), gi_offset);
535  }
536  }
537  for ( CTypeIterator<CID1blob_info> it(obj); it; ++it ) {
538  OffsetGi(it->SetGi(), gi_offset);
539  }
540 
541  // ID2
542  for ( CTypeIterator<CID2S_Seq_loc> it(obj); it; ++it ) {
543  CID2S_Seq_loc& loc = *it;
544  if ( loc.IsWhole_gi() ) {
545  OffsetGi(loc.SetWhole_gi(), gi_offset);
546  }
547  }
548  for ( CTypeIterator<CID2S_Gi_Range> it(obj); it; ++it ) {
549  OffsetGi(it->SetStart(), gi_offset);
550  }
551  for ( CTypeIterator<CID2S_Gi_Interval> it(obj); it; ++it ) {
552  OffsetGi(it->SetGi(), gi_offset);
553  }
554  for ( CTypeIterator<CID2S_Gi_Ints> it(obj); it; ++it ) {
555  OffsetGi(it->SetGi(), gi_offset);
556  }
557  for ( CTypeIterator<CID2S_Bioseq_Ids> it(obj); it; ++it ) {
558  NON_CONST_ITERATE ( CID2S_Bioseq_Ids::Tdata, it2, it->Set() ) {
559  if ( (*it2)->IsGi() ) {
560  OffsetGi((*it2)->SetGi(), gi_offset);
561  }
562  }
563  }
564  for ( CTypeIterator<CID2S_Chunk_Data> it(obj); it; ++it ) {
565  if ( it->SetId().IsGi() ) {
566  OffsetGi(it->SetId().SetGi(), gi_offset);
567  }
568  }
569 }
570 
571 
573 {
574  if ( !gi_offset ) {
575  return;
576  }
578  set_info.m_Seq_annot_InfoMap ) {
579  it->second.m_SNP_annot_Info->OffsetGi(gi_offset);
580  }
581 }
582 
583 
585 {
586  OffsetAllGis(obj, -GetGiOffset());
587 }
588 
589 
591 {
592  if ( TIntId gi_offset = GetGiOffset() ) {
593  OffsetAllGis(obj, gi_offset);
594  if ( set_info ) {
595  OffsetAllGis(*set_info, gi_offset);
596  }
597  }
598 }
599 
600 
602 
603 
604 static inline
605 bool s_CanBeWGSBlob(const CBlob_id& blob_id)
606 {
607  return !CProcessor_ExtAnnot::IsExtAnnot(blob_id);
608 }
609 
610 
612 
613 
614 namespace {
615  inline
616  CRef<CWriter::CBlobStream> OpenStream(CWriter* writer,
618  const CProcessor::TBlobId& blob_id,
619  CProcessor::TChunkId chunk_id,
620  const CProcessor* processor)
621  {
622  _ASSERT(writer);
623  _ASSERT(processor);
624  return writer->OpenBlobStream(result, blob_id, chunk_id, *processor);
625  /*
626  if ( chunk_id == kMain_ChunkId ) {
627  return writer->OpenBlobStream(result, blob_id, *processor);
628  }
629  else {
630  return writer->OpenChunkStream(result, blob_id, chunk_id,
631  *processor);
632  }
633  */
634  }
635 }
636 
637 
638 /////////////////////////////////////////////////////////////////////////////
639 // CProcessor_ID1
640 /////////////////////////////////////////////////////////////////////////////
641 
643  : CProcessor(dispatcher)
644 {
645 }
646 
647 
649 {
650 }
651 
652 
654 {
655  return eType_ID1;
656 }
657 
658 
660 {
661  static TMagic kMagic = s_GetMagic("ID1r");
662  return kMagic;
663 }
664 
665 
667  const TBlobId& blob_id,
668  TChunkId chunk_id,
669  CObjectIStream& obj_stream) const
670 {
671  CLoadLockBlob blob(result, blob_id, chunk_id);
672  if ( blob.IsLoadedChunk() ) {
673  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
674  "CProcessor_ID1: "
675  "double load of "<<blob_id<<'/'<<chunk_id);
676  }
677  CID1server_back reply;
679  CWriter* writer = GetWriter(result);
680  if ( writer ) {
681  guard.StartDelayBuffer(obj_stream);
682  }
683  SetSeqEntryReadHooks(obj_stream);
684  size_t data_size = 0;
685  {{
687 
688  obj_stream >> reply;
689 
690  data_size = size_t(obj_stream.GetStreamPos());
691  LogStat(r, blob_id,
693  "CProcessor_ID1: read data",
694  data_size);
695  }}
696 
698  if ( version >= 0 ) {
699  result.SetAndSaveBlobVersion(blob_id, version);
700  }
701 
702  TSeqEntryInfo entry = GetSeq_entry(result, blob_id, reply);
703  result.SetAndSaveBlobState(blob_id, entry.second);
704  CLoadLockSetter setter(blob);
705  if ( !setter.IsLoaded() ) {
706  if ( entry.first ) {
708  OffsetAllGisToOM(*entry.first);
709  setter.SetSeq_entry(*entry.first);
710  LogStat(r, blob_id,
712  "CProcessor_ID1: attached entry",
713  data_size);
714  }
715  setter.SetLoaded();
716  }
717 
718  if ( writer && version >= 0 ) {
719  SaveBlob(result, blob_id, chunk_id, writer,
720  guard.EndDelayBuffer());
721  }
722 }
723 
724 
727  const TBlobId& blob_id,
728  CID1server_back& reply) const
729 {
730  TSeqEntryInfo entry;
731  switch ( reply.Which() ) {
733  entry.first.Reset(&reply.SetGotseqentry());
734  break;
736  entry.second |= CBioseq_Handle::fState_dead;
737  entry.first.Reset(&reply.SetGotdeadseqentry());
738  break;
740  {{
741  const CID1blob_info& info = reply.GetGotsewithinfo().GetBlob_info();
742  if ( info.GetBlob_state() < 0 ) {
743  entry.second |= CBioseq_Handle::fState_dead;
744  }
745  if ( reply.GetGotsewithinfo().IsSetBlob() ) {
746  entry.first.Reset(&reply.SetGotsewithinfo().SetBlob());
747  }
748  else {
749  // no Seq-entry in reply, probably private data
750  entry.second |= CBioseq_Handle::fState_no_data;
751  }
752  if ( info.GetSuppress() & 5 ) { // suppressed(=1) & temporary(=4)
753  entry.second |=
754  (info.GetSuppress() == 4)
757  }
758  if ( info.GetWithdrawn() ) {
759  entry.second |=
762  }
763  if ( info.GetConfidential() ) {
764  entry.second |=
767  }
768  break;
769  }}
771  {{
772  int error = reply.GetError();
773  switch ( error ) {
774  case 1:
775  entry.second |=
778  break;
779  case 2:
780  entry.second |=
783  break;
784  case 10:
785  entry.second |= CBioseq_Handle::fState_no_data;
786  break;
787  case 100:
788  NCBI_THROW_FMT(CLoaderException, eConnectionFailed,
789  "ID1server-back.error "<<error);
790  default:
791  ERR_POST_X(1, "CId1Reader::GetMainBlob: "
792  "ID1server-back.error "<<error);
793  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
794  "CProcessor_ID1::GetSeq_entry: "
795  "ID1server-back.error "<<error);
796  }
797  break;
798  }}
799  default:
800  // no data
801  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
802  "CProcessor_ID1::GetSeq_entry: "
803  "bad ID1server-back type: "<<reply.Which());
804  }
805  return entry;
806 }
807 
808 
811 {
812  switch ( reply.Which() ) {
814  return abs(reply.GetGotblobinfo().GetBlob_state());
816  return abs(reply.GetGotsewithinfo().GetBlob_info().GetBlob_state());
817  default:
818  return -1;
819  }
820 }
821 
822 
824  const TBlobId& blob_id,
825  TChunkId chunk_id,
826  CWriter* writer,
827  CRef<CByteSource> byte_source) const
828 {
829  _ASSERT(writer && byte_source);
831  (OpenStream(writer, result, blob_id, chunk_id, this));
832  if ( !stream ) {
833  return;
834  }
835  try {
836  CWriter::WriteBytes(**stream, byte_source);
837  stream->Close();
838  }
839  catch ( CException& ) { // ignored
840  }
841 }
842 
843 
845  const TBlobId& blob_id,
846  TChunkId chunk_id,
847  CWriter* writer,
848  const CID1server_back& reply) const
849 {
850  _ASSERT(writer);
852  (OpenStream(writer, result, blob_id, chunk_id, this));
853  if ( !stream ) {
854  return;
855  }
856  try {
857  {{
858  CObjectOStreamAsnBinary obj_stream(**stream);
859  obj_stream << reply;
860  }}
861  stream->Close();
862  }
863  catch ( CException& ) { // ignored
864  }
865 }
866 
867 
868 /////////////////////////////////////////////////////////////////////////////
869 // CProcessor_ID1_SNP
870 /////////////////////////////////////////////////////////////////////////////
871 
873  : CProcessor_ID1(dispatcher)
874 {
875 }
876 
877 
879 {
880 }
881 
882 
884 {
885  return eType_ID1_SNP;
886 }
887 
888 
890 {
891  static TMagic kMagic = s_GetMagic("ID1S");
892  return kMagic;
893 }
894 
895 
897  const TBlobId& blob_id,
898  TChunkId chunk_id,
899  CObjectIStream& obj_stream) const
900 {
901  CLoadLockBlob blob(result, blob_id, chunk_id);
902  if ( blob.IsLoadedChunk() ) {
903  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
904  "CProcessor_ID1_SNP: "
905  "double load of "<<blob_id<<'/'<<chunk_id);
906  }
908  CID1server_back reply;
909  size_t data_size = 0;
910  {{
912 
914  Begin(reply),
915  *set_info);
916  data_size = size_t(obj_stream.GetStreamPos());
917  LogStat(r, blob_id,
919  "CProcessor_ID1: read SNP data",
920  data_size);
921  }}
922 
924  if ( version >= 0 ) {
925  result.SetAndSaveBlobVersion(blob_id, version);
926  }
927  TSeqEntryInfo entry = GetSeq_entry(result, blob_id, reply);
928  result.SetAndSaveBlobState(blob_id, entry.second);
929 
930  CWriter* writer = GetWriter(result);
931  if ( writer && version >= 0 ) {
932  if ( set_info->m_Seq_annot_InfoMap.empty() || !entry.first ) {
933  const CProcessor_ID1* prc =
934  dynamic_cast<const CProcessor_ID1*>
936  if ( prc ) {
937  prc->SaveBlob(result, blob_id, chunk_id, writer, reply);
938  }
939  }
940  else {
941  const CProcessor_St_SE_SNPT* prc =
942  dynamic_cast<const CProcessor_St_SE_SNPT*>
944  if ( prc ) {
945  prc->SaveSNPBlob(result, blob_id, chunk_id, writer,
946  *entry.first, entry.second, *set_info);
947  }
948  }
949  }
950 
951  CLoadLockSetter setter(blob);
952  if ( !setter.IsLoaded() ) {
953  if ( entry.first ) {
955  OffsetAllGisToOM(*entry.first, set_info);
956  setter.SetSeq_entry(*entry.first, set_info);
957  LogStat(r, blob_id,
959  "CProcessor_ID1: attached entry",
960  data_size);
961  }
962  setter.SetLoaded();
963  }
964 }
965 
966 
967 /////////////////////////////////////////////////////////////////////////////
968 // CProcessor_SE
969 /////////////////////////////////////////////////////////////////////////////
970 
972  : CProcessor(dispatcher)
973 {
974 }
975 
976 
978 {
979 }
980 
981 
983 {
984  return eType_Seq_entry;
985 }
986 
987 
989 {
990  static TMagic kMagic = s_GetMagic("SeqE");
991  return kMagic;
992 }
993 
994 
996  const TBlobId& blob_id,
997  TChunkId chunk_id,
998  CObjectIStream& obj_stream) const
999 {
1000  CLoadLockBlob blob(result, blob_id, chunk_id);
1001  CLoadLockSetter setter(blob);
1002  if ( setter.IsLoaded() ) {
1003  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1004  "CProcessor_SE: "
1005  "double load of "<<blob_id<<'/'<<chunk_id);
1006  }
1007  CRef<CSeq_entry> seq_entry(new CSeq_entry);
1008  {{
1010  CWriter* writer = x_GetWriterToSaveBlob(result, blob_id, setter, "SE");
1011  if ( writer ) {
1012  guard.StartDelayBuffer(obj_stream);
1013  }
1014 
1015  SetSeqEntryReadHooks(obj_stream);
1016  size_t data_size = 0;
1017  {{
1019 
1020  obj_stream >> *seq_entry;
1021 
1022  data_size = size_t(obj_stream.GetStreamPos());
1023  LogStat(r, blob_id,
1025  "CProcessor_SE: read seq-entry",
1026  data_size);
1027  }}
1028 
1029  {{
1031 
1032  OffsetAllGisToOM(*seq_entry);
1033  setter.SetSeq_entry(*seq_entry);
1034 
1035  LogStat(r, blob_id,
1037  "CProcessor_SE: attached entry",
1038  data_size);
1039  }}
1040 
1041  if ( chunk_id == kMain_ChunkId &&
1042  s_CanBeWGSBlob(blob_id) &&
1043  result.GetAddWGSMasterDescr() ) {
1045  }
1046  setter.SetLoaded();
1047 
1048  if ( writer ) {
1049  const CProcessor_St_SE* prc =
1050  dynamic_cast<const CProcessor_St_SE*>
1052  if ( prc ) {
1053  prc->SaveBlob(result, blob_id, chunk_id,
1054  setter.GetBlobState(), writer,
1055  guard.EndDelayBuffer());
1056  }
1057  }
1058  }}
1059 }
1060 
1061 
1063  const CBlob_id& blob_id,
1064  CLoadLockSetter& setter,
1065  const char* processor_name) const
1066 {
1067  if ( !result.IsLoadedBlobVersion(blob_id) ) {
1068  ERR_POST_X(4, "CProcessor_"<<processor_name<<"::ProcessObjStream: "
1069  "blob version is not set");
1070  return 0;
1071  }
1072  if ( setter.GetBlobState() & CBioseq_Handle::fState_no_data ) {
1073  ERR_POST_X(5, "CProcessor_"<<processor_name<<"::ProcessObjStream: "
1074  "state no_data is set");
1075  return 0;
1076  }
1077  return GetWriter(result);
1078 }
1079 
1080 
1081 /////////////////////////////////////////////////////////////////////////////
1082 // CProcessor_SE_SNP
1083 /////////////////////////////////////////////////////////////////////////////
1084 
1086  : CProcessor_SE(dispatcher)
1087 {
1088 }
1089 
1090 
1092 {
1093 }
1094 
1095 
1097 {
1098  return eType_Seq_entry_SNP;
1099 }
1100 
1101 
1103 {
1104  static TMagic kMagic = s_GetMagic("SESN");
1105  return kMagic;
1106 }
1107 
1108 
1110  const TBlobId& blob_id,
1111  TChunkId chunk_id,
1112  CObjectIStream& obj_stream) const
1113 {
1114  CLoadLockBlob blob(result, blob_id, chunk_id);
1115  CLoadLockSetter setter(blob);
1116  if ( setter.IsLoaded() ) {
1117  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1118  "CProcessor_SE_SNP: "
1119  "double load of "<<blob_id<<'/'<<chunk_id);
1120  }
1122  CRef<CSeq_entry> seq_entry(new CSeq_entry);
1123  size_t data_size = 0;
1124  {{
1125  CWriter* writer = x_GetWriterToSaveBlob(result, blob_id, setter, "SE_SNP");
1126 
1127  {{
1129 
1131  Begin(*seq_entry),
1132  *set_info);
1133  data_size = size_t(obj_stream.GetStreamPos());
1134 
1135  LogStat(r, blob_id,
1137  "CProcessor_SE_SNP: parse SNP data",
1138  data_size);
1139  }}
1140 
1141  if ( writer ) {
1142  if ( set_info->m_Seq_annot_InfoMap.empty() || !seq_entry ) {
1143  const CProcessor_St_SE* prc =
1144  dynamic_cast<const CProcessor_St_SE*>
1146  if ( prc ) {
1147  if ( seq_entry ) {
1148  prc->SaveBlob(result, blob_id, chunk_id,
1149  setter.GetBlobState(), writer, *seq_entry);
1150  }
1151  else {
1152  prc->SaveNoBlob(result, blob_id, chunk_id,
1153  setter.GetBlobState(), writer);
1154  }
1155  }
1156  }
1157  else {
1158  const CProcessor_St_SE_SNPT* prc =
1159  dynamic_cast<const CProcessor_St_SE_SNPT*>
1161  if ( prc ) {
1162  prc->SaveSNPBlob(result, blob_id, chunk_id, writer,
1163  *seq_entry, setter.GetBlobState(), *set_info);
1164  }
1165  }
1166  }
1167  }}
1168  {{
1170 
1171  OffsetAllGisToOM(*seq_entry, set_info);
1172  setter.SetSeq_entry(*seq_entry, set_info);
1173 
1174  LogStat(r, blob_id,
1176  "CProcessor_SE_SNP: attached SNP entry to OM",
1177  data_size);
1178  }}
1179 
1180  setter.SetLoaded();
1181 }
1182 
1183 
1184 /////////////////////////////////////////////////////////////////////////////
1185 // CProcessor_St_SE
1186 /////////////////////////////////////////////////////////////////////////////
1187 
1189  : CProcessor_SE(dispatcher)
1190 {
1191 }
1192 
1193 
1195 {
1196 }
1197 
1198 
1200 {
1201  return eType_St_Seq_entry;
1202 }
1203 
1204 
1206 {
1207  static TMagic kMagic = s_GetMagic("StSE");
1208  return kMagic;
1209 }
1210 
1211 
1213  const TBlobId& blob_id,
1214  TChunkId chunk_id,
1215  CObjectIStream& obj_stream) const
1216 {
1217  CLoadLockBlob blob(result, blob_id, chunk_id);
1218  TBlobState blob_state;
1219 
1220  {{
1222 
1223  blob_state = ReadBlobState(obj_stream);
1224 
1225  LogStat(r, blob_id,
1227  "CProcessor_St_SE: read state",
1228  obj_stream.GetStreamPos());
1229  }}
1230 
1231  result.SetAndSaveBlobState(blob_id, blob_state);
1232  if ( blob_state & CBioseq_Handle::fState_no_data ) {
1233  CLoadLockSetter setter(blob);
1234  if ( !setter.IsLoaded() ) {
1235  setter.SetLoaded();
1236  }
1237 
1238  CWriter* writer = GetWriter(result);
1239  if ( writer ) {
1240  const CProcessor_St_SE* prc =
1241  dynamic_cast<const CProcessor_St_SE*>
1243  if ( prc ) {
1244  prc->SaveNoBlob(result, blob_id, chunk_id,
1245  blob_state, writer);
1246  }
1247  }
1248  }
1249  else {
1250  CProcessor_SE::ProcessObjStream(result, blob_id, chunk_id, obj_stream);
1251  }
1252 }
1253 
1254 
1257 {
1258  return obj_stream.ReadInt4();
1259 }
1260 
1261 
1263  TBlobState blob_state) const
1264 {
1266  obj_stream.WriteInt4(blob_state);
1267 }
1268 
1269 
1272 {
1273  CObjectIStreamAsnBinary obj_stream(stream);
1274  return ReadBlobState(obj_stream);
1275 }
1276 
1277 
1279  TBlobState blob_state) const
1280 {
1281  CObjectOStreamAsnBinary obj_stream(stream);
1283  WriteBlobState(obj_stream, blob_state);
1284 }
1285 
1286 
1288  const TBlobId& blob_id,
1289  TChunkId chunk_id,
1290  TBlobState blob_state,
1291  CWriter* writer,
1292  CRef<CByteSource> byte_source) const
1293 {
1294  SaveBlob(result, blob_id, chunk_id, blob_state, writer, byte_source->Open());
1295 }
1296 
1297 
1299  const TBlobId& blob_id,
1300  TChunkId chunk_id,
1301  TBlobState blob_state,
1302  CWriter* writer,
1303  CRef<CByteSourceReader> reader) const
1304 {
1305  _ASSERT(writer && reader);
1307  (OpenStream(writer, result, blob_id, chunk_id, this));
1308  if ( !stream ) {
1309  return;
1310  }
1311  try {
1312  WriteBlobState(**stream, blob_state);
1313  CWriter::WriteBytes(**stream, reader);
1314  stream->Close();
1315  }
1316  catch ( CException& ) { // ignored
1317  }
1318 }
1319 
1320 
1322  const TBlobId& blob_id,
1323  TChunkId chunk_id,
1324  TBlobState blob_state,
1325  CWriter* writer,
1326  const TOctetStringSequence& data) const
1327 {
1328  _ASSERT(writer);
1330  (OpenStream(writer, result, blob_id, chunk_id, this));
1331  if ( !stream ) {
1332  return;
1333  }
1334  try {
1335  WriteBlobState(**stream, blob_state);
1336  CWriter::WriteBytes(**stream, data);
1337  stream->Close();
1338  }
1339  catch ( CException& ) { // ignored
1340  }
1341 }
1342 
1343 
1345  const TBlobId& blob_id,
1346  TChunkId chunk_id,
1347  TBlobState blob_state,
1348  CWriter* writer,
1349  const CSeq_entry& seq_entry) const
1350 {
1351  _ASSERT(writer);
1353  (OpenStream(writer, result, blob_id, chunk_id, this));
1354  if ( !stream ) {
1355  return;
1356  }
1357  try {
1358  {{
1359  CObjectOStreamAsnBinary obj_stream(**stream);
1361  WriteBlobState(obj_stream, blob_state);
1362  obj_stream << seq_entry;
1363  }}
1364  stream->Close();
1365  }
1366  catch ( CException& ) { // ignored
1367  }
1368 }
1369 
1370 
1372  const TBlobId& blob_id,
1373  TChunkId chunk_id,
1374  TBlobState blob_state,
1375  CWriter* writer) const
1376 {
1377  _ASSERT(writer);
1379  (OpenStream(writer, result, blob_id, chunk_id, this));
1380  if ( !stream ) {
1381  return;
1382  }
1383  try {
1384  WriteBlobState(**stream, blob_state);
1385  stream->Close();
1386  }
1387  catch ( CException& ) { // ignored
1388  }
1389 }
1390 
1391 
1392 /////////////////////////////////////////////////////////////////////////////
1393 // CProcessor_St_SE_SNPT
1394 /////////////////////////////////////////////////////////////////////////////
1395 
1397  : CProcessor_St_SE(d)
1398 {
1399 }
1400 
1401 
1403 {
1404 }
1405 
1406 
1408 {
1409  return eType_St_Seq_entry_SNPT;
1410 }
1411 
1412 
1414 {
1415  static TMagic kMagic = s_GetMagic("SEST");
1416  return kMagic;
1417 }
1418 
1419 
1421  const TBlobId& blob_id,
1422  TChunkId chunk_id,
1423  CNcbiIstream& stream) const
1424 {
1425  CLoadLockBlob blob(result, blob_id, chunk_id);
1426  CLoadLockSetter setter(blob);
1427  if ( setter.IsLoaded() ) {
1428  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1429  "CProcessor_St_SE_SNPT: "
1430  "double load of "<<blob_id<<'/'<<chunk_id);
1431  }
1432 
1433  TBlobState blob_state = ReadBlobState(stream);
1434  result.SetAndSaveBlobState(blob_id, blob_state);
1435 
1436  CRef<CSeq_entry> seq_entry(new CSeq_entry);
1438  Int8 data_size = 0;
1439 
1440  {{
1442  data_size = NcbiStreamposToInt8(stream.tellg());
1443 
1444  CSeq_annot_SNP_Info_Reader::Read(stream, Begin(*seq_entry), *set_info);
1445 
1446  data_size = NcbiStreamposToInt8(stream.tellg()) - data_size;
1447  LogStat(r, blob_id,
1449  "CProcessor_St_SE_SNPT: read SNP table",
1450  double(data_size));
1451  }}
1452 
1453  CWriter* writer = GetWriter(result);
1454  if ( writer ) {
1455  SaveSNPBlob(result, blob_id, chunk_id, writer, *seq_entry, blob_state, *set_info);
1456  }
1457  {{
1459 
1460  OffsetAllGisToOM(*seq_entry, set_info);
1461  setter.SetSeq_entry(*seq_entry, set_info);
1462 
1463  LogStat(r, blob_id,
1465  "CProcessor_St_SE_SNPT: attached SNP table",
1466  double(data_size));
1467  }}
1468  setter.SetLoaded();
1469 }
1470 
1471 
1473  const TBlobId& blob_id,
1474  TChunkId chunk_id,
1475  CWriter* writer,
1476  const CSeq_entry& seq_entry,
1477  TBlobState blob_state,
1478  const CTSE_SetObjectInfo& set_info) const
1479 {
1480  _ASSERT(writer);
1482  (OpenStream(writer, result, blob_id, chunk_id, this));
1483  if ( !stream ) {
1484  return;
1485  }
1486  try {
1487  WriteBlobState(**stream, blob_state);
1489  ConstBegin(seq_entry), set_info);
1490  stream->Close();
1491  }
1492  catch ( CException& ) { // ignored
1493  }
1494 }
1495 
1496 
1497 /////////////////////////////////////////////////////////////////////////////
1498 // CProcessor_ID2
1499 /////////////////////////////////////////////////////////////////////////////
1500 
1502  : CProcessor(d)
1503 {
1504 }
1505 
1506 
1508 {
1509 }
1510 
1511 
1513 {
1514  return eType_ID2;
1515 }
1516 
1517 
1519 {
1520  static TMagic kMagic = s_GetMagic("ID2s");
1521  return kMagic;
1522 }
1523 
1524 
1526  const TBlobId& blob_id,
1527  TChunkId chunk_id,
1528  CObjectIStream& obj_stream) const
1529 {
1530  TBlobState blob_state;
1532 
1533  {{
1535 
1536  blob_state = obj_stream.ReadInt4();
1537  obj_stream >> data;
1538 
1539  LogStat(r, blob_id,
1541  "CProcessor_ID2: read data",
1542  obj_stream.GetStreamPos());
1543  }}
1544 
1545  ProcessData(result, blob_id, blob_state, chunk_id, data);
1546 }
1547 
1548 
1550  const TBlobId& blob_id,
1551  TBlobState blob_state,
1552  TChunkId chunk_id,
1553  const CID2_Reply_Data& data,
1554  int split_version,
1555  const CID2_Reply_Data* skel) const
1556 {
1557  CLoadLockBlob blob(result, blob_id, chunk_id);
1558  if ( blob.IsLoadedChunk() ) {
1559  if ( chunk_id >= 0 ) {
1560  // the same chunk can be load multiple times by getblob request
1561  // do not treat this situation as an error since
1562  // the chunk was actually loaded successfully
1563  ERR_POST_X(4, Info << "CProcessor_ID2: "
1564  "double load of "<<blob_id<<'/'<<chunk_id);
1565  return;
1566  }
1567  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1568  "CProcessor_ID2: "
1569  "double load of "<<blob_id<<'/'<<chunk_id);
1570  }
1571 
1572  size_t data_size = 0;
1573  switch ( data.GetData_type() ) {
1575  {
1576  // plain seq-entry
1577  if ( split_version != 0 || skel ) {
1578  NCBI_THROW(CLoaderException, eLoaderFailed,
1579  "CProcessor_ID2: "
1580  "plain Seq-entry with extra ID2S-Split-Info");
1581  }
1582  if ( chunk_id != kMain_ChunkId && chunk_id != kDelayedMain_ChunkId ) {
1583  NCBI_THROW(CLoaderException, eLoaderFailed,
1584  "CProcessor_ID2: "
1585  "plain Seq-entry in chunk reply");
1586  }
1587  CRef<CSeq_entry> entry(new CSeq_entry);
1588 
1589  {{
1591 
1592  x_ReadData(data, Begin(*entry), data_size);
1593 
1594  LogStat(r, blob_id,
1596  "CProcessor_ID2: parsed Seq-entry",
1597  data_size);
1598  }}
1599 
1600  result.SetAndSaveBlobState(blob_id, blob_state);
1601  CLoadLockSetter setter(blob);
1602  if ( !setter.IsLoaded() ) {
1603  {{
1605 
1606  OffsetAllGisToOM(*entry);
1607  setter.SetSeq_entry(*entry);
1608 
1609  if ( s_CanBeWGSBlob(blob_id) &&
1610  result.GetAddWGSMasterDescr() ) {
1612  }
1613 
1614  LogStat(r, blob_id,
1616  "CProcessor_ID2: attached Seq-entry",
1617  data_size);
1618  }}
1619  setter.SetLoaded();
1620  }
1621 
1622  CWriter* writer = GetWriter(result);
1623  if ( writer ) {
1624  if ( data.GetData_format() == data.eData_format_asn_binary &&
1625  data.GetData_compression() == data.eData_compression_none &&
1626  !s_CacheRecompress() ) {
1627  // can save as simple Seq-entry
1628  const CProcessor_St_SE* prc =
1629  dynamic_cast<const CProcessor_St_SE*>
1631  if ( prc ) {
1632  prc->SaveBlob(result, blob_id, chunk_id,
1633  blob_state, writer, data.GetData());
1634  }
1635  }
1636  else {
1637  SaveData(result, blob_id, blob_state, chunk_id, writer, data);
1638  }
1639  }
1640  break;
1641  }
1643  {
1644  if ( chunk_id != kMain_ChunkId && chunk_id != kDelayedMain_ChunkId ) {
1645  NCBI_THROW(CLoaderException, eLoaderFailed,
1646  "CProcessor_ID2: "
1647  "plain ID2S-Split-Info in non-main reply");
1648  }
1649  CRef<CID2S_Split_Info> split_info(new CID2S_Split_Info);
1650 
1651  {{
1653 
1654  x_ReadData(data, Begin(*split_info), data_size);
1655 
1656  LogStat(r, blob_id,
1658  "CProcessor_ID2: parsed split info",
1659  data_size);
1660  }}
1661 
1662  bool with_skeleton = split_info->IsSetSkeleton();
1663  if ( !with_skeleton ) {
1664  // update skeleton field
1665  if ( !skel ) {
1666  NCBI_THROW(CLoaderException, eLoaderFailed,
1667  "CProcessor_ID2: "
1668  "ID2S-Split-Info without skeleton Seq-entry");
1669  }
1670 
1671  {{
1673 
1674  x_ReadData(*skel, Begin(split_info->SetSkeleton()), data_size);
1675 
1676  LogStat(r, blob_id,
1678  "CProcessor_ID2: parsed Seq-entry",
1679  data_size);
1680  }}
1681  }
1682 
1683  result.SetAndSaveBlobState(blob_id, blob_state);
1684  CLoadLockSetter setter(blob);
1685  if ( !setter.IsLoaded() ) {
1686  {{
1688 
1689  CTSE_LoadLock& lock = setter.GetTSE_LoadLock();
1690  lock->GetSplitInfo().SetSplitVersion(split_version);
1691  OffsetAllGisToOM(*split_info);
1692  CSplitParser::Attach(*lock, *split_info);
1693  if ( s_CanBeWGSBlob(blob_id) &&
1694  result.GetAddWGSMasterDescr() ) {
1696  }
1697 
1698  LogStat(r, blob_id,
1700  "CProcessor_ID2: attached entry",
1701  data_size);
1702  }}
1703 
1704  setter.SetLoaded();
1705  }
1706 
1707  CWriter* writer = GetWriter(result);
1708  if ( writer ) {
1709  if ( with_skeleton ) {
1710  const CProcessor_ID2_Split* prc =
1711  dynamic_cast<const CProcessor_ID2_Split*>
1713  if ( prc ) {
1714  prc->SaveSplitData(result, blob_id, blob_state, chunk_id,
1715  writer, split_version, data);
1716  }
1717  }
1718  else if ( skel ) {
1719  const CProcessor_ID2AndSkel* prc =
1720  dynamic_cast<const CProcessor_ID2AndSkel*>
1722  if ( prc ) {
1723  prc->SaveDataAndSkel(result, blob_id, blob_state, chunk_id,
1724  writer, split_version, data, *skel);
1725  }
1726  }
1727  }
1728  break;
1729  }
1731  {
1732  if ( chunk_id == kMain_ChunkId || chunk_id == kDelayedMain_ChunkId ) {
1733  NCBI_THROW(CLoaderException, eLoaderFailed,
1734  "CProcessor_ID2: "
1735  "ID2S-Chunk in main reply");
1736  }
1737  CLoadLockSetter setter(blob);
1738  if ( setter.IsLoaded() ) {
1739  break;
1740  }
1741  CRef<CID2S_Chunk> chunk(new CID2S_Chunk);
1742 
1743  {{
1745 
1746  x_ReadData(data, Begin(*chunk), data_size);
1747  LogStat(r, blob_id, chunk_id,
1749  "CProcessor_ID2: parsed split chunk",
1750  data_size);
1751  }}
1752 
1753  {{
1755 
1756  OffsetAllGisToOM(*chunk);
1757  CSplitParser::Load(setter.GetTSE_Chunk_Info(), *chunk);
1758 
1759  LogStat(r, blob_id, chunk_id,
1761  "CProcessor_ID2: attached chunk",
1762  data_size);
1763  }}
1764  setter.SetLoaded();
1765 
1766  CWriter* writer = GetWriter(result);
1767  if ( writer ) {
1768  SaveData(result, blob_id, blob_state, chunk_id, writer, data);
1769  }
1770  break;
1771  }
1772  default:
1773  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
1774  "CProcessor_ID2: "
1775  "invalid data type: "<<data.GetData_type());
1776  }
1777 }
1778 
1779 
1781  const TBlobId& blob_id,
1782  TBlobState blob_state,
1783  TChunkId chunk_id,
1784  CWriter* writer,
1785  const CID2_Reply_Data& data) const
1786 {
1787  _ASSERT(writer);
1789  (OpenStream(writer, result, blob_id, chunk_id, this));
1790  if ( !stream ) {
1791  return;
1792  }
1793  try {
1794  if ( s_CacheRecompress() ) {
1795  x_FixCompression(const_cast<CID2_Reply_Data&>(data));
1796  }
1797  {{
1798  CObjectOStreamAsnBinary obj_stream(**stream);
1799  SaveData(obj_stream, blob_state, data);
1800  }}
1801  stream->Close();
1802  }
1803  catch ( CException& ) { // ignored
1804  }
1805 }
1806 
1807 
1809  TBlobState blob_state,
1810  const CID2_Reply_Data& data) const
1811 {
1813  obj_stream.WriteInt4(blob_state);
1814  obj_stream << data;
1815 }
1816 
1817 
1819 {
1820  // TEMP: TODO: remove this
1821  if ( data.GetData_format() == CID2_Reply_Data::eData_format_xml &&
1822  data.GetData_compression()==CID2_Reply_Data::eData_compression_gzip ){
1823  // FIX old/wrong split fields
1825  data.SetData_compression(CID2_Reply_Data::eData_compression_nlmzip);
1826  if ( data.GetData_type() > CID2_Reply_Data::eData_type_seq_entry ) {
1827  data.SetData_type(data.GetData_type()+1);
1828  }
1829  }
1830 }
1831 
1832 
1834 {
1835  if (data.GetData_compression() != CID2_Reply_Data::eData_compression_none)
1836  return;
1837 
1838  CID2_Reply_Data new_data;
1839  {{
1840  COSSWriter writer(new_data.SetData());
1841  CWStream wstream(&writer);
1842  CCompressionOStream stream(wstream,
1845  ITERATE ( CID2_Reply_Data::TData, it, data.GetData() ) {
1846  stream.write(&(**it)[0], (*it)->size());
1847  }
1848  }}
1849  data.SetData().swap(new_data.SetData());
1850  data.SetData_compression(CID2_Reply_Data::eData_compression_gzip);
1851 }
1852 
1853 
1855 {
1856  x_FixDataFormat(const_cast<CID2_Reply_Data&>(data));
1858  switch ( data.GetData_format() ) {
1861  break;
1864  break;
1866  format = eSerial_Xml;
1867  break;
1868  default:
1869  NCBI_THROW(CLoaderException, eLoaderFailed,
1870  "CId2Reader::x_ReadData(): unknown data format");
1871  }
1872  unique_ptr<IReader> reader(new COSSReader(data.GetData()));
1873  unique_ptr<CNcbiIstream> stream;
1874  switch ( data.GetData_compression() ) {
1876  break;
1878  reader.reset(new CNlmZipReader(reader.release(),
1880  break;
1882  stream.reset(new CRStream(reader.release(),
1883  0, 0, CRWStreambuf::fOwnAll));
1884  stream.reset(new CCompressionIStream(*stream.release(),
1887  break;
1888  default:
1889  NCBI_THROW(CLoaderException, eLoaderFailed,
1890  "CId2Reader::x_ReadData(): unknown data compression");
1891  }
1892  if ( !stream.get() ) {
1893  stream.reset(new CRStream(reader.release(),
1894  0, 0, CRWStreambuf::fOwnAll));
1895  }
1896  unique_ptr<CObjectIStream> in;
1897  in.reset(CObjectIStream::Open(format, *stream.release(), eTakeOwnership));
1898  return in.release();
1899 }
1900 
1901 
1903  const CObjectInfo& object,
1904  size_t& data_size)
1905 {
1906  unique_ptr<CObjectIStream> in(x_OpenDataStream(data));
1907  switch ( data.GetData_type() ) {
1909  if ( object.GetTypeInfo() != CSeq_entry::GetTypeInfo() ) {
1910  NCBI_THROW(CLoaderException, eLoaderFailed,
1911  "CId2Reader::x_ReadData(): unexpected Seq-entry");
1912  }
1913  break;
1915  if ( object.GetTypeInfo() != CID2S_Split_Info::GetTypeInfo() ) {
1916  NCBI_THROW(CLoaderException, eLoaderFailed,
1917  "CId2Reader::x_ReadData(): unexpected ID2S-Split-Info");
1918  }
1919  break;
1921  if ( object.GetTypeInfo() != CID2S_Chunk::GetTypeInfo() ) {
1922  NCBI_THROW(CLoaderException, eLoaderFailed,
1923  "CId2Reader::x_ReadData(): unexpected ID2S-Chunk");
1924  }
1925  break;
1926  default:
1927  NCBI_THROW(CLoaderException, eLoaderFailed,
1928  "CId2Reader::x_ReadData(): unknown data type");
1929  }
1931  in->SetSkipUnknownMembers(eSerialSkipUnknown_Yes);
1932  in->SetSkipUnknownVariants(eSerialSkipUnknown_Yes);
1933  in->Read(object);
1934  data_size += size_t(in->GetStreamPos()); // in-memory size can't be bigger than size_t
1935 }
1936 
1937 
1939  CNcbiOstream& out_stream)
1940 {
1941  unique_ptr<CObjectIStream> in(x_OpenDataStream(data));
1942  unique_ptr<CObjectOStream> out(CObjectOStream::Open(eSerial_AsnText,
1943  out_stream));
1944  TTypeInfo type;
1945  switch ( data.GetData_type() ) {
1947  type = CSeq_entry::GetTypeInfo();
1948  break;
1950  type = CID2S_Split_Info::GetTypeInfo();
1951  break;
1953  type = CID2S_Chunk::GetTypeInfo();
1954  break;
1955  default:
1956  return;
1957  }
1958  CObjectStreamCopier copier(*in, *out);
1959  copier.Copy(type);
1960 }
1961 
1962 
1963 /////////////////////////////////////////////////////////////////////////////
1964 // CProcessor_ID2_Split
1965 /////////////////////////////////////////////////////////////////////////////
1966 
1968  : CProcessor_ID2(d)
1969 {
1970 }
1971 
1972 
1974 {
1975 }
1976 
1977 
1979 {
1980  return eType_ID2_Split;
1981 }
1982 
1983 
1985 {
1986  static TMagic kMagic = s_GetMagic("I2sp");
1987  return kMagic;
1988 }
1989 
1990 
1992  const TBlobId& blob_id,
1993  TChunkId chunk_id,
1994  CObjectIStream& obj_stream) const
1995 {
1996  TBlobState blob_state;
1997  TSplitVersion split_version;
1998  CID2_Reply_Data split_data;
1999 
2000  {{
2002 
2003  blob_state = obj_stream.ReadInt4();
2004  split_version = obj_stream.ReadInt4();
2005  obj_stream >> split_data;
2006 
2007  LogStat(r, blob_id,
2009  "CProcessor_ID2_Split: read skel",
2010  obj_stream.GetStreamPos());
2011  }}
2012 
2013  ProcessData(result, blob_id, blob_state, chunk_id,
2014  split_data, split_version);
2015 }
2016 
2017 
2019  const TBlobId& blob_id,
2020  TBlobState blob_state,
2021  TChunkId chunk_id,
2022  CWriter* writer,
2023  TSplitVersion split_version,
2024  const CID2_Reply_Data& split) const
2025 {
2026  _ASSERT(writer);
2028  (OpenStream(writer, result, blob_id, chunk_id, this));
2029  if ( !stream ) {
2030  return;
2031  }
2032  try {
2033  if ( s_CacheRecompress() ) {
2034  x_FixCompression(const_cast<CID2_Reply_Data&>(split));
2035  }
2036  {{
2037  CObjectOStreamAsnBinary obj_stream(**stream);
2038  SaveSplitData(obj_stream, blob_state, split_version, split);
2039  }}
2040  stream->Close();
2041  }
2042  catch ( CException& ) { // ignored
2043  }
2044 }
2045 
2046 
2048  TBlobState blob_state,
2049  TSplitVersion split_version,
2050  const CID2_Reply_Data& split) const
2051 {
2053  obj_stream.WriteInt4(blob_state);
2054  obj_stream.WriteInt4(split_version);
2055  obj_stream << split;
2056 }
2057 
2058 
2059 /////////////////////////////////////////////////////////////////////////////
2060 // CProcessor_ID2AndSkel
2061 /////////////////////////////////////////////////////////////////////////////
2062 
2064  : CProcessor_ID2(d)
2065 {
2066 }
2067 
2068 
2070 {
2071 }
2072 
2073 
2075 {
2076  return eType_ID2AndSkel;
2077 }
2078 
2079 
2081 {
2082  static TMagic kMagic = s_GetMagic("I2ss");
2083  return kMagic;
2084 }
2085 
2086 
2088  const TBlobId& blob_id,
2089  TChunkId chunk_id,
2090  CObjectIStream& obj_stream) const
2091 {
2092  TBlobState blob_state;
2093  TSplitVersion split_version;
2094  CID2_Reply_Data split_data, skel_data;
2095 
2096  {{
2098 
2099  blob_state = obj_stream.ReadInt4();
2100  split_version = obj_stream.ReadInt4();
2101  obj_stream >> split_data;
2102  obj_stream >> skel_data;
2103 
2104  LogStat(r, blob_id,
2106  "CProcessor_ID2AndSkel: read skel",
2107  obj_stream.GetStreamPos());
2108  }}
2109 
2110  ProcessData(result, blob_id, blob_state, chunk_id,
2111  split_data, split_version, ConstRef(&skel_data));
2112 }
2113 
2114 
2116  const TBlobId& blob_id,
2117  TBlobState blob_state,
2118  TChunkId chunk_id,
2119  CWriter* writer,
2120  TSplitVersion split_version,
2121  const CID2_Reply_Data& split,
2122  const CID2_Reply_Data& skel) const
2123 {
2124  _ASSERT(writer);
2126  (OpenStream(writer, result, blob_id, chunk_id, this));
2127  if ( !stream ) {
2128  return;
2129  }
2130  try {
2131  if ( s_CacheRecompress() ) {
2132  x_FixCompression(const_cast<CID2_Reply_Data&>(split));
2133  x_FixCompression(const_cast<CID2_Reply_Data&>(skel));
2134  }
2135  {{
2136  CObjectOStreamAsnBinary obj_stream(**stream);
2137  SaveDataAndSkel(obj_stream, blob_state, split_version,
2138  split, skel);
2139  }}
2140  stream->Close();
2141  }
2142  catch ( CException& ) { // ignored
2143  }
2144 }
2145 
2146 
2148  TBlobState blob_state,
2149  TSplitVersion split_version,
2150  const CID2_Reply_Data& split,
2151  const CID2_Reply_Data& skel) const
2152 {
2154  obj_stream.WriteInt4(blob_state);
2155  obj_stream.WriteInt4(split_version);
2156  obj_stream << split;
2157  obj_stream << skel;
2158 }
2159 
2160 
2161 /////////////////////////////////////////////////////////////////////////////
2162 // CProcessor_ExtAnnot
2163 /////////////////////////////////////////////////////////////////////////////
2164 
2165 
2167  : CProcessor(d)
2168 {
2169 }
2170 
2171 
2173 {
2174 }
2175 
2176 
2178 {
2179  return eType_ExtAnnot;
2180 }
2181 
2182 
2184 {
2185  static TMagic kMagic = s_GetMagic("EA26");
2186  return kMagic;
2187 }
2188 
2189 
2191  const TBlobId& blob_id,
2192  TChunkId chunk_id,
2193  CNcbiIstream& /*stream*/) const
2194 {
2195  Process(result, blob_id, chunk_id);
2196 }
2197 
2198 
2200 {
2201  switch ( blob_id.GetSubSat() ) {
2202  case eSubSat_SNP:
2203  case eSubSat_SNP_graph:
2204  case eSubSat_MGC:
2205  case eSubSat_HPRD:
2206  case eSubSat_tRNA:
2207  case eSubSat_STS:
2208  case eSubSat_microRNA:
2209  case eSubSat_Exon:
2210  return blob_id.GetSat() == eSat_ANNOT;
2211  case eSubSat_CDD:
2212  return blob_id.GetSat() == eSat_ANNOT_CDD;
2213  default:
2214  return false;
2215  }
2216 }
2217 
2218 
2220  TChunkId chunk_id)
2221 {
2222  return IsExtAnnot(blob_id) && chunk_id == kDelayedMain_ChunkId;
2223 }
2224 
2225 
2227  const TBlobId& blob_id,
2228  TChunkId chunk_id) const
2229 {
2230  if ( !IsExtAnnot(blob_id) || chunk_id != kMain_ChunkId ) {
2231  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
2232  "CProcessor_ExtAnnot: "
2233  "bad blob "<<blob_id<<'/'<<chunk_id);
2234  }
2235  CLoadLockBlob blob(result, blob_id, chunk_id);
2236  CLoadLockSetter setter(blob);
2237  if ( setter.IsLoaded() ) {
2238  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
2239  "CProcessor_ExtAnnot: "
2240  "double load of "<<blob_id<<'/'<<chunk_id);
2241  }
2242  // create special external annotations blob
2243  CAnnotName name;
2245  vector<SAnnotTypeSelector> more_types;
2246  string db_name;
2247  switch ( blob_id.GetSubSat() ) {
2248  case eSubSat_SNP:
2249  name.SetNamed("SNP");
2250  type.SetFeatSubtype(CSeqFeatData::eSubtype_variation);
2251  db_name = "Annot:SNP";
2252  break;
2253  case eSubSat_SNP_graph:
2254  name.SetNamed("SNP");
2255  type.SetAnnotType(CSeq_annot::C_Data::e_Graph);
2256  db_name = "Annot:SNP graph";
2257  break;
2258  case eSubSat_CDD:
2259  name.SetNamed("CDD");
2260  type.SetFeatSubtype(CSeqFeatData::eSubtype_region);
2261  more_types.push_back(SAnnotTypeSelector(CSeqFeatData::eSubtype_site));
2262  db_name = "Annot:CDD";
2263  break;
2264  case eSubSat_MGC:
2265  name.SetNamed("MGC");
2267  db_name = "Annot:MGC";
2268  break;
2269  case eSubSat_HPRD:
2270  name.SetNamed("HPRD");
2271  type.SetFeatSubtype(CSeqFeatData::eSubtype_site);
2272  db_name = "Annot:HPRD";
2273  break;
2274  case eSubSat_STS:
2275  name.SetNamed("STS");
2276  type.SetFeatSubtype(CSeqFeatData::eSubtype_STS);
2277  db_name = "Annot:STS";
2278  break;
2279  case eSubSat_tRNA:
2280  name.SetNamed("tRNA");
2281  type.SetFeatSubtype(CSeqFeatData::eSubtype_tRNA);
2282  db_name = "Annot:tRNA";
2283  break;
2284  case eSubSat_microRNA:
2285  name.SetNamed("other");
2286  type.SetFeatSubtype(CSeqFeatData::eSubtype_ncRNA);
2287  more_types.push_back(SAnnotTypeSelector(CSeqFeatData::eSubtype_otherRNA));
2288  db_name = "Annot:microRNA";
2289  break;
2290  case eSubSat_Exon:
2291  name.SetNamed("Exon");
2292  type.SetFeatSubtype(CSeqFeatData::eSubtype_exon);
2293  db_name = "Annot:Exon";
2294  break;
2295  default:
2296  _ASSERT(0 && "unknown annot type");
2297  break;
2298  }
2299  _ASSERT(!db_name.empty());
2300  if ( name.IsNamed() ) {
2301  setter.GetTSE_LoadLock()->SetName(name);
2302  }
2303 
2304  TGi gi = ConvertGiToOM(GI_FROM(Uint4, Uint4(blob_id.GetSatKey())));
2306  CSeq_id seq_id;
2307  seq_id.SetGeneral().SetDb(db_name);
2308  seq_id.SetGeneral().SetTag().SetId8(GI_TO(Int8, gi));
2310 
2312  chunk->x_AddAnnotType(name, type, gih);
2313  ITERATE ( vector<SAnnotTypeSelector>, it, more_types ) {
2314  chunk->x_AddAnnotType(name, *it, gih);
2315  }
2316  chunk->x_AddBioseqPlace(0);
2317  chunk->x_AddBioseqId(seh);
2318  setter.GetSplitInfo().AddChunk(*chunk);
2320 
2321  setter.SetLoaded();
2322 
2323  CWriter* writer = GetWriter(result);
2324  if ( writer ) {
2325  //m_Dispatcher->LoadBlobVersion(result, blob_id);
2327  (OpenStream(writer, result, blob_id, chunk_id, this));
2328  if ( stream ) {
2329  try {
2330  stream->Close();
2331  }
2332  catch ( CException& ) { // ignored
2333  }
2334  }
2335  }
2336 }
2337 
2338 
2340  : CProcessor(dispatcher)
2341 {
2342 }
2343 
2344 
2346 {
2347 }
2348 
2349 
2351 {
2352  return eType_AnnotInfo;
2353 }
2354 
2355 
2357 {
2358  static TMagic kMagic = s_GetMagic("NANT");
2359  return kMagic;
2360 }
2361 
2362 
2364  const CBlob_Info& info)
2365 {
2366  _ASSERT(info.IsSetAnnotInfo());
2367  const CBlob_id& blob_id = *info.GetBlob_id();
2368  CLoadLockBlob blob(result, blob_id);
2369  CLoadLockSetter setter(blob);
2370  if ( setter.IsLoaded() ) {
2371  NCBI_THROW_FMT(CLoaderException, eLoaderFailed,
2372  "CProcessor_AnnotInfo: "
2373  "double load of "<<blob_id);
2374  }
2375 
2377  const CBlob_Annot_Info::TAnnotInfo& annot_infos =
2378  info.GetAnnotInfo()->GetAnnotInfo();
2380  ITERATE ( CBlob_Annot_Info::TAnnotInfo, it, annot_infos ) {
2381  const CID2S_Seq_annot_Info& annot_info = **it;
2382  // create special external annotations blob
2383  CAnnotName name(annot_info.GetName());
2384  if ( name.IsNamed() && !ExtractZoomLevel(name.GetName(), 0, 0) ) {
2385  //setter.GetTSE_LoadLock()->SetName(name);
2386  names.insert(name.GetName());
2387  }
2388 
2389  vector<SAnnotTypeSelector> types;
2390  if ( annot_info.IsSetAlign() ) {
2392  }
2393  if ( annot_info.IsSetGraph() ) {
2395  }
2396  if ( annot_info.IsSetFeat() ) {
2397  ITERATE ( CID2S_Seq_annot_Info::TFeat, it, annot_info.GetFeat() ) {
2398  const CID2S_Feat_type_Info& finfo = **it;
2399  int feat_type = finfo.GetType();
2400  if ( feat_type == 0 ) {
2401  types.push_back(SAnnotTypeSelector
2403  }
2404  else if ( !finfo.IsSetSubtypes() ) {
2405  types.push_back(SAnnotTypeSelector
2406  (CSeqFeatData::E_Choice(feat_type)));
2407  }
2408  else {
2410  it2, finfo.GetSubtypes() ) {
2411  types.push_back(SAnnotTypeSelector
2412  (CSeqFeatData::ESubtype(*it2)));
2413  }
2414  }
2415  }
2416  }
2417 
2419  CSplitParser::x_ParseLocation(loc, annot_info.GetSeq_loc());
2420 
2421  ITERATE ( vector<SAnnotTypeSelector>, it, types ) {
2422  chunk->x_AddAnnotType(name, *it, loc);
2423  }
2424  }
2425  if ( names.size() == 1 ) {
2426  setter.GetTSE_LoadLock()->SetName(*names.begin());
2427  }
2428  setter.GetSplitInfo().AddChunk(*chunk);
2430 
2431  setter.SetLoaded();
2432 }
2433 
2434 
2435 namespace {
2436  class CCommandParseBlob : public CReadDispatcherCommand
2437  {
2438  public:
2439  CCommandParseBlob(CReaderRequestResult& result,
2441  const char* descr,
2442  const CBlob_id& blob_id,
2443  int chunk_id = -1)
2445  m_StatType(stat_type), m_Descr(descr),
2446  m_Blob_id(blob_id), m_ChunkId(chunk_id)
2447  {
2448  }
2449  bool IsDone(void) {
2450  return true;
2451  }
2452  bool Execute(CReader& reader) {
2453  return true;
2454  }
2455  string GetErrMsg(void) const {
2456  return string();
2457  }
2458  CGBRequestStatistics::EStatType GetStatistics(void) const
2459  {
2460  return m_StatType;
2461  }
2462  string GetStatisticsDescription(void) const
2463  {
2465  str << m_Descr << ' ' << m_Blob_id;
2466  if ( m_ChunkId >= 0 && m_ChunkId < kMax_Int )
2467  str << '.' << m_ChunkId;
2468  return CNcbiOstrstreamToString(str);
2469  }
2470  private:
2472  const string m_Descr;
2473  const CBlob_id& m_Blob_id;
2474  int m_ChunkId;
2475  };
2476 }
2477 
2478 
2480  const CBlob_id& blob_id,
2482  const char* descr,
2483  double size)
2484 {
2485  CCommandParseBlob cmd(recursion.GetResult(),
2486  stat_type, descr, blob_id);
2487  CReadDispatcher::LogStat(cmd, recursion, size);
2488 }
2489 
2490 
2492  const CBlob_id& blob_id,
2493  int chunk_id,
2495  const char* descr,
2496  double size)
2497 {
2498  CCommandParseBlob cmd(recursion.GetResult(),
2499  stat_type, descr, blob_id, chunk_id);
2500  CReadDispatcher::LogStat(cmd, recursion, size);
2501 }
2502 
2503 
User-defined methods of the data storage class.
void SetNamed(const string &name)
Definition: annot_name.hpp:73
const string & GetName(void) const
Definition: annot_name.hpp:62
bool IsNamed(void) const
Definition: annot_name.hpp:58
Class holding information about root of non-modifiable object hierarchy Do not use it directly.
Definition: iterator.hpp:58
vector< CConstRef< CID2S_Seq_annot_Info > > TAnnotInfo
TSat GetSat() const
Definition: blob_id.hpp:56
TSatKey GetSatKey() const
Definition: blob_id.hpp:64
TSubSat GetSubSat() const
Definition: blob_id.hpp:60
Definition: Dbtag.hpp:53
CID1blob_info –.
@ID1server_back.hpp User-defined methods of the data storage class.
CID1server_request –.
CID2S_Chunk –.
Definition: ID2S_Chunk.hpp:66
CID2S_Feat_type_Info –.
CID2S_Seq_annot_Info –.
CID2S_Seq_loc –.
CID2S_Split_Info –.
bool IsLoadedChunk(void) const
void SetSeq_entry(CSeq_entry &entry, CTSE_SetObjectInfo *set_info=0)
void SetLoaded(void)
CTSE_Split_Info & GetSplitInfo(void)
CTSE_Chunk_Info & GetTSE_Chunk_Info(void)
bool IsLoaded(void) const
CTSE_LoadLock & GetTSE_LoadLock(void)
TBlobState GetBlobState(void) const
Data loader exceptions, used by GenBank loader.
CNcbiOstrstreamToString class helps convert CNcbiOstrstream to a string Sample usage:
Definition: ncbistre.hpp:802
list< TOctetString * > TOctetStringSequence
virtual ERW_Result Flush(void)
Flush pending data (if any) down to the output device.
COSSWriter(TOctetStringSequence &out)
TOctetStringSequence & m_Output
vector< char > TOctetString
virtual ERW_Result Write(const void *buffer, size_t count, size_t *written)
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
CObjectIStreamAsnBinary –.
Definition: objistrasnb.hpp:59
CObjectIStream –.
Definition: objistr.hpp:93
CObjectInfo –.
Definition: objectinfo.hpp:597
CObjectOStreamAsnBinary –.
Definition: objostrasnb.hpp:58
CObjectOStream –.
Definition: objostr.hpp:83
CObjectStreamCopier –.
Definition: objcopy.hpp:71
CObjectTypeInfo –.
Definition: objectinfo.hpp:94
static bool TryStringPack(void)
Definition: pack_string.cpp:94
CProcessor_AnnotInfo(CReadDispatcher &dispatcher)
TMagic GetMagic(void) const
EType GetType(void) const
static void LoadBlob(CReaderRequestResult &result, const CBlob_Info &blob_info)
TMagic GetMagic(void) const
CProcessor_ExtAnnot(CReadDispatcher &dispatcher)
void Process(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id) const
EType GetType(void) const
void ProcessStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CNcbiIstream &stream) const
static bool IsExtAnnot(const TBlobId &blob_id)
EType GetType(void) const
Definition: processors.cpp:883
TMagic GetMagic(void) const
Definition: processors.cpp:889
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
Definition: processors.cpp:896
CProcessor_ID1_SNP(CReadDispatcher &dispatcher)
Definition: processors.cpp:872
TSeqEntryInfo GetSeq_entry(CReaderRequestResult &result, const TBlobId &blob_id, CID1server_back &reply) const
Definition: processors.cpp:726
CProcessor_ID1(CReadDispatcher &dispatcher)
Definition: processors.cpp:642
TBlobVersion GetVersion(const CID1server_back &reply) const
Definition: processors.cpp:810
~CProcessor_ID1(void)
Definition: processors.cpp:648
TMagic GetMagic(void) const
Definition: processors.cpp:659
void SaveBlob(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CWriter *writer, CRef< CByteSource > byte_source) const
Definition: processors.cpp:823
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
Definition: processors.cpp:666
pair< CRef< CSeq_entry >, int > TSeqEntryInfo
Definition: processors.hpp:67
EType GetType(void) const
Definition: processors.cpp:653
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
CProcessor_ID2AndSkel(CReadDispatcher &dispatcher)
void SaveDataAndSkel(CReaderRequestResult &result, const TBlobId &blob_id, TBlobState blob_state, TChunkId chunk_id, CWriter *writer, TSplitVersion split_version, const CID2_Reply_Data &split_data, const CID2_Reply_Data &skel_data) const
TMagic GetMagic(void) const
EType GetType(void) const
CProcessor_ID2_Split(CReadDispatcher &dispatcher)
void SaveSplitData(CReaderRequestResult &result, const TBlobId &blob_id, TBlobState blob_state, TChunkId chunk_id, CWriter *writer, TSplitVersion split_version, const CID2_Reply_Data &split_data) const
TMagic GetMagic(void) const
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
EType GetType(void) const
static CObjectIStream * x_OpenDataStream(const CID2_Reply_Data &data)
static void x_FixDataFormat(CID2_Reply_Data &data)
EType GetType(void) const
void SaveData(CReaderRequestResult &result, const TBlobId &blob_id, TBlobState blob_state, TChunkId chunk_id, CWriter *writer, const CID2_Reply_Data &data) const
TMagic GetMagic(void) const
static void x_ReadData(const CID2_Reply_Data &data, const CObjectInfo &object, size_t &data_size)
CProcessor_ID2(CReadDispatcher &dispatcher)
static void DumpDataAsText(const CID2_Reply_Data &data, CNcbiOstream &out)
void ProcessData(CReaderRequestResult &result, const TBlobId &blob_id, TBlobState blob_state, TChunkId chunk_id, const CID2_Reply_Data &data, TSplitVersion split_version=0, const CID2_Reply_Data *skel=0) const
static void x_FixCompression(CID2_Reply_Data &data)
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
TMagic GetMagic(void) const
CProcessor_SE_SNP(CReadDispatcher &dispatcher)
EType GetType(void) const
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
~CProcessor_SE(void)
Definition: processors.cpp:977
EType GetType(void) const
Definition: processors.cpp:982
CWriter * x_GetWriterToSaveBlob(CReaderRequestResult &result, const CBlob_id &blob_id, CLoadLockSetter &setter, const char *processor_name) const
CProcessor_SE(CReadDispatcher &dispatcher)
Definition: processors.cpp:971
TMagic GetMagic(void) const
Definition: processors.cpp:988
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
Definition: processors.cpp:995
EType GetType(void) const
void ProcessStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CNcbiIstream &stream) const
TMagic GetMagic(void) const
void SaveSNPBlob(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CWriter *writer, const CSeq_entry &seq_entry, TBlobState blob_state, const CTSE_SetObjectInfo &set_info) const
CProcessor_St_SE_SNPT(CReadDispatcher &dispatcher)
CProcessor_St_SE(CReadDispatcher &dispatcher)
void WriteBlobState(CNcbiOstream &stream, TBlobState blob_state) const
TBlobState ReadBlobState(CNcbiIstream &stream) const
TMagic GetMagic(void) const
list< TOctetString * > TOctetStringSequence
Definition: processors.hpp:179
void SaveBlob(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, TBlobState blob_state, CWriter *writer, CRef< CByteSource > byte_source) const
void SaveNoBlob(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, TBlobState blob_state, CWriter *writer) const
void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
EType GetType(void) const
CWriter * GetWriter(const CReaderRequestResult &result) const
Definition: processors.cpp:407
static bool TrySNPTable(void)
Definition: processors.cpp:335
unsigned TMagic
Definition: processor.hpp:84
virtual void ProcessObjStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CObjectIStream &obj_stream) const
Definition: processors.cpp:252
static bool TryStringPack(void)
Definition: processors.cpp:314
static void OffsetAllGisToOM(CBeginInfo obj, CTSE_SetObjectInfo *set_info=0)
Definition: processors.cpp:590
static int CollectStatistics(void)
Definition: processors.cpp:219
@ eType_AnnotInfo
Definition: processor.hpp:82
@ eType_St_Seq_entry
Definition: processor.hpp:73
@ eType_Seq_entry
Definition: processor.hpp:71
@ eType_ID2AndSkel
Definition: processor.hpp:76
@ eType_ExtAnnot
Definition: processor.hpp:81
@ eType_St_Seq_entry_SNPT
Definition: processor.hpp:74
@ eType_Seq_entry_SNP
Definition: processor.hpp:72
@ eType_ID2_Split
Definition: processor.hpp:78
static TGi ConvertGiToOM(TGi gi)
Definition: processor.hpp:144
static void RegisterAllProcessors(CReadDispatcher &dispatcher)
Definition: processors.cpp:299
virtual void ProcessStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, CNcbiIstream &stream) const
Definition: processors.cpp:237
static bool TrySNPSplit(void)
Definition: processors.cpp:328
static void OffsetAllGis(CBeginInfo obj, TIntId gi_offset)
Definition: processors.cpp:499
CProcessor(CReadDispatcher &dispatcher)
Definition: processors.cpp:226
virtual ~CProcessor(void)
Definition: processors.cpp:232
int TChunkId
Definition: processor.hpp:67
void ProcessBlobFromID2Data(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, const CID2_Reply_Data &data) const
Definition: processors.cpp:262
static void SetSeqEntryReadHooks(CObjectIStream &in)
Definition: processors.cpp:356
static TIntId GetGiOffset(void)
Definition: processors.cpp:417
static void OffsetGi(TGi &gi, TIntId gi_offset)
Definition: processor.hpp:120
static bool OffsetId(CSeq_id &id, TIntId gi_offset)
Definition: processors.cpp:429
static void LogStat(CReaderRequestResultRecursion &recursion, const CBlob_id &blob_id, CGBRequestStatistics::EStatType stat_type, const char *descr, double size)
static void SetSNPReadHooks(CObjectIStream &in)
Definition: processors.cpp:380
int TBlobState
Definition: processor.hpp:65
int TBlobVersion
Definition: processor.hpp:66
CReadDispatcher * m_Dispatcher
Definition: processor.hpp:168
static void OffsetAllGisFromOM(CBeginInfo obj)
Definition: processors.cpp:584
Note about the "buf_size" parameter for streams in this API.
Definition: rwstream.hpp:122
static void LogStat(CReadDispatcherCommand &command, CReaderRequestResultRecursion &recursion)
const CProcessor & GetProcessor(CProcessor::EType type) const
Definition: dispatcher.cpp:219
void InsertProcessor(CRef< CProcessor > processor)
Definition: dispatcher.cpp:194
CWriter * GetWriter(const CReaderRequestResult &result, CWriter::EType type) const
Definition: dispatcher.cpp:204
CReaderRequestResult & GetResult(void) const
CSafeStatic<>::
@ eSubtype_misc_difference
static void Write(CNcbiOstream &stream, const CSeq_annot_SNP_Info &snp_info)
Definition: reader_snp.cpp:653
static void Read(CNcbiIstream &stream, CSeq_annot_SNP_Info &snp_info)
Definition: reader_snp.cpp:664
static void Parse(CObjectIStream &in, CSeq_entry &tse, CTSE_SetObjectInfo &set_info)
Definition: reader_snp.cpp:301
Definition: Seq_entry.hpp:56
static void Attach(CTSE_Info &tse, const CID2S_Split_Info &split)
static void x_ParseLocation(TLocationSet &vec, const CID2S_Seq_loc &loc)
static void Load(CTSE_Chunk_Info &chunk, const CID2S_Chunk &data)
Guard class for CObjectIStream::StartDelayBuffer/EndDelayBuffer.
Definition: objistr.hpp:1177
vector< TLocation > TLocationSet
void x_AddBioseqPlace(TBioseq_setId id)
void x_AddAnnotType(const CAnnotName &annot_name, const SAnnotTypeSelector &annot_type, const TLocationId &location_id)
void x_AddBioseqId(const TBioseqId &id)
void SetName(const CAnnotName &name)
Definition: tse_info.cpp:333
CTSE_Split_Info & GetSplitInfo(void)
Definition: tse_info.cpp:1395
bool x_NeedsDelayedMainChunk(void) const
Definition: tse_info.cpp:1407
TSeq_annot_InfoMap m_Seq_annot_InfoMap
Definition: tse_info.hpp:163
void AddChunk(CTSE_Chunk_Info &chunk_info)
void SetSplitVersion(TSplitVersion version)
CTypeInfo class contains all information about C++ types (both basic and classes): members and layout...
Definition: typeinfo.hpp:76
Template class for iteration on objects of class C.
Definition: iterator.hpp:673
static void AddWGSMaster(CTSE_LoadLock &lock)
Definition: wgsmaster.cpp:577
Writer-based output stream.
Definition: rwstream.hpp:171
@ eBlobWriter
Definition: writer.hpp:65
virtual CRef< CBlobStream > OpenBlobStream(CReaderRequestResult &result, const TBlobId &blob_id, TChunkId chunk_id, const CProcessor &processor)=0
static void WriteBytes(CNcbiOstream &stream, CRef< CByteSource > bs)
Definition: writer.cpp:53
CZipStreamCompressor – zlib based compression stream processor.
Definition: zlib.hpp:763
CZipStreamDecompressor – zlib based decompression stream processor.
Definition: zlib.hpp:817
A very basic data-read interface.
A very basic data-write interface.
bool empty() const
Definition: map.hpp:149
const unsigned char kMagic[2]
std::ofstream out("events_result.xml")
main entry point for tests
static CS_COMMAND * cmd
Definition: ct_dynamic.c:26
static const struct name_t names[]
#define true
Definition: bool.h:35
static int type
Definition: getdata.c:31
static const char * str(char *buf, int n)
Definition: stats.c:84
static const struct type types[]
Definition: type.c:22
char data[12]
Definition: iconv.c:80
#define GI_FROM(T, value)
Definition: ncbimisc.hpp:1086
#define ITERATE(Type, Var, Cont)
ITERATE macro to sequence through container elements.
Definition: ncbimisc.hpp:815
Int8 TIntId
Definition: ncbimisc.hpp:999
#define NON_CONST_ITERATE(Type, Var, Cont)
Non constant version of ITERATE macro.
Definition: ncbimisc.hpp:822
#define ZERO_GI
Definition: ncbimisc.hpp:1088
#define GI_TO(T, gi)
Definition: ncbimisc.hpp:1085
@ eTakeOwnership
An object can take ownership of another.
Definition: ncbi_types.h:136
string
Definition: cgiapp.hpp:687
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
Definition: ncbidiag.hpp:550
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
Definition: ncbiexpt.hpp:704
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
void Info(CExceptionArgs_Base &args)
Definition: ncbiexpt.hpp:1185
C * SerialClone(const C &src)
Create on heap a clone of the source object.
Definition: serialbase.hpp:512
ESerialDataFormat
Data file format.
Definition: serialdef.hpp:71
@ eSerialSkipUnknown_Yes
do skip
Definition: serialdef.hpp:122
@ eSerial_AsnText
ASN.1 text.
Definition: serialdef.hpp:73
@ eSerial_Xml
XML.
Definition: serialdef.hpp:75
@ eSerial_AsnBinary
ASN.1 binary.
Definition: serialdef.hpp:74
static CSeq_id_Handle GetGiHandle(TGi gi)
Faster way to create a handle for a gi.
static CSeq_id_Handle GetHandle(const CSeq_id &id)
Normal way of getting a handle, works for any seq-id.
CConstBeginInfo ConstBegin(const C &obj)
Get starting point of non-modifiable object hierarchy.
Definition: iterator.hpp:1012
CBeginInfo Begin(C &obj)
Get starting point of object hierarchy.
Definition: iterator.hpp:1004
CRef< CByteSource > EndDelayBuffer(void)
Redirect call to protected CObjectIStream After this call guarding is finished.
TFlags SetFlags(TFlags flags)
static CObjectOStream * Open(ESerialDataFormat format, CNcbiOstream &outStream, bool deleteOutStream)
Create serial object writer and attach it to an output stream.
Definition: objostr.cpp:126
virtual void WriteInt4(Int4 data)=0
CNcbiStreampos GetStreamPos(void) const
Get the current stream position.
Definition: objistr.cpp:790
static CObjectIStream * Open(ESerialDataFormat format, CNcbiIstream &inStream, bool deleteInStream)
Create serial object reader and attach it to an input stream.
Definition: objistr.cpp:195
void StartDelayBuffer(CObjectIStream &istr)
Start deley buffer collection on a given CObjectIStream object.
virtual Int4 ReadInt4(void)
Definition: objistr.cpp:1715
void Copy(const CObjectTypeInfo &type)
Copy data.
Definition: objcopy.cpp:74
bool ExtractZoomLevel(const string &full_name, string *acc_ptr, int *zoom_level_ptr)
Extract optional zoom level suffix from named annotation string.
CConstRef< C > ConstRef(const C *object)
Template function for conversion of const object pointer to CConstRef.
Definition: ncbiobj.hpp:2024
#define NCBI_PARAM_TYPE(section, name)
Generate typename for a parameter from its {section, name} attributes.
Definition: ncbi_param.hpp:149
@ eParam_NoThread
Do not use per-thread values.
Definition: ncbi_param.hpp:418
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
#define kMax_Int
Definition: ncbi_limits.h:184
int64_t Int8
8-byte (64-bit) signed integer
Definition: ncbitype.h:104
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define END_SCOPE(ns)
End the previously defined scope.
Definition: ncbistl.hpp:75
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
#define BEGIN_SCOPE(ns)
Define a new scope.
Definition: ncbistl.hpp:72
virtual CRef< CByteSourceReader > Open(void)=0
ERW_Result
Result codes for I/O operations.
Int8 NcbiStreamposToInt8(NCBI_NS_STD::char_traits< char >::pos_type stream_pos)
Convert stream position to 64-bit int.
Definition: ncbistre.hpp:771
virtual ERW_Result PendingCount(size_t *count)=0
Via parameter "count" (which is guaranteed to be supplied non-NULL) return the number of bytes that a...
IO_PREFIX::ostream CNcbiOstream
Portable alias for ostream.
Definition: ncbistre.hpp:149
virtual ERW_Result Read(void *buf, size_t count, size_t *bytes_read=0)=0
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
Definition: ncbistre.hpp:146
@ eRW_Eof
End of data, should be considered permanent.
@ eRW_Success
Everything is okay, I/O completed.
NCBI_NS_STD::string::size_type SIZE_TYPE
Definition: ncbistr.hpp:132
#define NPOS
Definition: ncbistr.hpp:133
static TNumeric StringToNumeric(const CTempString str, TStringToNumFlags flags=0, int base=10)
Convert string to a numeric value.
Definition: ncbistr.hpp:330
static bool StartsWith(const CTempString str, const CTempString start, ECase use_case=eCase)
Check if a string starts with a specified prefix value.
Definition: ncbistr.hpp:5412
static bool EqualNocase(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2)
Case-insensitive equality of a substring with another string.
Definition: ncbistr.hpp:5353
static enable_if< is_arithmetic< TNumeric >::value||is_convertible< TNumeric, Int8 >::value, string >::type NumericToString(TNumeric value, TNumToStringFlags flags=0, int base=10)
Convert numeric value to string.
Definition: ncbistr.hpp:673
@ fConvErr_NoThrow
Do not throw an exception on error.
Definition: ncbistr.hpp:285
@ eNocase
Case insensitive compare.
Definition: ncbistr.hpp:1206
void SetTag(TTag &value)
Assign a value to Tag data member.
Definition: Dbtag_.cpp:66
const TDb & GetDb(void) const
Get the Db member data.
Definition: Dbtag_.hpp:220
void SetDb(const TDb &value)
Assign a value to Db data member.
Definition: Dbtag_.hpp:229
const TGotblobinfo & GetGotblobinfo(void) const
Get the variant data.
TBlob_state GetBlob_state(void) const
Get the Blob_state member data.
TGetgihist & SetGetgihist(void)
Select the variant.
TGetgirev & SetGetgirev(void)
Select the variant.
TGotgi & SetGotgi(void)
Select the variant.
E_Choice Which(void) const
Which variant is currently selected.
TError GetError(void) const
Get the variant data.
bool IsGotgi(void) const
Check if variant Gotgi is selected.
const TGotsewithinfo & GetGotsewithinfo(void) const
Get the variant data.
TGetgistate & SetGetgistate(void)
Select the variant.
E_Choice Which(void) const
Which variant is currently selected.
void SetBlob(TBlob &value)
Assign a value to Blob data member.
TGotseqentry & SetGotseqentry(void)
Select the variant.
TGotdeadseqentry & SetGotdeadseqentry(void)
Select the variant.
TGetseqidsfromgi & SetGetseqidsfromgi(void)
Select the variant.
TGotsewithinfo & SetGotsewithinfo(void)
Select the variant.
const TBlob_info & GetBlob_info(void) const
Get the Blob_info member data.
bool IsSetBlob(void) const
Check if a value has been assigned to Blob data member.
@ e_Getgihist
get an historical list of gis
@ e_Getgirev
get a revision history of gi
@ e_Getseqidsfromgi
get all Seq-ids of given gi
@ e_Getgistate
get a state of gi
TData & SetData(void)
Assign a value to Data data member.
list< vector< char > * > TData
list< CRef< C_E > > Tdata
const TSubtypes & GetSubtypes(void) const
Get the Subtypes member data.
bool IsSetSkeleton(void) const
Check if a value has been assigned to Skeleton data member.
const TSeq_loc & GetSeq_loc(void) const
Get the Seq_loc member data.
bool IsSetGraph(void) const
Check if a value has been assigned to Graph data member.
list< CRef< CID2S_Feat_type_Info > > TFeat
bool IsSetAlign(void) const
Check if a value has been assigned to Align data member.
bool IsSetSubtypes(void) const
Check if a value has been assigned to Subtypes data member.
void SetSkeleton(TSkeleton &value)
Assign a value to Skeleton data member.
bool IsWhole_gi(void) const
Check if variant Whole_gi is selected.
const TName & GetName(void) const
Get the Name member data.
const TFeat & GetFeat(void) const
Get the Feat member data.
bool IsSetFeat(void) const
Check if a value has been assigned to Feat data member.
TWhole_gi & SetWhole_gi(void)
Select the variant.
TType GetType(void) const
Get the Type member data.
E_Choice
Choice variants.
TGeneral & SetGeneral(void)
Select the variant.
Definition: Seq_id_.cpp:375
@ e_General
for other databases
Definition: Seq_id_.hpp:105
Definition of all error codes used in objtools libraries.
int i
static MDB_envinfo info
Definition: mdb_load.c:37
static int version
Definition: mdb_load.c:29
string Execute(const string &cmmd, const vector< string > &args, const string &data=kEmptyStr)
const struct ncbi::grid::netcache::search::fields::SIZE size
string s_Value(TValue value)
#define abs(a)
Definition: ncbi_heapmgr.c:130
const char * tag
T min(T x_, T y_)
static Format format
Definition: njn_ioutil.cpp:53
std::istream & in(std::istream &in_, double &x_)
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
void split(std::vector< std::string > *strVec, const std::string &str_, const std::string &split_)
@ kMain_ChunkId
Definition: blob_id.hpp:147
@ kDelayedMain_ChunkId
Definition: blob_id.hpp:149
static pcre_uint8 * buffer
Definition: pcretest.c:1051
static bool s_UseMemoryPool(void)
Definition: processors.cpp:342
BEGIN_LOCAL_NAMESPACE
Definition: processors.cpp:601
NCBI_PARAM_DEF_EX(bool, GENBANK, SNP_PACK_STRINGS, true, eParam_NoThread, GENBANK_SNP_PACK_STRINGS)
END_LOCAL_NAMESPACE
Definition: processors.cpp:611
static bool s_CacheRecompress(void)
Definition: processors.cpp:349
static bool s_CanBeWGSBlob(const CBlob_id &blob_id)
Definition: processors.cpp:605
static bool GetSeqId(const T &d, set< string > &labels, const string name="", bool detect=false, bool found=false)
Reader-writer based streams.
Definition: type.c:6
#define _ASSERT
else result
Definition: token2.c:20
ZLib Compression API.
Modified on Tue May 21 10:54:33 2024 by modify_doxy.py rev. 669887