NCBI C++ ToolKit
netstorage.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: netstorage.cpp 78888 2017-07-26 21:38:40Z sadyrovr $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Dmitry Kazimirov
27  *
28  * File Description:
29  * NetStorage implementation declarations.
30  *
31  */
32 
33 #include <ncbi_pch.hpp>
34 #include <corelib/rwstream.hpp>
37 
39 
40 
42 {
44 
45  ERW_Result Read(void* b, size_t c, size_t* r) override { return m_Impl->Read(b, c, r); }
46  ERW_Result PendingCount(size_t* c) override { return m_Impl->PendingCount(c); }
47  ERW_Result Write(const void* b, size_t c, size_t* w) override { return m_Impl->Write(b, c, w); }
48  ERW_Result Flush() override { return m_Impl->Flush(); }
49  void Close() override { m_Impl->Close(); }
50  void Abort() override { m_Impl->Abort(); }
51 
52 private:
54 };
55 
56 // Ignore empty writes to iostream (CXX-8936)
58 {
60 
61  ERW_Result Write(const void* buf, size_t count, size_t* written) override
62  {
63  if (count) return SEmbeddedStreamReaderWriter::Write(buf, count, written);
64 
65  if (written) *written = 0;
66  return eRW_Success;
67  }
68 };
69 
71 {
73  CNcbiIostream(0),
74  m_Object(impl),
75  m_Sb(rw, rw, 1, nullptr, CRWStreambuf::fLeakExceptions)
76  {
77  _ASSERT(impl);
78  _ASSERT(rw);
79  init(&m_Sb);
80  }
81 
83 
84 private:
87 };
88 
90 {
91  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Read() while writing " << GetLoc());
92 }
93 
95 {
96  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling PendingCount() while writing " << GetLoc());
97 }
98 
100 {
101  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Eof() while writing " << GetLoc());
102 }
103 
104 ERW_Result SNetStorageObjectIState::Write(const void*, size_t, size_t*)
105 {
106  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Write() while reading " << GetLoc());
107 }
108 
110 {
111  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Flush() while reading " << GetLoc());
112 }
113 
115 {
116  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling GetSize() while reading/writing " << GetLoc());
117 }
118 
120 {
121  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling GetAttributeList() while reading/writing " << GetLoc());
122 }
123 
124 string SNetStorageObjectIoState::GetAttribute(const string&) const
125 {
126  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling GetAttribute() while reading/writing " << GetLoc());
127 }
128 
129 void SNetStorageObjectIoState::SetAttribute(const string&, const string& )
130 {
131  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling SetAttribute() while reading/writing " << GetLoc());
132 }
133 
135 {
136  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling GetInfo() while reading/writing " << GetLoc());
137 }
138 
140 {
141  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling SetExpiration() while reading/writing " << GetLoc());
142 }
143 
145 {
146  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling FileTrack_Path() while reading/writing " << GetLoc());
147 }
148 
150 {
151  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Relocate() while reading/writing " << GetLoc());
152 }
153 
155 {
156  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Exists() while reading/writing " << GetLoc());
157 }
158 
160 {
161  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, "Calling Remove() while reading/writing " << GetLoc());
162 }
163 
165 {
166  if (api == eBuffer) {
167  if (mth == eRead) return "Read(buffer)";
168  if (mth == eWrite) return "Write(buffer)";
169  if (mth == eEof) return "Eof()";
170  }
171 
172  if (api == eIoStream) {
173  return "GetRWStream()";
174  }
175 
176  if (api == eIReaderIWriter) {
177  if (mth == eRead) return "GetReader()";
178  if (mth == eWrite) return "GetWriter()";
179  }
180 
181  if (api == eString) {
182  if (mth == eRead) return "Read(string)";
183  if (mth == eWrite) return "Write(string)";
184  }
185 
186  _ASSERT(false);
187  return ""; // Not reached
188 }
189 
190 void SNetStorageObjectIoMode::Throw(EApi api, EMth mth, string object_loc)
191 {
192  NCBI_THROW_FMT(CNetStorageException, eNotSupported,
193  "Calling " << ToString(api, mth) << " after " <<
194  ToString(m_Api, m_Mth) << " for " << object_loc);
195 }
196 
198 {
200  return *m_ReaderWriter;
201 }
202 
204 {
205  try {
206  if (m_Current) m_Current->Close();
207  }
208  NCBI_CATCH_ALL("Error while implicitly closing a NetStorage object.");
209 }
210 
212 {
214  return new SNetStorageObjectRWStream(this, m_IoStreamReaderWriter.get());
215 }
216 
218 {
220  m_IoMode.Reset();
221  return m_Current->Close();
222 }
223 
225 {
226  return m_Impl--->GetLoc();
227 }
228 
229 size_t CNetStorageObject::Read(void* buffer, size_t buf_size)
230 {
231  size_t bytes_read;
233  m_Impl--->Read(buffer, buf_size, &bytes_read);
234  return bytes_read;
235 }
236 
238 {
239  char buffer[64 * 1024];
240 
241  data->resize(0);
242  size_t bytes_read;
243 
245 
246  do {
247  m_Impl--->Read(buffer, sizeof(buffer), &bytes_read);
248  data->append(buffer, bytes_read);
249  } while (!m_Impl--->Eof());
250 
251  Close();
252 }
253 
255 {
257  return m_Impl->GetReaderWriter();
258 }
259 
261 {
263  return m_Impl--->Eof();
264 }
265 
266 void CNetStorageObject::Write(const void* buffer, size_t buf_size)
267 {
269  m_Impl--->Write(buffer, buf_size, NULL);
270 }
271 
272 void CNetStorageObject::Write(const string& data)
273 {
275  m_Impl--->Write(data.data(), data.length(), NULL);
276 }
277 
279 {
281  return m_Impl->GetReaderWriter();
282 }
283 
285 {
287  return m_Impl->GetRWStream();
288 }
289 
291 {
292  return m_Impl--->GetSize();
293 }
294 
296 {
297  return m_Impl--->GetAttributeList();
298 }
299 
300 string CNetStorageObject::GetAttribute(const string& attr_name) const
301 {
302  return m_Impl--->GetAttribute(attr_name);
303 }
304 
305 void CNetStorageObject::SetAttribute(const string& attr_name,
306  const string& attr_value)
307 {
308  m_Impl--->SetAttribute(attr_name, attr_value);
309 }
310 
312 {
313  return m_Impl--->GetInfo();
314 }
315 
317 {
318  m_Impl--->SetExpiration(ttl);
319 }
320 
322 {
323  m_Impl->Close();
324 }
325 
326 CNetStorage::CNetStorage(const string& init_string,
327  TNetStorageFlags default_flags) :
328  m_Impl(SNetStorage::CreateImpl(
329  SNetStorage::SConfig::Build(init_string), default_flags))
330 {
331 }
332 
334 {
335  return m_Impl->Create(flags);
336 }
337 
338 CNetStorageObject CNetStorage::Open(const string& object_loc)
339 {
340  return m_Impl->Open(object_loc);
341 }
342 
344 {
345  auto object = Open(object_loc);
346  return object--->Relocate(flags, cb);
347 }
348 
349 bool CNetStorage::Exists(const string& object_loc)
350 {
351  auto object = Open(object_loc);
352  return object--->Exists();
353 }
354 
356 {
357  auto object = Open(object_loc);
358  return object--->Remove();
359 }
360 
361 CNetStorageByKey::CNetStorageByKey(const string& init_string,
362  TNetStorageFlags default_flags) :
363  m_Impl(SNetStorage::CreateByKeyImpl(
364  SNetStorage::SConfig::Build(init_string), default_flags))
365 {
366 }
367 
368 CNetStorageObject CNetStorageByKey::Open(const string& unique_key,
370 {
371  SNetStorage::SLimits::Check<SNetStorage::SLimits::SUserKey>(unique_key);
372  return m_Impl->Open(unique_key, flags);
373 }
374 
375 string CNetStorageByKey::Relocate(const string& unique_key,
378 {
379  auto object = Open(unique_key, old_flags);
380  return object--->Relocate(flags, cb);
381 }
382 
384 {
385  auto object = Open(key, flags);
386  return object--->Exists();
387 }
388 
391 {
392  auto object = Open(key, flags);
393  return object--->Remove();
394 }
395 
396 inline
398 {
399  switch (code) {
403 
406 
409 
412 
417 
420  }
421 
423 }
424 
426  const CNetCacheException& prev_exception, const string& message)
427 {
429  ConvertErrCode(prev_exception.GetErrCode());
430  throw CNetStorageException(compile_info, &prev_exception, err_code, message);
431 }
432 
434  CNetStorageObject object)
435  : m_Path(object--->FileTrack_Path())
436 {
437 }
438 
439 void SNetStorage::SLimits::ThrowTooLong(const string& name, size_t max_length)
440 {
442  name << " exceeds maximum allowed length of " <<
443  max_length << " characters.");
444 }
445 
447  const string& value)
448 {
450  name << " contains illegal characters: " <<
452 }
453 
Incapsulate compile time information such as __FILE__, __LINE__, NCBI_MODULE, current function.
Definition: ncbidiag.hpp:65
NetCache internal exception.
Exception class for use by CNetStorage, CNetStorageByKey, and CNetStorageObject.
Definition: netstorage.hpp:67
Detailed information about a CNetStorage object.
Definition: netstorage.hpp:96
CNetStorageObject_FileTrack_Path(CNetStorageObject object)
Locks file path if object resides in FileTrack.
Definition: netstorage.cpp:433
Basic network-based data object I/O.
Definition: netstorage.hpp:167
Reader-writer-based stream buffer.
Definition: rwstreambuf.hpp:60
CTimeout – Timeout interval.
Definition: ncbitime.hpp:1693
virtual void Close()=0
virtual void Abort()=0
A very basic data-read interface.
CNetStorageException::EErrCode ConvertErrCode(CNetCacheException::TErrCode code)
Definition: netstorage.cpp:397
void g_ThrowNetStorageException(const CDiagCompileInfo &compile_info, const CNetCacheException &prev_exception, const string &message)
Definition: netstorage.cpp:425
static uch flags
static void DLIST_NAME() init(DLIST_LIST_TYPE *list)
Definition: dlist.tmpl.h:40
char data[12]
Definition: iconv.c:80
#define NULL
Definition: ncbistd.hpp:225
TErrCode GetErrCode(void) const
Get error code.
Definition: ncbiexpt.cpp:453
int TErrCode
Definition: ncbiexpt.hpp:889
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
#define NCBI_CATCH_ALL(message)
This macro is deprecated - use *_X or *_XX variant instead of it.
Definition: ncbiexpt.hpp:587
@ eAuthenticationError
If client is not allowed to run this operation.
@ eBlobClipped
The requested command is (yet) unknown.
@ eNotImplemented
Invalid response from the server.
@ eKeyFormatError
Server side error.
@ eServerError
Blob is not found.
@ eUnknownCommand
The requested command is not implemented.
@ eAccessDenied
Blob could not be read completely.
@ eBlobNotFound
Access denied.
string GetAttribute(const string &attr_name) const
Get the current value of the specified attribute.
Definition: netstorage.cpp:300
bool Eof(void)
Check if the last Read() has hit EOF.
Definition: netstorage.cpp:260
IReader & GetReader()
Get a reference to the IReader interface for reading the data stream of this object.
Definition: netstorage.cpp:254
bool Exists(const string &object_loc)
Check if the object addressed by 'object_loc' exists.
Definition: netstorage.cpp:349
string GetLoc(void) const
Return object locator.
Definition: netstorage.cpp:224
bool Exists(const string &key, TNetStorageFlags flags=0)
Check if a object with the specified key exists in the storage hinted by 'flags'.
Definition: netstorage.cpp:383
CNetRef< SNetStorageImpl > m_Impl
Definition: netstorage.hpp:369
CNetStorageByKey(EVoid)
Definition: netstorage.hpp:472
unsigned TNetStorageFlags
Bitwise OR of ENetStorageFlags.
Definition: netstorage.hpp:147
list< string > TAttributeList
Get list of names of all attributes set on this object.
Definition: netstorage.hpp:300
ENetStorageRemoveResult Remove(const string &key, TNetStorageFlags flags=0)
Remove a object addressed by a key and a set of flags.
Definition: netstorage.cpp:389
TAttributeList GetAttributeList() const
Definition: netstorage.cpp:295
CNetStorageObject Create(TNetStorageFlags flags=0)
Create new NetStorage object.
Definition: netstorage.cpp:333
CNcbiIostream * GetRWStream()
Get an iostream instance for reading the data stream of this NetStorage object as well as storing dat...
Definition: netstorage.cpp:284
void Write(const void *buffer, size_t buf_size)
Write data to the object (starting at the current position)
Definition: netstorage.cpp:266
size_t Read(void *buffer, size_t buf_size)
Read no more than 'buf_size' bytes of the object contents (starting at the current position)
Definition: netstorage.cpp:229
void SetExpiration(const CTimeout &ttl)
Updates expiration on the object.
Definition: netstorage.cpp:316
void Close(void)
Finalize and close the current object stream.
Definition: netstorage.cpp:321
Uint8 GetSize(void)
Return size of the object.
Definition: netstorage.cpp:290
void SetAttribute(const string &attr_name, const string &attr_value)
Set the new value for the specified attribute.
Definition: netstorage.cpp:305
CNetStorageObjectInfo GetInfo(void)
Return detailed information about the object.
Definition: netstorage.cpp:311
CNetStorageObject Open(const string &object_loc)
Open an existing NetStorage object for reading.
Definition: netstorage.cpp:338
CNetStorageObject Open(const string &unique_key, TNetStorageFlags flags=0)
Create a new object or open an existing object using the supplied unique key.
Definition: netstorage.cpp:368
string Relocate(const string &object_loc, TNetStorageFlags flags, TNetStorageProgressCb cb=TNetStorageProgressCb())
Relocate a object according to the specified combination of flags.
Definition: netstorage.cpp:343
IEmbeddedStreamWriter & GetWriter()
Get a reference to the IWriter interface for storing data into this object.
Definition: netstorage.cpp:278
ENetStorageRemoveResult Remove(const string &object_loc)
Remove the object addressed by 'object_loc'.
Definition: netstorage.cpp:355
string Relocate(const string &unique_key, TNetStorageFlags flags, TNetStorageFlags old_flags=0, TNetStorageProgressCb cb=TNetStorageProgressCb())
Relocate a object according to the specified combination of flags.
Definition: netstorage.cpp:375
CNetRef< SNetStorageObjectImpl > m_Impl
Definition: netstorage.hpp:168
function< void(CJsonNode)> TNetStorageProgressCb
Progress callback.
Definition: netstorage.hpp:348
CNetStorage(EVoid)
Definition: netstorage.hpp:369
CNetRef< SNetStorageByKeyImpl > m_Impl
Definition: netstorage.hpp:472
ENetStorageRemoveResult
Result returned by Remove() methods.
Definition: netstorage.hpp:356
@ eNotSupported
Feature is not supported.
Definition: netstorage.hpp:77
@ eServerError
NetStorage server error.
Definition: netstorage.hpp:74
@ eUnknown
Unknown error.
Definition: netstorage.hpp:79
@ eIOError
I/O error encountered while performing an op.
Definition: netstorage.hpp:73
@ eInvalidArg
Caller passed invalid arguments to the API.
Definition: netstorage.hpp:70
@ eNotExists
Illegal op applied to non-existent object.
Definition: netstorage.hpp:71
@ eAuthError
Authentication error (e.g. no FileTrack API key)
Definition: netstorage.hpp:72
uint64_t Uint8
8-byte (64-bit) unsigned integer
Definition: ncbitype.h:105
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
ERW_Result
Result codes for I/O operations.
virtual ERW_Result Flush(void)=0
Flush pending data (if any) down to the output device.
virtual ERW_Result PendingCount(size_t *count)=0
Via parameter "count" (which is guaranteed to be supplied non-NULL) return the number of bytes that a...
IO_PREFIX::iostream CNcbiIostream
Portable alias for iostream.
Definition: ncbistre.hpp:152
virtual ERW_Result Write(const void *buf, size_t count, size_t *bytes_written=0)=0
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
virtual ERW_Result Read(void *buf, size_t count, size_t *bytes_read=0)=0
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
@ eRW_Success
Everything is okay, I/O completed.
static string PrintableString(const CTempString str, TPrintableMode mode=fNewLine_Quote|fNonAscii_Passthru)
Get a printable version of the specified string.
Definition: ncbistr.cpp:3953
char * buf
const struct ncbi::grid::netcache::search::fields::KEY key
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
#define nullptr
Definition: ncbimisc.hpp:45
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
static pcre_uint8 * buffer
Definition: pcretest.c:1051
Reader-writer based streams.
virtual string GetLoc() const =0
ERW_Result Read(void *b, size_t c, size_t *r) override
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
Definition: netstorage.cpp:45
ERW_Result PendingCount(size_t *c) override
Via parameter "count" (which is guaranteed to be supplied non-NULL) return the number of bytes that a...
Definition: netstorage.cpp:46
ERW_Result Write(const void *b, size_t c, size_t *w) override
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
Definition: netstorage.cpp:47
SEmbeddedStreamReaderWriter(SNetStorageObjectImpl &impl)
Definition: netstorage.cpp:43
ERW_Result Flush() override
Flush pending data (if any) down to the output device.
Definition: netstorage.cpp:48
SNetStorageObjectImpl & m_Impl
Definition: netstorage.cpp:53
ERW_Result Write(const void *buf, size_t count, size_t *written) override
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
Definition: netstorage.cpp:61
SIoStreamEmbeddedStreamReaderWriter(SNetStorageObjectImpl &impl)
Definition: netstorage.cpp:59
ERW_Result Write(const void *buf, size_t count, size_t *written) final
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
Definition: netstorage.cpp:104
ERW_Result Flush() final
Flush pending data (if any) down to the output device.
Definition: netstorage.cpp:109
INetStorageObjectState * m_Current
SNetStorageObjectIoMode m_IoMode
unique_ptr< IEmbeddedStreamReaderWriter > m_IoStreamReaderWriter
unique_ptr< IEmbeddedStreamReaderWriter > m_ReaderWriter
CNcbiIostream * GetRWStream()
Definition: netstorage.cpp:211
IEmbeddedStreamReaderWriter & GetReaderWriter()
Definition: netstorage.cpp:197
void Throw(EApi api, EMth mth, string object_loc)
Definition: netstorage.cpp:190
static string ToString(EApi api, EMth mth)
Definition: netstorage.cpp:164
void SetExpiration(const CTimeout &ttl) final
Definition: netstorage.cpp:139
string FileTrack_Path() final
Definition: netstorage.cpp:144
string GetAttribute(const string &name) const final
Definition: netstorage.cpp:124
list< string > GetAttributeList() const final
Definition: netstorage.cpp:119
ENetStorageRemoveResult Remove() final
Definition: netstorage.cpp:159
CNetStorageObjectInfo GetInfo() final
Definition: netstorage.cpp:134
string Relocate(TNetStorageFlags flags, TNetStorageProgressCb cb) final
Definition: netstorage.cpp:149
void SetAttribute(const string &name, const string &value) final
Definition: netstorage.cpp:129
ERW_Result PendingCount(size_t *count) final
Via parameter "count" (which is guaranteed to be supplied non-NULL) return the number of bytes that a...
Definition: netstorage.cpp:94
ERW_Result Read(void *buf, size_t count, size_t *read) final
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
Definition: netstorage.cpp:89
~SNetStorageObjectRWStream() override
Definition: netstorage.cpp:82
SNetStorageObjectRWStream(SNetStorageObjectImpl *impl, IEmbeddedStreamReaderWriter *rw)
Definition: netstorage.cpp:72
CNetStorageObject m_Object
Definition: netstorage.cpp:85
static void ThrowTooLong(const string &, size_t)
Definition: netstorage.cpp:439
static void ThrowIllegalChars(const string &, const string &)
Definition: netstorage.cpp:446
Definition: inftrees.h:24
#define _ASSERT
Modified on Sat Apr 13 11:50:18 2024 by modify_doxy.py rev. 669887