NCBI C++ ToolKit
object.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: object.cpp 89271 2020-03-13 12:16:51Z gouriano $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Rafael Sadyrov
27  *
28  */
29 
30 #include <ncbi_pch.hpp>
31 #include "object.hpp"
32 
33 #include <array>
34 #include <sstream>
35 #include <typeinfo>
36 
37 
39 
40 
42 {
43 
45  m_ObjectLoc(loc),
46  m_Context(context),
47  m_NotFound(m_ObjectLoc, fsm),
48  m_IsOpened(is_opened)
49 {
50  const bool primary_nc = location == eNFL_NetCache;
51  const bool primary_ft = location == eNFL_FileTrack;
52  const bool secondary_nc = flags & (fNST_NetCache | fNST_Fast);
53  const bool secondary_ft = flags & (fNST_FileTrack | fNST_Persistent);
54  const bool movable = flags & fNST_Movable;
55 
56  if (primary_ft || secondary_ft || movable) {
57  unique_ptr<CFileTrack> ft(new CFileTrack(m_ObjectLoc, fsm, m_Context, &m_CancelRelocate));
58  if (ft->Init()) m_Locations.emplace_back(ft.release());
59  }
60 
61  if (primary_nc || secondary_nc || movable) {
62  unique_ptr<CNetCache> nc(new CNetCache(m_ObjectLoc, fsm, m_Context, &m_CancelRelocate));
63  if (nc->Init()) m_Locations.emplace_back(nc.release());
64  }
65 
66  if (primary_nc || (!primary_ft && !secondary_ft && secondary_nc)) {
67  m_Locations.reverse();
68  }
69 
70  if (m_Locations.size()) return;
71 
72  // No locations
73  const bool ft = m_Context->filetrack_api;
74  const bool nc = m_Context->icache_client;
75 
76  ostringstream os;
77 
78  os << "No backends to use for locator=\"" << m_ObjectLoc.GetLocator() << "\"";
79 
80  if (secondary_ft && secondary_nc) {
81  os << ", requested FileTrack+NetCache";
82  } else if (secondary_ft) {
83  os << ", requested FileTrack";
84  } else if (secondary_nc) {
85  os << ", requested NetCache";
86  } else if (flags && !movable) {
87  os << ", zero requested backends";
88  }
89 
90  os << " and ";
91 
92  if (ft && nc) {
93  os << "configured FileTrack+NetCache";
94  } else if (ft) {
95  os << "configured FileTrack";
96  } else if (nc) {
97  os << "configured NetCache";
98  } else {
99  os << "no configured backends";
100  }
101 
102  NCBI_THROW(CNetStorageException, eInvalidArg, os.str());
103 }
104 
105 
106 template <class TCaller>
107 auto CObj::Meta(TCaller caller) -> decltype(caller(nullptr))
108 {
109  auto i = m_Locations.begin();
110  auto last = prev(m_Locations.end());
111 
112  for (;;) {
113  try {
114  return caller(i->get());
115  }
116  catch (CNetStorageException& e) {
118  }
119  catch (int) {
120  // Just a signal to try next location
121  }
122 
123  if (i++ == last) break;
124 
125  // Move failed location to the end
126  m_Locations.splice(m_Locations.end(), m_Locations, m_Locations.begin());
127  }
128 
129  return caller(&m_NotFound);
130 }
131 
132 
134 {
135  return Meta([](ILocation* l) { return l->GetSize(); });
136 }
137 
138 
139 list<string> CObj::GetAttributeList() const
140 {
141  NCBI_THROW(CNetStorageException, eNotSupported,
142  "Attribute support is only implemented in NetStorage server.");
143 }
144 
145 
146 string CObj::GetAttribute(const string&) const
147 {
148  NCBI_THROW(CNetStorageException, eNotSupported,
149  "Attribute support is only implemented in NetStorage server.");
150 }
151 
152 
153 void CObj::SetAttribute(const string&, const string&)
154 {
155  NCBI_THROW(CNetStorageException, eNotSupported,
156  "Attribute support is only implemented in NetStorage server.");
157 }
158 
159 
161 {
162  return Meta([](ILocation* l) { return l->GetInfo(); });
163 }
164 
165 
167 {
168  return Meta([&](ILocation* l) { return l->SetExpiration(ttl); });
169 }
170 
171 
173 {
174  return Meta([&](ILocation* l) { return l->FileTrack_Path(); });
175 }
176 
177 
178 pair<string, string> CObj::GetUserInfo()
179 {
180  return Meta([&](ILocation* l) { return l->GetUserInfo(); });
181 }
182 
183 
184 void s_Check(ERW_Result result, const char* what)
185 {
186  switch (result) {
187  case eRW_Success:
188  case eRW_Eof:
189  return;
190 
191  case eRW_Timeout:
193  "Timeout during " << what << " on relocate");
194 
195  case eRW_Error:
197  "IO error during " << what << " on relocate");
198 
199  case eRW_NotImplemented:
200  NCBI_THROW_FMT(CNetStorageException, eNotSupported,
201  "Non-implemented operation called during " << what << " on relocate");
202  }
203 }
204 
205 
207 {
208  const size_t max = m_Context->relocate_chunk;
209  size_t current = 0;
210  size_t total = 0;
211  size_t bytes_read;
213 
214  // Use Read() to detect the current location
216  auto rw_state = StartRead(buffer.data(), buffer.size(), &bytes_read, &result);
217  s_Check(result, "reading");
218 
219  _ASSERT(rw_state); // Cannot be a nullptr if result check passed
220 
221  // Selector can only be cloned after location is detected
222  CObj* new_obj;
223  CNetStorageObject new_file(Clone(flags, &new_obj));
224 
225  auto& src = *m_Locations.front();
226  auto& dst = *new_obj->m_Locations.front();
227 
228  if (typeid(src) == typeid(dst)) {
229  rw_state->Close();
230  return new_obj->m_ObjectLoc.GetLocator();
231  }
232 
233  for (;;) {
234  current += bytes_read;
235 
236  const char *data = buffer.data();
237  size_t bytes_written;
238 
239  do {
240  s_Check(new_file--->Write(data, bytes_read, &bytes_written), "writing");
241  data += bytes_written;
242  bytes_read -= bytes_written;
243  }
244  while (bytes_read);
245 
246  if (rw_state->Eof()) break;
247 
248  if (current >= max) {
249  total += current;
250  current = 0;
251 
252  if (cb) {
254  progress.SetInteger("BytesRelocated", total);
255  progress.SetString("Message", "Relocating");
256 
257  // m_CancelRelocate may only be set to true inside the callback
258  if (cb(progress), m_CancelRelocate) {
259  m_CancelRelocate = false;
260  new_file--->Abort();
261  rw_state->Abort();
262  NCBI_THROW(CNetStorageException, eInterrupted,
263  "Request to interrupt Relocate has been received.");
264  }
265  }
266  }
267 
268  s_Check(rw_state->Read(buffer.data(), buffer.size(), &bytes_read), "reading");
269  }
270 
271  new_file--->Close();
272  rw_state->Close();
273  Remove();
274  return new_file--->GetLoc();
275 }
276 
277 
279 {
280 }
281 
282 
284 {
285  return Meta([&](ILocation* l) { return l->Exists(); });
286 }
287 
288 
290 {
291  return Meta([&](ILocation* l) { return l->Remove(); });
292 }
293 
294 
296 {
297 }
298 
299 
301 {
302 }
303 
304 
305 string CObj::GetLoc() const
306 {
307  return m_ObjectLoc.GetLocator();
308 }
309 
310 
311 ERW_Result CObj::Read(void* buf, size_t count, size_t* bytes_read)
312 {
314  StartRead(buf, count, bytes_read, &result);
315  return result;
316 }
317 
318 
319 bool CObj::Eof()
320 {
321  return false;
322 }
323 
324 
325 ERW_Result CObj::Write(const void* buf, size_t count, size_t* bytes_written)
326 {
327  // If object already exists, it will be overwritten in the same backend
328  if (m_IsOpened) Exists();
329 
331  EnterState(m_Locations.front()->StartWrite(buf, count, bytes_written, &result));
332 
333  if (m_IsOpened) {
334  CObj* old_obj;
335  CNetStorageObject guard(Clone(fNST_Movable, &old_obj));
336  old_obj->RemoveOldCopyIfExists(m_Locations.front().get());
337  }
338 
339  m_IsOpened = true;
340  return result;
341 }
342 
343 
344 INetStorageObjectState* CObj::StartRead(void* buf, size_t count, size_t* bytes_read, ERW_Result* result)
345 {
346  INetStorageObjectState* rw_state = nullptr;
347 
348  auto f = [&](ILocation* l)
349  {
350  rw_state = l->StartRead(buf, count, bytes_read, result);
351 
352  if (!rw_state) throw 0;
353 
354  EnterState(rw_state);
355  return rw_state;
356  };
357 
358  return Meta(f);
359 }
360 
361 
363 {
364  auto f = [&current](ILocation* l)
365  {
366  // Do not remove object from current location
367  if (typeid(*l) == typeid(*current)) throw 0;
368 
369  return l->Remove();
370  };
371 
372  Meta(f);
373 }
374 
375 
377 {
378  return m_ObjectLoc;
379 }
380 
381 
383 {
384  m_Locations.front()->SetLocator();
385 }
386 
387 
389 {
390  _ASSERT(copy);
391 
392  if (!flags) flags = m_Context->default_flags;
393 
396 
397  auto l = [&](CObj& state) { *copy = &state; };
398  return SNetStorageObjectImpl::CreateAndStart<CObj>(l, m_Context, loc, flags);
399 }
400 
401 
402 }
403 
JSON node abstraction.
void SetString(const string &key, const string &value)
Set a JSON object element to the specified string value.
void SetInteger(const string &key, Int8 value)
Set a JSON object element to the specified integer value.
static CJsonNode NewObjectNode()
Create a new JSON object node.
Exception class for use by CNetStorage, CNetStorageByKey, and CNetStorageObject.
Definition: netstorage.hpp:67
Detailed information about a CNetStorage object.
Definition: netstorage.hpp:96
void SetStorageAttrFlags(TNetStorageAttrFlags flags)
string GetLocator() const
Basic network-based data object I/O.
Definition: netstorage.hpp:167
CTimeout – Timeout interval.
Definition: ncbitime.hpp:1693
virtual void Abort()=0
void Abort() override
Definition: object.cpp:300
SNetStorageObjectImpl * Clone(TNetStorageFlags flags, CObj **copy)
Definition: object.cpp:388
CNetStorageObjectLoc & Locator() override
Definition: object.cpp:376
ERW_Result Read(void *, size_t, size_t *) override
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
Definition: object.cpp:311
void RemoveOldCopyIfExists(ILocation *current)
Definition: object.cpp:362
string GetAttribute(const string &) const override
Definition: object.cpp:146
bool Exists() override
Definition: object.cpp:283
ERW_Result Write(const void *, size_t, size_t *) override
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
Definition: object.cpp:325
void SetAttribute(const string &, const string &) override
Definition: object.cpp:153
void CancelRelocate() override
Definition: object.cpp:278
list< unique_ptr< ILocation > > m_Locations
Definition: object.hpp:89
CNetStorageObjectInfo GetInfo() override
Definition: object.cpp:160
string FileTrack_Path() override
Definition: object.cpp:172
list< string > GetAttributeList() const override
Definition: object.cpp:139
CRef< SContext > m_Context
Definition: object.hpp:88
void Close() override
Definition: object.cpp:295
void SetExpiration(const CTimeout &) override
Definition: object.cpp:166
CNetStorageObjectLoc m_ObjectLoc
Definition: object.hpp:87
CObj(SNetStorageObjectImpl &fsm, SContext *context, const CNetStorageObjectLoc &loc, TNetStorageFlags flags, bool is_opened=false, ENetStorageObjectLocation location=eNFL_Unknown)
Definition: object.cpp:44
string Relocate(TNetStorageFlags, TNetStorageProgressCb cb) override
Definition: object.cpp:206
auto Meta(TCaller caller) -> decltype(caller(nullptr))
Definition: object.cpp:107
ENetStorageRemoveResult Remove() override
Definition: object.cpp:289
bool Eof() override
Definition: object.cpp:319
pair< string, string > GetUserInfo() override
Definition: object.cpp:178
Uint8 GetSize() override
Definition: object.cpp:133
string GetLoc() const override
Definition: object.cpp:305
INetStorageObjectState * StartRead(void *, size_t, size_t *, ERW_Result *)
Definition: object.cpp:344
virtual void SetExpiration(const CTimeout &)=0
virtual pair< string, string > GetUserInfo()=0
virtual CNetStorageObjectInfo GetInfo()=0
virtual string FileTrack_Path()=0
virtual ENetStorageRemoveResult Remove()=0
static uch flags
static DLIST_TYPE *DLIST_NAME() last(DLIST_LIST_TYPE *list)
Definition: dlist.tmpl.h:51
static DLIST_TYPE *DLIST_NAME() prev(DLIST_LIST_TYPE *list, DLIST_TYPE *item)
Definition: dlist.tmpl.h:61
static const char location[]
Definition: config.c:97
char data[12]
Definition: iconv.c:80
TErrCode GetErrCode(void) const
Get error code.
Definition: ncbiexpt.cpp:453
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
Definition: ncbiexpt.hpp:704
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
unsigned TNetStorageFlags
Bitwise OR of ENetStorageFlags.
Definition: netstorage.hpp:147
ENetStorageObjectLocation
Enumeration that indicates the current location of the object.
Definition: netstorage.hpp:86
function< void(CJsonNode)> TNetStorageProgressCb
Progress callback.
Definition: netstorage.hpp:348
ENetStorageRemoveResult
Result returned by Remove() methods.
Definition: netstorage.hpp:356
@ fNST_Persistent
Definition: netstorage.hpp:135
@ fNST_Fast
Definition: netstorage.hpp:134
@ fNST_NetCache
Use NetCache as the primary storage.
Definition: netstorage.hpp:130
@ fNST_FileTrack
Use FileTrack as the primary storage.
Definition: netstorage.hpp:131
@ fNST_Movable
Allow the object to move between storages.
Definition: netstorage.hpp:138
@ eNotExists
Illegal op applied to non-existent object.
Definition: netstorage.hpp:71
@ eNFL_FileTrack
Definition: netstorage.hpp:90
@ eNFL_NetCache
Definition: netstorage.hpp:89
uint64_t Uint8
8-byte (64-bit) unsigned integer
Definition: ncbitype.h:105
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
ERW_Result
Result codes for I/O operations.
@ eRW_NotImplemented
Action / information is not available.
@ eRW_Eof
End of data, should be considered permanent.
@ eRW_Error
Unrecoverable error, no retry possible.
@ eRW_Timeout
Timeout expired, try again later.
@ eRW_Success
Everything is okay, I/O completed.
char * buf
int i
void s_Check(ERW_Result result, const char *what)
Definition: object.cpp:184
T max(T x_, T y_)
void copy(Njn::Matrix< S > *matrix_, const Njn::Matrix< T > &matrix0_)
Definition: njn_matrix.hpp:613
double f(double x_, const double &y_)
Definition: njn_root.hpp:188
static pcre_uint8 * buffer
Definition: pcretest.c:1051
void EnterState(INetStorageObjectState *state)
#define _ASSERT
else result
Definition: token2.c:20
static CS_CONTEXT * context
Definition: will_convert.c:21
Modified on Wed Jul 24 17:21:37 2024 by modify_doxy.py rev. 669887