NCBI C++ ToolKit
netstorageobjectloc.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: netstorageobjectloc.cpp 94953 2021-09-23 18:35:13Z sadyrovr $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Dmitry Kazimirov
27  *
28  * File Description:
29  * Implementation of the unified network blob storage API.
30  *
31  */
32 
33 #include <ncbi_pch.hpp>
34 
37 
38 #include <connect/ncbi_socket.h>
39 
40 #include <corelib/ncbi_base64.h>
41 #include <corelib/ncbiargs.hpp>
42 
43 #include <time.h>
44 #include <string.h>
45 
46 
47 #define NCBI_USE_ERRCODE_X NetStorage_Common
48 
49 #define FILETRACK_STORAGE_CODE "FT"
50 #define NETCACHE_STORAGE_CODE "NC"
51 
52 #define STORAGE_INFO_CUE 0
53 
55 
58  const string& app_domain,
59  Uint8 random_number,
60  EFileTrackSite ft_site) :
61  m_CompoundIDPool(cid_pool),
62  m_LocatorFlags(x_StorageFlagsToLocatorFlags(flags, ft_site)),
63  m_AppDomain(app_domain),
64  m_Timestamp(time(NULL)),
65  m_Random(random_number),
66  m_ShortUniqueKey(MakeShortUniqueKey()),
67  m_UniqueKey(MakeUniqueKey()),
68  m_Dirty(true)
69 {
70 
71 }
72 
75  const string& app_domain,
76  const string& unique_key,
77  EFileTrackSite ft_site) :
78  m_CompoundIDPool(cid_pool),
79  m_LocatorFlags(x_StorageFlagsToLocatorFlags(flags, ft_site) | fLF_HasUserKey),
80  m_AppDomain(app_domain),
81  m_ShortUniqueKey(unique_key),
82  m_UniqueKey(MakeUniqueKey()),
83  m_Dirty(true)
84 {
85 }
86 
87 #define INVALID_LOC_ERROR_MSG "Invalid NetStorage object locator"
88 
89 #define THROW_INVALID_LOC_ERROR(cid, msg) \
90  NCBI_THROW_FMT(CNetStorageException, eInvalidArg, \
91  msg " '" << cid.ToString() << '\'')
92 
93 #define VERIFY_FIELD_EXISTS(field) \
94  if (!(field)) { \
95  THROW_INVALID_LOC_ERROR(cid, INVALID_LOC_ERROR_MSG); \
96  }
97 
99 {
100  if (location.length() == 2) {
101  if (location.data()[0] == FILETRACK_STORAGE_CODE[0] &&
102  location.data()[1] == FILETRACK_STORAGE_CODE[1])
103  return eNFL_FileTrack;
104  if (location.data()[0] == NETCACHE_STORAGE_CODE[0] &&
105  location.data()[1] == NETCACHE_STORAGE_CODE[1])
106  return eNFL_NetCache;
107  }
108  return eNFL_Unknown;
109 }
110 
112  const string& object_loc) :
113  m_CompoundIDPool(cid_pool),
114  m_Dirty(false),
115  m_Locator(object_loc)
116 {
117  auto cid = m_CompoundIDPool.FromString(object_loc);
118  Parse(cid, false);
119 }
120 
121 void CNetStorageObjectLoc::SetServiceName(const string& service_name)
122 {
123  if (service_name.empty() ||
124  strchr(service_name.c_str(), ':') != NULL)
126  else {
127  m_ServiceName = service_name;
129  }
130  m_Dirty = true;
131 }
132 
134 {
136  loc.Parse(cid, true);
137  return loc.m_ServiceName;
138 }
139 
140 void CNetStorageObjectLoc::Parse(CCompoundID cid, bool service_name_only)
141 {
142  // Check the ID class.
143  switch (cid.GetClass()) {
146  "Unsupported NetStorage object locator version");
148  break;
149  default:
151  }
152 
153  // Get locator flags.
154  CCompoundIDField field = cid.GetFirst(eCIT_Flags);
155  VERIFY_FIELD_EXISTS(field);
157 
158  // Restore NetStorage service name.
160  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
161  m_ServiceName = field.GetServiceName();
162  }
163 
164  if (service_name_only) return;
165 
166  // Restore object ID.
168  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
169  m_ObjectID = field.GetID();
170  }
171 
172  // Get the domain name.
173  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
174  m_AppDomain = field.GetString();
175 
176  m_SubKey.clear();
177  m_Version = 0;
178 
179  // Restore object identification.
181  // Get the unique object key.
182  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
183  m_ShortUniqueKey = field.GetString();
184 
186  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
187  m_SubKey = field.GetString();
188 
190  // null is stored as 0 (0 is not stored as it's the default)
191  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
192  auto version = static_cast<int>(field.GetInteger());
194  }
195  }
196  } else {
197  // Get object creation timestamp.
198  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
199  m_Timestamp = field.GetTimestamp();
200  // Get the random ID.
201  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
202  m_Random = (Uint8) field.GetRandom() << (sizeof(Uint4) * 8);
203  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
204  m_Random |= field.GetRandom();
206  }
207 
209 
210  // Not used, though has to be read to be backward-compatible
212  field = field.GetNextNeighbor();
213  }
214 
215  // Find storage info (optional).
216  for (field = cid.GetFirst(eCIT_Cue); field; field = field.GetNextHomogeneous()) {
217  if (field.GetCue() == STORAGE_INFO_CUE) {
218  // Restore object location.
219  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
222 
223  // Restore storage-specific info.
224  if (m_Location == eNFL_NetCache) {
225  // Not used, though has to be read to be backward-compatible
226  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
227 
228  // Get the service name.
229  VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
231  }
232 
233  break;
234  }
235  }
236 }
237 
238 void CNetStorageObjectLoc::SetLocation(const string& nc_service_name)
239 {
240  // If NetCache
241  if (!nc_service_name.empty()) {
242  if (m_Location == eNFL_NetCache) return;
243 
246  } else {
247  if (m_Location == eNFL_FileTrack) return;
248 
251  }
252 
253  m_Dirty = true;
254  m_NCServiceName = nc_service_name;
255 }
256 
258 {
262 }
263 
265 {
267  result += '-';
269 
271  result += '-';
273  }
274 
275  return result;
276 }
277 
279 {
280  // Allocate a new CompoundID object.
282 
283  // Save locator flags.
284  // If its location is not set yet, add "Movable", so the locator can be used
287 
288  // Save NetStorage service name.
291 
292  // Save object ID.
294  cid.AppendID(m_ObjectID);
295 
296  // Save the domain name.
298 
299  // Save object identification
301  // Save the unique object key.
303 
305  cid.AppendString(m_SubKey);
306 
308  // null is stored as 0 (0 is not stored as it's the default)
310  }
311  }
312  } else {
313  // Save object creation timestamp.
315  // Save the random ID.
316  cid.AppendRandom((Uint4)(m_Random >> (sizeof(Uint4) * 8)));
317  cid.AppendRandom((Uint4) m_Random);
318  }
319 
320  // Not used, though has to be written to be backward-compatible
322  cid.AppendInteger(0);
323 
324  // Save storage info (optional).
325  if (m_Location != eNFL_Unknown) {
327 
328  // Save object location.
330 
331  switch (m_Location) {
332  case eNFL_NetCache:
333  // Not used, though has to be written to be backward-compatible
334  cid.AppendFlags(0);
335  // Save the service name.
337  break;
338  default:
339  break;
340  }
341  }
342 
343  // Now pack it all up.
344  m_Locator = cid.ToString();
345 
346  m_Dirty = false;
347 }
348 
350 {
352 
354  flags |= fNST_Movable;
359 
360  return flags;
361 }
362 
364 {
365  const auto new_locator_flags = (m_LocatorFlags & eLF_FieldFlags) | x_StorageFlagsToLocatorFlags(flags);
366 
367  if (new_locator_flags != m_LocatorFlags) {
368  m_Dirty = true;
369  m_LocatorFlags = new_locator_flags;
370  }
371 }
372 
375  TNetStorageAttrFlags storage_flags,
376  EFileTrackSite ft_site)
377 {
378  TLocatorFlags locator_flags = 0;
379 
380  if (storage_flags & fNST_Movable)
381  locator_flags |= fLF_Movable;
382  if (storage_flags & fNST_Cacheable)
383  locator_flags |= fLF_Cacheable;
384  if (storage_flags & fNST_NoMetaData)
385  locator_flags |= fLF_NoMetaData;
386 
387  if (ft_site == eFileTrack_DevSite)
388  locator_flags |= fLF_DevEnv;
389  else if (ft_site == eFileTrack_QASite)
390  locator_flags |= fLF_QAEnv;
391 
392  return locator_flags;
393 }
394 
396 {
398  ToJSON(root);
399  return root;
400 }
401 
403 {
404  root.SetInteger("Version", 1 +
406 
407  root.SetString("Environment",
408  m_LocatorFlags & fLF_DevEnv ? "dev/test" :
409  m_LocatorFlags & fLF_QAEnv ? "QA" : "production");
410 
412  root.SetString("ServiceName", m_ServiceName);
413 
415  root.SetString("Cache", m_AppDomain);
416  root.SetString("ObjectKey", m_ShortUniqueKey);
417  root.SetString("ObjectSubKey", m_SubKey);
418 
419  if (m_Version.IsNull()) {
420  root.SetNull("ObjectVersion");
421  } else {
422  root.SetInteger("ObjectVersion", m_Version);
423  }
424  } else {
425  root.SetString("ObjectKey", m_UniqueKey);
426  }
427 
428  CJsonNode storage_flags(CJsonNode::NewObjectNode());
429 
430  storage_flags.SetBoolean("Movable",
431  (m_LocatorFlags & fLF_Movable) != 0);
432  storage_flags.SetBoolean("Cacheable",
433  (m_LocatorFlags & fLF_Cacheable) != 0);
434  storage_flags.SetBoolean("NoMetaData",
435  (m_LocatorFlags & fLF_NoMetaData) != 0);
436 
437  root.SetByKey("StorageFlags", storage_flags);
438 
439  if (!m_LocationCode.empty())
440  root.SetString("DefaultLocation", m_LocationCode);
441 
442  CJsonNode storage_info(CJsonNode::NewObjectNode());
443 
444  switch (m_Location) {
445  case eNFL_NetCache:
446  storage_info.SetString("ServiceName", m_NCServiceName);
447  root.SetByKey("NetCache", storage_info);
448  break;
449  default:
450  break;
451  }
452 }
453 
455 {
457  {
458  { "production", eFileTrack_ProdSite },
459  { "prod", eFileTrack_ProdSite },
460  { "submit", eFileTrack_ProdSite },
461  { "development", eFileTrack_DevSite },
462  { "dev", eFileTrack_DevSite },
463  { "dsubmit", eFileTrack_DevSite },
464  { "qa", eFileTrack_QASite },
465  { "qsubmit", eFileTrack_QASite },
466  };
467 
468  auto i = p.find(ft_site_name);
469 
470  if (i == p.end()) {
471  NCBI_THROW_FMT(CArgException, eInvalidArg, "unrecognized FileTrack site '" << ft_site_name << '\'');
472  }
473 
474  return i->second;
475 }
476 
477 string CNetStorageObjectLoc::Create(const string& service_name, const string& cache_name,
478  const string& key, const string& subkey, const TVersion& version)
479 {
480  CCompoundIDPool id_pool;
482  CNetStorageObjectLoc loc(id_pool, fNST_NoMetaData, cache_name, key, ft_site);
483  loc.SetLocation(service_name);
484  loc.m_SubKey = subkey;
485  loc.m_Version = version;
487  return loc.GetLocator();
488 }
489 
CArgException –.
Definition: ncbiargs.hpp:120
Compound ID field – an element of the compound ID that has a type and a value.
string GetString() const
Return the string value that this field contains.
CCompoundIDField GetNextHomogeneous()
Return the next field of the same type.
CCompoundIDField GetNextNeighbor()
Return the next immediately adjacent field.
Uint4 GetRandom() const
Return the random number value that this field contains.
Uint8 GetCue() const
Return the application-specific numeric tag value that this field contains.
string GetServiceName() const
Return the LBSM service name that this field contains.
Int8 GetTimestamp() const
Return the UNIX timestamp that this field contains.
Int8 GetInteger() const
Return the integer value that this field contains.
string GetDatabaseName() const
Return the database name that this field contains.
Uint8 GetID() const
Return the ID value that this field contains.
Uint8 GetFlags() const
Return the combination of binary flags stored in this field.
Pool of recycled CCompoundID objects.
CCompoundID NewID(ECompoundIDClass new_id_class)
Create and return a new CCompoundID objects.
CCompoundID FromString(const string &cid)
Unpack the base64-encoded ID and return a CCompoundID object for field extraction.
Base64-encoded ID string that contains extractable typed fields.
void AppendDatabaseName(const string &db_name)
Append an eCIT_DatabaseName field at the end of this compound ID.
void AppendRandom(Uint4 random_number)
Append an eCIT_Random field at the end of this compound ID.
void AppendCue(Uint8 tag)
Append an eCIT_Cue field at the end of this compound ID.
void AppendString(const string &string_value)
Append an eCIT_String field at the end of this compound ID.
CCompoundIDField GetFirst(ECompoundIDFieldType field_type)
Return the first field of the specified type or NULL if this compound ID contains no fields of such t...
string ToString()
Pack the ID and return its string representation.
void AppendTimestamp(Int8 timestamp)
Append an eCIT_Timestamp field at the end of this compound ID.
void AppendID(Uint8 id)
Append an eCIT_ID field at the end of this compound ID.
void AppendFlags(Uint8 flags)
Append an eCIT_Flags field at the end of this compound ID.
ECompoundIDClass GetClass() const
One of the registered ID classes.
void AppendServiceName(const string &service_name)
Append an eCIT_ServiceName field at the end of this compound ID.
void AppendInteger(Int8 number)
Append an eCIT_Integer field at the end of this compound ID.
JSON node abstraction.
void SetString(const string &key, const string &value)
Set a JSON object element to the specified string value.
void SetNull(const string &key)
Set a JSON object element to the specified null value.
void SetBoolean(const string &key, bool value)
Set a JSON object element to the specified boolean value.
void SetInteger(const string &key, Int8 value)
Set a JSON object element to the specified integer value.
void SetByKey(const string &key, CJsonNode::TInstance value)
For a JSON object node, insert a new element or update an existing element.
static CJsonNode NewObjectNode()
Create a new JSON object node.
void SetStorageAttrFlags(TNetStorageAttrFlags flags)
void Parse(CCompoundID cid, bool service_name_only)
static string Create(const string &service_name, const string &cache_name, const string &key, const string &subkey, const TVersion &version=null)
static TLocatorFlags x_StorageFlagsToLocatorFlags(TNetStorageAttrFlags storage_flags, EFileTrackSite ft_site=eFileTrack_ProdSite)
TNetStorageAttrFlags GetStorageAttrFlags() const
void SetLocatorFlags(TLocatorFlags flags)
string GetLocator() const
CCompoundIDPool m_CompoundIDPool
ENetStorageObjectLocation m_Location
TLocatorFlags m_LocatorFlags
string MakeShortUniqueKey() const
void SetLocation(const string &nc_service_name)
CNullable< int > TVersion
string MakeUniqueKey() const
void ClearLocatorFlags(TLocatorFlags flags)
string GetServiceName() const
static EFileTrackSite ParseFileTrackSite(const string &ft_site_name)
void SetServiceName(const string &service_name)
EFileTrackSite GetFileTrackSite() const
const_iterator end() const
Definition: map.hpp:152
const_iterator find(const key_type &key) const
Definition: map.hpp:153
Definition: map.hpp:338
@ eCIT_Cue
Definition: compound_id.hpp:78
@ eCIT_Flags
Definition: compound_id.hpp:76
@ eCIC_NetStorageObjectLocV1
Old version of NetStorageObjectLoc.
Definition: compound_id.hpp:54
@ eCIC_NetStorageObjectLoc
Definition: compound_id.hpp:55
static uch flags
#define true
Definition: bool.h:35
#define false
Definition: bool.h:36
static const char location[]
Definition: config.c:97
bool IsNull(void) const
Check if the object is unassigned.
Definition: ncbimisc.hpp:686
const TValue & GetValue(void) const
Get a const reference to the current value.
Definition: ncbimisc.hpp:703
#define NULL
Definition: ncbistd.hpp:225
static const string & GetHostRole(void)
Get host role (DEV/QA/TRY/PROD) from /etc/ncbi/role.
Definition: ncbidiag.cpp:3091
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
Definition: ncbiexpt.hpp:719
ENetStorageObjectLocation
Enumeration that indicates the current location of the object.
Definition: netstorage.hpp:86
@ fNST_NoMetaData
Do not use NetStorage relational database to track ownership & changes.
Definition: netstorage.hpp:140
@ fNST_Cacheable
Has no effect at the moment.
Definition: netstorage.hpp:139
@ fNST_Movable
Allow the object to move between storages.
Definition: netstorage.hpp:138
@ eNFL_FileTrack
Definition: netstorage.hpp:90
@ eNFL_NetCache
Definition: netstorage.hpp:89
@ eNFL_Unknown
Definition: netstorage.hpp:87
uint32_t Uint4
4-byte (32-bit) unsigned integer
Definition: ncbitype.h:103
uint64_t Uint8
8-byte (64-bit) unsigned integer
Definition: ncbitype.h:105
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
static enable_if< is_arithmetic< TNumeric >::value||is_convertible< TNumeric, Int8 >::value, string >::type NumericToString(TNumeric value, TNumToStringFlags flags=0, int base=10)
Convert numeric value to string.
Definition: ncbistr.hpp:673
Definition of all error codes used in connect services library (xconnserv.lib and others).
int i
static int version
Definition: mdb_load.c:29
const struct ncbi::grid::netcache::search::fields::KEY key
const struct ncbi::grid::netcache::search::fields::SUBKEY subkey
Defines command line argument related classes.
#define FILETRACK_STORAGE_CODE
#define VERIFY_FIELD_EXISTS(field)
#define NETCACHE_STORAGE_CODE
#define STORAGE_INFO_CUE
ENetStorageObjectLocation s_LocationCodeToLocation(const string &location)
#define THROW_INVALID_LOC_ERROR(cid, msg)
#define INVALID_LOC_ERROR_MSG
else result
Definition: token2.c:20
Modified on Mon Apr 22 04:07:32 2024 by modify_doxy.py rev. 669887