NCBI C++ ToolKit
blob_storage.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef BLOB_STORAGE__HPP
2 #define BLOB_STORAGE__HPP
3 
4 /* $Id: blob_storage.hpp 101703 2024-01-29 15:34:46Z saprykin $
5  * ===========================================================================
6  *
7  * PUBLIC DOMAIN NOTICE
8  * National Center for Biotechnology Information
9  *
10  * This software/database is a "United States Government Work" under the
11  * terms of the United States Copyright Act. It was written as part of
12  * the author's official duties as a United States Government employee and
13  * thus cannot be copyrighted. This software/database is freely available
14  * to the public for use. The National Library of Medicine and the U.S.
15  * Government have not placed any restriction on its use or reproduction.
16  *
17  * Although all reasonable efforts have been taken to ensure the accuracy
18  * and reliability of the software and data, the NLM and the U.S.
19  * Government do not and cannot warrant the performance or results that
20  * may be obtained by using this software or data. The NLM and the U.S.
21  * Government disclaim all warranties, express or implied, including
22  * warranties of performance, merchantability or fitness for any particular
23  * purpose.
24  *
25  * Please cite the author in any work or product based on this material.
26  *
27  * ===========================================================================
28  *
29  * Authors: Sergey Satskiy, Dmitrii Saprykin
30  *
31  * File Description:
32  *
33  * The functionality not directly related to blob operations
34  *
35  */
36 
37 #include <corelib/ncbistd.hpp>
38 #include <corelib/ncbireg.hpp>
39 
40 #include <chrono>
41 #include <optional>
42 
45 #include "IdCassScope.hpp"
46 
48 
50 
51 // Note: must be in sync with the Cassandra SAT_INFO.SAT2KEYSPACE schema type values
60 };
61 
62 enum class ECassSatInfoFlags : Int4
63 {
64  // Requires WebCookie verification befor blob access
65  eSecureSat = 1,
66 };
67 
69 {
71  static const char* const kChunkTableDefault;
72  static const char* const kChunkTableBig;
73 };
74 
75 class CSatInfoSchema;
76 struct SSatInfoEntry final
77 {
79  public:
80  static constexpr int32_t kInvalidSat{-100};
81 
82  SSatInfoEntry() = default;
84  SSatInfoEntry(SSatInfoEntry const&) = default;
87 
88  // Satellite id
90 
91  // Satellite schema type
93 
94  // Keyspace name to read data
95  string keyspace;
96 
97  // Service name to connect to satellite
98  string service;
99 
100  // Connection to access secure satellite
101  shared_ptr<CCassConnection> connection;
102 
103  // Bitmask of satellite properties
104  // @see ECassSatInfoFlags
106 
107  /// Is satellite requires secure access. If True connection should be acquired through
108  /// GetSecureConnection() method.
109  ///
110  /// @return
111  /// True if satellite requires secure access
112  /// False - otherwise (data is public)
113  bool IsSecureSat() const
114  {
115  return flags & static_cast<Int4>(ECassSatInfoFlags::eSecureSat);
116  }
117 
118  /// Get string representation for debug
119  ///
120  /// @return
121  /// String representation for debug
122  string ToString(string const& prefix) const;
123 
124  /// Get secure satellite connection
125  ///
126  /// @param username
127  /// User name
128  ///
129  /// @throws When IsSecureSat() == false
130  ///
131  /// @return
132  /// Connection if username is in allowed users list
133  /// nullptr - otherwise
134  shared_ptr<CCassConnection> GetSecureConnection(string const& username) const;
135 
136  /// Get public satellite connection
137  ///
138  /// @throws When IsSecureSat() == true
139  ///
140  /// @return
141  /// Connection for public satellites
142  shared_ptr<CCassConnection> GetConnection() const;
143 
144  private:
145  // Connection to access secure satellite
146  shared_ptr<CCassConnection> m_SecureConnection;
147 
148  // List of usernames for eSecureSat satellite.
150 };
151 
153 {
154  eUndefined = 0,
155  eSatInfoUpdated = 1,
156  eSatInfoUnchanged = 2,
164 };
165 
167 {
168  eUndefined = 0,
169  eMessagesUpdated = 1,
170  eMessagesUnchanged = 2,
173 };
174 
176 
177 class CSatInfoSchema final
178 {
179  public:
181 
182  CSatInfoSchema() = default;
183 
184  /// Get blob keyspace connection by sat id
185  ///
186  /// @param sat
187  /// Blob sat id
188  ///
189  /// @return
190  /// Configuration entry or empty optional
191  /// if sat does not exists or has non blob schema type
192  optional<SSatInfoEntry> GetBlobKeyspace(int32_t sat) const;
193 
194  /// Get list of BioseqNA keyspaces connections
195  vector<SSatInfoEntry> GetNAKeyspaces() const;
196 
197  /// Get connection to resolver keyspace
199 
200  /// Get connection to IPG keyspace
201  optional<SSatInfoEntry> GetIPGKeyspace() const;
202 
203  /// Get max id value for existing blob sat
205 
206  /// Print internal state of CSatInfoSchema
207  ///
208  /// @for_tests, @for_debug
209  string ToString() const;
210 
211  private:
212  shared_ptr<CCassConnection> x_GetConnectionByService(string const& service, string const& registry_section) const;
213  shared_ptr<CCassConnection> x_GetConnectionByConnectionPoint(string const& connection_point) const;
214 
215  optional<ESatInfoRefreshSchemaResult> x_AddConnection(
216  shared_ptr<CCassConnection> const& connection,
217  string const& registry_section,
218  string const& service,
219  bool is_default
220  );
221  optional<ESatInfoRefreshSchemaResult> x_AddSatInfoEntry(
222  SSatInfoEntry entry,
223  shared_ptr<CSatInfoSchema> const& old_schema,
224  shared_ptr<IRegistry const> const& registry,
225  string const& registry_section,
226  set<string> const& secure_users
227  );
228  optional<ESatInfoRefreshSchemaResult> x_ResolveConnectionByServiceName(
229  string service,
230  shared_ptr<CSatInfoSchema> const& old_schema,
231  shared_ptr<IRegistry const> const& registry,
232  string const& registry_section,
233  shared_ptr<CCassConnection>& connection
234  );
235  optional<ESatInfoRefreshSchemaResult> x_ResolveServiceName(
236  string const& service, string const& registry_section, vector<string>& connection_points);
237 
238  // Mapping {sat => Connection to blob keyspace}
240 
241  // Mapping {sat => set<string>}. List of usernames for eSecureSat satellites.
243 
244  // List of BioseqNA connections
245  vector<SSatInfoEntry> m_BioseqNaKeyspaces;
246 
247  // Connection to resolver keyspace
249 
250  // Connection to IPG keyspace
251  optional<SSatInfoEntry> m_IPGKeyspace;
252 
253  // Host:port => ClusterConnection
255 
256  // ServiceName from service column => ClusterConnection
258 
259  // Default cluster connection
260  shared_ptr<CCassConnection> m_DefaultConnection;
261 
262  // Default cluster registry section name
264 
265  optional<chrono::milliseconds> m_ResolveTimeout{nullopt};
266 };
267 
269 {
270  public:
271 
272  /// @param sat_info_keyspace
273  /// Name of Cassandra keyspace to use as a source of configuration
274  /// @param domain
275  /// Name of configuration domain for that provider
276  /// @param sat_info_connection
277  /// Connection to Cassandra cluster hosting {sat_info_keyspace}
278  /// @param registry
279  /// Configuration registry used to create new cluster connections
280  /// @param registry_section
281  /// Registry section name used as a template creating new cluster connection
282  /// {service} entry of registry section gets replaced by {service} from {sat_info3.sat2keyspace} table
284  string const& sat_info_keyspace,
285  string const& domain,
286  shared_ptr<CCassConnection> sat_info_connection,
287  shared_ptr<IRegistry const> registry,
288  string const& registry_section
289  );
290 
291  /// Overrides data retrieval timeout
292  ///
293  /// @param timeout
294  /// Timeout in milliseconds
295  void SetTimeout(chrono::milliseconds timeout)
296  {
297  m_Timeout = timeout;
298  }
299 
300  /// Changes configuration domain for existing provider
301  ///
302  /// @param registry_section
303  /// Registry section name used as a template creating new cluster connection for secure satellite access
304  /// and secure satellite user list storage
305  /// {service} entry of registry section gets replaced by {service} from {sat_info3.sat2keyspace} table
306  ///
307  /// @not_thread_safe
308  void SetSecureSatRegistrySection(string const& registry_section)
309  {
310  m_SecureSatRegistrySection = registry_section;
311  }
312 
313  /// Changes configuration domain for existing provider
314  ///
315  /// @param domain
316  /// Name of configuration domain for that provider
317  ///
318  /// @for_tests, @not_thread_safe
319  void SetDomain(string const& domain)
320  {
321  m_Domain = domain;
322  }
323 
324  /// Should existence of {resolver} keyspace be checked?
325  ///
326  /// @param value
327  /// true - Resolver keyspace is required to be present in configuration domain
328  /// false - Resolver keyspace is optional (e.g. for blob daemon or ASN.1 dumpers)
329  /// {default} - true
330  ///
331  /// @not_thread_safe
333  {
335  }
336 
337  /// Changes Cassandra connection used to communicate with sat_info3
338  ///
339  /// @used in genbank-blobdaemon project
340  ///
341  /// @thread_safe
342  void SetSatInfoConnection(shared_ptr<CCassConnection> sat_info_connection);
343 
344  /// Get blob keyspace connection by sat id
345  ///
346  /// @param sat
347  /// Blob sat id
348  ///
349  /// @return
350  /// Configuration entry or empty optional
351  /// if sat does not exists or has non blob schema type
352  ///
353  /// @thread_safe
354  optional<SSatInfoEntry> GetBlobKeyspace(int32_t sat) const;
355 
356  /// Get list of BioseqNA keyspaces connections
357  ///
358  /// @thread_safe
359  vector<SSatInfoEntry> GetNAKeyspaces() const;
360 
361  /// Get connection to resolver keyspace
362  ///
363  /// @thread_safe
365 
366  /// Get connection to IPG keyspace
367  ///
368  /// @thread_safe
369  optional<SSatInfoEntry> GetIPGKeyspace() const;
370 
371  /// Get max id value for existing blob sat
372  ///
373  /// @thread_safe
375 
376  /// Get configured message by name
377  ///
378  /// @param name
379  /// Message name
380  ///
381  /// @thread_safe
382  string GetMessage(string const& name) const;
383 
384  /// Refresh information for configuration database {sat_info3.sat2keyspace}
385  ///
386  /// @param appy
387  /// - true to apply changes
388  /// - false to check if configuration database content changed
389  ///
390  /// @return
391  /// Operation result
392  ///
393  /// @non_thread_safe - Should be called from a single control thread
395 
396  /// Refresh information for messages database {sat_info3.messages}
397  ///
398  /// @param appy
399  /// - true to apply changes
400  /// - false to check if configuration database content changed
401  ///
402  /// @return
403  /// Operation result
404  ///
405  /// @non_thread_safe - Should be called from a single control thread
407 
408  /// Get configuration schema snapshot
409  ///
410  /// @thread_safe
411  shared_ptr<CSatInfoSchema> GetSchema() const;
412 
413  /// Get messages snapshot
414  ///
415  /// @thread_safe
416  shared_ptr<CPSGMessages> GetMessages() const;
417 
418  /// Get detailed message for last refresh operation (common for RefreshSchema and RefreshMessages).
419  /// Empty string in case there were no errors
420  ///
421  /// @thread_safe
422  string GetRefreshErrorMessage() const;
423 
424  private:
425  shared_ptr<CCassConnection> x_GetSatInfoConnection() const;
426  void x_SetRefreshErrorMessage(string const& message);
427  optional<ESatInfoRefreshSchemaResult> x_PopulateNewSchema(
428  shared_ptr<CSatInfoSchema>& new_schema,
429  shared_ptr<CSatInfoSchema> const& old_schema,
430  vector<SSatInfoEntry>&& sat_info,
431  map<int32_t, set<string>>&& secure_users
432  );
433 
435  string m_Domain;
436  shared_ptr<CCassConnection> m_SatInfoConnection;
437 
438  shared_ptr<IRegistry const> m_Registry;
441 
442  shared_ptr<CSatInfoSchema> m_SatInfoSchema;
443  shared_ptr<CPSGMessages> m_SatInfoMessages;
444  size_t m_SatInfoHash{0};
445 
446  shared_ptr<string> m_RefreshErrorMessage;
448 
449  optional<chrono::milliseconds> m_Timeout{nullopt};
450 };
451 
453 
454 #endif
#define END_IDBLOB_SCOPE
Definition: IdCassScope.hpp:40
#define BEGIN_IDBLOB_SCOPE
Definition: IdCassScope.hpp:39
shared_ptr< CPSGMessages > m_SatInfoMessages
ESatInfoRefreshMessagesResult RefreshMessages(bool apply)
Refresh information for messages database {sat_info3.messages}.
optional< SSatInfoEntry > GetBlobKeyspace(int32_t sat) const
Get blob keyspace connection by sat id.
shared_ptr< CSatInfoSchema > GetSchema() const
Get configuration schema snapshot.
ESatInfoRefreshSchemaResult RefreshSchema(bool apply)
Refresh information for configuration database {sat_info3.sat2keyspace}.
CSatInfoSchemaProvider(string const &sat_info_keyspace, string const &domain, shared_ptr< CCassConnection > sat_info_connection, shared_ptr< IRegistry const > registry, string const &registry_section)
vector< SSatInfoEntry > GetNAKeyspaces() const
Get list of BioseqNA keyspaces connections.
optional< chrono::milliseconds > m_Timeout
shared_ptr< CCassConnection > x_GetSatInfoConnection() const
void SetResolverKeyspaceRequired(bool value)
Should existence of {resolver} keyspace be checked?
shared_ptr< CPSGMessages > GetMessages() const
Get messages snapshot.
void SetSatInfoConnection(shared_ptr< CCassConnection > sat_info_connection)
Changes Cassandra connection used to communicate with sat_info3.
string GetMessage(string const &name) const
Get configured message by name.
SSatInfoEntry GetResolverKeyspace() const
Get connection to resolver keyspace.
shared_ptr< IRegistry const > m_Registry
void SetSecureSatRegistrySection(string const &registry_section)
Changes configuration domain for existing provider.
shared_ptr< string > m_RefreshErrorMessage
shared_ptr< CSatInfoSchema > m_SatInfoSchema
string GetRefreshErrorMessage() const
Get detailed message for last refresh operation (common for RefreshSchema and RefreshMessages).
void SetTimeout(chrono::milliseconds timeout)
Overrides data retrieval timeout.
shared_ptr< CCassConnection > m_SatInfoConnection
void x_SetRefreshErrorMessage(string const &message)
optional< ESatInfoRefreshSchemaResult > x_PopulateNewSchema(shared_ptr< CSatInfoSchema > &new_schema, shared_ptr< CSatInfoSchema > const &old_schema, vector< SSatInfoEntry > &&sat_info, map< int32_t, set< string >> &&secure_users)
int32_t GetMaxBlobKeyspaceSat() const
Get max id value for existing blob sat.
optional< SSatInfoEntry > GetIPGKeyspace() const
Get connection to IPG keyspace.
void SetDomain(string const &domain)
Changes configuration domain for existing provider.
map< string, shared_ptr< CCassConnection > > m_Service2Cluster
shared_ptr< CCassConnection > x_GetConnectionByService(string const &service, string const &registry_section) const
optional< ESatInfoRefreshSchemaResult > x_AddConnection(shared_ptr< CCassConnection > const &connection, string const &registry_section, string const &service, bool is_default)
map< string, shared_ptr< CCassConnection > > m_Point2Cluster
optional< SSatInfoEntry > GetIPGKeyspace() const
Get connection to IPG keyspace.
optional< ESatInfoRefreshSchemaResult > x_AddSatInfoEntry(SSatInfoEntry entry, shared_ptr< CSatInfoSchema > const &old_schema, shared_ptr< IRegistry const > const &registry, string const &registry_section, set< string > const &secure_users)
optional< ESatInfoRefreshSchemaResult > x_ResolveServiceName(string const &service, string const &registry_section, vector< string > &connection_points)
int32_t GetMaxBlobKeyspaceSat() const
Get max id value for existing blob sat.
vector< SSatInfoEntry > GetNAKeyspaces() const
Get list of BioseqNA keyspaces connections.
optional< SSatInfoEntry > GetBlobKeyspace(int32_t sat) const
Get blob keyspace connection by sat id.
shared_ptr< CCassConnection > m_DefaultConnection
CSatInfoSchema()=default
map< int32_t, set< string > > m_SecureSatUsers
shared_ptr< CCassConnection > x_GetConnectionByConnectionPoint(string const &connection_point) const
SSatInfoEntry m_ResolverKeyspace
vector< SSatInfoEntry > m_BioseqNaKeyspaces
string ToString() const
Print internal state of CSatInfoSchema.
optional< ESatInfoRefreshSchemaResult > x_ResolveConnectionByServiceName(string service, shared_ptr< CSatInfoSchema > const &old_schema, shared_ptr< IRegistry const > const &registry, string const &registry_section, shared_ptr< CCassConnection > &connection)
SSatInfoEntry GetResolverKeyspace() const
Get connection to resolver keyspace.
map< int32_t, SSatInfoEntry > m_BlobKeyspaces
optional< SSatInfoEntry > m_IPGKeyspace
optional< chrono::milliseconds > m_ResolveTimeout
string m_DefaultRegistrySection
static CMemoryRegistry registry
Definition: cn3d_tools.cpp:81
Include a standard set of the NCBI C++ Toolkit most basic headers.
Int4 int32_t
Int8 int64_t
int32_t Int4
4-byte (32-bit) signed integer
Definition: ncbitype.h:102
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
Process information in the NCBI Registry, including working with configuration files.
ESatInfoRefreshSchemaResult
ESatInfoRefreshMessagesResult
ECassSatInfoFlags
USING_NCBI_SCOPE
ECassSchemaType
@ eIPGSchema
@ eBlobVer2Schema
@ eMaxSchema
@ eUnknownSchema
@ eResolverSchema
@ eBlobVer1Schema
@ eNamedAnnotationsSchema
static const char * prefix[]
Definition: pcregrep.c:405
static const char *const kChunkTableDefault
SBlobStorageConstants()=delete
static const char *const kChunkTableBig
SSatInfoEntry(SSatInfoEntry &&)=default
SSatInfoEntry()=default
static constexpr int32_t kInvalidSat
set< string > m_SecureUsers
ECassSchemaType schema_type
friend CSatInfoSchema
shared_ptr< CCassConnection > GetConnection() const
Get public satellite connection.
bool IsSecureSat() const
Is satellite requires secure access.
string ToString(string const &prefix) const
Get string representation for debug.
SSatInfoEntry & operator=(SSatInfoEntry const &)=default
shared_ptr< CCassConnection > GetSecureConnection(string const &username) const
Get secure satellite connection.
SSatInfoEntry(SSatInfoEntry const &)=default
shared_ptr< CCassConnection > connection
SSatInfoEntry & operator=(SSatInfoEntry &&)=default
shared_ptr< CCassConnection > m_SecureConnection
Modified on Sun Apr 21 03:37:45 2024 by modify_doxy.py rev. 669887