39 #pragma warning (disable: 4191)
61 #define NCBI_USE_ERRCODE_X ConnServ_NetCache
73 return address.
host ? &address :
nullptr;
106 eAuthenticationError,
"Client name is too short or empty");
112 const bool prolong_on_write =
registry.
Get(sections,
"prolong_blob_lifetime_on_write",
true);
113 const bool create_on_write =
registry.
Get(sections,
"create_blob_on_write",
true);
122 m_FlagsOnWrite = (prolong_on_write ? 0 : 1) | (create_on_write ? 0 : 2);
124 if (allowed_services.empty())
return;
128 vector<string> services;
132 for (
auto& service : services) {
147 if (server_props->mirroring_checked) {
151 string version_info(connection.
Exec(
m_Auth +
"\r\nVERSION",
false));
153 server_props->mirroring_checked =
true;
159 if (field->name ==
"mirrored" && field->value ==
"true")
160 server_props->mirrored =
true;
170 static const char s_BlobNotFoundMsg[] =
"BLOB not found";
172 sizeof(s_BlobNotFoundMsg) - 1) == 0) {
173 if (strstr(err_msg.c_str(),
"AGE=") !=
NULL) {
182 static const char s_AccessDenied[] =
"Access denied";
184 sizeof(s_AccessDenied) - 1) == 0)
187 static const char s_UnknownCommandMsg[] =
"Unknown command";
189 sizeof(s_UnknownCommandMsg) - 1) == 0)
200 ": WARNING: " << warn_msg);
206 const string& service,
const string& client_name,
208 m_NetScheduleAPI(ns_api),
209 m_DefaultParameters(
eVoid)
213 string ns_client_name;
224 registry_builder, sections, ns_client_name);
225 Init(registry_builder, sections);
229 m_DefaultParameters(
eVoid)
236 m_ServiceMap(parent->m_ServiceMap),
237 m_TempDir(parent->m_TempDir),
238 m_CacheInput(parent->m_CacheInput),
239 m_CacheOutput(parent->m_CacheOutput),
240 m_NetScheduleAPI(parent->m_NetScheduleAPI),
241 m_DefaultParameters(parent->m_DefaultParameters)
265 if (!password.empty()) {
266 cmd->append(
" pass=\"");
268 cmd->append(1,
'\"');
273 cmd->append(
" age=");
297 string result(cmd_base +
key.StripKeyExtensions());
305 string::size_type pos = exec_result.
response.find(
"AGE=");
307 if (pos == string::npos) {
310 "No AGE field in " << cmd_name <<
311 " output: \"" << exec_result.
response <<
"\"");
340 bool multiline_output,
344 const string& key_service_name =
key.GetServiceName();
346 bool key_has_service_name = !key_service_name.empty();
350 if (key_has_service_name && key_service_name != service.
GetServiceName()) {
355 key_service_name <<
" is not in the allowed services");
361 bool mirroring_allowed =
365 if (
key.GetVersion() == 3) {
369 key.GetKey() <<
": NetCache key version 3 "
370 "requires an LBSM service name.");
388 if (!mirroring_allowed)
399 exec_result, &mirror_traversal, error_handling);
405 if (mirroring_allowed) {
412 key.GetKey() <<
": unable to find a NetCache server "
413 "by the checksum from this key.");
424 if (key_has_service_name && mirroring_allowed) {
428 primary_server, server_check);
431 &mirror_traversal, error_handling);
440 if (key_has_service_name) {
442 key.GetKey() <<
": NetCache server " <<
444 "accessed because it is not registered for the service.");
450 key.GetKey() <<
": NetCache server " <<
452 "accessed because it is not registered for the allowed "
462 const string& conf_section ,
493 kEmptyStr, service_name, client_name, ns_api))
499 m_Impl->m_DefaultParameters.LoadNamedParameters(parameters);
514 const string& blob_id(nc_writer->
GetBlobID());
516 const bool write_existing_blob = !blob_id.empty();
518 string stripped_blob_id;
520 if (write_existing_blob) {
523 stripped_blob_id =
key.StripKeyExtensions();
524 cmd.append(stripped_blob_id);
532 if (write_existing_blob)
541 if (backup ==
NULL) {
542 LOG_POST(
Info <<
"Fallback server address is not configured.");
548 ". Connecting to backup server " << backup->
AsString() <<
".");
560 "Unexpected server response: " << exec_result.
response);
568 "Invalid server response. Empty key.");
571 if (write_existing_blob) {
572 if (exec_result.
response != stripped_blob_id) {
576 "Server created " << exec_result.
response <<
577 " in response to PUT3 \"" << stripped_blob_id <<
"\"");
594 bool server_check_hint =
true;
596 if (!server_check_hint)
610 return exec_result.
conn;
619 string actual_key(
key);
663 m_Impl->MakeCmd(
"HASB ",
key, ¶meters),
665 ¶meters).response[0] ==
'1';
680 m_Impl->MakeCmd(
"GSIZ ",
key, ¶meters),
682 ¶meters).response));
697 m_Impl->MakeCmd(
"RMV2 ",
key, ¶meters),
701 catch (std::exception& e) {
702 ERR_POST(
"Could not remove blob \"" << blob_id <<
"\": " << e.what());
705 ERR_POST(
"Could not remove blob \"" << blob_id <<
"\"");
714 string cmd(
"GETMETA " +
key.StripKeyExtensions());
715 cmd.append(
m_Impl->m_Service->m_ServerPool->m_EnforcedServer.host == 0 ?
722 m_Impl->AppendClientIPSessionIDHitID(&
cmd);
725 cmd,
true, ¶meters));
727 output->SetNetCacheCompatMode();
739 if (
output.ReadLine(line)) {
742 while (
output.ReadLine(line))
761 m_Impl->AppendClientIPSessionIDPasswordAgeHitID(&
cmd, ¶meters);
763 m_Impl->ExecMirrorAware(key_obj,
cmd,
false, ¶meters);
769 return m_Impl->GetPartReader(
key, 0, 0, blob_size, optional);
773 size_t offset,
size_t part_size,
size_t* blob_size,
777 blob_size, optional);
792 &blob_size, optional));
796 m_Impl->ReadBuffer(*reader,
const_cast<char*
>(
buffer.data()),
797 blob_size,
NULL, blob_size);
820 size_t offset,
size_t part_size,
825 string stripped_blob_id(
key.StripKeyExtensions());
827 const char* cmd_name;
830 if (
offset == 0 && part_size == 0) {
832 cmd = cmd_name + stripped_blob_id;
834 cmd_name =
"GETPART ";
835 cmd = cmd_name + stripped_blob_id +
' ' +
866 if (max_age > 0 && actual_age_ptr !=
NULL)
870 exec_result, blob_size_ptr, ¶meters);
882 size_t x_blob_size = 0;
884 unique_ptr<IReader> reader(
GetData(
key, &x_blob_size, optional));
885 if (reader.get() == 0)
889 *blob_size = x_blob_size;
891 return m_Impl->ReadBuffer(*reader,
892 (
char*)
buf, buf_size, n_read, x_blob_size);
899 size_t x_blob_size = 0;
901 unique_ptr<IReader> reader(
GetData(
key, &x_blob_size, optional));
902 if (reader.get() == 0)
905 buffer.resize_mem(x_blob_size);
906 return m_Impl->ReadBuffer(*reader,
907 (
char*)
buffer.data(), x_blob_size,
NULL, x_blob_size);
934 return m_Impl->m_CompoundIDPool;
940 m_Impl->m_CompoundIDPool = compound_id_pool;
952 size_t total_bytes_read = 0;
954 while (buf_size > 0) {
957 total_bytes_read += bytes_read;
958 buf_ptr += bytes_read;
959 buf_size -= bytes_read;
960 }
else if (rw_res ==
eRW_Eof) {
964 "Error while reading BLOB");
969 *n_read = total_bytes_read;
971 return total_bytes_read == blob_size ?
998 int patch_level = -1)
1015 if (params && (driver.empty() || driver ==
m_DriverName) &&
1048 _ASSERT(0 &&
"Cannot create an unitialized CBlobStorage_NetCache object.");
1057 NCBI_CATCH_ALL(
"CBlobStorage_NetCache_Impl::~CBlobStorage_NetCache()");
Pool of recycled CCompoundID objects.
TParent::TDriverList TDriverList
CNetCacheAPICF(const string &driver_name=kNetCacheAPIDriverName, int patch_level=-1)
Construction.
void GetDriverVersions(TDriverList &info_list) const
Versions of the interface exported by the factory.
virtual TInterface * CreateInstance(const string &driver=kEmptyStr, CVersionInfo version=NCBI_INTERFACE_VERSION(IFace), const TPluginManagerParamTree *params=0) const
Create instance of TDriver.
IClassFactory< SNetCacheAPIImpl > TParent
TParent::SDriverInfo TDriverInfo
CVersionInfo m_DriverVersionInfo
Exception thrown when the requested blob is older than the requested age.
NetCache internal exception.
void OnWarningImpl(const string &warn_msg, CNetServer &server) override
void OnErrorImpl(const string &err_msg, CNetServer &server) override
TPropCreator GetPropCreator() const override
void OnConnected(CNetServerConnection &connection) override
void SetAuthString(const string &auth)
INetServerConnectionListener * Clone() override
string Exec(const string &cmd, bool multiline_output=false, const STimeout *timeout=NULL)
Execute remote command 'cmd', wait for the reply, check that it starts with 'OK:',...
unsigned short GetPort() const
SExecResult ExecWithRetry(const string &cmd, bool multiline_output=false)
Execute remote command 'cmd', wait for the reply, check if it starts with 'OK:', and return the remai...
string GetServerAddress() const
const string & GetServiceName() const
CNetServer GetServer(unsigned host, unsigned short port)
CNetServer::SExecResult FindServerAndExec(const string &cmd, bool multiline_output=false)
bool IsLoadBalanced() const
CNetServiceIterator Iterate(EIterationMode mode=eSortByLoad)
Note about the "buf_size" parameter for streams in this API.
@ fLeakExceptions
Exceptions leaked out.
@ fOwnReader
Own the underlying reader.
@ fOwnWriter
Own the underlying writer.
Reallocable memory buffer (no memory copy overhead) Mimics vector<>, without the overhead of explicit...
definition of a Culling tree
Writer-based output stream.
A very basic data-read interface.
virtual CNetServer BeginIteration()
CNetServer::TInstance m_PrimaryServer
virtual CNetServer NextServer()
CNetServiceIterator m_Iterator
bool m_PrimaryServerCheck
static SQLCHAR output[256]
static const char * str(char *buf, int n)
#define ITERATE(Type, Var, Cont)
ITERATE macro to sequence through container elements.
static CRequestContext & GetRequestContext(void)
Shortcut to CDiagContextThreadData::GetThreadData().GetRequestContext()
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
bool IsSetClientIP(void) const
#define ERR_POST(message)
Error posting with file, line number information but without error codes.
#define LOG_POST(message)
This macro is deprecated and it's strongly recomended to move in all projects (except tests) to macro...
TErrCode GetErrCode(void) const
Get error code.
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
void Warning(CExceptionArgs_Base &args)
void AddToMessage(const string &add_msg)
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
virtual const char * what(void) const noexcept
Standard report (includes full backlog).
#define NCBI_CATCH_ALL(message)
This macro is deprecated - use *_X or *_XX variant instead of it.
void Info(CExceptionArgs_Base &args)
static Uint4 CalculateChecksum(const string &host, unsigned short port)
Calculate and return the CRC32 checksum generated from the string "host:port".
void SetUseCompoundID(bool use_compound_id)
IReader * GetPartReader(const string &key, size_t offset, size_t part_size, size_t *blob_size=NULL, const CNamedParameterList *optional=NULL)
Get a pointer to the IReader interface to read a portion of the blob contents.
unsigned * GetActualBlobAgePtr() const
virtual void Reset()
Close all streams and connections.
static string KeyToCompoundID(const string &key_str, CCompoundIDPool id_pool)
unsigned TNCKeyFlags
Binary OR of ENCKeyFlag.
CNetCacheAPI::EMirroringMode GetMirroringMode() const
const string & GetBlobID() const
CNcbiOstream * CreateOStream(string &key, const CNamedParameterList *optional=NULL)
Create a stream object for sending data to a blob.
void SetServerCheck(ESwitch server_check)
virtual bool IsKeyValid(const string &str)
Check if a given string is a valid key.
std::string GetPassword() const
void SetMirroringMode(CNetCacheAPI::EMirroringMode mirroring_mode)
const char *const kNetCacheAPIDriverName
size_t GetBlobSize(const string &blob_id, const CNamedParameterList *optional=NULL)
Returns the size of the BLOB identified by the "key" parameter.
void ReadPart(const string &key, size_t offset, size_t part_size, string &buffer, const CNamedParameterList *optional=NULL)
Read a part of the blob pointed to by "key" and store its contents in "buffer".
bool GetUseCompoundID() const
unique_ptr< CNcbiOstream > m_OStream
virtual void DeleteBlob(const string &data_id)
Delete a blob.
CCompoundIDPool GetCompoundIDPool()
void NCBI_EntryPoint_xnetcacheapi(CPluginManager< SNetCacheAPIImpl >::TDriverInfoList &info_list, CPluginManager< SNetCacheAPIImpl >::EEntryPointRequest method)
CNetRef< SNetCacheAPIImpl > m_Impl
static bool IsValidKey(const char *key_str, size_t key_len, CCompoundIDPool::TInstance id_pool=NULL)
virtual string GetBlobAsString(const string &data_id)
Get a blob content as a string.
bool GetServerCheck(ESwitch *server_check) const
void SetCompoundIDPool(CCompoundIDPool::TInstance compound_id_pool)
void Remove(const string &blob_id, const CNamedParameterList *optional=NULL)
Remove BLOB by key.
EAppRegistry
Defines how this object must be initialized.
unique_ptr< CNcbiIstream > m_IStream
CNetServerMultilineCmdOutput GetBlobInfo(const string &blob_id, const CNamedParameterList *optional=NULL)
Return a CNetServerMultilineCmdOutput object for reading meta information about the specified blob.
string PutData(const void *buf, size_t size, const CNamedParameterList *optional=NULL)
Put BLOB to server.
bool HasBlob(const string &blob_id, const CNamedParameterList *optional=NULL)
Check if the BLOB identified by the key "key" exists.
void PrintBlobInfo(const string &blob_key, const CNamedParameterList *optional=NULL)
Print meta information about the specified blob.
void LoadNamedParameters(const CNamedParameterList *optional)
size_t CheckBlobSize(Uint8 blob_size)
void WriteBufferAndClose(const char *buf_ptr, size_t buf_size)
virtual ~CBlobStorage_NetCache()
CNcbiIstream * GetIStream(const string &key, size_t *blob_size=NULL, const CNamedParameterList *optional=NULL)
Create an istream object for reading blob data.
virtual CNcbiOstream & CreateOStream(string &data_id, ELockMode lock_mode=eLockNoWait)
Get an output stream to a blob.
virtual CNcbiIstream & GetIStream(const string &data_id, size_t *blob_size_ptr=0, ELockMode lock_mode=eLockWait)
Get an input stream to a blob.
void SetDefaultParameters(const CNamedParameterList *parameters)
Override defaults used by this object.
unsigned GetMaxBlobAge() const
void ProlongBlobLifetime(const string &blob_key, unsigned ttl, const CNamedParameterList *optional=NULL)
IReader * GetReader(const string &key, size_t *blob_size=NULL, const CNamedParameterList *optional=NULL)
Get a pointer to the IReader interface to read blob contents.
virtual string CreateEmptyBlob()
Create an new blob.
IReader * GetData(const string &key, size_t *blob_size=NULL, const CNamedParameterList *optional=NULL)
Retrieve BLOB from server by key.
EReadResult
Status of GetData() call.
void SetBlobID(const string &blob_id)
void ReadData(const string &key, string &buffer, const CNamedParameterList *optional=NULL)
Read the blob pointed to by "key" and store its contents in "buffer".
string StripKeyExtensions() const
If the blob key has been parsed successfully, this method returns a trimmed "base" version of the key...
CNetCacheAdmin GetAdmin()
void SetServerCheckHint(bool server_check_hint)
bool GetServerCheckHint(bool *server_check_hint) const
void SetCachingMode(CNetCacheAPI::ECachingMode caching_mode)
static void AddExtensions(string &blob_id, const string &service_name, TNCKeyFlags flags, unsigned ver=1)
Unconditionally append a service name to the specified string.
@ fNCKey_NoServerCheck
Disable the check for whether the server IP is still in service.
@ fNCKey_SingleServer
Mark this blob as not mirrored.
@ eReadPart
Read part of the BLOB (buffer capacity)
@ eNotFound
BLOB not found or error.
@ eReadComplete
The whole BLOB has been read.
@ eAccessDenied
Blob could not be read completely.
@ eBlobNotFound
Access denied.
ncbi::TDriverList TDriverList
static void NCBI_EntryPointImpl(TDriverInfoList &info_list, EEntryPointRequest method)
Entry point implementation.
#define NCBI_INTERFACE_VERSION(iface)
Macro to construct CVersionInfo class using interface name (relies on CInterfaceVersion class)
ncbi::SDriverInfo SDriverInfo
list< SDriverInfo > TDriverInfoList
List of driver information.
EEntryPointRequest
Actions performed by the entry point.
uint32_t Uint4
4-byte (32-bit) unsigned integer
uint64_t Uint8
8-byte (64-bit) unsigned integer
virtual const string & Get(const string §ion, const string &name, TFlags flags=0) const
Get the parameter value.
#define END_NCBI_SCOPE
End previously defined NCBI scope.
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
static string ntoa(unsigned int host)
BSD-like API. NB: when int, "host" must be in network byte order.
ERW_Result
Result codes for I/O operations.
IO_PREFIX::ostream CNcbiOstream
Portable alias for ostream.
virtual ERW_Result Read(void *buf, size_t count, size_t *bytes_read=0)=0
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
@ eRW_Eof
End of data, should be considered permanent.
@ eRW_Success
Everything is okay, I/O completed.
static string PrintableString(const CTempString str, TPrintableMode mode=fNewLine_Quote|fNonAscii_Passthru)
Get a printable version of the specified string.
static int CompareNocase(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2)
Case-insensitive compare of a substring with another string.
static list< string > & Split(const CTempString str, const CTempString delim, list< string > &arr, TSplitFlags flags=0, vector< SIZE_TYPE > *token_pos=NULL)
Split a string using specified delimiters.
static string IntToString(int value, TNumToStringFlags flags=0, int base=10)
Convert int to string.
static SIZE_TYPE FindCase(const CTempString str, const CTempString pattern, SIZE_TYPE start, SIZE_TYPE end, EOccurrence which=eFirst)
Find the pattern in the specified range of a string using a case sensitive search.
static bool StartsWith(const CTempString str, const CTempString start, ECase use_case=eCase)
Check if a string starts with a specified prefix value.
static Uint8 StringToUInt8(const CTempString str, TStringToNumFlags flags=0, int base=10)
Convert string to Uint8.
static unsigned int StringToUInt(const CTempString str, TStringToNumFlags flags=0, int base=10)
Convert string to unsigned int.
static int strncmp(const char *s1, const char *s2, size_t n)
String compare up to specified number of characters.
static enable_if< is_arithmetic< TNumeric >::value||is_convertible< TNumeric, Int8 >::value, string >::type NumericToString(TNumeric value, TNumToStringFlags flags=0, int base=10)
Convert numeric value to string.
static string UInt8ToString(Uint8 value, TNumToStringFlags flags=0, int base=10)
Convert UInt8 to string.
@ fAllowTrailingSymbols
Ignore trailing non-numerics characters.
@ fSplit_MergeDelimiters
Merge adjacent delimiters.
const TArgs & GetArgs(void) const
Get the const list of arguments.
enum ENcbiSwitch ESwitch
Aux.
@ eNonCompatible
major, minor does not match
#define NCBI_XCONNECT_EXPORT
Definition of all error codes used in connect services library (xconnserv.lib and others).
const string version
version string
const struct ncbi::grid::netcache::search::fields::SIZE size
const struct ncbi::grid::netcache::search::fields::KEY key
Magic spell ;-) needed for some weird compilers... very empiric.
Static variables safety - create on demand, destroy on application termination.
Defines: CTimeFormat - storage class for time format.
@ eVoid
To create a void (uninitialized) instance of a component.
void g_AppendClientIPAndSessionID(string &cmd, const CRequestContext &req)
void g_AppendHitID(string &cmd, CRequestContext &req)
#define CONNSERV_THROW_FMT(exception_class, err_code, server, message)
Helper classes and templates to implement plugins.
Defines CRequestContext class for NCBI C++ diagnostic API.
Reader-writer based streams.
CNetCacheAPIExt GetServer(CNetServer::TInstance server)
Meaningful information encoded in the NetCache key.
CNetServerConnection conn
function< INetServerProperties *()> TPropCreator
static const TAddress * Get()
void AppendClientIPSessionIDHitID(string *cmd)
void AppendClientIPSessionID(string *cmd, CRequestContext &req)
CNetCacheServerListener * GetListener()
void AppendClientIPSessionIDPasswordAgeHitID(string *cmd, const CNetCacheAPIParameters *parameters)
CNetCacheAPIParameters m_DefaultParameters
void AppendHitID(string *cmd, CRequestContext &req)
CNetCacheReader * GetPartReader(const string &blob_id, size_t offset, size_t part_size, size_t *blob_size, const CNamedParameterList *optional)
string MakeCmd(const char *cmd_base, const CNetCacheKey &key, const CNetCacheAPIParameters *parameters)
SNetServiceMap m_ServiceMap
unsigned x_ExtractBlobAge(const CNetServer::SExecResult &exec_result, const char *cmd_name)
CNetServer::SExecResult ExecMirrorAware(const CNetCacheKey &key, const string &cmd, bool multiline_output, const CNetCacheAPIParameters *parameters, SNetServiceImpl::EServerErrorHandling error_handling=SNetServiceImpl::eRethrowServerErrors)
static CNetCacheAPI::EReadResult ReadBuffer(IReader &reader, char *buf_ptr, size_t buf_size, size_t *n_read, size_t blob_size)
CCompoundIDPool m_CompoundIDPool
void Init(CSynRegistry ®istry, const SRegSynonyms §ions)
virtual CNetServerConnection InitiateWriteCmd(CNetCacheWriter *nc_writer, const CNetCacheAPIParameters *parameters)
void WriteLine(const string &line)
CRef< SNetServerInPool > m_ServerInPool
CRef< TProperties > Get()
CNetServer GetServer(SSocketAddress server_address)
static SNetServiceImpl * Create(const string &api_name, const string &service_name, const string &client_name, INetServerConnectionListener *listener, CSynRegistry ®istry, SRegSynonyms §ions, const string &ns_client_name=kEmptyStr)
CNetServer::SExecResult FindServerAndExec(const string &cmd, bool multiline_output)
const string & GetClientName() const
void IterateUntilExecOK(const string &cmd, bool multiline_output, CNetServer::SExecResult &exec_result, IServiceTraversal *service_traversal, EServerErrorHandling error_handling)
bool IsInService(CNetServer::TInstance server)
bool IsAllowed(const string &service_name) const
void AddToAllowed(const string &service_name)
CNetService GetServiceByName(const string &service_name, SNetServiceImpl *prototype)
static SSocketAddress Parse(const string &address, SHost::EName name=SHost::EName::eResolved)