NCBI C++ ToolKit
ns_server.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef NETSCHEDULE_SERVER__HPP
2 #define NETSCHEDULE_SERVER__HPP
3 
4 /* $Id: ns_server.hpp 84290 2018-10-31 14:21:21Z satskyse $
5  * ===========================================================================
6  *
7  * PUBLIC DOMAIN NOTICE
8  * National Center for Biotechnology Information
9  *
10  * This software/database is a "United States Government Work" under the
11  * terms of the United States Copyright Act. It was written as part of
12  * the author's official duties as a United States Government employee and
13  * thus cannot be copyrighted. This software/database is freely available
14  * to the public for use. The National Library of Medicine and the U.S.
15  * Government have not placed any restriction on its use or reproduction.
16  *
17  * Although all reasonable efforts have been taken to ensure the accuracy
18  * and reliability of the software and data, the NLM and the U.S.
19  * Government do not and cannot warrant the performance or results that
20  * may be obtained by using this software or data. The NLM and the U.S.
21  * Government disclaim all warranties, express or implied, including
22  * warranties of performance, merchantability or fitness for any particular
23  * purpose.
24  *
25  * Please cite the author in any work or product based on this material.
26  *
27  * ===========================================================================
28  *
29  * Authors: Anatoliy Kuznetsov, Victor Joukov
30  *
31  * File Description: NetScheduler threaded server
32  *
33  */
34 
35 #include <string>
36 #include <connect/server.hpp>
38 
39 #include "ns_server_misc.hpp"
40 #include "ns_server_params.hpp"
41 #include "ns_queue.hpp"
42 #include "ns_alert.hpp"
43 #include "ns_start_ids.hpp"
44 
46 
47 class CQueueDataBase;
48 
49 const unsigned int kSubmitCounterInitialValue = 1000000000;
50 
51 
52 //////////////////////////////////////////////////////////////////////////
53 /// NetScheduler threaded server
54 ///
55 /// @internal
57 {
58 public:
59  CNetScheduleServer(const string & dbpath, bool diskless);
60  virtual ~CNetScheduleServer();
61 
62  const bool & IsLog() const
63  { return m_LogFlag; }
64  const bool & IsLogBatchEachJob() const
65  { return m_LogBatchEachJobFlag; }
66  const bool & IsLogNotificationThread() const
67  { return m_LogNotificationThreadFlag; }
68  const bool & IsLogCleaningThread() const
69  { return m_LogCleaningThreadFlag; }
70  const bool & IsLogExecutionWatcherThread() const
72  const bool & IsLogStatisticsThread() const
73  { return m_LogStatisticsThreadFlag; }
74  const unsigned int & GetStatInterval() const
75  { return m_StatInterval; }
76  const unsigned int & GetJobCountersInterval() const
77  { return m_JobCountersInterval; }
78  bool GetRefuseSubmits() const
79  { return m_RefuseSubmits; }
80  void SetRefuseSubmits(bool val)
81  { m_RefuseSubmits = val; }
82  unsigned GetInactivityTimeout(void) const
83  { return m_InactivityTimeout; }
84  string & GetHost()
85  { return m_Host; }
86  unsigned GetPort() const
87  { return m_Port; }
88  unsigned GetDeleteBatchSize(void) const
89  { return m_DeleteBatchSize; }
90  bool GetUseHostname(void) const
91  { return m_UseHostname; }
92  bool GetDiskless(void) const
93  { return m_Diskless; }
94  unsigned GetMarkdelBatchSize(void) const
95  { return m_MarkdelBatchSize; }
96  unsigned GetScanBatchSize(void) const
97  { return m_ScanBatchSize; }
98  double GetPurgeTimeout(void) const
99  { return m_PurgeTimeout; }
100  unsigned GetHostNetAddr() const
101  { return m_HostNetAddr; }
102  const CTime & GetStartTime(void) const
103  { return m_StartTime; }
105  { return m_BackgroundHost; }
106  unsigned int GetMaxClientData(void) const
107  { return m_MaxClientData; }
108  string GetNodeID(void) const
109  { return m_NodeID; }
110  string GetSessionID(void) const
111  { return m_SessionID; }
113  { return m_AffRegistrySettings; }
115  { return m_GroupRegistrySettings; }
117  { return m_ScopeRegistrySettings; }
118  bool IsDrainShutdown(void) const
120  bool WasDBDrained(void) const
121  { return m_DBDrained; }
122  void SetDrainShutdown(void)
123  { m_CurrentSubmitsCounter.Add(-1*static_cast<int>
125  unsigned int GetCurrentSubmitsCounter(void)
126  { return m_CurrentSubmitsCounter.Get(); }
128  { return m_CurrentSubmitsCounter.Add(1); }
130  { return m_CurrentSubmitsCounter.Add(-1); }
132  { return m_AdminHosts; }
134  { return m_CompoundIDPool; }
135  void SetJobsStartID(const string & qname, unsigned int value)
136  { m_StartIDs.Set(qname, value); }
137  unsigned int GetJobsStartID(const string & qname)
138  { return m_StartIDs.Get(qname); }
139  void LoadJobsStartIDs(void)
140  { m_StartIDs.Load(); }
142  { m_StartIDs.Serialize(); }
143 
144 
146  CJsonNode SetNSParameters(const SNS_Parameters & new_params,
147  bool limited);
149 
150  virtual bool ShutdownRequested(void);
151  void SetShutdownFlag(int signum = 0, bool db_was_drained = false);
152  void SetQueueDB(CQueueDataBase* qdb);
153 
154  // Queue handling
155  unsigned int Configure(const IRegistry & reg,
156  CJsonNode & diff);
157  unsigned CountActiveJobs() const;
158  CRef<CQueue> OpenQueue(const string & name);
160  const string & qname,
161  const string & qclass,
162  const string & description);
164  const string & qname);
165  SQueueParameters QueueInfo(const string & qname) const;
166  string GetQueueNames(const string & sep) const;
167  string PrintTransitionCounters(void);
168  string PrintJobsStat(const CNSClientId & client);
169  string GetQueueClassesInfo(void) const;
170  string GetQueueClassesConfig(void) const;
171  string GetQueueInfo(void) const;
172  string GetQueueConfig(void) const;
173  string GetLinkedSectionConfig(void) const;
174  string GetServiceToQueueSectionConfig(void) const;
175  string ResolveService(const string & service) const;
176  void GetServices(map<string, string> & services) const;
177 
178  bool AdminHostValid(unsigned host) const;
179  bool IsAdminClientName(const string & name) const;
180  bool ShouldPerfLogTransitions(const string & queue_name,
181  const string & class_name) const;
182 
183  void InitNodeID(const string & db_path);
184 
185  static CNetScheduleServer* GetInstance(void);
186  string GetAdminClientNames(void) const;
187  string GetStateTransitionPerfLogQueues(void) const;
188  string GetStateTransitionPerfLogClasses(void) const;
189 
190  string GetAlerts(void) const;
191  string SerializeAlerts(void) const;
192  enum EAlertAckResult AcknowledgeAlert(const string & id,
193  const string & user);
195  const string & user);
196  void RegisterAlert(EAlertType alert_type,
197  const string & message);
198  void SetRAMConfigFileChecksum(const string & checksum);
199  string GetRAMConfigFileChecksum(void) const
200  { return m_RAMConfigFileChecksum; }
201  void SetDiskConfigFileChecksum(const string & checksum);
202  string GetDiskConfigFileChecksum(void) const
203  { return m_DiskConfigFileChecksum; }
206  bool AnybodyCanReconfigure(void) const
207  { return m_AnybodyCanReconfigure; }
208  unsigned int GetReserveDumpSpace(void) const
209  { return m_ReserveDumpSpace; }
210  map<string, int> GetPauseQueues(void) const;
211  vector<string> GetRefuseSubmitQueues(void) const;
212  string GetDataPath(void) const;
213 
214 protected:
215  virtual void Exit();
216 
217 private:
218  // API for background threads
220  // Host name where server runs
221  string m_Host;
222  unsigned m_Port;
223  unsigned m_HostNetAddr;
224  mutable bool m_Shutdown;
225  int m_SigNum; // Shutdown signal number
226  // Time to wait for the client (seconds)
227  unsigned int m_InactivityTimeout;
230 
231  // Log related flags
232  bool m_LogFlag;
238 
241 
243 
244  // Support for shutdown with drain
247 
248 
249  // Purge() related parameters
250  unsigned int m_DeleteBatchSize; // Max # for erasing
251  unsigned int m_MarkdelBatchSize; // Max # for marking
252  unsigned int m_ScanBatchSize; // Max # of scanned
253  double m_PurgeTimeout; // Time between purges
254  unsigned int m_StatInterval;
255  unsigned int m_JobCountersInterval;
256 
257  unsigned int m_MaxClientData;
258 
259  string m_NodeID; // From the ini file
260  string m_SessionID; // Generated
261 
265 
266  // List of admin stations
268 
270 
272  vector<string> m_AdminClientNames;
273 
277 
279 
282 
284 
286 
289 
291 
292  unsigned int m_ReserveDumpSpace;
293 
294 private:
295  string x_GenerateGUID(void) const;
296  CJsonNode x_SetAdminClientNames(const string & client_names);
297  CJsonNode x_SetFromList(const string & from, vector<string> & to,
298  CRWLock & lock);
299 
300 #if defined(_DEBUG) && !defined(NDEBUG)
301 private:
309 
310 public:
312  { return debug_fd_count; }
314  { return debug_mem_count; }
316  { return debug_write_delay; }
318  { return debug_conn_drop_before_write; }
320  { return debug_conn_drop_after_write; }
322  { return debug_reply_with_garbage; }
323  string GetDebugGarbage(void) const
324  { return debug_garbage; }
325 #endif
326 };
327 
328 
330 
331 #endif
332 
CAtomicCounter –.
Definition: ncbicntr.hpp:71
Pool of recycled CCompoundID objects.
CFastMutex –.
Definition: ncbimtx.hpp:667
JSON node abstraction.
unsigned int Get(const string &qname)
void Set(const string &qname, unsigned int value)
void Load(void)
void Serialize(void) const
CNcbiRegistry –.
Definition: ncbireg.hpp:913
Host for background threads.
NetScheduler threaded server.
Definition: ns_server.hpp:57
const CNetScheduleAccessList & GetAdminHosts(void) const
Definition: ns_server.hpp:131
unsigned int GetReserveDumpSpace(void) const
Definition: ns_server.hpp:208
unsigned int m_MaxClientData
Definition: ns_server.hpp:257
string ResolveService(const string &service) const
Definition: ns_server.cpp:502
string GetSessionID(void) const
Definition: ns_server.hpp:110
string GetQueueNames(const string &sep) const
Definition: ns_server.cpp:434
CFastMutex m_ServicesLock
Definition: ns_server.hpp:280
string GetDebugGarbage(void) const
Definition: ns_server.hpp:323
string GetDiskConfigFileChecksum(void) const
Definition: ns_server.hpp:202
unsigned int m_ReserveDumpSpace
Definition: ns_server.hpp:292
void SetJobsStartID(const string &qname, unsigned int value)
Definition: ns_server.hpp:135
SErrorEmulatorParameter GetDebugConnDropBeforeWrite(void) const
Definition: ns_server.hpp:317
unsigned GetHostNetAddr() const
Definition: ns_server.hpp:100
void GetServices(map< string, string > &services) const
Definition: ns_server.cpp:515
map< string, string > m_Services
Definition: ns_server.hpp:281
string GetAdminClientNames(void) const
Definition: ns_server.cpp:539
string & GetHost()
Definition: ns_server.hpp:84
CJsonNode x_SetFromList(const string &from, vector< string > &to, CRWLock &lock)
Definition: ns_server.cpp:743
SErrorEmulatorParameter GetDebugFDCount(void) const
Definition: ns_server.hpp:311
unsigned int m_JobCountersInterval
Definition: ns_server.hpp:255
bool m_AnybodyCanReconfigure
Definition: ns_server.hpp:290
unsigned GetInactivityTimeout(void) const
Definition: ns_server.hpp:82
static CNetScheduleServer * GetInstance(void)
Definition: ns_server.cpp:690
CNetScheduleBackgroundHost m_BackgroundHost
Definition: ns_server.hpp:219
unsigned GetScanBatchSize(void) const
Definition: ns_server.hpp:96
string SerializeAlerts(void) const
Definition: ns_server.cpp:621
unsigned int m_ScanBatchSize
Definition: ns_server.hpp:252
string GetStateTransitionPerfLogClasses(void) const
Definition: ns_server.cpp:570
string GetNodeID(void) const
Definition: ns_server.hpp:108
bool AdminHostValid(unsigned host) const
Definition: ns_server.cpp:522
bool GetRefuseSubmits() const
Definition: ns_server.hpp:78
unsigned int DecrementCurrentSubmitsCounter(void)
Definition: ns_server.hpp:129
unsigned GetPort() const
Definition: ns_server.hpp:86
unsigned int m_StatInterval
Definition: ns_server.hpp:254
void SetAnybodyCanReconfigure(bool val)
Definition: ns_server.hpp:204
static CNetScheduleServer * sm_netschedule_server
Definition: ns_server.hpp:269
CJsonNode x_SetAdminClientNames(const string &client_names)
SErrorEmulatorParameter GetDebugConnDropAfterWrite(void) const
Definition: ns_server.hpp:319
void RegisterAlert(EAlertType alert_type, const string &message)
Definition: ns_server.cpp:643
unsigned int GetJobsStartID(const string &qname)
Definition: ns_server.hpp:137
bool m_LogCleaningThreadFlag
Definition: ns_server.hpp:235
SErrorEmulatorParameter debug_write_delay
Definition: ns_server.hpp:304
const bool & IsLogNotificationThread() const
Definition: ns_server.hpp:66
SNSRegistryParameters m_ScopeRegistrySettings
Definition: ns_server.hpp:264
CNSAlerts m_Alerts
Definition: ns_server.hpp:278
string GetStateTransitionPerfLogQueues(void) const
Definition: ns_server.cpp:554
vector< string > m_AdminClientNames
Definition: ns_server.hpp:272
unsigned int m_DeleteBatchSize
Definition: ns_server.hpp:250
void SetDrainShutdown(void)
Definition: ns_server.hpp:122
vector< string > GetRefuseSubmitQueues(void) const
Definition: ns_server.cpp:793
unsigned int IncrementCurrentSubmitsCounter(void)
Definition: ns_server.hpp:127
string m_DiskConfigFileChecksum
Definition: ns_server.hpp:288
enum EAlertAckResult AcknowledgeAlert(const string &id, const string &user)
Definition: ns_server.cpp:628
string PrintTransitionCounters(void)
Definition: ns_server.cpp:440
const bool & IsLogExecutionWatcherThread() const
Definition: ns_server.hpp:70
CNetScheduleServer(const string &dbpath, bool diskless)
NetScheduler threaded server implementation.
Definition: ns_server.cpp:48
SNSRegistryParameters GetAffRegistrySettings(void) const
Definition: ns_server.hpp:112
void CreateDynamicQueue(const CNSClientId &client, const string &qname, const string &qclass, const string &description)
Definition: ns_server.cpp:411
const unsigned int & GetJobCountersInterval() const
Definition: ns_server.hpp:76
const unsigned int & GetStatInterval() const
Definition: ns_server.hpp:74
const bool & IsLogBatchEachJob() const
Definition: ns_server.hpp:64
SErrorEmulatorParameter GetDebugWriteDelay(void) const
Definition: ns_server.hpp:315
SNSRegistryParameters m_AffRegistrySettings
Definition: ns_server.hpp:262
unsigned int m_InactivityTimeout
Definition: ns_server.hpp:227
CJsonNode SetNSParameters(const SNS_Parameters &new_params, bool limited)
Definition: ns_server.cpp:101
CNetScheduleAccessList m_AdminHosts
Definition: ns_server.hpp:267
unsigned int GetMaxClientData(void) const
Definition: ns_server.hpp:106
string GetAlerts(void) const
Definition: ns_server.cpp:615
string GetQueueClassesInfo(void) const
Definition: ns_server.cpp:452
void SetShutdownFlag(int signum=0, bool db_was_drained=false)
Definition: ns_server.cpp:382
SErrorEmulatorParameter debug_fd_count
Definition: ns_server.hpp:302
string GetQueueInfo(void) const
Definition: ns_server.cpp:464
void SetQueueDB(CQueueDataBase *qdb)
Definition: ns_server.cpp:375
virtual bool ShutdownRequested(void)
Runs synchronously between iterations.
Definition: ns_server.cpp:369
const bool & IsLogCleaningThread() const
Definition: ns_server.hpp:68
CQueueDataBase * m_QueueDB
Definition: ns_server.hpp:228
SQueueParameters QueueInfo(const string &qname) const
Definition: ns_server.cpp:428
unsigned GetMarkdelBatchSize(void) const
Definition: ns_server.hpp:94
bool m_LogStatisticsThreadFlag
Definition: ns_server.hpp:237
void SetDiskConfigFileChecksum(const string &checksum)
Definition: ns_server.cpp:781
unsigned GetDeleteBatchSize(void) const
Definition: ns_server.hpp:88
string GetServiceToQueueSectionConfig(void) const
Definition: ns_server.cpp:482
void AddDefaultListener(IServer_ConnectionFactory *factory)
Definition: ns_server.cpp:92
CNSStartIDs m_StartIDs
Definition: ns_server.hpp:283
unsigned CountActiveJobs() const
Definition: ns_server.cpp:399
SErrorEmulatorParameter debug_mem_count
Definition: ns_server.hpp:303
bool GetUseHostname(void) const
Definition: ns_server.hpp:90
virtual void Exit()
Cleanup the server.
Definition: ns_server.cpp:696
void LoadJobsStartIDs(void)
Definition: ns_server.hpp:139
map< string, int > GetPauseQueues(void) const
Definition: ns_server.cpp:787
CRWLock m_STPerfLogQCLock
Definition: ns_server.hpp:274
string x_GenerateGUID(void) const
Definition: ns_server.cpp:732
unsigned int Configure(const IRegistry &reg, CJsonNode &diff)
Definition: ns_server.cpp:393
SErrorEmulatorParameter GetDebugMemCount(void) const
Definition: ns_server.hpp:313
const CTime & GetStartTime(void) const
Definition: ns_server.hpp:102
bool WasDBDrained(void) const
Definition: ns_server.hpp:120
virtual ~CNetScheduleServer()
Definition: ns_server.cpp:86
bool IsAdminClientName(const string &name) const
Definition: ns_server.cpp:528
CAtomicCounter m_CurrentSubmitsCounter
Definition: ns_server.hpp:245
bool IsDrainShutdown(void) const
Definition: ns_server.hpp:118
bool ShouldPerfLogTransitions(const string &queue_name, const string &class_name) const
Definition: ns_server.cpp:587
SErrorEmulatorParameter debug_reply_with_garbage
Definition: ns_server.hpp:307
SErrorEmulatorParameter GetDebugReplyWithGarbage(void) const
Definition: ns_server.hpp:321
double GetPurgeTimeout(void) const
Definition: ns_server.hpp:98
void SetRefuseSubmits(bool val)
Definition: ns_server.hpp:80
SNSRegistryParameters m_GroupRegistrySettings
Definition: ns_server.hpp:263
void DeleteDynamicQueue(const CNSClientId &client, const string &qname)
Definition: ns_server.cpp:420
bool m_LogExecutionWatcherThreadFlag
Definition: ns_server.hpp:236
string GetLinkedSectionConfig(void) const
Definition: ns_server.cpp:476
void InitNodeID(const string &db_path)
Definition: ns_server.cpp:654
string m_RAMConfigFileChecksum
Definition: ns_server.hpp:287
CCompoundIDPool GetCompoundIDPool(void) const
Definition: ns_server.hpp:133
unsigned m_HostNetAddr
Definition: ns_server.hpp:223
const bool & IsLogStatisticsThread() const
Definition: ns_server.hpp:72
string GetRAMConfigFileChecksum(void) const
Definition: ns_server.hpp:199
string GetDataPath(void) const
Definition: ns_server.cpp:799
CJsonNode ReadServicesConfig(const CNcbiRegistry &reg)
Definition: ns_server.cpp:291
unsigned int GetCurrentSubmitsCounter(void)
Definition: ns_server.hpp:125
string PrintJobsStat(const CNSClientId &client)
Definition: ns_server.cpp:446
SErrorEmulatorParameter debug_conn_drop_before_write
Definition: ns_server.hpp:305
SErrorEmulatorParameter debug_conn_drop_after_write
Definition: ns_server.hpp:306
bool AnybodyCanReconfigure(void) const
Definition: ns_server.hpp:206
bool GetDiskless(void) const
Definition: ns_server.hpp:92
void SetRAMConfigFileChecksum(const string &checksum)
Definition: ns_server.cpp:775
const bool & IsLog() const
Definition: ns_server.hpp:62
SNSRegistryParameters GetScopeRegistrySettings(void) const
Definition: ns_server.hpp:116
unsigned int m_MarkdelBatchSize
Definition: ns_server.hpp:251
bool m_LogNotificationThreadFlag
Definition: ns_server.hpp:234
string GetQueueConfig(void) const
Definition: ns_server.cpp:470
string GetQueueClassesConfig(void) const
Definition: ns_server.cpp:458
void SerializeJobsStartIDs(void)
Definition: ns_server.hpp:141
CRWLock m_AdminClientsLock
Definition: ns_server.hpp:271
vector< string > m_StateTransitionPerfLogClasses
Definition: ns_server.hpp:276
CRef< CQueue > OpenQueue(const string &name)
Definition: ns_server.cpp:405
CCompoundIDPool m_CompoundIDPool
Definition: ns_server.hpp:285
SNSRegistryParameters GetGroupRegistrySettings(void) const
Definition: ns_server.hpp:114
vector< string > m_StateTransitionPerfLogQueues
Definition: ns_server.hpp:275
CBackgroundHost & GetBackgroundHost()
Definition: ns_server.hpp:104
CRWLock –.
Definition: ncbimtx.hpp:953
CRef –.
Definition: ncbiobj.hpp:618
CServer::
Definition: server.hpp:97
CTime –.
Definition: ncbitime.hpp:296
IRegistry –.
Definition: ncbireg.hpp:73
IServer_ConnectionFactory::
Definition: server.hpp:388
TValue Add(int delta) THROWS_NONE
Atomically add value (=delta), and return new counter value.
Definition: ncbicntr.hpp:278
TValue Get(void) const THROWS_NONE
Get atomic counter value.
Definition: ncbicntr.hpp:168
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
T signum(T x_)
EAlertAckResult
Definition: ns_alert.hpp:63
EAlertType
Definition: ns_alert.hpp:46
const unsigned int kSubmitCounterInitialValue
Definition: ns_server.hpp:49
Framework to create multithreaded network servers with thread-per-request scheduling.
static CNamedPipeClient * client
Modified on Sun May 19 04:44:22 2024 by modify_doxy.py rev. 669887