NCBI C++ ToolKit
grid_client.hpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 #ifndef CONNECT_SERVICES_GRID__GRID_CLIENT__HPP
2 #define CONNECT_SERVICES_GRID__GRID_CLIENT__HPP
3 
4 /* $Id: grid_client.hpp 87146 2019-07-30 15:47:21Z sadyrovr $
5  * ===========================================================================
6  *
7  * PUBLIC DOMAIN NOTICE
8  * National Center for Biotechnology Information
9  *
10  * This software/database is a "United States Government Work" under the
11  * terms of the United States Copyright Act. It was written as part of
12  * the author's official duties as a United States Government employee and
13  * thus cannot be copyrighted. This software/database is freely available
14  * to the public for use. The National Library of Medicine and the U.S.
15  * Government have not placed any restriction on its use or reproduction.
16  *
17  * Although all reasonable efforts have been taken to ensure the accuracy
18  * and reliability of the software and data, the NLM and the U.S.
19  * Government do not and cannot warrant the performance or results that
20  * may be obtained by using this software or data. The NLM and the U.S.
21  * Government disclaim all warranties, express or implied, including
22  * warranties of performance, merchantability or fitness for any particular
23  * purpose.
24  *
25  * Please cite the author in any work or product based on this material.
26  *
27  * ===========================================================================
28  *
29  * Authors: Maxim Didenko, Dmitry Kazimirov
30  *
31  */
32 
33 /// @file grid_client.hpp
34 /// NetSchedule Framework specs.
35 ///
36 
37 
41 
42 #include <connect/connect_export.h>
43 
44 #include <corelib/ncbistre.hpp>
45 
46 
48 
49 /** @addtogroup NetScheduleClient
50  *
51  * @{
52  */
53 
54 
55 class CGridClient;
56 
57 /// Grid Job Batch Submitter
58 ///
60 {
61 public:
62  /// Set a job's input This string will be sent to
63  /// then the job is submitted.
64  ///
65  /// This method can be used to send a short data to the worker node.
66  /// To send a large data use GetOStream method. Don't call this
67  /// method after GetOStream method is called.
68  ///
69  void SetJobInput(const string& input);
70 
71  /// Get a stream where a client can write an input
72  /// data for the remote job
73  ///
74  CNcbiOstream& GetOStream();
75 
76  /// Set a job mask
77  ///
78  void SetJobMask(CNetScheduleAPI::TJobMask mask);
79 
80  /// Set a job affinity
81  ///
82  void SetJobAffinity(const string& affinity);
83 
84  void PrepareNextJob();
85 
86  /// Submit a batch to the queue
87  ///
88  void Submit(const string& job_group = kEmptyStr);
89 
90  ///
91  void Reset();
92 
93  const vector<CNetScheduleJob>& GetBatch() const { return m_Jobs; }
94 private:
95  /// Only CGridClient can create an instance of this class
96  friend class CGridClient;
97 
98  void CheckIfBatchAlreadySubmitted();
99  void CheckIfBatchSubmittedAndPrepareNextJob();
100 
102 
104  vector<CNetScheduleJob> m_Jobs;
105  size_t m_JobIndex;
108 
109  /// The copy constructor and the assignment operator
110  /// are prohibited
113 };
114 
115 /// Grid Client (the submitter).
116 ///
118 {
119 public:
120  enum ECleanUp {
121  eAutomaticCleanup = 0,
122  eManualCleanup
123  };
124 
126  eProgressMsgOn = 0,
127  eProgressMsgOff
128  };
129 
130  /// Constructor
131  ///
132  /// @param ns_client
133  /// NetSchedule client - an instance of CNetScheduleSubmitter.
134  /// @param storage
135  /// NetSchedule storage
136  /// @param cleanup
137  /// if true the grid client will automatically remove
138  /// a job's input data from a storage when the job is
139  /// done or canceled
140  ///
142  IBlobStorage& storage,
143  ECleanUp cleanup,
144  EProgressMsg progress_msg);
145 
146  /// Constructor
147  ///
148  /// @param ns_client
149  /// NetSchedule client - an instance of CNetScheduleSubmitter.
150  /// @param nc_client
151  /// NetCache client - an instance of CNetCacheAPI.
152  /// @param cleanup
153  /// if true the grid client will automatically remove
154  /// a job's input data from a storage when the job is
155  /// done or canceled
156  ///
158  CNetCacheAPI::TInstance nc_client,
159  ECleanUp cleanup,
160  EProgressMsg progress_msg);
161 
162  /// Get a job's output string.
163  ///
164  /// This string can be used in two ways.
165  /// 1. It can be an output data from a remote job (if that data is short)
166  /// If it is so don't use GetIStream method.
167  /// 2. It holds a key to a data stored in an external data storage.
168  /// (NetCache) In this case use GetIStream method to get a stream with
169  /// a job's result.
170  ///
171  const string& GetJobOutput();
172 
173  /// Get a job's input sting
174  const string& GetJobInput();
175 
176  /// Get a job's return code
177  //
178  int GetReturnCode();
179 
180  /// If something bad has happened this method will return an
181  /// explanation
182  ///
183  const string& GetErrorMessage();
184 
185  /// Get a job status
186  ///
187  CNetScheduleAPI::EJobStatus GetStatus();
188 
189  /// Get a stream with a job's result. Stream is based on network
190  /// data storage (NetCache). Size of the input data can be determined
191  /// using GetInputBlobSize
192  ///
194 
195  /// Get the size of an input stream
196  ///
197  size_t GetBlobSize() const { return m_BlobSize; }
198 
199  /// Get a job interim message
200  ///
201  string GetProgressMessage();
202 
203  /// Set a job's input This string will be sent to
204  /// then the job is submitted.
205  ///
206  /// This method can be used to send a short data to the worker node.
207  /// To send a large data use GetOStream method. Don't call this
208  /// method after GetOStream method is called.
209  ///
210  void SetJobInput(const string& input);
211 
212  /// Get a stream where a client can write an input
213  /// data for the remote job
214  ///
215  CNcbiOstream& GetOStream();
216 
217  void CloseStream();
218 
220 
221  void SetJobGroup(const string& group) {m_Job.group = group;}
222 
223  void SetJobAffinity(const string& affinity) {m_Job.affinity = affinity;}
224 
225  /// Submit a job to the queue
226  ///
227  /// @return a job key
228  string Submit(const string& affinity = kEmptyStr);
229 
230  /// Submit a job to the queue
231  ///
232  CNetScheduleAPI::EJobStatus SubmitAndWait(unsigned wait_time);
233 
234  /// Wait for job to finish its execution (done/failed/canceled).
235  /// @sa CNetScheduleAPI
236  ///
237  CNetScheduleAPI::EJobStatus WaitForJob(const string& job_key,
238  unsigned wait_time);
239 
240  CNetScheduleJob& GetJob() {return m_Job;}
241 
242  /// Get a job submitter
243  ///
244  CGridJobBatchSubmitter& GetJobBatchSubmitter();
245 
246  void SetJobKey(const string& job_key);
247 
248  /// Cancel Job
249  ///
250  /// @param job_key
251  /// Job key
252  ///
253  void CancelJob(const string& job_key);
254 
256  {
257  return m_NetScheduleSubmitter;
258  }
259  CNetCacheAPI& GetNetCacheAPI() { return m_NetCacheAPI; }
260 
261  size_t GetMaxServerInputSize();
262 
263 private:
264  void RemoveDataBlob(const string&);
265 
268 
272 
274 
275  void x_ProlongBlobLifetime(const string& blob_key, unsigned ttl);
276  bool x_ProlongJobFieldLifetime(const string& job_field, unsigned ttl);
277  void x_RenewAllJobBlobs(unsigned ttl);
278  CNetScheduleAPI::EJobStatus x_CheckAllJobBlobs(
279  CNetScheduleAPI::EJobStatus status, time_t job_exptime);
280  void x_GetJobDetails();
281 
282  size_t m_BlobSize = 0;
285  bool m_JobDetailsRead = false;
286 
287  /// The copy constructor and the assignment operator
288  /// are prohibited
291 };
292 
293 /// Grid Client exception
294 ///
296 {
297 public:
298  enum EErrCode {
300  };
301 
302  virtual const char* GetErrCodeString(void) const override
303  {
304  switch (GetErrCode())
305  {
306  case eBatchAlreadySubmitted: return "eBatchAlreadySubmitted";
307  default: return CException::GetErrCodeString();
308  }
309  }
310 
312 };
313 
314 /////////////////////////////////////////////////////////////////////////////
315 
317 
318 #endif // CONNECT_SERVICES_GRID__GRID_CLIENT__HPP
ncbi::TMaskedQueryRegions mask
Grid Client exception.
Grid Client (the submitter).
Grid Job Batch Submitter.
Definition: grid_client.hpp:60
Client API for NetCache server.
Smart pointer to the job submission part of the NetSchedule API.
Blob Storage interface.
static void cleanup(void)
Definition: ct_dynamic.c:30
TErrCode GetErrCode(void) const
Get error code.
Definition: ncbiexpt.cpp:453
virtual const char * GetErrCodeString(void) const
Get error code interpreted as text.
Definition: ncbiexpt.cpp:444
SGridRead m_GridRead
CNetCacheAPI m_NetCacheAPI
CGridJobBatchSubmitter & operator=(CGridJobBatchSubmitter &)
EJobStatus
Job status codes.
CGridClient & m_GridClient
bool m_AutoCleanUp
CGridJobBatchSubmitter(const CGridJobBatchSubmitter &)
The copy constructor and the assignment operator are prohibited.
CGridClient(const CGridClient &)
The copy constructor and the assignment operator are prohibited.
CNetScheduleSubmitter GetNetScheduleSubmitter()
void SetJobAffinity(const string &affinity)
NCBI_EXCEPTION_DEFAULT(CGridClientException, CException)
CNetScheduleJob & GetJob()
CNetCacheAPI & GetNetCacheAPI()
void SetJobMask(CNetScheduleAPI::TJobMask mask)
SGridWrite m_GridWrite
CNetScheduleJob m_Job
bool m_UseProgress
virtual const char * GetErrCodeString(void) const override
Get error code interpreted as text.
vector< CNetScheduleJob > m_Jobs
CNetScheduleSubmitter m_NetScheduleSubmitter
CGridJobBatchSubmitter m_JobBatchSubmitter
CGridClient & operator=(const CGridClient &)
size_t GetBlobSize() const
Get the size of an input stream.
const vector< CNetScheduleJob > & GetBatch() const
Definition: grid_client.hpp:93
void SetJobGroup(const string &group)
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
IO_PREFIX::ostream CNcbiOstream
Portable alias for ostream.
Definition: ncbistre.hpp:149
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
Definition: ncbistre.hpp:146
#define kEmptyStr
Definition: ncbistr.hpp:123
#define NCBI_XCONNECT_EXPORT
static int input()
NCBI C++ stream class wrappers for triggering between "new" and "old" C++ stream libraries.
NetCache client specs.
NetSchedule client specs.
Job description.
unique_ptr< CObjectIStream > GetIStream(string path, ESerialDataFormat serial_format)
Modified on Wed May 22 11:30:29 2024 by modify_doxy.py rev. 669887