NCBI C++ ToolKit
grid_cgiapp.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: grid_cgiapp.cpp 95599 2021-11-30 15:30:23Z sadyrovr $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Maxim Didenko
27  *
28  * File Description:
29  *
30  */
31 
32 #include <ncbi_pch.hpp>
33 #include <corelib/ncbistr.hpp>
34 #include <corelib/ncbi_system.hpp>
35 
37 #include <misc/error_codes.hpp>
38 
39 #include <vector>
40 
41 
42 #define NCBI_USE_ERRCODE_X Misc_GridCgi
43 
45 
46 struct SExceptionMessage
47 {
49 
50  string operator()(const string& what)
51  {
53  // Must correspond to TExceptionMessage
54  auto message = m_Registry->GetString("CGI", "Exception_Message", "Some exception was thrown (not shown for safety reasons)");
55  return message.empty() ? what : message;
56  }
57 
58 private:
60 };
61 
62 
64  : m_Page(page), m_CgiContext(ctx), m_NeedRenderPage(true)
65 {
66  const CCgiRequest& req = ctx.GetRequest();
67  string query_string = req.GetProperty(eCgi_QueryString);
69 }
70 
72 {
73 }
74 
76 {
77  string url = m_CgiContext.GetSelfURL();
78  bool first = true;
80  for (it = m_PersistedEntries.begin();
81  it != m_PersistedEntries.end(); ++it) {
82  const string& name = it->first;
83  const string& value = it->second;
84  if (!name.empty() && !value.empty()) {
85  if (first) {
86  url += '?';
87  first = false;
88  }
89  else
90  url += '&';
91  url += name + '=' + NStr::URLEncode(value);
92  }
93  }
94  return url;
95 }
96 
98 {
99  string ret;
101  for (it = m_PersistedEntries.begin();
102  it != m_PersistedEntries.end(); ++it) {
103  const string& name = it->first;
104  const string& value = it->second;
105  ret += "<INPUT TYPE=\"HIDDEN\" NAME=\"" + name
106  + "\" VALUE=\"" + value + "\">\n";
107  }
108  return ret;
109 }
110 
111 void CGridCgiContext::SetJobKey(const string& job_key)
112 {
113  DefinePersistentEntry("job_key", job_key);
114 
115 }
116 const string& CGridCgiContext::GetJobKey(void) const
117 {
118  return GetEntryValue("job_key");
119 }
120 
121 const string& CGridCgiContext::GetEntryValue(const string& entry_name) const
122 {
124  if (it != m_PersistedEntries.end())
125  return it->second;
126  return kEmptyStr;
127 }
128 
129 void CGridCgiContext::PullUpPersistentEntry(const string& entry_name)
130 {
131  string value = kEmptyStr;
133  if (NStr::CompareNocase(entry_name, eit->first) == 0 ) {
134  string v = eit->second;
135  if (!v.empty())
136  value = v;
137  }
138  }
139  if (value.empty()) {
141  ITERATE(TCgiEntries, eit, entries) {
142  if (NStr::CompareNocase(entry_name, eit->first) == 0 ) {
143  string v = eit->second;
144  if (!v.empty())
145  value = v;
146  }
147  }
148  }
149  DefinePersistentEntry(entry_name, value);
150 }
151 void CGridCgiContext::DefinePersistentEntry(const string& entry_name,
152  const string& value)
153 {
154  if (value.empty()) {
156  m_PersistedEntries.find(entry_name);
157  if (it != m_PersistedEntries.end())
159  } else {
160  m_PersistedEntries[entry_name] = value;
161  }
162 }
163 
165 {
167 }
168 
170 {
171  m_CgiContext.GetResponse().out() << is.rdbuf();
172  m_NeedRenderPage = false;
173 }
174 
175 /////////////////////////////////////////////////////////////////////////////
176 // CGridCgiSampleApplication::
177 //
178 
179 
181 {
182  // Standard CGI framework initialization
184  InitGridClient();
185 }
186 
188 {
189  string query_string = request.GetProperty(eCgi_QueryString);
191  CCgiRequest::ParseEntries(query_string, entries);
192  return entries.find("job_key") == entries.end();
193 }
194 
196 {
197  auto& config = GetConfig();
198 
199  // Must correspond to TServConn_ErrorOnUnexpectedReply
200  if (!config.HasEntry("netservice_api", "error_on_unexpected_reply")) {
201  config.Set("netservice_api", "error_on_unexpected_reply", "true");
202  }
203 
204  m_RefreshDelay =
205  GetConfig().GetInt("grid_cgi", "refresh_delay", 5, IRegistry::eReturn);
206  m_FirstDelay =
207  GetConfig().GetInt("grid_cgi", "expect_complete", 5, IRegistry::eReturn);
208  if (m_FirstDelay > 20 ) m_FirstDelay = 20;
209  if (m_FirstDelay < 0) m_FirstDelay = 0;
210 
211  bool automatic_cleanup =
212  GetConfig().GetBool("grid_cgi", "automatic_cleanup", true, IRegistry::eReturn);
213  bool use_progress =
214  GetConfig().GetBool("grid_cgi", "use_progress", true, IRegistry::eReturn);
215 
216  if (!m_NSClient) {
219  }
220  if (!m_NetCacheAPI)
222 
224  automatic_cleanup?
227  use_progress?
230 
231 }
232 
234 {
235  OnJobFailed("NetSchedule Queue is busy", ctx);
236 }
237 
238 const string kGridCgiForm = "<FORM METHOD=\"GET\" ACTION=\"<@SELF_URL@>\">\n"
239  "<@HIDDEN_FIELDS@>\n<@STAT_VIEW@>\n"
240  "</FORM>";
241 
243 {
244  SExceptionMessage exception_message = &GetRWConfig();
245 
246  // Given "CGI context", get access to its "HTTP request" and
247  // "HTTP response" sub-objects
248  //const CCgiRequest& request = ctx.GetRequest();
249  CCgiResponse& response = ctx.GetResponse();
250  m_Response = &response;
251 
252 
253  // Create a HTML page (using template HTML file "grid_cgi_sample.html")
254  unique_ptr<CHTMLPage> page;
255  try {
256  page.reset(new CHTMLPage(GetPageTitle(), GetPageTemplate()));
257  CHTMLText* stat_view = new CHTMLText(kGridCgiForm);
258  page->AddTagMap("VIEW", stat_view);
259  } catch (exception& e) {
260  ERR_POST_X(1, "Failed to create " << GetPageTitle()
261  << " HTML page: " << e.what());
262  return 2;
263  }
264  CGridCgiContext grid_ctx(*page, ctx);
265  grid_ctx.PullUpPersistentEntry("job_key");
266  grid_ctx.PullUpPersistentEntry("Cancel");
267  string job_key = grid_ctx.GetEntryValue("job_key");
268  try {
269  try {
270  OnBeginProcessRequest(grid_ctx);
271 
272  if (!job_key.empty()) {
273 
274  bool finished = x_CheckJobStatus(grid_ctx);
275  if (x_JobStopRequested(grid_ctx))
276  GetGridClient().CancelJob(job_key);
277 
278  if (finished)
279  grid_ctx.Clear();
280  else
281  RenderRefresh(*page, grid_ctx.GetSelfURL(), m_RefreshDelay);
282  }
283  else {
284  if (CollectParams(grid_ctx)) {
285  bool finished = false;
286  CGridClient& grid_client(GetGridClient());
287  // Submit a job
288  try {
289  PrepareJobData(grid_client);
290  job_key = grid_client.Submit();
291  grid_ctx.SetJobKey(job_key);
292 
293  unsigned long wait_time = m_FirstDelay*1000;
294  unsigned long sleep_time = 6;
295  unsigned long total_sleep_time = 0;
296  while ( total_sleep_time < wait_time) {
297  SleepMilliSec(sleep_time);
298  finished = x_CheckJobStatus(grid_ctx);
299  if (finished) break;
300  total_sleep_time += sleep_time;
301  sleep_time += sleep_time/3;
302  }
303  if( !finished ) {
304  OnJobSubmitted(grid_ctx);
305  RenderRefresh(*page, grid_ctx.GetSelfURL(), m_RefreshDelay);
306  }
307  }
308  catch (CNetScheduleException& ex) {
309  if (ex.GetErrCode() ==
311  OnQueueIsBusy(grid_ctx);
312  else
313  OnJobFailed(exception_message(ex.what()), grid_ctx);
314  finished = true;
315  }
316  catch (exception& ex) {
317  OnJobFailed(exception_message(ex.what()), grid_ctx);
318  finished = true;
319  }
320  if (finished)
321  grid_ctx.Clear();
322 
323  }
324  else {
325  ShowParamsPage(grid_ctx);
326  }
327  }
328  } // try
329  catch (/*CNetServiceException*/ exception& ex) {
330  OnJobFailed(exception_message(ex.what()), grid_ctx);
331  }
332  CHTMLPlainText* self_url =
333  new CHTMLPlainText(grid_ctx.GetSelfURL(),true);
334  page->AddTagMap("SELF_URL", self_url);
335  CHTMLPlainText* hidden_fields =
336  new CHTMLPlainText(grid_ctx.GetHiddenFields(),true);
337  page->AddTagMap("HIDDEN_FIELDS", hidden_fields);
338 
339  OnEndProcessRequest(grid_ctx);
340  } //try
341  catch (exception& e) {
342  ERR_POST_X(2, "Failed to populate " << GetPageTitle()
343  << " HTML page: " << e.what());
344  return 3;
345  }
346 
347  if (!grid_ctx.NeedRenderPage())
348  return 0;
349  // Compose and flush the resultant HTML page
350  try {
351  response.WriteHeader();
352  page->Print(response.out(), CNCBINode::eHTML);
353  } catch (exception& e) {
354  ERR_POST_X(3, "Failed to compose/send " << GetPageTitle()
355  <<" HTML page: " << e.what());
356  return 4;
357  }
358 
359 
360  return 0;
361 }
362 
364 {
365  if (JobStopRequested())
366  return true;
367  if (!ctx.GetEntryValue("Cancel").empty())
368  return true;
369  return false;
370 }
371 
373  const string& url,
374  int idelay)
375 {
376  if (idelay >= 0) {
377  CHTMLText* redirect = new CHTMLText(
378  "<META HTTP-EQUIV=Refresh "
379  "CONTENT=\"<@REDIRECT_DELAY@>; URL=<@REDIRECT_URL@>\">");
380  page.AddTagMap("REDIRECT", redirect);
381 
382  CHTMLPlainText* delay = new CHTMLPlainText(NStr::IntToString(idelay));
383  page.AddTagMap("REDIRECT_DELAY",delay);
384  }
385 
386  CHTMLPlainText* h_url = new CHTMLPlainText(url,true);
387  page.AddTagMap("REDIRECT_URL",h_url);
388  m_Response->SetHeaderValue("NCBI-RCGI-RetryURL", url);
389 }
390 
391 
393 {
394  string job_key = grid_ctx.GetEntryValue("job_key");
395  CGridClient& grid_client(GetGridClient());
396  grid_client.SetJobKey(job_key);
397 
399  status = grid_client.GetStatus();
400  grid_ctx.SetJobInput(grid_client.GetJobInput());
401  grid_ctx.SetJobOutput(grid_client.GetJobOutput());
402 
403  bool finished = false;
404  bool save_result = false;
405  grid_ctx.GetCGIContext().GetResponse().
406  SetHeaderValue("NCBI-RCGI-JobStatus", CNetScheduleAPI::StatusToString(status));
407  switch (status) {
409  // a job is done
410  OnJobDone(grid_client, grid_ctx);
411  save_result = true;
412  finished = true;
413  break;
415  // a job has failed
416  OnJobFailed(grid_client.GetErrorMessage(), grid_ctx);
417  finished = true;
418  break;
419 
421  // A job has been canceled
422  OnJobCanceled(grid_ctx);
423  finished = true;
424  break;
425 
427  // A lost job
428  //cerr << "|" << job_key << "|" << endl;
429  OnJobFailed("Job is not found.", grid_ctx);
430  finished = true;
431  break;
432 
434  // A job is in the Netscheduler's Queue
435  OnJobPending(grid_ctx);
436  break;
437 
439  // A job is being processed by a worker node
440  grid_ctx.SetJobProgressMessage(grid_client.GetProgressMessage());
441  OnJobRunning(grid_ctx);
442  break;
443 
444  default:
445  _ASSERT(0);
446  }
447  SetRequestId(job_key,save_result);
448  return finished;
449 }
450 
451 
452 
453 /////////////////////////////////////////////////////////////////////////////
454 
CCgiRequest::
Definition: ncbicgi.hpp:685
Grid CGI Context Context in which a request is processed.
Definition: grid_cgiapp.hpp:68
Grid Client (the submitter).
CHTMLPage –.
Definition: page.hpp:161
CNcbiRegistry –.
Definition: ncbireg.hpp:913
Client API for NetCache server.
Client API for NCBI NetSchedule server.
NetSchedule internal exception.
void erase(iterator pos)
Definition: map.hpp:167
container_type::const_iterator const_iterator
Definition: map.hpp:53
const_iterator begin() const
Definition: map.hpp:151
const_iterator end() const
Definition: map.hpp:152
void clear()
Definition: map.hpp:169
const_iterator find(const key_type &key) const
Definition: map.hpp:153
static CMemoryRegistry registry
Definition: cn3d_tools.cpp:81
The NCBI C++ standard methods for dealing with std::string.
CS_CONTEXT * ctx
Definition: t0006.c:12
#define true
Definition: bool.h:35
static DLIST_TYPE *DLIST_NAME() first(DLIST_LIST_TYPE *list)
Definition: dlist.tmpl.h:46
const string kGridCgiForm
NetSchedule Framework specs.
const CNcbiRegistry & GetConfig(void) const
Get the application's cached configuration parameters (read-only).
#define ITERATE(Type, Var, Cont)
ITERATE macro to sequence through container elements.
Definition: ncbimisc.hpp:815
CNcbiRegistry & GetRWConfig(void)
Get the application's cached configuration parameters, accessible for read-write for an application's...
const CCgiResponse & GetResponse(void) const
Definition: cgictx.hpp:388
const CCgiRequest & GetRequest(void) const
Definition: cgictx.hpp:374
void Init(void) override
This method is called on the CGI application initialization – before starting to process a HTTP reque...
void SetRequestId(const string &rid, bool is_done)
Definition: cgiapp.cpp:1412
const string & GetSelfURL(ESelfUrlPort) const
Using HTTP environment variables, compose the CGI's own URL as: SCHEME://SERVER_NAME[:SERVER_PORT]/SC...
Definition: cgictx.hpp:217
const TCgiEntries & GetEntries(void) const
Get a set of entries(decoded) received from the client.
Definition: ncbicgi.hpp:1181
static SIZE_TYPE ParseEntries(const string &str, TCgiEntries &entries)
Decode the URL-encoded(FORM or ISINDEX) string "str" into a set of entries <"name",...
Definition: ncbicgi.cpp:1530
void SetHeaderValue(const string &name, const string &value)
Definition: ncbicgir.cpp:151
CNcbiOstream & out(void) const
Get output stream. Throw exception if GetOutput() is NULL.
Definition: ncbicgir.cpp:257
const string & GetProperty(ECgiProp prop) const
Get value of a "standard" property (return empty string if not defined)
Definition: ncbicgi.cpp:1432
CNcbiOstream & WriteHeader(void) const
Write HTTP response header to the output stream.
Definition: ncbicgir.hpp:396
@ eCgi_QueryString
Definition: ncbicgi.hpp:399
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
Definition: ncbidiag.hpp:550
TErrCode GetErrCode(void) const
Get error code.
Definition: ncbiexpt.cpp:453
virtual const char * what(void) const noexcept
Standard report (includes full backlog).
Definition: ncbiexpt.cpp:342
virtual void AddTagMap(const string &name, BaseTagMapper *mapper)
Tag mappers.
Definition: page.cpp:289
@ eHTML
Definition: node.hpp:109
CCgiContext & m_CgiContext
TPersistedEntries m_PersistedEntries
CNcbiRegistry * m_Registry
Definition: cgi2rcgi.cpp:240
EJobStatus
Job status codes.
TCgiEntries m_ParsedQueryString
virtual void OnJobCanceled(CGridCgiContext &)
This method is called if job was canceled during its execution.
void SetJobProgressMessage(const string &msg)
CNetScheduleSubmitter GetSubmitter()
Create an instance of CNetScheduleSubmitter.
void SetCompleteResponse(CNcbiIstream &is)
virtual void PrepareJobData(CGridClient &grid_client)=0
This method is called when a job is ready to be send to a the queue.
void DefinePersistentEntry(const string &entry_name, const string &value)
Definition: cgi2rcgi.cpp:430
void PullUpPersistentEntry(const string &entry_name)
Save this entry as a cookie add it to serf url.
Definition: cgi2rcgi.cpp:405
void Clear()
Remove all persisted entries from cookie and self url.
Definition: cgi2rcgi.cpp:452
const string & GetJobInput()
Get a job's input sting.
void SetJobInput(const string &input)
bool x_JobStopRequested(const CGridCgiContext &) const
virtual void Init(void)
This method is called on the CGI application initialization – before starting to process a HTTP reque...
virtual void OnQueueIsBusy(CGridCgiContext &)
This method is call when a job couldn't be submitted because of NetSchedule queue is full.
const string & GetEntryValue(const string &entry_name) const
Get a value from a CGI request.
void SetJobKey(const string &job_key)
virtual int ProcessRequest(CCgiContext &ctx)
Do not override this method yourself! – it includes all the GRIDCGI specific machinery.
virtual void OnBeginProcessRequest(CGridCgiContext &)
This method is call at the very beginning of the request processing.
virtual bool JobStopRequested(void) const
When job is still running this method is called to check if cancel has been requested via the user in...
const string & GetErrorMessage()
If something bad has happened this method will return an explanation.
CNetScheduleAPI::EJobStatus GetStatus()
Get a job status.
virtual string GetPageTitle(void) const =0
Return page name.
CCgiContext & GetCGIContext()
Get CGI Context.
Definition: grid_cgiapp.hpp:98
virtual void OnJobPending(CGridCgiContext &)
This method is call when a job is in NetSchedule queue and is waiting for a worker node.
virtual void OnJobDone(CGridClient &grid_client, CGridCgiContext &ctx)=0
This method is call when a worker node finishes its job and result is ready to be retrieved.
bool x_CheckJobStatus(CGridCgiContext &grid_ctx)
unique_ptr< CGridClient > m_GridClient
virtual void ShowParamsPage(CGridCgiContext &ctx) const =0
Show a page with input data.
void InitGridClient()
Initialize grid client.
string GetSelfURL(void) const
Get Self URL.
Definition: grid_cgiapp.cpp:75
string GetProgressMessage()
Get a job interim message.
CCgiResponse * m_Response
static string StatusToString(EJobStatus status)
Printable status type.
virtual void OnJobRunning(CGridCgiContext &)
This method is call when a job is taken by a worker node to be processed.
bool NeedRenderPage() const
void RenderRefresh(CHTMLPage &page, const string &url, int delay)
virtual bool CollectParams(CGridCgiContext &ctx)=0
Collect parameters from HTML form If this method returns false that means that input parameters were ...
void CancelJob(const string &job_key)
Cancel Job.
Definition: grid_client.cpp:70
void SetJobKey(const string &job_key)
string Submit(const string &affinity=kEmptyStr)
Submit a job to the queue.
const string & GetJobKey(void) const
Get Current job key.
CGridClient & GetGridClient(void)
Get a Grid Client.
virtual void OnJobSubmitted(CGridCgiContext &)
This method is called just after a job has been submitted.
virtual void OnJobFailed(const string &, CGridCgiContext &)
This method is called when worker node reported a failure.
virtual string GetProgramVersion(void) const =0
Get program version (like: MyProgram v.
const string & GetJobOutput()
Get a job's output string.
void SetProgramVersion(const string &pv)
Set program version (like: MyProgram v.
virtual void OnEndProcessRequest(CGridCgiContext &)
This method is call at the very end of the request processing.
void SetJobOutput(const string &output)
CNetScheduleAPI m_NSClient
virtual string GetPageTemplate(void) const =0
Return a name of a file this HTML page template.
string GetHiddenFields() const
Definition: grid_cgiapp.cpp:97
virtual bool IsCachingNeeded(const CCgiRequest &request) const
CNetCacheAPI m_NetCacheAPI
CGridCgiContext(CHTMLPage &page, CCgiContext &ctx)
Definition: grid_cgiapp.cpp:63
@ eDone
Job is ready (computed successfully)
@ eCanceled
Explicitly canceled.
@ eRunning
Running on a worker node.
@ eJobNotFound
No such job.
@ ePending
Waiting for execution.
@ eFailed
Failed to run (execution timeout)
virtual bool GetBool(const string &section, const string &name, bool default_value, TFlags flags=0, EErrAction err_action=eThrow) const
Get boolean value of specified parameter name.
Definition: ncbireg.cpp:391
virtual int GetInt(const string &section, const string &name, int default_value, TFlags flags=0, EErrAction err_action=eThrow) const
Get integer value of specified parameter name.
Definition: ncbireg.cpp:362
virtual string GetString(const string &section, const string &name, const string &default_value, TFlags flags=0) const
Get the parameter string value.
Definition: ncbireg.cpp:321
@ eReturn
Return default value.
Definition: ncbireg.hpp:203
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
IO_PREFIX::istream CNcbiIstream
Portable alias for istream.
Definition: ncbistre.hpp:146
#define kEmptyStr
Definition: ncbistr.hpp:123
static int CompareNocase(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2)
Case-insensitive compare of a substring with another string.
Definition: ncbistr.cpp:219
static string IntToString(int value, TNumToStringFlags flags=0, int base=10)
Convert int to string.
Definition: ncbistr.hpp:5084
static string URLEncode(const CTempString str, EUrlEncode flag=eUrlEnc_SkipMarkChars)
URL-encode string.
Definition: ncbistr.cpp:6062
const GenericPointer< typename T::ValueType > T2 value
Definition: pointer.h:1227
void SleepMilliSec(unsigned long ml_sec, EInterruptOnSignal onsignal=eRestartOnSignal)
SExceptionMessage(CNcbiRegistry *registry=nullptr)
Definition: grid_cgiapp.cpp:48
string operator()(const string &what)
Definition: grid_cgiapp.cpp:50
#define _ASSERT
static wxAcceleratorEntry entries[3]
Modified on Fri May 24 14:57:04 2024 by modify_doxy.py rev. 669887