NCBI C++ ToolKit
ncbi_dispd.c
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: ncbi_dispd.c 98190 2022-10-09 00:18:19Z lavr $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Anton Lavrentiev
27  *
28  * File Description:
29  * Low-level API to resolve an NCBI service name to server meta-addresses
30  * with the use of the NCBI network dispatcher (DISPD).
31  *
32  */
33 
34 #include "ncbi_ansi_ext.h"
35 #include "ncbi_comm.h"
36 #include "ncbi_dispd.h"
37 #include "ncbi_lb.h"
38 #include "ncbi_priv.h"
40 #include <ctype.h>
41 #include <stdlib.h>
42 
43 #ifdef fabs
44 # undef fabs
45 #endif /*fabs*/
46 #define fabs(v) ((v) < 0.0 ? -(v) : (v))
47 
48 #define NCBI_USE_ERRCODE_X Connect_LBSM
49 
50 /* Lower bound of up-to-date/out-of-date ratio */
51 #define DISPD_STALE_RATIO_OK 0.8
52 /* Default rate increase 20% if svc runs locally */
53 #define DISPD_LOCAL_BONUS 1.2
54 
55 
56 #ifdef __cplusplus
57 extern "C" {
58 #endif /*__cplusplus*/
59 
61 static int/*bool*/ s_Update (SERV_ITER, const char*, int);
62 static void s_Reset (SERV_ITER);
63 static void s_Close (SERV_ITER);
64 
65 static const SSERV_VTable kDispdOp = {
66  s_GetNextInfo, 0/*Feedback*/, s_Update, s_Reset, s_Close, "DISPD"
67 };
68 
69 #ifdef __cplusplus
70 } /* extern "C" */
71 #endif /*__cplusplus*/
72 
73 
74 struct SDISPD_Data {
75  int code; /* last HTTP code */
76  short/*bool*/ fail; /* no more connects */
77  short/*bool*/ eof; /* no more resolves */
80  size_t n_cand;
81  size_t a_cand;
82  size_t n_skip;
83 };
84 
85 
86 static int/*bool*/ s_AddServerInfo(struct SDISPD_Data* data, SSERV_Info* info)
87 {
88  size_t n;
89  const char* name = SERV_NameOfInfo(info);
90  /* First check that the new server info updates an existing one */
91  for (n = 0; n < data->n_cand; ++n) {
92  if (strcasecmp(name, SERV_NameOfInfo(data->cand[n].info)) == 0
93  && SERV_EqualInfo(info, data->cand[n].info)) {
94  /* Replace older version */
95  free((void*) data->cand[n].info);
96  data->cand[n].info = info;
97  data->cand[n].status = info->rate;
98  return 1/*success*/;
99  }
100  }
101  /* Next, add new service to the list */
102  if (data->n_cand == data->a_cand) {
103  SLB_Candidate* temp;
104  n = data->a_cand + 10;
105  temp = (SLB_Candidate*)(data->cand
106  ? realloc(data->cand, n * sizeof(*temp))
107  : malloc ( n * sizeof(*temp)));
108  if (!temp)
109  return 0/*failure*/;
110  data->cand = temp;
111  data->a_cand = n;
112  }
113  data->cand[data->n_cand].info = info;
114  data->cand[data->n_cand].status = info->rate;
115  data->n_cand++;
116  return 1/*success*/;
117 }
118 
119 
120 #ifdef __cplusplus
121 extern "C" {
122  static EHTTP_HeaderParse s_ParseHeader(const char*, void*, int);
123 }
124 #endif /*__cplusplus*/
125 
126 static EHTTP_HeaderParse s_ParseHeader(const char* header,
127  void* user_data,
128  int server_error)
129 {
130  SERV_ITER iter = (SERV_ITER) user_data;
131  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
132  if (server_error) {
133  if (server_error == 400 || server_error == 403 || server_error == 404)
134  data->fail = 1/*true*/;
135  data->code = server_error;
136  } else if (sscanf(header, "%*s %d", &data->code) < 1) {
137  data->eof = 1/*true*/;
138  data->code = -1/*failure*/;
139  return eHTTP_HeaderError;
140  }
141  /* a check for empty document */
142  if (!SERV_Update(iter, header, server_error) || data->code == 204)
143  data->eof = 1/*true*/;
144  return eHTTP_HeaderSuccess;
145 }
146 
147 
148 #ifdef __cplusplus
149 extern "C" {
150  static int s_Adjust(SConnNetInfo*, void*, unsigned int);
151 }
152 #endif /*__cplusplus*/
153 
154 /*ARGSUSED*/
155 static int/*bool*/ s_Adjust(SConnNetInfo* net_info,
156  void* iter,
157  unsigned int unused)
158 {
159  struct SDISPD_Data* data = (struct SDISPD_Data*)((SERV_ITER) iter)->data;
160  return data->fail ? 0/*no more tries*/ : 1/*may try again*/;
161 }
162 
163 
164 static void s_Resolve(SERV_ITER iter)
165 {
166  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
167  SConnNetInfo* net_info = data->net_info;
168  EIO_Status status = eIO_Success;
169  CONNECTOR c = 0;
170  CONN conn = 0;
171  char* s;
172 
173  assert(!(data->eof | data->fail));
174  assert(!!net_info->firewall == !!(iter->types & fSERV_Firewall));
175  assert(!!net_info->stateless == !!(iter->types & fSERV_Stateless));
176 
177  /* Obtain additional header information */
178  if ((!(s = SERV_Print(iter, 0/*net_info*/, 0))
180  &&
182  iter->ok_down && iter->ok_suppressed
183  ? "Dispatch-Mode: PROMISCUOUS\r\n"
184  : iter->ok_down
185  ? "Dispatch-Mode: OK_DOWN\r\n"
186  : iter->ok_suppressed
187  ? "Dispatch-Mode: OK_SUPPRESSED\r\n"
188  : "Dispatch-Mode: INFORMATION_ONLY\r\n")
189  &&
191  ? "Client-Mode: REVERSE_DNS\r\n"
192  : !net_info->stateless
193  ? "Client-Mode: STATEFUL_CAPABLE\r\n"
194  : "Client-Mode: STATELESS_ONLY\r\n")) {
196  iter/*user_data*/, s_Adjust, 0/*cleanup*/);
197  }
198  if (s) {
200  free(s);
201  }
202  if (c && (status = CONN_Create(c, &conn)) == eIO_Success
203  /* send all the HTTP data... */
204  && (status = CONN_Flush(conn)) == eIO_Success) {
205  /* ...then trigger the header callback */
206  CONN_Close(conn);
207  } else {
208  const char* url = ConnNetInfo_URL(net_info);
210  ("[%s] Unable to create %s network dispatcher%s%s%s: %s",
211  iter->name, c ? "connection to" : "connector for",
212  url ? " at \"" : "", url ? url : "", &"\""[!url],
213  IO_StatusStr(c ? status : eIO_Unknown)));
214  if (url)
215  free((void*) url);
216  if (conn)
217  CONN_Close(conn);
218  else if (c && c->destroy)
219  c->destroy(c);
220  }
221 }
222 
223 
224 static int/*bool*/ s_Update(SERV_ITER iter, const char* text, int code)
225 {
226  static const char server_info[] = "Server-Info-";
227  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
228  int/*bool*/ failure;
229 
230  if (strncasecmp(text, server_info, sizeof(server_info) - 1) == 0
231  && isdigit((unsigned char) text[sizeof(server_info) - 1])) {
232  const char* name;
233  SSERV_Info* info;
234  unsigned int d1;
235  char* s;
236  int d2;
237 
238  text += sizeof(server_info) - 1;
239  if (sscanf(text, "%u: %n", &d1, &d2) < 1 || d1 < 1)
240  return 0/*not updated*/;
241  if (iter->ismask || iter->reverse_dns) {
242  char* c;
243  if (!(s = strdup(text + d2)))
244  return 0/*not updated*/;
245  name = s;
246  while (*name && isspace((unsigned char)(*name)))
247  ++name;
248  if (!*name) {
249  free(s);
250  return 0/*not updated*/;
251  }
252  for (c = s + (name - s); *c; ++c) {
253  if (isspace((unsigned char)(*c)))
254  break;
255  }
256  *c++ = '\0';
257  d2 += (int)(c - s);
258  } else {
259  s = 0;
260  name = "";
261  }
262  info = SERV_ReadInfoEx(text + d2, name, 0);
263  if (s)
264  free(s);
265  if (info) {
266  if (info->time != NCBI_TIME_INFINITE)
267  info->time += iter->time; /* expiration time now */
268  if (s_AddServerInfo(data, info))
269  return 1/*updated*/;
270  free(info);
271  }
272  } else if (((failure = strncasecmp(text, HTTP_DISP_FAILURES,
273  sizeof(HTTP_DISP_FAILURES) - 1) == 0)
275  sizeof(HTTP_DISP_MESSAGES) - 1) == 0) &&
276  isspace((unsigned char) text[sizeof(HTTP_DISP_FAILURES) - 1])) {
277  assert(sizeof(HTTP_DISP_FAILURES) == sizeof(HTTP_DISP_MESSAGES));
278 #if defined(_DEBUG) && !defined(NDEBUG)
279  if (data->net_info->debug_printout) {
280  text += sizeof(HTTP_DISP_FAILURES) - 1;
281  while (*text && isspace((unsigned char)(*text)))
282  ++text;
284  ("[%s] %s", iter->name, text));
285  }
286 #endif /*_DEBUG && !NDEBUG*/
287  if (failure) {
288  if (code)
289  data->fail = 1/*true*/;
290  return 1/*updated*/;
291  }
292  /* NB: a mere message does not constitute an update */
293  }
294 
295  return 0/*not updated*/;
296 }
297 
298 
299 static int/*bool*/ s_IsUpdateNeeded(struct SDISPD_Data *data, TNCBI_Time now)
300 {
301  double status = 0.0, total = 0.0;
302 
303  if (data->n_cand) {
304  size_t i = 0;
305  while (i < data->n_cand) {
306  const SSERV_Info* info = data->cand[i].info;
307  total += fabs(info->rate);
308  if (info->time < now) {
309  if (i < --data->n_cand) {
310  memmove(data->cand + i, data->cand + i + 1,
311  sizeof(*data->cand)*(data->n_cand - i));
312  }
313  free((void*) info);
314  } else {
315  status += fabs(info->rate);
316  ++i;
317  }
318  }
319  }
320  return total == 0.0 ? 1/*true*/ : status / total < DISPD_STALE_RATIO_OK;
321 }
322 
323 
324 static SLB_Candidate* s_GetCandidate(void* user_data, size_t n)
325 {
326  struct SDISPD_Data* data = (struct SDISPD_Data*) user_data;
327  return n < data->n_cand ? &data->cand[n] : 0;
328 }
329 
330 
331 static SSERV_Info* s_GetNextInfo(SERV_ITER iter, HOST_INFO* host_info)
332 {
333  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
334  SSERV_Info* info;
335  size_t n;
336 
337  assert(data);
338 
339  if (!data->fail && iter->n_skip < data->n_skip)
340  data->eof = 0/*false*/;
341  data->n_skip = iter->n_skip;
342 
343  if (s_IsUpdateNeeded(data, iter->time)) {
344  if (!(data->eof | data->fail))
345  s_Resolve(iter);
346  if (!data->n_cand)
347  return 0;
348  }
349 
351  info = (SSERV_Info*) data->cand[n].info;
352  info->rate = data->cand[n].status;
353  if (n < --data->n_cand) {
354  memmove(data->cand + n, data->cand + n + 1,
355  (data->n_cand - n) * sizeof(*data->cand));
356  }
357 
358  if (host_info)
359  *host_info = 0;
360  data->n_skip++;
361 
362  return info;
363 }
364 
365 
366 static void s_Reset(SERV_ITER iter)
367 {
368  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
369  data->eof = data->fail = 0/*false*/;
370  if (data->cand) {
371  size_t i;
372  assert(data->a_cand && data->n_cand <= data->a_cand);
373  for (i = 0; i < data->n_cand; ++i)
374  free((void*) data->cand[i].info);
375  data->n_cand = 0;
376  }
377  data->n_skip = iter->n_skip;
378 }
379 
380 
381 static void s_Close(SERV_ITER iter)
382 {
383  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
384  iter->data = 0;
385  assert(data && !data->n_cand); /*s_Reset() had to be called before*/
386  if (data->cand)
387  free(data->cand);
388  ConnNetInfo_Destroy(data->net_info);
389  free(data);
390 }
391 
392 
393 /***********************************************************************
394  * EXTERNAL
395  ***********************************************************************/
396 
398  const SConnNetInfo* net_info,
399  SSERV_Info** info)
400 {
401  struct SDISPD_Data* data;
402 
403  assert(iter && net_info && !iter->data && !iter->op);
404  if (!(data = (struct SDISPD_Data*) calloc(1, sizeof(*data))))
405  return 0;
406  iter->data = data;
407 
408  data->net_info = ConnNetInfo_Clone(net_info);
409  if (!ConnNetInfo_SetupStandardArgs(data->net_info, iter->name)) {
410  s_Close(iter);
411  return 0;
412  }
413  data->net_info->scheme = eURL_Https;
414  data->net_info->req_method = eReqMethod_Get;
415  if (iter->types & fSERV_Stateless)
416  data->net_info->stateless = 1/*true*/;
417  if ((iter->types & fSERV_Firewall) && !data->net_info->firewall)
418  data->net_info->firewall = eFWMode_Adaptive;
419 
421  "User-Agent: NCBIServiceDispatcher/"
423 #ifdef NCBI_CXX_TOOLKIT
424  " (CXX Toolkit)"
425 #else
426  " (C Toolkit)"
427 #endif /*NCBI_CXX_TOOLKIT*/
428  "\r\n");
429 
430  if (g_NCBI_ConnectRandomSeed == 0) {
433  }
434 
435  data->n_skip = iter->n_skip;
436  iter->op = &kDispdOp; /*SERV_Update() [from HTTP callback] expects*/
437  s_Resolve(iter);
438  iter->op = 0;
439 
440  if (!data->n_cand && (data->fail
441  || !(data->net_info->stateless &&
442  data->net_info->firewall))) {
444  ("SERV_DISPD_Open(\"%s\"): Service not found", iter->name));
445  s_Reset(iter);
446  s_Close(iter);
447  return 0;
448  }
449 
450  /* call GetNextInfo subsequently if info is actually needed */
451  if (info)
452  *info = 0;
453  return &kDispdOp;
454 }
static CS_CONNECTION * conn
Definition: ct_dynamic.c:25
static int failure
Definition: t0019.c:11
char data[12]
Definition: iconv.c:80
CONNECTOR HTTP_CreateConnectorEx(const SConnNetInfo *net_info, THTTP_Flags flags, FHTTP_ParseHeader parse_header, void *user_data, FHTTP_Adjust adjust, FHTTP_Cleanup cleanup)
Create new CONNECTOR structure to hit the specified URL using HTTP with either POST / GET (or ANY) me...
EIO_Status CONN_Flush(CONN conn)
Explicitly flush connection from any pending data written by CONN_Write().
EIO_Status CONN_Create(CONNECTOR connector, CONN *conn)
Same as CONN_CreateEx() called with 0 in the "flags" parameter.
EHTTP_HeaderParse
The extended version HTTP_CreateConnectorEx() is able to track the HTTP response chain and also chang...
EIO_Status CONN_Close(CONN conn)
Close the connection and destroy all relevant internal data.
@ fHTTP_Flushable
Connector will really flush on Flush()
@ eHTTP_HeaderSuccess
Parse succeeded, retain server status.
@ eHTTP_HeaderError
Parse failed, treat as a server error.
struct SSERV_IterTag * SERV_ITER
Iterator through the servers.
Definition: ncbi_service.h:66
int SERV_EqualInfo(const SSERV_Info *info1, const SSERV_Info *info2)
@ fSERV_Firewall
@ fSERV_Stateless
Stateless servers only.
Definition: ncbi_service.h:81
EBFWMode firewall
int ConnNetInfo_ExtendUserHeader(SConnNetInfo *net_info, const char *header)
unsigned int TNCBI_Time
Definition: ncbi_types.h:145
int ConnNetInfo_SetupStandardArgs(SConnNetInfo *net_info, const char *service)
int ConnNetInfo_OverrideUserHeader(SConnNetInfo *net_info, const char *header)
SConnNetInfo * ConnNetInfo_Clone(const SConnNetInfo *net_info)
EIO_Status
I/O status.
Definition: ncbi_core.h:132
unsigned stateless
#define NCBI_TIME_INFINITE
Definition: ncbi_types.h:147
char * ConnNetInfo_URL(const SConnNetInfo *net_info)
const char * IO_StatusStr(EIO_Status status)
Get the text form of an enum status value.
Definition: ncbi_core.c:56
void ConnNetInfo_DeleteUserHeader(SConnNetInfo *net_info, const char *header)
void ConnNetInfo_Destroy(SConnNetInfo *net_info)
@ eLOG_Error
Definition: ncbi_core.h:297
@ eLOG_Note
Definition: ncbi_core.h:294
@ eLOG_Warning
Definition: ncbi_core.h:296
@ eLOG_Trace
Definition: ncbi_core.h:293
@ eURL_Https
@ eIO_Success
everything is fine, no error occurred
Definition: ncbi_core.h:133
@ eIO_Unknown
unknown I/O error (likely fatal but can retry)
Definition: ncbi_core.h:139
@ eReqMethod_Get
@ eFWMode_Adaptive
Regular firewall ports first, then fallback.
unsigned int
A callback function used to compare two keys in a database.
Definition: types.hpp:1210
int i
if(yy_accept[yy_current_state])
yy_size_t n
static const CS_INT unused
Definition: long_binary.c:20
static void text(MDB_val *v)
Definition: mdb_dump.c:62
static MDB_envinfo info
Definition: mdb_load.c:37
#define strdup
Definition: ncbi_ansi_ext.h:70
#define strncasecmp
#define strcasecmp
#define NCBI_DISP_VERSION
Definition: ncbi_comm.h:38
#define HTTP_DISP_FAILURES
Definition: ncbi_comm.h:40
#define HTTP_DISP_MESSAGES
Definition: ncbi_comm.h:41
#define DISPD_LOCAL_BONUS
Definition: ncbi_dispd.c:53
static int s_IsUpdateNeeded(struct SDISPD_Data *data, TNCBI_Time now)
Definition: ncbi_dispd.c:299
static SLB_Candidate * s_GetCandidate(void *user_data, size_t n)
Definition: ncbi_dispd.c:324
#define fabs(v)
Definition: ncbi_dispd.c:46
static void s_Close(SERV_ITER)
Definition: ncbi_dispd.c:381
static void s_Reset(SERV_ITER)
Definition: ncbi_dispd.c:366
#define DISPD_STALE_RATIO_OK
Definition: ncbi_dispd.c:51
static int s_Adjust(SConnNetInfo *net_info, void *iter, unsigned int unused)
Definition: ncbi_dispd.c:155
static SSERV_Info * s_GetNextInfo(SERV_ITER, HOST_INFO *)
Definition: ncbi_dispd.c:331
static int s_Update(SERV_ITER, const char *, int)
Definition: ncbi_dispd.c:224
const SSERV_VTable * SERV_DISPD_Open(SERV_ITER iter, const SConnNetInfo *net_info, SSERV_Info **info)
Definition: ncbi_dispd.c:397
static EHTTP_HeaderParse s_ParseHeader(const char *header, void *user_data, int server_error)
Definition: ncbi_dispd.c:126
static int s_AddServerInfo(struct SDISPD_Data *data, SSERV_Info *info)
Definition: ncbi_dispd.c:86
static void s_Resolve(SERV_ITER iter)
Definition: ncbi_dispd.c:164
static const SSERV_VTable kDispdOp
Definition: ncbi_dispd.c:65
size_t LB_Select(SERV_ITER iter, void *data, FGetCandidate get_candidate, double bonus)
Definition: ncbi_lb.c:62
unsigned int g_NCBI_ConnectRandomSeed
Definition: ncbi_priv.c:51
#define CORE_LOGF_X(subcode, level, fmt_args)
Definition: ncbi_priv.h:150
#define NCBI_CONNECT_SRAND_ADDEND
Definition: ncbi_priv.h:343
#define CORE_LOGF(level, fmt_args)
Definition: ncbi_priv.h:158
SSERV_Info * SERV_ReadInfoEx(const char *str, const char *name, int lazy)
const char * SERV_NameOfInfo(const SSERV_Info *info)
int SERV_Update(SERV_ITER iter, const char *text, int code)
char * SERV_Print(SERV_ITER iter, const SConnNetInfo *net_info, int but_last)
#define NCBI_CXX_TOOLKIT
Definition: ncbiconf_msvc.h:16
int isspace(Uchar c)
Definition: ncbictype.hpp:69
int isdigit(Uchar c)
Definition: ncbictype.hpp:64
#define memmove(a, b, c)
#define assert(x)
Definition: srv_diag.hpp:58
Connector specification.
short eof
Definition: ncbi_dispd.c:77
SConnNetInfo * net_info
Definition: ncbi_dispd.c:78
size_t n_skip
Definition: ncbi_dispd.c:82
size_t a_cand
Definition: ncbi_dispd.c:81
SLB_Candidate * cand
Definition: ncbi_dispd.c:79
size_t n_cand
Definition: ncbi_dispd.c:80
short fail
Definition: ncbi_dispd.c:76
unsigned ok_suppressed
Definition: ncbi_servicep.h:99
unsigned ok_down
Definition: ncbi_servicep.h:96
const char * name
Definition: ncbi_servicep.h:89
unsigned ismask
Definition: ncbi_servicep.h:95
const SSERV_VTable * op
unsigned reverse_dns
TSERV_TypeOnly types
Definition: ncbi_servicep.h:94
TNCBI_Time time
Definition: ncbi_servicep.h:91
Definition: inftrees.h:24
void free(voidpf ptr)
voidp malloc(uInt size)
voidp calloc(uInt items, uInt size)
Modified on Tue May 28 05:54:26 2024 by modify_doxy.py rev. 669887