NCBI C++ ToolKit
ncbi_http_connector.c
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: ncbi_http_connector.c 101304 2023-11-28 18:43:41Z lavr $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Anton Lavrentiev, Denis Vakatov
27  *
28  * File Description:
29  * Implement CONNECTOR for the HTTP-based network connection
30  *
31  * See in "ncbi_connector.h" for the detailed specification of the underlying
32  * connector("CONNECTOR", "SConnectorTag") methods and structures.
33  *
34  */
35 
36 #include "ncbi_ansi_ext.h"
37 #include "ncbi_comm.h"
38 #include "ncbi_priv.h"
39 #include "ncbi_servicep.h"
40 #include <connect/ncbi_base64.h>
42 #include <ctype.h>
43 #include <errno.h>
44 #include <stdlib.h>
45 
46 #define NCBI_USE_ERRCODE_X Connect_HTTP
47 
48 
49 #define HTTP_SOAK_READ_SIZE 16384
50 
51 
52 /***********************************************************************
53  * INTERNAL -- Auxiliary types and static functions
54  ***********************************************************************/
55 
56 
57 /* Connection states, see table below
58  */
59 enum EConnState {
65  eCS_DoneBody = 5, /* NB: |eCS_ReadBody */
66  eCS_Discard = 7, /* NB: |eCS_DoneBody */
67  eCS_Eom = 0xF /* NB: |eCS_Discard */
68 };
69 typedef unsigned EBConnState; /* packed EConnState */
70 
71 
72 /* Whether the connector is allowed to connect
73  * NB: In order to be able to connect, fCC_Once must be set.
74  */
76  fCC_None, /* 0 */
77  fCC_Once, /* 1 */
78  fCC_Unlimited /* 0|2 */
79 };
80 typedef unsigned EBCanConnect; /* packed ECanConnect */
81 
82 typedef unsigned EBSwitch; /* packed ESwitch */
83 
84 typedef enum {
85  eEM_Drop, /* 0 */
86  eEM_Wait, /* 1 */
87  eEM_Read, /* 2 */
88  eEM_Flush /* 1|2 */
90 
91 
92 typedef enum {
93  eRetry_None = 0, /* 0 */
94  eRetry_Unused = 1, /* yet */
95  eRetry_Redirect = 2, /* 2 */
96  eRetry_Redirect303 = 3, /* 2|1 */
97  eRetry_Authenticate = 4, /* 4 */
98  eRetry_ProxyAuthenticate = 5 /* 4|1 */
100 
101 typedef struct {
103  const char* data;
104 } SRetry;
105 
106 
107 /* All internal data necessary to perform the (re)connect and I/O.
108  *
109  * The following connection states are defined: | sock?
110  * --------------+--------------------------------------------------+---------
111  * NotInitiated | HTTP request needs to be (re-)initiated |
112  * WriteRequest | HTTP request body is being written | != NULL
113  * FlushRequest | HTTP request is being completed and flushed out | != NULL
114  * ReadHeader | HTTP response header is being read | != NULL
115  * ReadBody | HTTP response body is being read | != NULL
116  * DoneBody | HTTP body is all read out of the connection | != NULL
117  * Discard | HTTP data (if any) to be discarded | != NULL
118  * Eom | HTTP message has been completed (end of message) |
119  * --------------+--------------------------------------------------+---------
120  */
121 typedef struct {
122  SConnNetInfo* net_info; /* network configuration parameters */
123  FHTTP_ParseHeader parse_header; /* callback to parse HTTP reply header */
124  void* user_data; /* user data handle for callbacks (CB) */
125  FHTTP_Adjust adjust; /* on-the-fly net_info adjustment CB */
126  FHTTP_Cleanup cleanup; /* cleanup callback */
127 
128  THTTP_Flags flags; /* as passed to constructor */
129  EBSwitch unsafe_redir:2; /* if unsafe redirects are allowed */
130  EBSwitch error_header:2; /* only err.HTTP header on SOME debug */
131  EBCanConnect can_connect:2; /* whether more connections permitted */
132  EBConnState conn_state:4; /* connection state per table above */
133  unsigned auth_done:1; /* website authorization sent */
134  unsigned proxy_auth_done:1; /* proxy authorization sent */
135  unsigned skip_host:1; /* do *not* add the "Host:" header tag */
136  unsigned keepalive:1; /* keep-alive connection */
137  unsigned chunked:1; /* if writing/reading chunked, HTTP/1.1*/
138  unsigned entity:1; /* if there's entity payload (body)/1.1*/
139  unsigned reused:1; /* if connection was re-used */
140  unsigned retry:1; /* if the request is being re-tried */
141  unsigned reserved:14;
142  unsigned char unused[3];
143  unsigned char minor_fault; /* incr each minor failure since major */
144  unsigned short major_fault; /* incr each major failure since open */
145  unsigned short http_code; /* last HTTP response code */
146 
147  SOCK sock; /* socket; NULL if not connected */
148  const STimeout* o_timeout; /* NULL(infinite), dflt or ptr to next */
149  STimeout oo_timeout; /* storage for (finite) open timeout */
150  const STimeout* w_timeout; /* NULL(infinite), dflt or ptr to next */
151  STimeout ww_timeout; /* storage for a (finite) write tmo */
152 
153  BUF http; /* storage for HTTP reply */
154  BUF r_buf; /* storage to accumulate input data */
155  BUF w_buf; /* storage to accumulate output data */
156  size_t w_len; /* pending message body size */
157 
158  TNCBI_BigCount expected; /* expected to receive before end/EOF */
159  TNCBI_BigCount received; /* actually received so far */
161 
162 
163 static const char kHttpHostTag[] = "Host: ";
164 static const STimeout kZeroTimeout = {0, 0};
165 
166 
167 /* NCBI messaging support */
168 static int s_MessageIssued = 0;
170 
171 
172 static int/*bool*/ x_UnsafeRedirectOK(SHttpConnector* uuu)
173 {
174  if (uuu->unsafe_redir == eDefault) {
175  if (!(uuu->flags & fHTTP_UnsafeRedirects)) {
176  char val[32];
178  "HTTP_UNSAFE_REDIRECTS",
179  val, sizeof(val), 0);
181  } else
182  uuu->unsafe_redir = eOn;
183  }
184  return uuu->unsafe_redir == eOn ? 1/*true*/ : 0/*false*/;
185 }
186 
187 
188 typedef enum {
189  /* Negative code: NOP; positive: Error */
190  eHTTP_AuthMissing = -3, /* No auth info available */
191  eHTTP_AuthUnsafe = -2, /* Auth would be unsafe */
192  eHTTP_AuthDone = -1, /* Auth already sent */
193  eHTTP_AuthOK = 0, /* Success */
194  eHTTP_AuthError = 1, /* Auth can't/doesn't work */
195  eHTTP_AuthIllegal = 3, /* Auth won't work now */
196  eHTTP_AuthNotSupp = 2, /* Unknown auth parameters */
197  eHTTP_AuthConnect = 4 /* Auth with CONNECT */
199 
200 
202  ERetry auth,
203  int/*bool*/ retry)
204 {
205  static const char kAuthTagTemplate[] = "Proxy-Authorization: Basic ";
206  char buf[80 + (CONN_USER_LEN + CONN_PASS_LEN)*3], *s;
207  size_t taglen, userlen, passlen, len, n;
208  const char *tag, *user, *pass;
209 
210  switch (auth) {
211  case eRetry_Authenticate:
212  if (uuu->auth_done)
213  return eHTTP_AuthDone;
214  tag = kAuthTagTemplate + 6;
215  taglen = sizeof(kAuthTagTemplate) - 7;
216  user = uuu->net_info->user;
217  pass = uuu->net_info->pass;
218  break;
220  if (uuu->proxy_auth_done)
221  return eHTTP_AuthDone;
222  if (!uuu->net_info->http_proxy_host[0] ||
223  !uuu->net_info->http_proxy_port) {
224  return retry ? eHTTP_AuthError : eHTTP_AuthMissing;
225  }
226  tag = kAuthTagTemplate;
227  taglen = sizeof(kAuthTagTemplate) - 1;
228  user = uuu->net_info->http_proxy_user;
229  pass = uuu->net_info->http_proxy_pass;
230  break;
231  default:
232  assert(0);
233  return eHTTP_AuthError;
234  }
235  assert(tag && user && pass);
236  if (!*user)
237  return eHTTP_AuthMissing;
238  if (retry && uuu->entity)
239  return eHTTP_AuthIllegal;
240  if (auth == eRetry_Authenticate) {
241  if (uuu->net_info->scheme != eURL_Https
242  && !x_UnsafeRedirectOK(uuu)) {
243  return eHTTP_AuthUnsafe;
244  }
245  uuu->auth_done = 1/*true*/;
246  } else
247  uuu->proxy_auth_done = 1/*true*/;
248  userlen = strlen(user);
249  passlen = strlen(pass);
250  s = buf + sizeof(buf) - passlen;
251  if (passlen > 2 && pass[0] == '[' && pass[passlen - 1] == ']') {
252  if (BASE64_Decode(pass + 1, passlen - 2, &len, s, passlen, &n)
253  && len == passlen - 2) {
254  len = n;
255  } else
256  len = 0;
257  } else
258  len = 0;
259  s -= userlen;
260  memcpy(--s, user, userlen);
261  s[userlen++] = ':';
262  if (!len) {
263  memcpy(s + userlen, pass, passlen);
264  len = userlen + passlen;
265  } else
266  len += userlen;
267  /* usable room */
268  userlen = (size_t)(s - buf);
269  assert(userlen > taglen);
270  userlen -= taglen + 1;
271  passlen = 0;
272  BASE64_Encode(s, len, &n, buf + taglen, userlen, &passlen, &passlen/*0*/);
273  if (len != n || buf[taglen + passlen])
274  return eHTTP_AuthError;
275  memcpy(buf, tag, taglen);
277 }
278 
279 
280 #ifdef __GNUC__
281 inline
282 #endif /*__GNUC__*/
283 static int/*bool*/ x_SameScheme(EBURLScheme scheme1, EBURLScheme scheme2)
284 {
285  if (!scheme1)
286  scheme1 = eURL_Http;
287  if (!scheme2)
288  scheme2 = eURL_Http;
289  return scheme1 == scheme2 ? 1/*true*/ : 0/*false*/;
290 }
291 
292 
293 static int/*bool*/ x_SameHost(const char* host1, const char* host2)
294 {
295  char buf1[CONN_HOST_LEN+1], buf2[CONN_HOST_LEN+1];
296  unsigned int ip1, ip2;
297 
298  if (strcasecmp(host1, host2) == 0)
299  return 1/*true*/;
300  if (!(ip1 = SOCK_gethostbyname(host1)) || ip1 == (unsigned int)(-1))
301  return 0/*false*/;
302  if (!(ip2 = SOCK_gethostbyname(host2)) || ip2 == (unsigned int)(-1))
303  return 0/*false*/;
304  if (ip1 == ip2)
305  return 1/*true*/;
306  SOCK_gethostbyaddr(ip1, buf1, sizeof(buf1));
307  SOCK_gethostbyaddr(ip2, buf2, sizeof(buf2));
308  return *buf1 && strcasecmp(buf1, buf2) == 0 ? 1/*true*/ : 0/*false*/;
309 }
310 
311 
312 static unsigned short x_PortForScheme(unsigned port, EBURLScheme scheme)
313 {
314  if (port)
315  return port;
316  switch (scheme) {
317  case eURL_Http:
318  break;
319  case eURL_Https:
320  return CONN_PORT_HTTPS;
321  default:
322  assert(!scheme);
323  break;
324  }
325  return CONN_PORT_HTTP;
326 }
327 
328 
329 #ifdef __GNUC__
330 inline
331 #endif /*__GNUC__*/
332 static int/*bool*/ x_SamePort(unsigned short port1, EBURLScheme scheme1,
333  unsigned short port2, EBURLScheme scheme2)
334 {
335  return x_PortForScheme(port1, scheme1) == x_PortForScheme(port2, scheme2)
336  ? 1/*true*/ : 0/*false*/;
337 }
338 
339 
340 static int/*bool*/ s_CallAdjust(SHttpConnector* uuu, unsigned int arg)
341 {
342  int retval;
344  if (!net_info)
345  return 0/*failure*/;
346  retval = uuu->adjust(uuu->net_info, uuu->user_data, arg);
347  if (retval/*advisory of no change if < 0 but we don't trust it :-)*/) {
348  int same_host = -1/*undef*/;
349  if (uuu->sock) {
350  int/*bool*/ close = 0/*false*/;
352  net_info->http_proxy_host)
353  || (uuu->net_info->http_proxy_host[0] &&
354  uuu->net_info->http_proxy_port !=
355  net_info->http_proxy_port)){
356  close = 1/*true*/;
357  } else if (net_info->http_proxy_host[0] &&
358  net_info->http_proxy_port) {
359  if (net_info->scheme == eURL_Https) {
360  if (uuu->net_info->scheme != eURL_Https)
361  close = 1/*true*/;
362  else if (!(same_host = !strcasecmp(uuu->net_info->host,
363  net_info->host)) ||
364  !x_SamePort(uuu->net_info->port,
365  uuu->net_info->scheme,
366  net_info->port,
367  net_info->scheme)) {
368  close = 1/*true*/;
369  }
370  } else if (!net_info->http_proxy_only)
371  close = 1/*true*/;
372  /* connection reused with HTTP and w/CONNECT: HTTP -> HTTPS */
373  } else if (!x_SameScheme(uuu->net_info->scheme,
374  net_info->scheme) ||
375  !(same_host = !strcasecmp(uuu->net_info->host,
376  net_info->host)) ||
377  !x_SamePort(uuu->net_info->port,
378  uuu->net_info->scheme,
379  net_info->port,
380  net_info->scheme)) {
381  close = 1/*true*/;
382  }
383  if (close) {
384  SOCK_Destroy(uuu->sock);
385  uuu->sock = 0;
386  }
387  }
388  if (!same_host || uuu->net_info->port != net_info->port
389  || (same_host < 0
390  && strcasecmp(uuu->net_info->host,
391  net_info->host) != 0)) {
392  /* drop the flag on host / port replaced */
393  uuu->skip_host = 0/*false*/;
394  }
395  }
396  ConnNetInfo_Destroy(net_info);
397  return retval;
398 }
399 
400 
401 
402 #ifdef __GNUC__
403 inline
404 #endif /*__GNUC__*/
405 static int/*bool*/ x_SameOrDefPort(unsigned short port1, EBURLScheme scheme1,
406  unsigned short port2, EBURLScheme scheme2)
407 {
408  return port1 == port2
409  || ((!port1 || port1 == x_PortForScheme(0, scheme1)) &&
410  (!port2 || port2 == x_PortForScheme(0, scheme2)))
411  ? 1/*true*/ : 0/*false*/;
412 }
413 
414 
415 /* NB: treatment of 'host_from' and 'host_to' is not symmetrical */
416 static int/*bool*/ x_RedirectOK(EBURLScheme scheme_to,
417  const char* host_to,
418  unsigned short port_to,
419  EBURLScheme scheme_from,
420  const char* host_from,
421  unsigned short port_from)
422 {
423  char buf1[CONN_HOST_LEN+1], buf2[CONN_HOST_LEN+1];
424  unsigned int ip1, ip2;
425  if (!x_SameOrDefPort(port_to, scheme_to, port_from, scheme_from))
426  return 0/*false*/;
427  if (strcasecmp(host_to, host_from) == 0)
428  return 1/*true*/;
429  if (!SOCK_isipEx(host_from, 1/*full-quad*/))
430  return 0/*false*/;
431  if ((ip1 = SOCK_gethostbyname(host_from)) == (unsigned int)(-1))
432  ip1 = 0;
433  if (!ip1 || !SOCK_gethostbyaddr(ip1, buf1, sizeof(buf1)))
434  strncpy0(buf1, host_from, sizeof(buf1) - 1);
435  else if (strcasecmp(buf1, host_to) == 0)
436  return 1/*true*/;
437  if ((ip2 = SOCK_gethostbyname(host_to)) == (unsigned int)(-1))
438  ip2 = 0;
439  if (ip1/*&& ip2*/ && ip1 == ip2)
440  return 1/*true*/;
441  if (!ip2 || !SOCK_gethostbyaddr(ip2, buf2, sizeof(buf2)))
442  strncpy0(buf2, host_to, sizeof(buf2) - 1);
443  return strcasecmp(buf1, buf2) == 0 ? 1/*true*/ : 0/*false*/;
444 }
445 
446 
447 #ifdef __GNUC__
448 inline
449 #endif /*__GNUC__*/
450 static int/*bool*/ x_IsWriteThru(const SHttpConnector* uuu)
451 {
452  return !uuu->net_info->http_version || !(uuu->flags & fHTTP_WriteThru)
453  ? 0/*false*/ : 1/*true*/;
454 }
455 
456 
457 typedef enum {
464 
465 
466 static EHTTP_Redirect x_Redirect(SHttpConnector* uuu, const SRetry* retry)
467 {
468  EBURLScheme scheme = uuu->net_info->scheme;
469  EReqMethod req_method = (EReqMethod) uuu->net_info->req_method;
470  char host[sizeof(uuu->net_info->host)];
471  unsigned short port = uuu->net_info->port;
472  int/*bool*/ unsafe;
473 
474  strcpy(host, uuu->net_info->host);
475  if (req_method == eReqMethod_Any)
476  req_method = BUF_Size(uuu->w_buf) ? eReqMethod_Post : eReqMethod_Get;
477  ConnNetInfo_SetArgs(uuu->net_info, ""); /*arguments not inherited*/
478 
479  if (!ConnNetInfo_ParseURL(uuu->net_info, retry->data))
480  return eHTTP_RedirectError;
481 
482  unsafe = scheme == eURL_Https && uuu->net_info->scheme != eURL_Https
483  ? 1/*true*/ : 0/*false*/;
484 
485  if (req_method == eReqMethod_Put ||
486  req_method == eReqMethod_Post ||
487  req_method == eReqMethod_Delete) {
488  if (uuu->net_info->req_method == eReqMethod_Post
489  && retry->mode == eRetry_Redirect303) {
491  BUF_Erase(uuu->w_buf);
492  } else {
493  if (x_IsWriteThru(uuu) && BUF_Size(uuu->w_buf))
494  return eHTTP_RedirectInvalid;
495  if (!unsafe && !x_RedirectOK(uuu->net_info->scheme,
496  uuu->net_info->host,
497  uuu->net_info->port,
498  scheme,
499  host,
500  port)) {
501  unsafe = 1/*true*/;
502  }
503  }
504  }
505 
506  if (unsafe && !x_UnsafeRedirectOK(uuu))
507  return eHTTP_RedirectUnsafe;
508 
509  if ((uuu->flags & fHTTP_AdjustOnRedirect) && uuu->adjust) {
510  if (!s_CallAdjust(uuu, 0))
511  return eHTTP_RedirectError;
512  } else if (port != uuu->net_info->port ||
513  strcasecmp(uuu->net_info->host, host) != 0) {
514  /* drop the flag on host / port replaced */
515  uuu->skip_host = 0/*false*/;
516  }
517 
518  return eHTTP_RedirectOK;
519 }
520 
521 
522 /* Try to fix connection parameters (called for an unconnected connector) */
524  const SRetry* retry,
525  EIO_Status status,
526  EExtractMode extract)
527 {
528  const char* msg;
529 
530  assert(status != eIO_Success);
531  assert(!retry || !retry->data || *retry->data);
532  assert(!uuu->sock && uuu->can_connect != fCC_None);
533 
534  uuu->retry = 0;
535  if (uuu->reused) {
536  assert(!uuu->sock && (!retry || !retry->data));
537  if (uuu->entity)
538  return eIO_Unknown;
539  uuu->retry = 1;
540  return eIO_Success;
541  }
542  if (!retry || !retry->mode || uuu->minor_fault > 5) {
543  uuu->minor_fault = 0;
544  uuu->major_fault++;
545  } else
546  uuu->minor_fault++;
547 
548  if (uuu->major_fault >= uuu->net_info->max_try) {
549  msg = extract != eEM_Drop && uuu->major_fault > 1
550  ? "[HTTP%s%s] Too many failed attempts (%hu), giving up" : "";
551  } else if (retry && retry->mode) {
552  int secure = uuu->net_info->scheme == eURL_Https ? 1/*T*/ : 0/*F*/;
553  char* url = ConnNetInfo_URL(uuu->net_info);
554  int fail = 0;
555  switch (retry->mode) {
556  case eRetry_Redirect:
557  if (uuu->entity/*FIXME*/)
558  fail = eHTTP_RedirectInvalid;
559  /*FALLTHRU*/
560  case eRetry_Redirect303:
561  if (!fail) {
563  fail = eHTTP_RedirectTunnel;
564  else if (!retry->data || *retry->data == '?')
565  fail = eHTTP_RedirectError;
566  else
567  fail = x_Redirect(uuu, retry);
568  }
569  if (fail) {
570  const char* reason;
571  switch (fail) {
573  reason = "Invalid";
574  status = eIO_Unknown;
575  break;
577  reason = "Prohibited";
578  status = eIO_NotSupported;
579  break;
580  case eHTTP_RedirectError:
581  reason = "Cannot";
582  status = eIO_Unknown;
583  break;
585  reason = "Spurious tunnel";
586  status = eIO_InvalidArg;
587  break;
588  default:
589  reason = "Unknown failure of";
590  status = eIO_Unknown;
591  assert(0);
592  break;
593  }
594  assert(status != eIO_Success);
596  ("[HTTP%s%s] %s %s%s to %s%s%s",
597  url ? "; " : "",
598  url ? url : "",
599  reason,
600  fail == eHTTP_RedirectUnsafe && secure
601  ? "insecure " : "",
602  fail > eHTTP_RedirectError
603  || retry->mode != eRetry_Redirect303
604  ? "redirect" : "submission",
605  retry->data ? "\"" : "<",
606  retry->data ? retry->data : "NULL",
607  retry->data ? "\"" : ">"));
608  } else {
610  ("[HTTP%s%s] %s \"%s\"",
611  url ? "; " : "",
612  url ? url : "",
613  retry->mode == eRetry_Redirect303
614  ? "Finishing submission with" : "Redirecting to",
615  retry->data));
616  status = eIO_Success;
617  }
618  break;
619  case eRetry_Authenticate:
621  fail = eHTTP_AuthConnect;
622  /*FALLTHRU*/
624  if (!fail) {
625  if (!retry->data
626  || strncasecmp(retry->data, "basic",
627  strcspn(retry->data, " \t")) != 0) {
628  fail = eHTTP_AuthNotSupp;
629  } else
630  fail = x_Authenticate(uuu, retry->mode, 1/*retry*/);
631  }
632  if (fail) {
633  const char* reason;
634  switch (fail) {
635  case eHTTP_AuthConnect:
636  reason = "not allowed with CONNECT";
637  status = eIO_Unknown;
638  break;
639  case eHTTP_AuthNotSupp:
640  reason = "not implemented";
641  status = eIO_NotSupported;
642  break;
643  case eHTTP_AuthIllegal:
644  reason = "cannot be done at this point";
645  status = eIO_InvalidArg;
646  break;
647  case eHTTP_AuthDone:
648  case eHTTP_AuthError:
649  reason = "failed";
650  status = eIO_Unknown;
651  break;
652  case eHTTP_AuthUnsafe:
653  reason = "prohibited";
654  status = eIO_NotSupported;
655  break;
656  case eHTTP_AuthMissing:
657  reason = "required";
658  status = eIO_Unknown;
659  break;
660  default:
661  reason = "unknown failure";
662  status = eIO_NotSupported;
663  assert(0);
664  break;
665  }
666  assert(status != eIO_Success);
668  ("[HTTP%s%s] %s %s %c%s%c",
669  url ? "; " : "",
670  url ? url : "",
671  retry->mode == eRetry_Authenticate
672  ? "Authorization" : "Proxy authorization",
673  reason,
674  "(["[!retry->data],
675  retry->data ? retry->data : "NULL",
676  ")]"[!retry->data]));
677  } else {
679  ("[HTTP%s%s] Authorizing%s",
680  url ? "; " : "",
681  url ? url : "",
682  retry->mode == eRetry_Authenticate
683  ? "" : " proxy"));
684  status = eIO_Success;
685  }
686  break;
687  default:
689  ("[HTTP%s%s] Unknown retry mode #%u",
690  url ? "; " : "",
691  url ? url : "", (unsigned int) retry->mode));
692  status = eIO_InvalidArg;
693  assert(0);
694  break;
695  }
696  if (url)
697  free(url);
698  if (status != eIO_Success)
699  uuu->can_connect = fCC_None;
700  return status;
701  } else if (uuu->adjust && !s_CallAdjust(uuu, uuu->major_fault)) {
702  msg = extract != eEM_Drop && uuu->major_fault > 1
703  ? "[HTTP%s%s] Retry attempts (%hu) exhausted, giving up" : "";
704  } else
705  return eIO_Success;
706 
707  assert(msg);
708  if (*msg) {
709  char* url = ConnNetInfo_URL(uuu->net_info);
711  (msg,
712  url ? "; " : "",
713  url ? url : "", uuu->major_fault));
714  if (url)
715  free(url);
716  }
717  uuu->can_connect = fCC_None;
718  return status;
719 }
720 
721 
722 static char* x_HostPort(size_t reserve, const char* host, unsigned short xport)
723 {
724  size_t hostlen = strlen(host), portlen;
725  char* hostport, port[16];
726 
727  if (!xport) {
728  portlen = 1;
729  port[0] = '\0';
730  } else
731  portlen = (size_t) sprintf(port, ":%hu", xport) + 1;
732  hostport = (char*) malloc(reserve + hostlen + portlen);
733  if (hostport) {
734  memcpy(hostport + reserve, host, hostlen);
735  hostlen += reserve;
736  memcpy(hostport + hostlen, port, portlen);
737  }
738  return hostport;
739 }
740 
741 
742 static int/*bool*/ x_SetHttpHostTag(SConnNetInfo* net_info)
743 {
744  char* tag;
745  int/*bool*/ retval;
746  unsigned short port = net_info->port;
747 
748  if (port && port == x_PortForScheme(0, net_info->scheme))
749  port = 0;
750  tag = x_HostPort(sizeof(kHttpHostTag)-1, net_info->host, port);
751  if (tag) {
752  memcpy(tag, kHttpHostTag, sizeof(kHttpHostTag)-1);
753  retval = ConnNetInfo_OverrideUserHeader(net_info, tag);
754  free(tag);
755  } else
756  retval = 0/*failure*/;
757  return retval;
758 }
759 
760 
761 static void x_SetRequestIDs(SConnNetInfo* net_info)
762 {
764  assert(!id || *id);
765  if (id) {
766  char* tag;
767  size_t len = strlen(id);
768  if (!(tag = (char*) realloc(id, ++len + sizeof(HTTP_NCBI_SID)))) {
770  free(id);
771  } else {
772  memmove(tag + sizeof(HTTP_NCBI_SID), tag, len);
773  memcpy (tag, HTTP_NCBI_SID, sizeof(HTTP_NCBI_SID) - 1);
774  tag[sizeof(HTTP_NCBI_SID) - 1] = ' ';
776  free(tag);
777  }
778  }
780  assert(!id || *id);
781  if (id) {
782  char* tag;
783  size_t len = strlen(id);
784  if (!(tag = (char*) realloc(id, ++len + sizeof(HTTP_NCBI_PHID)))) {
786  free(id);
787  } else {
788  memmove(tag + sizeof(HTTP_NCBI_PHID), tag, len);
789  memcpy (tag, HTTP_NCBI_PHID, sizeof(HTTP_NCBI_PHID) - 1);
790  tag[sizeof(HTTP_NCBI_PHID) - 1] = ' ';
792  free(tag);
793  }
794  }
795 }
796 
797 
798 /*ARGSUSED*/
799 static int s_TunnelAdjust(SConnNetInfo* net_info, void* data, unsigned int arg)
800 {
801  SHttpConnector* uuu = (SHttpConnector*) data;
802  uuu->minor_fault = 0;
803  uuu->major_fault++;
804  return -1/*noop*/;
805 }
806 
807 
808 /* Connect to the HTTP server, specified by uuu->net_info's "port:host".
809  * Return eIO_Success only if socket connection has succeeded and uuu->sock
810  * is non-zero. If unsuccessful, try to adjust uuu->net_info with s_Adjust(),
811  * and then re-try the connection attempt.
812  */
814  const STimeout* timeout,
815  EExtractMode extract)
816 {
817  EIO_Status status;
818  SOCK sock;
819 
821 
822  uuu->http_code = 0;
823  if (!(uuu->can_connect & fCC_Once)) {
824  if (extract == eEM_Read && uuu->net_info->max_try
825  && uuu->can_connect == fCC_None) {
826  char* url = ConnNetInfo_URL(uuu->net_info);
828  ("[HTTP%s%s] Connector is no longer usable",
829  url ? "; " : "",
830  url ? url : ""));
831  if (url)
832  free(url);
833  }
834  return eIO_Unknown;
835  }
836 
837  if (uuu->conn_state == eCS_Eom) {
838  if (uuu->adjust) {
839  int retval = s_CallAdjust(uuu, (unsigned int)(-1));
840  if (!retval)
841  return eIO_Unknown;
842  if (retval < 0)
843  return eIO_Closed;
844  } else
845  return eIO_Closed;
847  }
848 
849  uuu->entity = 0;
850  uuu->chunked = 0;
851  /* the re-try loop... */
852  for (;;) {
857  sock = uuu->sock;
858  uuu->sock = 0;
859  uuu->reused = sock ? 1/*true*/ : 0/*false*/;
860  if ((!sock || !SOCK_IsSecure(sock))
862  && uuu->net_info->scheme == eURL_Https
863  && uuu->net_info->http_proxy_host[0]
864  && uuu->net_info->http_proxy_port
865  && !uuu->net_info->http_proxy_only) {
866  SConnNetInfo* net_info = ConnNetInfo_Clone(uuu->net_info);
867  uuu->reused = 0/*false*/;
868  if (!net_info) {
869  status = eIO_Unknown;
870  break;
871  }
872  net_info->scheme = eURL_Http;
873  net_info->user[0] = '\0';
874  net_info->pass[0] = '\0';
875  if (net_info->port == 0)
876  net_info->port = CONN_PORT_HTTPS;
877  net_info->firewall = 0/*false*/;
879  status = HTTP_CreateTunnelEx(net_info, fHTTP_NoUpread,
880  0, 0, uuu, s_TunnelAdjust, &sock);
881  assert((status == eIO_Success) ^ !sock);
882  ConnNetInfo_Destroy(net_info);
883  } else
884  status = eIO_Success;
885  if (status == eIO_Success) {
886  EReqMethod req_method = (EReqMethod) uuu->net_info->req_method;
887  int/*bool*/ reset_user_header;
888  char* http_user_header;
889  const char* host;
890  unsigned short port;
891  const char* path;
892  const char* args;
893  char* temp;
894  size_t len;
895 
896  /* RFC7230 now requires Host: for CONNECT just as well */
897  if (!uuu->skip_host && !x_SetHttpHostTag(uuu->net_info)) {
898  status = eIO_Unknown;
899  break;
900  }
901  if (x_IsWriteThru(uuu)) {
902  assert(req_method != eReqMethod_Connect);
903  if (req_method == eReqMethod_Any) {
904  req_method = BUF_Size(uuu->w_buf)
906  : eReqMethod_Get;
907  }
908  if (req_method != eReqMethod_Head &&
909  req_method != eReqMethod_Get) {
911  (uuu->net_info, "Transfer-Encoding: chunked")) {
912  status = eIO_Unknown;
913  break;
914  }
915  uuu->chunked = 1;
916  }
917  len = (size_t)(-1L);
918  } else
919  len = BUF_Size(uuu->w_buf);
920  if (req_method == eReqMethod_Connect
921  || (uuu->net_info->scheme != eURL_Https
922  && uuu->net_info->http_proxy_host[0]
923  && uuu->net_info->http_proxy_port)) {
924  if (uuu->net_info->http_push_auth &&
926  || (req_method != eReqMethod_Connect &&
927  x_Authenticate(uuu, eRetry_Authenticate, 0) > 0))) {
928  status = eIO_Unknown;
929  break;
930  }
931  host = uuu->net_info->http_proxy_host;
932  port = uuu->net_info->http_proxy_port;
933  path = ConnNetInfo_URL(uuu->net_info);
934  if (!path) {
935  status = eIO_Unknown;
936  break;
937  }
938  if (req_method == eReqMethod_Connect) {
939  /* Tunnel (RFC2817) */
941  assert(!uuu->net_info->http_version);
942  if (!len) {
943  args = 0;
944  } else if (!(temp = (char*) malloc(len))
945  || BUF_Peek(uuu->w_buf, temp, len) != len) {
946  if (temp)
947  free(temp);
948  status = eIO_Unknown;
949  free((void*) path);
950  break;
951  } else
952  args = temp;
953  } else {
954  /* Proxied HTTP */
955  assert(uuu->net_info->scheme == eURL_Http);
956  if (uuu->flags & fHCC_UrlEncodeArgs) {
957  /* args added to path not valid(unencoded), remove */
958  if ((temp = (char*) strchr(path, '?')) != 0)
959  *temp = '\0';
960  args = ConnNetInfo_GetArgs(uuu->net_info);
961  if (args && (!*args || *args == '#'))
962  args = 0;
963  } else
964  args = 0;
965  }
966  } else {
967  /* Direct HTTP[S] or tunneled HTTPS */
968  if (uuu->net_info->http_push_auth &&
969  x_Authenticate(uuu, eRetry_Authenticate, 0) > 0) {
970  status = eIO_Unknown;
971  break;
972  }
973  host = uuu->net_info->host;
974  port = uuu->net_info->port;
975  args = ConnNetInfo_GetArgs(uuu->net_info);
976  if (*args == '#' || !(uuu->flags & fHCC_UrlEncodeArgs)) {
977  path = uuu->net_info->path;
978  args = 0;
979  } else if (!(path = strndup(uuu->net_info->path, (size_t)
980  (args - uuu->net_info->path) -
981  !(args == uuu->net_info->path)
982  /*'?'*/))) {
983  status = eIO_Unknown;
984  break;
985  } else if (!*args)
986  args = 0;
987  }
988 
989  /* encode args (obsolete feature) */
990  if (req_method != eReqMethod_Connect && args) {
991  size_t args_len = strcspn(args, "#");
992  size_t size = args_len * 3;
993  size_t rd_len, wr_len;
994  assert((uuu->flags & fHCC_UrlEncodeArgs) && args_len > 0);
995  if (!(temp = (char*) malloc(size + strlen(args+args_len) +1))){
996  int error = errno;
997  temp = ConnNetInfo_URL(uuu->net_info);
999  ("[HTTP%s%s] Out of memory encoding"
1000  " URL arguments (%lu bytes)",
1001  temp ? "; " : "",
1002  temp ? temp : "",
1003  (unsigned long)(size + strlen
1004  (args + args_len) +1)));
1005  if (path != uuu->net_info->path)
1006  free((void*) path);
1007  status = eIO_Unknown;
1008  break;
1009  }
1010  URL_Encode(args, args_len, &rd_len, temp, size, &wr_len);
1011  assert(rd_len == args_len);
1012  assert(wr_len <= size);
1013  strcpy(temp + wr_len, args + args_len);
1014  args = temp;
1015  }
1016 
1017  /* NCBI request IDs */
1018  if (!(uuu->flags & fHTTP_NoAutomagicSID))
1019  x_SetRequestIDs(uuu->net_info);
1020 
1021  /* identify the connector in the User-Agent: header tag */
1022  http_user_header = (uuu->net_info->http_user_header
1024  : 0);
1025  if (!uuu->net_info->http_user_header == !http_user_header) {
1027  (uuu->net_info, "User-Agent: NCBIHttpConnector"
1028 #ifdef NCBI_CXX_TOOLKIT
1029  " (CXX Toolkit)"
1030 #else
1031  " (C Toolkit)"
1032 #endif /*NCBI_CXX_TOOLKIT*/
1033  );
1034  if (!uuu->net_info->http_version
1035  && req_method != eReqMethod_Connect) {
1037  "Connection: keep-alive");
1038  }
1039  reset_user_header = 1;
1040  } else
1041  reset_user_header = 0;
1042 
1043  if (uuu->net_info->debug_printout) {
1046  uuu->retry ? eLOG_Trace : eLOG_Note,
1047  CORE_GetLOG());
1048  CORE_UNLOCK;
1049  }
1050  uuu->retry = 0;
1051 
1052  /* connect & send HTTP header */
1053  if (uuu->net_info->scheme == eURL_Https)
1054  flags |= fSOCK_Secure;
1055  if (!(uuu->flags & fHTTP_NoUpread))
1057 
1058  status = URL_ConnectEx(host, port, path, args,
1059  req_method | (uuu->net_info->http_version
1060  ? eReqMethod_v1
1061  : 0), len,
1062  uuu->o_timeout, timeout,
1063  uuu->net_info->http_user_header,
1064  uuu->net_info->credentials,
1065  flags, &sock);
1066  /* FIXME: remember if "sock" reused here; and not cause major fault
1067  * if failed later; but simply re-try w/o calling adjust. */
1068 
1069  if (reset_user_header) {
1071  uuu->net_info->http_user_header = http_user_header;
1072  }
1073 
1074  if (path != uuu->net_info->path)
1075  free((void*) path);
1076  if (args)
1077  free((void*) args);
1078 
1079  if (sock) {
1080  assert(status == eIO_Success);
1081  uuu->w_len = req_method != eReqMethod_Connect
1082  ? BUF_Size(uuu->w_buf) : 0;
1083  break;
1084  }
1085  } else
1086  assert(!sock);
1087 
1088  assert(status != eIO_Success);
1089  /* connection failed, try another server */
1090  if ((status = s_Adjust(uuu, 0, status, extract)) != eIO_Success)
1091  break;
1092  }
1093 
1094  if (status == eIO_Success) {
1096  uuu->sock = sock;
1097  assert(uuu->sock);
1098  } else {
1099  if (sock) {
1100  SOCK_Abort(sock);
1101  SOCK_Destroy(sock);
1102  }
1103  assert(!uuu->sock);
1104  if (status == eIO_Closed)
1105  status = eIO_Unknown;
1106  }
1107  return status;
1108 }
1109 
1110 
1111 /* Unconditionally drop the connection w/o any wait */
1113 {
1114  assert(uuu->sock);
1115  if (!(uuu->conn_state & eCS_ReadBody) || uuu->conn_state == eCS_Discard)
1116  SOCK_Abort(uuu->sock);
1117  else
1119  SOCK_Close(uuu->sock);
1120  uuu->sock = 0;
1121  uuu->retry = 0;
1122  uuu->conn_state = state;
1123 }
1124 
1125 
1126 typedef struct {
1129 } XBUF_PeekCBCtx;
1130 
1131 
1132 static size_t x_WriteBuf(void* data, const void* buf, size_t size)
1133 {
1134  XBUF_PeekCBCtx* ctx = (XBUF_PeekCBCtx*) data;
1135  size_t written;
1136 
1137  assert(buf && size);
1138  assert(ctx->status == eIO_Success);
1139  ctx->status = SOCK_Write(ctx->sock, buf, size,
1140  &written, eIO_WritePlain);
1141  return written;
1142 }
1143 
1144 
1145 /* Connect to the server specified by uuu->net_info, then compose and form
1146  * relevant HTTP header, and flush the accumulated output data(uuu->w_buf)
1147  * after the HTTP header.
1148  * If connection/write unsuccessful, retry to reconnect and send the data
1149  * again until permitted by s_Adjust().
1150  */
1152  const STimeout* timeout,
1153  EExtractMode extract)
1154 {
1155  EIO_Status status = eIO_Success;
1156 
1157  for (;;) {
1158  char what[80];
1159  int error;
1160  size_t off;
1161  char* url;
1162 
1163  assert(status == eIO_Success);
1164 
1165  if ((uuu->conn_state == eCS_NotInitiated || uuu->conn_state == eCS_Eom)
1166  && (status = s_Connect(uuu, timeout, extract)) != eIO_Success) {
1167  break;
1168  }
1169 
1170  if (uuu->w_len) {
1172  ctx.sock = uuu->sock;
1173  ctx.status = eIO_Success/*NB:==status*/;
1174  off = BUF_Size(uuu->w_buf) - uuu->w_len;
1176  SOCK_SetTimeout(ctx.sock, eIO_Write, uuu->w_timeout);
1177  uuu->w_len -= BUF_PeekAtCB(uuu->w_buf, off,
1178  x_WriteBuf, &ctx, uuu->w_len);
1179  status = ctx.status;
1180  } else
1181  off = 0;
1182 
1183  if (status == eIO_Success) {
1184  if (uuu->w_len)
1185  continue;
1186  if (uuu->conn_state > eCS_FlushRequest)
1187  break;
1188  if (!uuu->chunked)
1190  if (uuu->conn_state == eCS_FlushRequest) {
1191  /* "flush" */
1192  status = SOCK_Write(uuu->sock, 0, 0, 0, eIO_WritePlain);
1193  if (status == eIO_Success) {
1194  uuu->conn_state = eCS_ReadHeader;
1195  uuu->keepalive = uuu->net_info->http_version;
1196  uuu->expected = (TNCBI_BigCount)(-1L);
1197  uuu->received = 0;
1198  uuu->chunked = 0;
1199  BUF_Erase(uuu->http);
1200 #if !defined(NCBI_OS_MSWIN) && !defined(NCBI_OS_CYGWIN)
1201  if (!uuu->keepalive)
1202  ; //status = SOCK_Shutdown(uuu->sock, eIO_Write);
1203 #endif /*NCBI_OS_MSWIN*/
1204  break;
1205  }
1206  } else
1207  break;
1208  }
1209 
1210  if (status == eIO_Timeout
1211  && (extract == eEM_Wait
1212  || (timeout && !(timeout->sec | timeout->usec)))) {
1213  break;
1214  }
1215 
1216  error = errno;
1217  if (uuu->w_len && uuu->conn_state == eCS_WriteRequest) {
1218  if (!x_IsWriteThru(uuu)) {
1219  sprintf(what, "write request body at offset %lu",
1220  (unsigned long) off);
1221  } else
1222  strcpy(what, "write request body");
1223  } else {
1224  if (uuu->w_len)
1225  strcpy(what, "finalize request body");
1226  else
1227  strcpy(what, "finalize request");
1228  }
1229 
1230  url = ConnNetInfo_URL(uuu->net_info);
1232  ("[HTTP%s%s] Cannot %s (%s)",
1233  url ? "; " : "",
1234  url ? url : "", what, IO_StatusStr(status)));
1235  if (url)
1236  free(url);
1237 
1238  /* write failed; close and try to use another server, if possible */
1240  assert(status != eIO_Success);
1241  if ((status = s_Adjust(uuu, 0, status, extract)) != eIO_Success)
1242  break;
1243  }
1244 
1245  return status;
1246 }
1247 
1248 
1249 static int/*bool*/ x_IsValidParam(const char* param, size_t paramlen)
1250 {
1251  const char* e = (const char*) memchr(param, '=', paramlen);
1252  size_t len;
1253  if (!e || e == param)
1254  return 0/*false*/;
1255  if ((len = (size_t)(++e - param)) >= paramlen)
1256  return 0/*false*/;
1257  assert(!isspace((unsigned char)(*param)));
1258  if (strcspn(param, " \t") < len)
1259  return 0/*false*/;
1260  if (*e == '\'' || *e == '"') {
1261  /* a quoted string */
1262  assert(len < paramlen);
1263  len = paramlen - len;
1264  if (!(e = (const char*) memchr(e + 1, *e, --len)))
1265  return 0/*false*/;
1266  e++/*skip the quote*/;
1267  } else
1268  e += strcspn(e, " \t");
1269  if (e != param + paramlen && e + strspn(e, " \t") != param + paramlen)
1270  return 0/*false*/;
1271  return 1/*true*/;
1272 }
1273 
1274 
1275 static int/*bool*/ x_IsValidChallenge(const char* text, size_t len)
1276 {
1277  /* Challenge must contain a scheme name token and a non-empty param
1278  * list (comma-separated pairs token={token|quoted_string}), with at
1279  * least one parameter being named "realm=".
1280  */
1281  size_t word = strcspn(text, " \t");
1282  int retval = 0/*false*/;
1283  if (word < len) {
1284  /* 1st word is always the scheme name */
1285  const char* param = text + word;
1286  for (param += strspn(param, " \t"); param < text + len;
1287  param += strspn(param, ", \t")) {
1288  size_t paramlen = (size_t)(text + len - param);
1289  const char* c = (const char*) memchr(param, ',', paramlen);
1290  if (c)
1291  paramlen = (size_t)(c - param);
1292  if (!x_IsValidParam(param, paramlen))
1293  return 0/*false*/;
1294  if (paramlen > 6 && strncasecmp(param, "realm=", 6) == 0)
1295  retval = 1/*true, but keep scanning*/;
1296  param += c ? ++paramlen : paramlen;
1297  }
1298  }
1299  return retval;
1300 }
1301 
1302 
1303 static size_t x_PushbackBuf(void* data, const void* buf, size_t size)
1304 {
1305  XBUF_PeekCBCtx* ctx = (XBUF_PeekCBCtx*) data;
1306 
1307  assert(buf && size);
1308  assert(ctx->status == eIO_Success);
1309  ctx->status = SOCK_Pushback(ctx->sock, buf, size);
1310  return ctx->status != eIO_Success ? 0 : size;
1311 }
1312 
1313 
1314 static int/*bool*/ x_Pushback(SOCK sock, BUF buf)
1315 {
1316  size_t size = BUF_Size(buf);
1318  ctx.sock = sock;
1319  ctx.status = eIO_Success;
1320  return !(size ^ BUF_PeekAtCB(buf, 0, x_PushbackBuf, &ctx, size));
1321 }
1322 
1323 
1325 {
1326  int n;
1327  BUF buf;
1328  char* str;
1329  size_t size;
1330  TNCBI_BigCount chunk;
1331  EIO_Status status;
1332 
1333  buf = 0;
1334  str = 0;
1335  size = 0;
1336  for (;;) {
1337  size_t off;
1338  status = SOCK_StripToPattern(uuu->sock, "\r\n", 2, &buf, &off);
1339  if (status != eIO_Success || (size += off) != BUF_Size(buf))
1340  break;
1341  if (size > 2) {
1342  if (!(str = (char*) malloc(size + 1)))
1343  break;
1344  verify(BUF_Peek(buf, str, size) == size);
1345  if (!first && (str[0] != '\r' || str[1] != '\n')) {
1346  status = eIO_NotSupported/*protocol error*/;
1347  free(str);
1348  str = 0;
1349  break;
1350  }
1351  assert(str[size - 1] == '\n');
1352  str[size] = '\0';
1353  break;
1354  }
1355  }
1356 
1357  if (!str
1358  || sscanf(str, "%" NCBI_BIGCOUNT_FORMAT_SPEC_HEX "%n", &chunk, &n) < 1
1359  || (!isspace((unsigned char) str[n]) && str[n] != ';')) {
1360  int error = errno/*iff memory allocation failure*/;
1361  char* url = ConnNetInfo_URL(uuu->net_info);
1362  const char* err;
1363  char errbuf[256];
1364  if (!str) {
1365  if (status == eIO_Success) {
1366  if (BUF_Size(buf) != size) {
1367  sprintf(errbuf, "Partial read %lu out of %lu",
1368  (unsigned long) BUF_Size(buf),
1369  (unsigned long) size);
1370  err = errbuf;
1371  } else
1372  err = strerror(error);
1373  } else {
1374  err = status != eIO_NotSupported
1375  ? IO_StatusStr(status)
1376  : "Protocol error";
1377  }
1378  } /* else "err" is unused */
1379  CORE_LOGF_X(23, eLOG_Error,
1380  ("[HTTP%s%s] Cannot read chunk size: %s%.*s%s",
1381  url ? "; " : "",
1382  url ? url : "",
1383  &"\""[!str],
1384  !str ? (int) strlen(err) : (int)(size - (first ? 2 : 4)),
1385  !str ? err : str + (first ? 0 : 2),
1386  &"\""[!str]));
1387  if (url)
1388  free(url);
1389  if (str)
1390  free(str);
1391  if (status == eIO_Closed || !x_Pushback(uuu->sock, buf))
1392  status = eIO_Unknown;
1393  BUF_Destroy(buf);
1394  return status ? status : eIO_Unknown;
1395  }
1396 
1397  free(str);
1398  BUF_Destroy(buf);
1399  uuu->received = 0;
1400  uuu->expected = chunk;
1401  return eIO_Success;
1402 }
1403 
1404 
1406 {
1407  EIO_Status status;
1408  BUF buf = 0;
1409  do {
1410  size_t n;
1411  status = SOCK_StripToPattern(uuu->sock, "\r\n", 2, &buf, &n);
1412  if (n == 2) {
1413  /*last trailer*/
1414  BUF_Destroy(buf);
1415  uuu->conn_state = eCS_Discard;
1416  return eIO_Closed;
1417  }
1418  } while (status == eIO_Success);
1419  if (status == eIO_Closed) {
1420  char* url = ConnNetInfo_URL(uuu->net_info);
1421  CORE_LOGF_X(25, eLOG_Error,
1422  ("[HTTP%s%s] Cannot read chunk tail",
1423  url ? "; " : "",
1424  url ? url : ""));
1425  if (url)
1426  free(url);
1427  status = eIO_Unknown;
1428  } else if (!x_Pushback(uuu->sock, buf))
1429  status = eIO_Unknown;
1430  BUF_Destroy(buf);
1431  return status;
1432 }
1433 
1434 
1435 /* If "size"==0, then "buf" is a pointer to a BUF to receive the current body.
1436  * Otherwise, "buf" is a non-NULL destination buffer.
1437  */
1439  void* buf, size_t size,
1440  size_t* n_read, EIO_ReadMethod how)
1441 {
1442  BUF* xxx;
1443  EIO_Status status;
1444 
1445  assert(buf && n_read && !*n_read && uuu->conn_state == eCS_ReadBody);
1446 
1447  if (!uuu->chunked || uuu->expected > uuu->received) {
1448  if (!size) {
1449  size = uuu->expected != (TNCBI_BigCount)(-1L)
1450  ? uuu->expected - uuu->received
1452  if (!size) {
1453  assert(!uuu->chunked);
1454  uuu->conn_state = eCS_DoneBody;
1455  return eIO_Closed;
1456  }
1457  if (size > HTTP_SOAK_READ_SIZE)
1459  xxx = (BUF*) buf;
1460  if (!(buf = (void*) malloc(size))) {
1461  int error = errno;
1462  char* url = ConnNetInfo_URL(uuu->net_info);
1464  ("[HTTP%s%s] Cannot allocate response chunk"
1465  " (%lu byte%s)",
1466  url ? "; " : "",
1467  url ? url : "",
1468  (unsigned long) size, &"s"[size == 1]));
1469  if (url)
1470  free(url);
1471  return eIO_Unknown;
1472  }
1473  } else if (uuu->chunked) {
1474  assert(uuu->expected != (TNCBI_BigCount)(-1L));
1475  if (size > uuu->expected - uuu->received)
1476  size = uuu->expected - uuu->received;
1477  xxx = 0;
1478  } else if (uuu->expected == uuu->received) {
1479  uuu->conn_state = eCS_DoneBody;
1480  return eIO_Closed;
1481  } else
1482  xxx = 0;
1483  status = SOCK_Read(uuu->sock, buf, size, n_read, how);
1484  if (xxx && !BUF_AppendEx(xxx, buf, size, buf, *n_read)) {
1485  int error = errno;
1486  char* url = ConnNetInfo_URL(uuu->net_info);
1488  ("[HTTP%s%s] Cannot collect response body",
1489  url ? "; " : "",
1490  url ? url : ""));
1491  if (url)
1492  free(url);
1493  free(buf);
1494  status = eIO_Unknown;
1495  }
1496  if (status == eIO_Closed)
1497  uuu->conn_state = eCS_Eom;
1498  return status;
1499  }
1500 
1501  if ((status = x_ReadChunkHead(uuu, !uuu->expected)) != eIO_Success)
1502  return status;
1503 
1504  if (uuu->expected) {
1505  assert(uuu->expected != (TNCBI_BigCount)(-1L) && !uuu->received);
1506  if (size > uuu->expected)
1507  size = uuu->expected;
1508  return s_ReadData(uuu, buf, size, n_read, how);
1509  }
1510 
1511  uuu->conn_state = eCS_DoneBody;
1512  return x_ReadChunkTail(uuu);
1513 }
1514 
1515 
1516 static int/*bool*/ x_ErrorHeaderOnly(SHttpConnector* uuu)
1517 {
1518  if (uuu->error_header == eDefault) {
1519  char val[32];
1521  "HTTP_ERROR_HEADER_ONLY",
1522  val, sizeof(val), 0);
1524  }
1525  return uuu->error_header == eOn ? 1/*true*/ : 0/*false*/;
1526 }
1527 
1528 
1529 /* Read and parse HTTP header */
1531  SRetry* retry,
1532  EExtractMode extract)
1533 {
1534  enum EHTTP_Tag {
1535  eHTTP_NcbiMsg = 1 << 0,
1536  eHTTP_NcbiSid = 1 << 1,
1537  eHTTP_Location = 1 << 2,
1538  eHTTP_Connection = 1 << 3,
1539  eHTTP_Authenticate = 1 << 4,
1540  eHTTP_ContentLength = 1 << 5,
1541  eHTTP_TransferEncoding = 1 << 6
1542  };
1543  typedef unsigned short THTTP_Tags; /* Bitwise-OR of EHTTP_Tag */
1544  char* url = 0, *hdr, *s;
1545  EHTTP_HeaderParse header_parse;
1546  int http_code;
1547  size_t size, n;
1548  EIO_Status status;
1549  int fatal;
1550  THTTP_Tags tags;
1551 
1552  assert(uuu->sock && uuu->conn_state == eCS_ReadHeader);
1553  memset(retry, 0, sizeof(*retry));
1554  /*retry->mode = eRetry_None;
1555  retry->data = 0;*/
1556 
1557  /* line by line HTTP header input */
1558  for (;;) {
1559  /* do we have full header yet? */
1560  if ((size = BUF_Size(uuu->http)) >= 4) {
1561  if (!(hdr = (char*) malloc(size + 1))) {
1562  int error = errno;
1563  assert(!url);
1564  url = ConnNetInfo_URL(uuu->net_info);
1566  ("[HTTP%s%s] Cannot allocate header"
1567  " (%lu bytes)",
1568  url ? "; " : "",
1569  url ? url : "",
1570  (unsigned long) size));
1571  if (url)
1572  free(url);
1573  uuu->reused = 0;
1574  return eIO_Unknown;
1575  }
1576  verify(BUF_Peek(uuu->http, hdr, size) == size);
1577  if (memcmp(&hdr[size - 4], "\r\n\r\n", 4) == 0) {
1578  /* full header captured */
1579  hdr[size] = '\0';
1580  break;
1581  }
1582  free(hdr);
1583  }
1584 
1585  status = SOCK_StripToPattern(uuu->sock, "\r\n", 2, &uuu->http, &n);
1586 
1587  if (status != eIO_Success || size + n != BUF_Size(uuu->http)) {
1588  ELOG_Level level;
1589  if (status == eIO_Timeout) {
1590  const STimeout* tmo = SOCK_GetTimeout(uuu->sock, eIO_Read);
1591  if (!tmo)
1592  level = eLOG_Error;
1593  else if (extract == eEM_Wait)
1594  return status;
1595  else if (tmo->sec | tmo->usec)
1596  level = eLOG_Warning;
1597  else
1598  level = eLOG_Trace;
1599  } else
1600  level = status && uuu->reused ? eLOG_Trace : eLOG_Error;
1601  assert(!url);
1602  url = ConnNetInfo_URL(uuu->net_info);
1603  CORE_LOGF_X(8, level,
1604  ("[HTTP%s%s] Cannot %s header (%s)",
1605  url ? "; " : "",
1606  url ? url : "",
1607  status != eIO_Success ? "read" : "scan",
1608  IO_StatusStr(status != eIO_Success
1609  ? status : eIO_Unknown)));
1610  if (url)
1611  free(url);
1612  if (status != eIO_Success)
1613  return status;
1614  uuu->reused = 0;
1615  return eIO_Unknown;
1616  }
1617  }
1618  /* the entire header has been read in */
1619  uuu->conn_state = eCS_ReadBody;
1620  BUF_Erase(uuu->http);
1621  uuu->reused = 0;
1622  assert(hdr);
1623 
1624  /* HTTP status must come on the first line of the response */
1625  fatal = 0/*false*/;
1626  if (sscanf(hdr, "HTTP/%*d.%*d %d ", &http_code) != 1 || !http_code)
1627  http_code = -1;
1628  uuu->http_code = (unsigned short) http_code;
1629  if (http_code < 200 || 299 < http_code) {
1630  if (http_code == 304)
1631  /*void*/;
1632  else if (http_code == 301 || http_code == 302 || http_code == 307)
1633  retry->mode = eRetry_Redirect;
1634  else if (http_code == 303)
1635  retry->mode = eRetry_Redirect303;
1636  else if (http_code == 401)
1637  retry->mode = eRetry_Authenticate;
1638  else if (http_code == 407)
1639  retry->mode = eRetry_ProxyAuthenticate;
1640  else if (http_code <= 0 ||
1641  http_code == 400 || http_code == 403 ||
1642  http_code == 404 || http_code == 405 ||
1643  http_code == 406 || http_code == 410) {
1644  fatal = 1/*true*/;
1645  }
1646  } else
1647  http_code = 0/*no server error*/;
1648 
1650  && (http_code || !x_ErrorHeaderOnly(uuu))) {
1651  /* HTTP header gets printed as part of data logging when
1652  uuu->net_info->debug_printout == eDebugPrintout_Data. */
1653  const char* header_header;
1654  if (!http_code || http_code == 304)
1655  header_header = "HTTP header";
1656  else if (fatal)
1657  header_header = "HTTP header (fatal)";
1658  else if (uuu->flags & fHTTP_KeepHeader)
1659  header_header = "HTTP header (error)";
1660  else if (retry->mode == eRetry_Redirect)
1661  header_header = "HTTP header (redirect)";
1662  else if (retry->mode == eRetry_Redirect303)
1663  header_header = "HTTP header (see other)";
1664  else if (retry->mode & eRetry_Authenticate)
1665  header_header = "HTTP header (authentication)";
1666  else if (retry->mode)
1667  header_header = 0, assert(0);
1668  else
1669  header_header = "HTTP header (retriable server error)";
1670  assert(!url);
1671  url = ConnNetInfo_URL(uuu->net_info);
1673  ("[HTTP%s%s] %s",
1674  url ? "; " : "",
1675  url ? url : "", header_header));
1676  }
1677 
1678  if (!(uuu->flags & fHTTP_KeepHeader)) {
1679  if (fatal) {
1680  assert(http_code);
1681  if (!uuu->net_info->debug_printout && !uuu->parse_header) {
1682  char text[40];
1683  if (!url)
1684  url = ConnNetInfo_URL(uuu->net_info);
1685  if (http_code > 0)
1686  sprintf(text, "%d", http_code);
1687  else
1688  strcpy(text, "occurred");
1689  CORE_LOGF_X(22, eLOG_Error,
1690  ("[HTTP%s%s] Fatal error %s",
1691  url ? "; " : "",
1692  url ? url : "", text));
1693  }
1694  if (!uuu->adjust)
1695  uuu->net_info->max_try = 0;
1696  }
1697 
1698  header_parse = uuu->parse_header
1699  ? uuu->parse_header(hdr, uuu->user_data, http_code)
1701 
1702  if (header_parse == eHTTP_HeaderError) {
1703  retry->mode = eRetry_None;
1705  && !http_code/*i.e. was okay*/ && x_ErrorHeaderOnly(uuu)) {
1706  if (!url)
1707  url = ConnNetInfo_URL(uuu->net_info);
1708  CORE_DATAF_X(9, eLOG_Note, hdr, size,
1709  ("[HTTP%s%s] HTTP header (parse error)",
1710  url ? "; " : "",
1711  url ? url : ""));
1712  }
1713  } else if (header_parse == eHTTP_HeaderComplete) {
1714  /* i.e. stop processing */
1715  retry->mode = eRetry_None;
1716  http_code = 0/*fake success*/;
1717  }
1718  } else {
1719  header_parse = eHTTP_HeaderSuccess;
1720  retry->mode = eRetry_None;
1721  }
1722 
1723  tags = (THTTP_Tags)
1724  (eHTTP_NcbiMsg | eHTTP_Connection
1725  | (retry->mode & eRetry_Redirect ? eHTTP_Location :
1726  retry->mode & eRetry_Authenticate ? eHTTP_Authenticate : 0)
1727  | (uuu->flags & fHTTP_NoAutomagicSID ? 0 : eHTTP_NcbiSid));
1728  if (uuu->http_code / 100 != 1
1729  && uuu->http_code != 204 && uuu->http_code != 304) {
1730  tags |= eHTTP_ContentLength | eHTTP_TransferEncoding;
1731  }
1732 
1733  /* NB: the loop may clobber "hdr" unless fHTTP_KeepHeader is set */
1734  for (s = strchr(hdr, '\n'); s && *s; s = strchr(s + 1, '\n')) {
1735  if (tags & eHTTP_NcbiMsg) {
1736  /* parse NCBI message */
1737  static const char kNcbiMsgTag[] = "\n" HTTP_NCBI_MESSAGE;
1738  if (strncasecmp(s, kNcbiMsgTag, sizeof(kNcbiMsgTag) - 1) == 0) {
1739  char* msg = s + sizeof(kNcbiMsgTag) - 1, *e;
1740  while (*msg && isspace((unsigned char)(*msg)))
1741  ++msg;
1742  if (!(e = strchr(msg, '\r')) && !(e = strchr(msg, '\n')))
1743  break;
1744  if (e > msg) do {
1745  if (!isspace((unsigned char) e[-1]))
1746  break;
1747  } while (--e > msg);
1748  n = (size_t)(e - msg);
1749  if (n) {
1750  char c = msg[n];
1751  msg[n] = '\0';
1752  if (s_MessageHook) {
1753  if (s_MessageIssued <= 0) {
1754  s_MessageIssued = 1;
1755  s_MessageHook(msg);
1756  }
1757  } else {
1758  s_MessageIssued = -1;
1760  ("[NCBI-MESSAGE] %s", msg));
1761  }
1762  msg[n] = c;
1763  }
1764  tags &= (THTTP_Tags)(~eHTTP_NcbiMsg);
1765  continue;
1766  }
1767  }
1768  if (tags & eHTTP_NcbiSid) {
1769  /* parse NCBI SID */
1770  static const char kNcbiSidTag[] = "\n" HTTP_NCBI_SID;
1771  if (strncasecmp(s, kNcbiSidTag, sizeof(kNcbiSidTag) - 1) == 0) {
1772  char* sid = s + sizeof(kNcbiSidTag) - 1, *e;
1773  while (*sid && isspace((unsigned char)(*sid)))
1774  ++sid;
1775  if (!(e = strchr(sid, '\r')) && !(e = strchr(sid, '\n')))
1776  break;
1777  if (e > sid) do {
1778  if (!isspace((unsigned char) e[-1]))
1779  break;
1780  } while (--e > sid);
1781  n = (size_t)(e - sid);
1782  if (n) {
1783  char c = sid[n];
1784  sid[n] = '\0';
1786  sid[n] = c;
1787  }
1788  tags &= (THTTP_Tags)(~eHTTP_NcbiSid);
1789  continue;
1790  }
1791  }
1792  if (tags & eHTTP_Location) {
1793  /* parse "Location" pointer */
1794  static const char kLocationTag[] = "\nLocation:";
1795  assert(http_code);
1796  if (strncasecmp(s, kLocationTag, sizeof(kLocationTag) - 1) == 0) {
1797  char* loc = s + sizeof(kLocationTag) - 1, *e;
1798  while (*loc && isspace((unsigned char)(*loc)))
1799  ++loc;
1800  if (!(e = strchr(loc, '\r')) && !(e = strchr(loc, '\n')))
1801  break;
1802  if (e > loc) do {
1803  if (!isspace((unsigned char) e[-1]))
1804  break;
1805  } while (--e > loc);
1806  n = (size_t)(e - loc);
1807  if (n) {
1808  memmove(hdr, loc, n);
1809  hdr[n] = '\0';
1810  retry->data = hdr;
1811  }
1812  tags &= (THTTP_Tags)(~eHTTP_Location);
1813  continue;
1814  }
1815  }
1816  if (tags & eHTTP_Connection) {
1817  /* parse "Connection" tag */
1818  static const char kConnectionTag[] = "\nConnection:";
1819  if (strncasecmp(s, kConnectionTag, sizeof(kConnectionTag)-1) == 0){
1820  char* con = s + sizeof(kConnectionTag) - 1, *e;
1821  while (*con && isspace((unsigned char)(*con)))
1822  ++con;
1823  if (!(e = strchr(con, '\r')) && !(e = strchr(con, '\n')))
1824  break;
1825  if (e > con) do {
1826  if (!isspace((unsigned char) e[-1]))
1827  break;
1828  } while (--e > con);
1829  while ((n = (size_t)(e - con)) > 0) {
1830  const char* c = (const char*) memchr(con, ',', n);
1831  size_t m;
1832  if (c > con) {
1833  do {
1834  if (!isspace((unsigned char) c[-1]))
1835  break;
1836  } while (--c > con);
1837  m = (size_t)(c - con);
1838  } else
1839  m = n;
1840  if (m == 5 && strncasecmp(con, "close", 5) == 0) {
1841  uuu->keepalive = 0;
1842  break;
1843  }
1844  if (m == 10 && strncasecmp(con, "keep-alive", 10) == 0) {
1845  uuu->keepalive = 1;
1846  break;
1847  }
1848  if (m == n)
1849  break;
1850  con += m + 1;
1851  while (con < e && isspace((unsigned char)(*con)))
1852  ++con;
1853  }
1854  tags &= (THTTP_Tags)(~eHTTP_Connection);
1855  continue;
1856  }
1857  }
1858  if (tags & eHTTP_Authenticate) {
1859  /* parse "Authenticate"/"Proxy-Authenticate" tags */
1860  static const char kAuthenticateTag[] = "-Authenticate:";
1861  n = retry->mode == eRetry_Authenticate ? 4 : 6;
1862  assert(http_code);
1863  if (((retry->mode == eRetry_Authenticate
1864  && strncasecmp(s, "\nWWW", n) == 0) ||
1865  (retry->mode == eRetry_ProxyAuthenticate
1866  && strncasecmp(s, "\nProxy", n) == 0)) &&
1867  strncasecmp(s + n,
1868  kAuthenticateTag, sizeof(kAuthenticateTag)-1)==0) {
1869  char* txt = s + n + sizeof(kAuthenticateTag) - 1, *e;
1870  while (*txt && isspace((unsigned char)(*txt)))
1871  ++txt;
1872  if (!(e = strchr(txt, '\r')) && !(e = strchr(txt, '\n')))
1873  break;
1874  if (e > txt) do {
1875  if (!isspace((unsigned char) e[-1]))
1876  break;
1877  } while (--e > txt);
1878  n = (size_t)(e - txt);
1879  if (n && x_IsValidChallenge(txt, n)) {
1880  memmove(hdr, txt, n);
1881  hdr[n] = '\0';
1882  retry->data = hdr;
1883  }
1884  tags &= (THTTP_Tags)(~eHTTP_Authenticate);
1885  continue;
1886  }
1887  }
1888  if (tags & eHTTP_ContentLength) {
1889  /* parse "Content-Length" for non-HEAD/CONNECT */
1890  static const char kContentLenTag[] = "\nContent-Length:";
1891  if (strncasecmp(s, kContentLenTag, sizeof(kContentLenTag)-1) == 0){
1892  if (!uuu->chunked
1893  && uuu->net_info->req_method != eReqMethod_Head
1894  && uuu->net_info->req_method != eReqMethod_Connect) {
1895  const char* len = s + sizeof(kContentLenTag) - 1, *e;
1896  int tmp;
1897  while (*len && isspace((unsigned char)(*len)))
1898  ++len;
1899  if (!(e = strchr(len, '\r')) && !(e = strchr(len, '\n')))
1900  break;
1901  if (e > len) do {
1902  if (!isspace((unsigned char) e[-1]))
1903  break;
1904  } while (--e > len);
1905  if (e == len
1906  || sscanf(len, "%" NCBI_BIGCOUNT_FORMAT_SPEC "%n",
1907  &uuu->expected, &tmp) < 1
1908  || len + tmp != e) {
1909  uuu->expected = (TNCBI_BigCount)(-1L)/*no checks*/;
1910  }
1911  }
1912  tags &= (THTTP_Tags)(~eHTTP_ContentLength);
1913  continue;
1914  }
1915  }
1916  if (tags & eHTTP_TransferEncoding) {
1917  static const char kTransferEncodingTag[] = "\nTransfer-Encoding:";
1918  if (strncasecmp(s, kTransferEncodingTag,
1919  sizeof(kTransferEncodingTag)-1) == 0) {
1920  const char* te = s + sizeof(kTransferEncodingTag) - 1, *e;
1921  while (*te && isspace((unsigned char)(*te)))
1922  ++te;
1923  if (!(e = strchr(te, '\r')) && !(e = strchr(te, '\n')))
1924  break;
1925  if (e > te) do {
1926  if (!isspace((unsigned char) e[-1]))
1927  break;
1928  } while(--e > te);
1929  n = (size_t)(e - te);
1930  if (n == 7 && strncasecmp(&e[-7], "chunked", 7) == 0
1931  && (isspace((unsigned char) e[-8])
1932  || e[-8] == ':' || e[-8] == ',')) {
1933  uuu->chunked = 1;
1934  }
1935  uuu->expected = 0;
1936  tags &= (THTTP_Tags)
1937  (~(eHTTP_ContentLength | eHTTP_TransferEncoding));
1938  if (!uuu->net_info->http_version) {
1939  if (!url)
1940  url = ConnNetInfo_URL(uuu->net_info);
1942  ("[HTTP%s%s] Chunked transfer encoding"
1943  " with HTTP/1.0",
1944  url ? "; " : "",
1945  url ? url : ""));
1946  }
1947  continue;
1948  }
1949  }
1950  if (!tags)
1951  break;
1952  }
1953  if (uuu->keepalive && uuu->expected == (TNCBI_BigCount)(-1L)) {
1954  if (uuu->net_info->http_version
1955  || uuu->net_info->req_method != eReqMethod_Head) {
1956  uuu->keepalive = 0;
1957  } else
1958  uuu->expected = 0;
1959  }
1960 
1961  if (uuu->flags & fHTTP_KeepHeader) {
1962  assert(retry->mode == eRetry_None);
1963  if (!BUF_AppendEx(&uuu->r_buf, hdr, 0, hdr, size)) {
1964  int error = errno;
1965  if (!url)
1966  url = ConnNetInfo_URL(uuu->net_info);
1968  ("[HTTP%s%s] Cannot save HTTP header",
1969  url ? "; " : "",
1970  url ? url : ""));
1971  free(hdr);
1972  }
1973  if (url)
1974  free(url);
1975  return eIO_Success;
1976  }
1977 
1978  if (!retry->data)
1979  free(hdr);
1980 
1981  if (!http_code || http_code == 304) {
1982  if (url)
1983  free(url);
1984  return header_parse == eHTTP_HeaderError ? eIO_Unknown : eIO_Success;
1985  }
1986  assert(header_parse != eHTTP_HeaderComplete);
1987 
1988  if (uuu->net_info->debug_printout
1989  || header_parse == eHTTP_HeaderContinue) {
1990  if (http_code > 0/*real error, w/only a very short body expected*/)
1992  do {
1993  n = 0;
1994  status = s_ReadData(uuu, &uuu->http, 0, &n, eIO_ReadPlain);
1995  uuu->received += n;
1996  } while (status == eIO_Success);
1997  if (header_parse == eHTTP_HeaderContinue && status == eIO_Closed) {
1998  if (url)
1999  free(url);
2000  return retry->mode ? status/*eIO_Closed*/ : eIO_Success;
2001  }
2002  } else
2003  status = eIO_Success/*NB: irrelevant*/;
2004 
2006  || header_parse == eHTTP_HeaderContinue) {
2007  const char* err = status != eIO_Closed ? IO_StatusStr(status) : 0;
2008  assert(status != eIO_Success);
2009  assert(!err || *err);
2010  if (!url)
2011  url = ConnNetInfo_URL(uuu->net_info);
2012  if (header_parse == eHTTP_HeaderContinue) {
2013  assert(err/*status != eIO_Closed*/);
2015  ("[HTTP%s%s] Server error message incomplete (%s)",
2016  url ? "; " : "",
2017  url ? url : "", err));
2018  } else if (!(size = BUF_Size(uuu->http))) {
2019  CORE_LOGF_X(12, err && !(uuu->flags & fHTTP_SuppressMessages)
2021  ("[HTTP%s%s] No error message received from server"
2022  "%s%s%s",
2023  url ? "; " : "",
2024  url ? url : "",
2025  err ? " (" : "",
2026  err ? err : "", &")"[!err]));
2027  } else if ((s = (char*) malloc(size)) != 0) {
2028  n = BUF_Read(uuu->http, s, size);
2029  if (n != size) {
2030  CORE_LOGF_X(13, eLOG_Error,
2031  ("[HTTP%s%s] Cannot extract server error message"
2032  " from buffer entirely (%lu/%lu)",
2033  url ? "; " : "",
2034  url ? url : "",
2035  (unsigned long) n, (unsigned long) size));
2036  }
2037  if (n) {
2038  CORE_DATAF_X(14, eLOG_Note, s, n,
2039  ("[HTTP%s%s] Server error message%s%s%s",
2040  url ? "; " : "",
2041  url ? url : "",
2042  err ? " (" : "",
2043  err ? err : "", &")"[!err]));
2044  }
2045  free(s);
2046  } else {
2047  CORE_LOGF_X(15, eLOG_Error,
2048  ("[HTTP%s%s] Cannot allocate server error message"
2049  " (%lu byte%s)",
2050  url ? "; " : "",
2051  url ? url : "",
2052  (unsigned long) size, &"s"[size == 1]));
2053  }
2054  }
2055 
2056  if (url)
2057  free(url);
2058  if (header_parse != eHTTP_HeaderContinue)
2059  BUF_Erase(uuu->http);
2060  return eIO_Unknown;
2061 }
2062 
2063 
2064 /* Prepare connector for reading. Open socket if necessary and make an initial
2065  * connect and send, re-trying if possible until success.
2066  * Return codes:
2067  * eIO_Success = success, connector is ready for reading (uuu->sock != NULL);
2068  * eIO_Timeout = maybe (check uuu->sock) connected and no data available yet;
2069  * other code = error, not connected (uuu->sock == NULL).
2070  * NB: On error, uuu->r_buf may be updated to contain new pending data!
2071  */
2073  const STimeout* timeout,
2074  EExtractMode extract)
2075 {
2076  EIO_Status status;
2077 
2078  for (;;) {
2079  EIO_Status adjust;
2080  SRetry retry;
2081 
2082  if ((status = s_ConnectAndSend(uuu, timeout, extract)) != eIO_Success)
2083  break;
2084  assert(!uuu->w_len);
2085 
2086  if (uuu->conn_state == eCS_WriteRequest) {
2087  assert(x_IsWriteThru(uuu) && uuu->chunked);
2088  BUF_Erase(uuu->w_buf);
2089  if (!BUF_Write(&uuu->w_buf, "0\r\n\r\n", 5)) {
2090  status = eIO_Unknown;
2091  break;
2092  }
2093  uuu->w_len = 5;
2095 
2096  status = s_ConnectAndSend(uuu, timeout, extract);
2097  if (status != eIO_Success)
2098  break;
2099  assert(!uuu->w_len);
2100  }
2102 
2103  if (extract == eEM_Flush)
2104  return eIO_Success;
2105 
2106  if (extract == eEM_Wait
2107  && (uuu->conn_state & eCS_DoneBody) == eCS_DoneBody) {
2108  return eIO_Closed;
2109  }
2110 
2111  /* set read timeout before leaving (s_Read() expects it set) */
2112  SOCK_SetTimeout(uuu->sock, eIO_Read, timeout);
2113 
2114  if (uuu->conn_state & eCS_ReadBody)
2115  return eIO_Success;
2116 
2117  assert(uuu->sock && uuu->conn_state == eCS_ReadHeader);
2118  if ((status = s_ReadHeader(uuu, &retry, extract)) == eIO_Success) {
2119  assert((uuu->conn_state & eCS_ReadBody) && !retry.mode);
2120  /* pending output data no longer needed */
2121  BUF_Erase(uuu->w_buf);
2122  break;
2123  }
2124 
2125  assert(status != eIO_Timeout || !retry.mode);
2126  /* if polling then bail out with eIO_Timeout */
2127  if (status == eIO_Timeout
2128  && (extract == eEM_Wait
2129  || (timeout && !(timeout->sec | timeout->usec)))) {
2130  assert(!retry.data);
2131  return eIO_Timeout;
2132  }
2133 
2134  /* HTTP header read error; disconnect and retry */
2135  assert(status != eIO_Success);
2137  adjust = s_Adjust(uuu, &retry, status, extract);
2138  if (retry.data)
2139  free((void*) retry.data);
2140  if (adjust != eIO_Success) {
2141  if (adjust != eIO_Closed)
2142  status = adjust;
2143  break;
2144  }
2145  }
2146 
2147  if (BUF_Size(uuu->http) && !BUF_Splice(&uuu->r_buf, uuu->http))
2148  BUF_Erase(uuu->http);
2149  assert(!BUF_Size(uuu->http));
2150  return status;
2151 }
2152 
2153 
2154 /* NB: Sets the EOM read state */
2155 static void x_Close(SHttpConnector* uuu)
2156 {
2157  /* since this is merely an acknowledgement, it will be "instant" */
2159  SOCK_Destroy(uuu->sock);
2160  uuu->sock = 0;
2161  uuu->conn_state = eCS_Eom;
2162 }
2163 
2164 
2165 /* Read non-header data from connection */
2167  size_t size, size_t* n_read)
2168 {
2169  EIO_Status status = eIO_Success;
2170 
2171  assert(uuu->conn_state & eCS_ReadBody);
2173 
2174  if ((uuu->conn_state & eCS_DoneBody) == eCS_DoneBody) {
2175  if (uuu->conn_state == eCS_Eom)
2176  return eIO_Closed;
2177  if (uuu->chunked) {
2178  if (uuu->conn_state != eCS_Discard)
2179  status = x_ReadChunkTail(uuu);
2180  if (uuu->conn_state == eCS_Discard) {
2181  if (uuu->keepalive)
2182  uuu->conn_state = eCS_Eom;
2183  else
2184  x_Close(uuu);
2185  } else if (status != eIO_Closed)
2186  x_Close(uuu);
2187  } else
2188  uuu->conn_state = eCS_Eom;
2189  return status ? status : eIO_Closed;
2190  }
2191  assert(uuu->sock && size && n_read && !*n_read);
2192  assert(uuu->received <= uuu->expected);
2193 
2194  if (uuu->net_info->req_method == eReqMethod_Head
2195  || uuu->http_code / 100 == 1
2196  || uuu->http_code == 204
2197  || uuu->http_code == 304) {
2198  uuu->conn_state = eCS_Discard;
2199  status = eIO_Closed;
2200  } else if (!uuu->net_info->http_version
2201  && (uuu->flags & fHCC_UrlDecodeInput)) {
2202  /* read and URL-decode */
2203  size_t n_peeked, n_decoded;
2204  TNCBI_BigCount remain = uuu->expected - uuu->received;
2205  size_t peek_size = size > remain ? (size_t)(remain + 1) : size;
2206  void* peek_buf = malloc(peek_size *= 3);
2207 
2208  /* peek the data */
2209  status= SOCK_Read(uuu->sock,peek_buf,peek_size,&n_peeked,eIO_ReadPeek);
2210  if (status == eIO_Success) {
2211  if (URL_DecodeEx(peek_buf,n_peeked,&n_decoded,buf,size,n_read,"")){
2212  /* decode, then discard successfully decoded data from input */
2213  if (n_decoded) {
2214  SOCK_Read(uuu->sock,0,n_decoded,&n_peeked,eIO_ReadPersist);
2215  assert(n_peeked == n_decoded);
2216  uuu->received += n_decoded;
2217  } else {
2218  assert(!*n_read);
2219  if (size) {
2220  /* if at EOF and remaining data cannot be decoded */
2221  status = SOCK_Status(uuu->sock, eIO_Read);
2222  if (status == eIO_Closed)
2223  status = eIO_Unknown;
2224  }
2225  }
2226  } else
2227  status = eIO_Unknown;
2228  if (status != eIO_Success) {
2229  char* url = ConnNetInfo_URL(uuu->net_info);
2230  CORE_LOGF_X(16, eLOG_Error,
2231  ("[HTTP%s%s] Cannot URL-decode data: %s",
2232  url ? "; " : "",
2233  url ? url : "", IO_StatusStr(status)));
2234  if (url)
2235  free(url);
2236  }
2237  } else
2238  assert(!n_peeked);
2239  if (peek_buf)
2240  free(peek_buf);
2241  } else {
2242  /* just read, with no URL-decoding */
2243  status = s_ReadData(uuu, buf, size, n_read, eIO_ReadPlain);
2244  uuu->received += *n_read;
2245  }
2246 
2247  if (status == eIO_Closed) {
2248  if (!uuu->keepalive)
2249  x_Close(uuu);
2250  else if (uuu->conn_state == eCS_Discard)
2251  uuu->conn_state = eCS_Eom;
2252  }
2253 
2254  if (uuu->expected != (TNCBI_BigCount)(-1L)) {
2255  const char* how = 0;
2256  if (uuu->received < uuu->expected) {
2257  if (status == eIO_Closed) {
2258  status = eIO_Unknown;
2259  how = "Premature EOM in";
2260  }
2261  } else if (uuu->expected < uuu->received) {
2262  if (!uuu->net_info->http_version
2263  && (uuu->flags & fHCC_UrlDecodeInput)) {
2264  assert(*n_read);
2265  --(*n_read);
2266  } else {
2267  TNCBI_BigCount excess = uuu->received - uuu->expected;
2268  assert(*n_read >= excess);
2269  *n_read -= (size_t) excess;
2270  }
2271  uuu->conn_state = eCS_Discard;
2272  status = eIO_Unknown;
2273  how = "Got too much";
2274  } else if (!uuu->chunked && uuu->keepalive)
2275  uuu->conn_state = eCS_DoneBody;
2276  if (how) {
2277  char* url = ConnNetInfo_URL(uuu->net_info);
2279  ("[HTTP%s%s] %s data (received "
2280  "%" NCBI_BIGCOUNT_FORMAT_SPEC " vs. "
2281  "%" NCBI_BIGCOUNT_FORMAT_SPEC " expected)",
2282  url ? "; " : "",
2283  url ? url : "", how,
2284  uuu->received,
2285  uuu->expected != (TNCBI_BigCount)(-1L) ?
2286  uuu->expected : 0));
2287  if (url)
2288  free(url);
2289  }
2290  }
2291  return status;
2292 }
2293 
2294 
2295 /* Reset/readout input data and close socket */
2297  const STimeout* timeout,
2298  EExtractMode extract)
2299 {
2300  EIO_Status status = eIO_Success;
2301 
2302  assert(!(extract & eEM_Wait)); /* here it's only either drop or read */
2303 
2304  BUF_Erase(uuu->http);
2305  if (extract == eEM_Drop)
2306  BUF_Erase(uuu->r_buf);
2307  else if (uuu->conn_state != eCS_Eom
2308  && (status = s_PreRead(uuu, timeout, extract)) == eIO_Success) {
2309  char* x_buf = 0;
2310  do {
2311  if (x_buf || (x_buf = (char*) malloc(HTTP_SOAK_READ_SIZE)) != 0){
2312  size_t x_read = 0;
2313  status = s_Read(uuu, x_buf, HTTP_SOAK_READ_SIZE, &x_read);
2314  if (x_read < HTTP_SOAK_READ_SIZE / 2) {
2315  if (!BUF_Write(&uuu->r_buf, x_buf, x_read))
2316  status = eIO_Unknown;
2317  } else {
2318  if (!BUF_AppendEx(&uuu->r_buf,
2319  x_buf, HTTP_SOAK_READ_SIZE,
2320  x_buf, x_read)) {
2321  status = eIO_Unknown;
2322  } else
2323  x_buf = 0;
2324  }
2325  } else
2326  status = eIO_Unknown;
2327  } while (status == eIO_Success);
2328  if (x_buf)
2329  free(x_buf);
2330  if (status == eIO_Closed)
2331  status = eIO_Success;
2332  }
2333 
2334  /* s_PreRead() might have dropped the connection already */
2335  if (uuu->sock && (extract == eEM_Drop || !uuu->keepalive))
2336  s_DropConnection(uuu, eCS_Eom);
2337  uuu->can_connect &= ~fCC_Once;
2338  return status;
2339 }
2340 
2341 
2343  const STimeout* timeout)
2344 {
2345  /* NOTE: the real connect will be performed on the first "READ", or
2346  * "FLUSH/CLOSE", or on "WAIT" on read -- see in "s_ConnectAndSend" */
2347  assert(!uuu->sock);
2348 
2349  /* store timeouts for later use */
2350  if (timeout) {
2351  uuu->oo_timeout = *timeout;
2352  uuu->o_timeout = &uuu->oo_timeout;
2353  uuu->ww_timeout = *timeout;
2354  uuu->w_timeout = &uuu->ww_timeout;
2355  } else {
2356  uuu->o_timeout = kInfiniteTimeout;
2357  uuu->w_timeout = kInfiniteTimeout;
2358  }
2359 
2360  /* reset the auto-reconnect/state/re-try/auth features */
2361  uuu->can_connect = (uuu->flags & fHTTP_AutoReconnect
2364  uuu->major_fault = 0;
2365  uuu->minor_fault = 0;
2366  uuu->auth_done = 0;
2367  uuu->proxy_auth_done = 0;
2368  uuu->retry = 0;
2369 }
2370 
2371 
2373 {
2374  assert(!uuu->sock);
2375  if (uuu->cleanup)
2376  uuu->cleanup(uuu->user_data);
2378  BUF_Destroy(uuu->r_buf);
2379  BUF_Destroy(uuu->w_buf);
2380  BUF_Destroy(uuu->http);
2381  free(uuu);
2382 }
2383 
2384 
2385 /***********************************************************************
2386  * INTERNAL -- "s_VT_*" functions for the "virt. table" of connector methods
2387  ***********************************************************************/
2388 
2389 #ifdef __cplusplus
2390 extern "C" {
2391 #endif /* __cplusplus */
2392  static const char* s_VT_GetType (CONNECTOR connector);
2393  static char* s_VT_Descr (CONNECTOR connector);
2394  static EIO_Status s_VT_Open (CONNECTOR connector,
2395  const STimeout* timeout);
2396  static EIO_Status s_VT_Wait (CONNECTOR connector,
2397  EIO_Event event,
2398  const STimeout* timeout);
2399  static EIO_Status s_VT_Write (CONNECTOR connector,
2400  const void* buf,
2401  size_t size,
2402  size_t* n_written,
2403  const STimeout* timeout);
2404  static EIO_Status s_VT_Flush (CONNECTOR connector,
2405  const STimeout* timeout);
2406  static EIO_Status s_VT_Read (CONNECTOR connector,
2407  void* buf,
2408  size_t size,
2409  size_t* n_read,
2410  const STimeout* timeout);
2411  static EIO_Status s_VT_Status (CONNECTOR connector,
2412  EIO_Event dir);
2413  static EIO_Status s_VT_Close (CONNECTOR connector,
2414  const STimeout* timeout);
2415  static void s_Setup (CONNECTOR connector);
2416  static void s_Destroy (CONNECTOR connector);
2417 #ifdef __cplusplus
2418 } /* extern "C" */
2419 #endif /* __cplusplus */
2420 
2421 
2422 /*ARGSUSED*/
2423 static const char* s_VT_GetType
2424 (CONNECTOR connector)
2425 {
2426  return "HTTP";
2427 }
2428 
2429 
2430 static char* s_VT_Descr
2431 (CONNECTOR connector)
2432 {
2433  return ConnNetInfo_URL(((SHttpConnector*) connector->handle)->net_info);
2434 }
2435 
2436 
2438 (CONNECTOR connector,
2439  const STimeout* timeout)
2440 {
2441  s_OpenHttpConnector((SHttpConnector*) connector->handle, timeout);
2442  return eIO_Success;
2443 }
2444 
2445 
2447 (CONNECTOR connector,
2448  EIO_Event event,
2449  const STimeout* timeout)
2450 {
2451  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2452  EIO_Status status;
2453 
2454  assert(event == eIO_Read || event == eIO_Write);
2455  switch (event) {
2456  case eIO_Read:
2457  if (BUF_Size(uuu->r_buf))
2458  return eIO_Success;
2459  if (uuu->can_connect == fCC_None)
2460  return eIO_Closed;
2461  status = s_PreRead(uuu, timeout, eEM_Wait);
2462  if (BUF_Size(uuu->r_buf))
2463  return eIO_Success;
2464  if (status != eIO_Success)
2465  return status;
2466  assert(uuu->sock);
2467  status = SOCK_Status(uuu->sock, eIO_Read);
2468  if (status != eIO_Success)
2469  return status;
2470  return SOCK_Wait(uuu->sock, eIO_Read, timeout);
2471  case eIO_Write:
2472  if (uuu->can_connect == fCC_None)
2473  return eIO_Closed;
2474  if (!x_IsWriteThru(uuu)) {
2475  return uuu->sock && uuu->can_connect == fCC_Once
2476  ? eIO_Closed : eIO_Success;
2477  }
2478  if (!uuu->sock && !BUF_Size(uuu->w_buf))
2479  return eIO_Success;
2480  if (!uuu->sock || uuu->conn_state < eCS_ReadHeader) {
2481  status = s_ConnectAndSend(uuu, timeout, eEM_Flush);
2482  if (status != eIO_Success)
2483  return status;
2484  } else
2485  return uuu->can_connect == fCC_Once ? eIO_Closed : eIO_Success;
2486  assert(uuu->sock);
2487  assert(uuu->conn_state < eCS_ReadHeader && !uuu->w_len);
2488  return uuu->conn_state < eCS_FlushRequest
2489  ? SOCK_Wait(uuu->sock, eIO_Write, timeout)
2490  : eIO_Success;
2491  default:
2492  assert(0);
2493  break;
2494  }
2495  return eIO_InvalidArg;
2496 }
2497 
2498 
2500 (CONNECTOR connector,
2501  const void* buf,
2502  size_t size,
2503  size_t* n_written,
2504  const STimeout* timeout)
2505 {
2506  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2507  EIO_Status status;
2508 
2509  assert(!*n_written);
2510 
2511  /* store the write timeout */
2512  if (timeout) {
2513  uuu->ww_timeout = *timeout;
2514  uuu->w_timeout = &uuu->ww_timeout;
2515  } else
2516  uuu->w_timeout = kInfiniteTimeout;
2517 
2518  /* if trying to write after a request then close the socket first */
2519  if (uuu->conn_state > eCS_WriteRequest) {
2520  status = s_Disconnect(uuu, timeout,
2521  uuu->flags & fHTTP_DropUnread
2522  ? eEM_Drop : eEM_Read);
2523  if (status != eIO_Success)
2524  return status;
2526  }
2527  if (uuu->can_connect == fCC_None)
2528  return eIO_Closed; /* no more connects permitted */
2529  uuu->can_connect |= fCC_Once;
2530 
2531  /* check if writing is at all legitimate */
2532  if (size && (uuu->net_info->req_method == eReqMethod_Head ||
2533  uuu->net_info->req_method == eReqMethod_Get)) {
2534  char* url = ConnNetInfo_URL(uuu->net_info);
2535  CORE_LOGF_X(24, eLOG_Error,
2536  ("[HTTP%s%s] Illegal write (%lu byte%s) with %s",
2537  url ? "; " : "",
2538  url ? url : "",
2539  (unsigned long) size, &"s"[size == 1],
2541  ? "GET" : "HEAD"));
2542  if (url)
2543  free(url);
2544  return eIO_Closed;
2545  }
2546 
2547  /* write-through with HTTP/1.1 */
2548  if (x_IsWriteThru(uuu)) {
2549  if (BUF_Size(uuu->w_buf)) {
2550  status = s_ConnectAndSend(uuu, timeout, eEM_Flush);
2551  if (status != eIO_Success)
2552  return status;
2553  }
2554  assert(!uuu->sock || (uuu->conn_state == eCS_WriteRequest
2555  && !uuu->w_len));
2556  if (size) {
2557  char prefix[80];
2558  int n = sprintf(prefix, "%" NCBI_BIGCOUNT_FORMAT_SPEC_HEX "\r\n",
2559  (TNCBI_BigCount) size);
2560  BUF_Erase(uuu->w_buf);
2561  if (!BUF_Write(&uuu->w_buf, prefix, (size_t) n) ||
2562  !BUF_Write(&uuu->w_buf, buf, size) ||
2563  !BUF_Write(&uuu->w_buf, "\r\n", 2)) {
2564  BUF_Erase(uuu->w_buf);
2565  return eIO_Unknown;
2566  }
2567  *n_written = size;
2568  size += (size_t) n + 2;
2569  uuu->w_len = size;
2570  uuu->entity = 1;
2571  }
2572  return eIO_Success;
2573  }
2574 
2575  /* accumulate all output in a memory buffer */
2576  if (size && !uuu->net_info->http_version
2577  && (uuu->flags & fHCC_UrlEncodeOutput)) {
2578  /* with URL-encoding */
2579  size_t dst_size = 3 * size;
2580  void* dst = malloc(dst_size);
2581  URL_Encode(buf, size, n_written, dst, dst_size, &dst_size);
2582  if (!*n_written
2583  || !BUF_AppendEx(&uuu->w_buf, dst, 0, dst, dst_size)) {
2584  if (dst)
2585  free(dst);
2586  return eIO_Unknown;
2587  }
2588  } else {
2589  /* "as is" (without URL-encoding) */
2590  if (!BUF_Write(&uuu->w_buf, buf, size))
2591  return eIO_Unknown;
2592  *n_written = size;
2593  }
2594  return eIO_Success;
2595 }
2596 
2597 
2599 (CONNECTOR connector,
2600  const STimeout* timeout)
2601 {
2602  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2603  EIO_Status status;
2604 
2605  if (!uuu->sock && uuu->can_connect == fCC_None)
2606  return eIO_Closed;
2607 
2608  if (timeout) {
2609  uuu->ww_timeout = *timeout;
2610  uuu->w_timeout = &uuu->ww_timeout;
2611  } else
2612  uuu->w_timeout = timeout;
2613 
2614  if (!(uuu->flags & fHTTP_Flushable))
2615  return eIO_Success;
2616 
2617  if (uuu->conn_state & eCS_ReadBody)
2618  return eIO_Success;
2619  if (uuu->sock
2620  && !(x_IsWriteThru(uuu) && uuu->conn_state < eCS_ReadHeader)) {
2621  return eIO_Success;
2622  }
2623  status = x_IsWriteThru(uuu)
2624  ? s_ConnectAndSend(uuu, timeout, eEM_Flush)
2625  : s_PreRead (uuu, timeout, eEM_Flush);
2626  return BUF_Size(uuu->r_buf) ? eIO_Success : status;
2627 }
2628 
2629 
2631 (CONNECTOR connector,
2632  void* buf,
2633  size_t size,
2634  size_t* n_read,
2635  const STimeout* timeout)
2636 {
2637  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2638  EExtractMode extract = BUF_Size(uuu->r_buf) ? eEM_Flush : eEM_Read;
2639  EIO_Status status = uuu->can_connect & fCC_Once
2640  ? s_PreRead(uuu, timeout, extract) : eIO_Unknown;
2641  size_t x_read = BUF_Read(uuu->r_buf, buf, size);
2642 
2643  assert(n_read && !*n_read);
2644  if (x_read < size && extract == eEM_Read && status == eIO_Success) {
2645  status = s_Read(uuu, (char*) buf + x_read, size - x_read, n_read);
2646  *n_read += x_read;
2647  } else
2648  *n_read = x_read;
2649  return extract == eEM_Read ? status : eIO_Success;
2650 }
2651 
2652 
2654 (CONNECTOR connector,
2655  EIO_Event dir)
2656 {
2657  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2658  return uuu->sock ? SOCK_Status(uuu->sock, dir) :
2659  (uuu->can_connect == fCC_None ? eIO_Closed : eIO_Success);
2660 }
2661 
2662 
2664 (CONNECTOR connector,
2665  const STimeout* timeout)
2666 {
2667  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2668 
2669  /* Send the accumulated output data(if any) to server, then close the
2670  * socket. Regardless of the flush, clear both input and output buffers.
2671  */
2672  if ((uuu->can_connect & fCC_Once)
2673  && ((!uuu->sock && BUF_Size(uuu->w_buf))
2674  || (uuu->flags & fHTTP_Flushable))) {
2675  /* "WRITE" mode and data (or just flag) is still pending */
2676  s_PreRead(uuu, timeout, eEM_Drop);
2677  }
2678  s_Disconnect(uuu, timeout, eEM_Drop);
2679  assert(!uuu->sock);
2680 
2681  /* clear pending output data, if any */
2682  BUF_Erase(uuu->w_buf);
2683  return eIO_Success;
2684 }
2685 
2686 
2687 static void s_Setup
2688 (CONNECTOR connector)
2689 {
2690  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2691  SMetaConnector* meta = connector->meta;
2692 
2693  /* initialize virtual table */
2694  CONN_SET_METHOD(meta, get_type, s_VT_GetType, connector);
2695  CONN_SET_METHOD(meta, descr, s_VT_Descr, connector);
2696  CONN_SET_METHOD(meta, open, s_VT_Open, connector);
2697  CONN_SET_METHOD(meta, wait, s_VT_Wait, connector);
2698  CONN_SET_METHOD(meta, write, s_VT_Write, connector);
2699  CONN_SET_METHOD(meta, flush, s_VT_Flush, connector);
2700  CONN_SET_METHOD(meta, read, s_VT_Read, connector);
2701  CONN_SET_METHOD(meta, status, s_VT_Status, connector);
2702  CONN_SET_METHOD(meta, close, s_VT_Close, connector);
2704 }
2705 
2706 
2707 static void s_Destroy
2708 (CONNECTOR connector)
2709 {
2710  SHttpConnector* uuu = (SHttpConnector*) connector->handle;
2711  connector->handle = 0;
2712 
2714  free(connector);
2715 }
2716 
2717 
2718 /* NB: per the standard, the HTTP tag name is mis-spelled as "Referer" */
2719 static int/*bool*/ x_FixupUserHeader(SConnNetInfo* net_info,
2720  int /*bool*/ has_ref,
2721  int* /*bool*/ has_sid)
2722 {
2723  int/*bool*/ has_host = 0/*false*/;
2724  const char* s;
2725 
2726  if ((s = net_info->http_user_header) != 0) {
2727  int/*bool*/ first = 1/*true*/;
2728  while (*s) {
2729  if (has_ref < 0/*unset*/
2730  && strncasecmp(s, &"\nReferer:"[first], 9 - !!first) == 0) {
2731  has_ref = 1/*true*/;
2732  } else if (strncasecmp(s, &"\nHost:"[first], 6 - !!first) == 0) {
2733  has_host = 1/*true*/;
2734  } else if (strncasecmp(s, &"\nCAF"[first], 4 - !!first) == 0
2735  && (s[4 - first] == '-' || s[4 - !!first] == ':')) {
2736  char* caftag = strndup(s + !first, strcspn(s + !first, " \t"));
2737  if (caftag) {
2738  size_t cafoff = (size_t)(s - net_info->http_user_header);
2739  ConnNetInfo_DeleteUserHeader(net_info, caftag);
2740  free(caftag);
2741  if (!(s = net_info->http_user_header) || !*(s += cafoff))
2742  break;
2743  continue;
2744  }
2745  } else if (!*has_sid
2746  && strncasecmp(s, &"\n" HTTP_NCBI_SID[first],
2747  sizeof(HTTP_NCBI_SID) - !!first) == 0) {
2748  *has_sid = 1/*true*/;
2749  }
2750  if (!(s = strchr(++s, '\n')))
2751  break;
2752  first = 0/*false*/;
2753  }
2754  }
2755  s = CORE_GetAppName();
2756  if (s && *s) {
2757  char buf[128];
2758  sprintf(buf, "User-Agent: %.80s", s);
2759  ConnNetInfo_ExtendUserHeader(net_info, buf);
2760  }
2761  if ((s = net_info->http_referer) != 0) {
2762  char* ref;
2763  if (has_ref <= 0 && *s) {
2764  size_t len = strlen(s);
2765  if ((ref = (char*) realloc((char*) s, 10 + len)) != 0) {
2766  memmove(ref + 9, ref, len + 1);
2767  memcpy(ref, "Referer: ", 9);
2768  ConnNetInfo_AppendUserHeader(net_info, ref);
2769  } else
2770  ref = (char*) s;
2771  } else
2772  ref = (char*) s;
2773  net_info->http_referer = 0;
2774  assert(ref);
2775  free(ref);
2776  }
2777  if (net_info->external)
2779  return has_host;
2780 }
2781 
2782 
2784 (const SConnNetInfo* net_info,
2785  const char* user_header,
2786  int/*bool*/ tunnel,
2788  void* user_data,
2789  FHTTP_Adjust adjust,
2790  SHttpConnector** http)
2791 {
2792  SConnNetInfo* xxx;
2793  SHttpConnector* uuu;
2794  int/*bool*/ ref;
2795  int/*bool*/ sid;
2796 
2797  *http = 0;
2798  xxx = (net_info
2799  ? ConnNetInfo_Clone(net_info)
2801  if (!xxx)
2802  return eIO_Unknown;
2803 
2804  if (xxx->req_method >= eReqMethod_v1) {
2805  xxx->req_method &= ~eReqMethod_v1;
2806  xxx->http_version = 1;
2807  }
2808  if (xxx->http_version && (flags & fHTTP_PushAuth))
2809  xxx->http_push_auth = 1;
2810  if (!tunnel) {
2811  if (xxx->req_method == eReqMethod_Connect
2812  || (xxx->scheme != eURL_Unspec &&
2813  xxx->scheme != eURL_Https &&
2814  xxx->scheme != eURL_Http)) {
2815  ConnNetInfo_Destroy(xxx);
2816  return eIO_InvalidArg;
2817  }
2818  if (xxx->scheme == eURL_Unspec)
2819  xxx->scheme = eURL_Http;
2820  ConnNetInfo_SetFrag(xxx, "");
2821  }
2822 
2823  if (user_header && *user_header
2824  && !ConnNetInfo_OverrideUserHeader(xxx, user_header)) {
2825  ConnNetInfo_Destroy(xxx);
2826  return eIO_Unknown;
2827  }
2828 
2829  if (tunnel) {
2830  if (!xxx->http_proxy_host[0] || !xxx->http_proxy_port
2831  || xxx->http_proxy_only) {
2832  ConnNetInfo_Destroy(xxx);
2833  return eIO_InvalidArg;
2834  }
2836  xxx->http_version = 0;
2837  xxx->path[0] = '\0';
2838  if (xxx->http_referer) {
2839  free((void*) xxx->http_referer);
2840  xxx->http_referer = 0;
2841  }
2842  ConnNetInfo_DeleteUserHeader(xxx, "Referer:");
2843  ref = 0/*false*/;
2844  } else
2845  ref = -1/*unset*/;
2846 
2847  if (!(uuu = (SHttpConnector*) malloc(sizeof(SHttpConnector)))) {
2848  ConnNetInfo_Destroy(xxx);
2849  return eIO_Unknown;
2850  }
2851 
2852  if (xxx->max_try < 1 || (flags & fHTTP_NoAutoRetry))
2853  xxx->max_try = 1;
2854 
2855  /* initialize internal data structure */
2856  uuu->net_info = xxx;
2857 
2858  uuu->parse_header = 0;
2859  uuu->user_data = user_data;
2860  uuu->adjust = adjust;
2861  uuu->cleanup = 0;
2862 
2863  sid = flags & fHTTP_NoAutomagicSID ? 1 : tunnel;
2864  uuu->skip_host = x_FixupUserHeader(xxx, ref, &sid) ? 1 : 0;
2865  if (sid)
2867  uuu->flags = flags;
2868 
2870  uuu->error_header = eDefault;
2871  uuu->can_connect = fCC_None; /* will be properly set at open */
2872 
2873  uuu->reserved = 0;
2874  memset(uuu->unused, 0, sizeof(uuu->unused));
2875 
2876  uuu->sock = 0;
2877  uuu->o_timeout = kDefaultTimeout; /* deliberately bad values here... */
2878  uuu->w_timeout = kDefaultTimeout; /* ...must be reset prior to use */
2879  uuu->http = 0;
2880  uuu->r_buf = 0;
2881  uuu->w_buf = 0;
2882  uuu->w_len = 0;
2883 
2884  if (tunnel)
2885  s_OpenHttpConnector(uuu, xxx->timeout);
2886  /* else there are some unintialized fields left -- they are inited later */
2887 
2888  *http = uuu;
2889  return eIO_Success;
2890 }
2891 
2892 
2894 (const SConnNetInfo* net_info,
2895  const char* user_header,
2897  FHTTP_ParseHeader parse_header,
2898  void* user_data,
2899  FHTTP_Adjust adjust,
2901 {
2902  SHttpConnector* uuu;
2903  CONNECTOR ccc;
2904 
2905  if (s_CreateHttpConnector(net_info, user_header, 0/*regular*/,
2906  flags, user_data, adjust, &uuu) != eIO_Success) {
2907  assert(!uuu);
2908  return 0;
2909  }
2910  assert(uuu);
2911 
2912  if (!(ccc = (SConnector*) malloc(sizeof(SConnector)))) {
2914  return 0;
2915  }
2916 
2917  /* initialize additional internal data structure */
2918  uuu->parse_header = parse_header;
2919  uuu->cleanup = cleanup;
2920 
2921  /* enable an override from outside */
2922  if (!uuu->unsafe_redir)
2923  uuu->unsafe_redir = eDefault;
2924 
2925  /* initialize connector structure */
2926  ccc->handle = uuu;
2927  ccc->next = 0;
2928  ccc->meta = 0;
2929  ccc->setup = s_Setup;
2930  ccc->destroy = s_Destroy;
2931 
2932  return ccc;
2933 }
2934 
2935 
2936 /***********************************************************************
2937  * EXTERNAL -- the connector's "constructor"
2938  ***********************************************************************/
2939 
2941 (const SConnNetInfo* net_info,
2942  const char* user_header,
2944 {
2945  return s_CreateConnector(net_info, user_header, flags, 0, 0, 0, 0);
2946 }
2947 
2948 
2950 (const SConnNetInfo* net_info,
2952  FHTTP_ParseHeader parse_header,
2953  void* user_data,
2954  FHTTP_Adjust adjust,
2956 {
2957  return s_CreateConnector(net_info, 0/*user_header*/, flags,
2958  parse_header, user_data, adjust, cleanup);
2959 }
2960 
2961 
2963 (const SConnNetInfo* net_info,
2965  const void* init_data,
2966  size_t init_size,
2967  void* user_data,
2968  FHTTP_Adjust adjust,
2969  SOCK* sock)
2970 {
2971  unsigned short http_code;
2972  EIO_Status status;
2973  SHttpConnector* uuu;
2974 
2975  if (!sock)
2976  return eIO_InvalidArg;
2977 
2978  status = s_CreateHttpConnector(net_info, 0/*user_header*/, 1/*tunnel*/,
2980  user_data, adjust, &uuu);
2981  if (status != eIO_Success) {
2982  assert(!uuu);
2983  return status;
2984  }
2985  uuu->sock = *sock;
2986  *sock = 0;
2987  assert(uuu && !BUF_Size(uuu->w_buf));
2988  if (!init_size || BUF_Prepend(&uuu->w_buf, init_data, init_size)) {
2989  status = s_PreRead(uuu, uuu->net_info->timeout, eEM_Wait);
2990  if (status == eIO_Success) {
2991  assert(uuu->conn_state == eCS_ReadBody);
2992  assert(uuu->http_code / 100 == 2);
2993  assert(uuu->sock);
2994  *sock = uuu->sock;
2995  uuu->sock = 0;
2996  http_code = 0;
2997  } else
2998  http_code = uuu->http_code;
2999  } else {
3000  http_code = 0;
3001  status = eIO_Unknown;
3002  }
3003  if (uuu->sock)
3004  s_DropConnection(uuu, eCS_Eom);
3006  switch (http_code) {
3007  case 503:
3008  return eIO_NotSupported;
3009  case 426:
3010  case 404:
3011  return eIO_InvalidArg;
3012  case 403:
3013  return eIO_Closed;
3014  default:
3015  break;
3016  }
3017  return status;
3018 }
3019 
3020 
3022 (const SConnNetInfo* net_info,
3024  SOCK* sock)
3025 {
3026  return HTTP_CreateTunnelEx(net_info, flags, 0, 0, 0, 0, sock);
3027 }
3028 
3029 
3031 {
3032  if (hook) {
3033  if (hook != s_MessageHook)
3034  s_MessageIssued = s_MessageIssued ? -1 : -2;
3035  } else if (s_MessageIssued < -1)
3036  s_MessageIssued = 0;
3037  s_MessageHook = hook;
3038 }
static void fatal(const char *msg,...)
Definition: attributes.c:18
static void cleanup(void)
Definition: ct_dynamic.c:30
static uch flags
int close(int fd)
Definition: connection.cpp:45
CS_CONTEXT * ctx
Definition: t0006.c:12
static DLIST_TYPE *DLIST_NAME() first(DLIST_LIST_TYPE *list)
Definition: dlist.tmpl.h:46
@ eOff
Definition: ncbi_types.h:110
@ eDefault
Definition: ncbi_types.h:112
@ eOn
Definition: ncbi_types.h:111
void BUF_Erase(BUF buf)
Definition: ncbi_buffer.c:461
int BUF_Write(BUF *pBuf, const void *data, size_t size)
Definition: ncbi_buffer.c:224
int BUF_Prepend(BUF *pBuf, const void *data, size_t size)
Definition: ncbi_buffer.c:218
size_t BUF_Size(BUF buf)
Definition: ncbi_buffer.c:84
int BUF_AppendEx(BUF *pBuf, void *base, size_t alloc_size, void *data, size_t size)
Definition: ncbi_buffer.c:138
size_t BUF_Read(BUF buf, void *data, size_t size)
Definition: ncbi_buffer.c:414
int BUF_Splice(BUF *dst, BUF src)
Definition: ncbi_buffer.c:477
size_t BUF_Peek(BUF buf, void *data, size_t size)
Definition: ncbi_buffer.c:408
size_t BUF_PeekAtCB(BUF buf, size_t pos, size_t(*callback)(void *cbdata, const void *buf, size_t size), void *cbdata, size_t size)
void BUF_Destroy(BUF buf)
Definition: ncbi_buffer.c:500
void(* FHTTP_NcbiMessageHook)(const char *message)
EIO_Status HTTP_CreateTunnel(const SConnNetInfo *net_info, THTTP_Flags flags, SOCK *sock)
Same as HTTP_CreateTunnelEx(net_info, flags, 0, 0, 0, 0, sock)
FSetupVTable setup
init meta, may not be NULL
#define CONN_SET_METHOD(meta, method, function, connector)
CONNECTOR HTTP_CreateConnectorEx(const SConnNetInfo *net_info, THTTP_Flags flags, FHTTP_ParseHeader parse_header, void *user_data, FHTTP_Adjust adjust, FHTTP_Cleanup cleanup)
Create new CONNECTOR structure to hit the specified URL using HTTP with either POST / GET (or ANY) me...
unsigned int THTTP_Flags
Bitwise OR of EHTTP_Flag.
int(* FHTTP_Adjust)(SConnNetInfo *net_info, void *user_data, unsigned int failure_count)
#define CONN_SET_DEFAULT_TIMEOUT(meta, timeout)
void(* FHTTP_Cleanup)(void *user_data)
CONNECTOR next
linked list
CONNECTOR HTTP_CreateConnector(const SConnNetInfo *net_info, const char *user_header, THTTP_Flags flags)
Same as HTTP_CreateConnector(net_info, flags, 0, 0, 0, 0) with the passed "user_header" overriding th...
void HTTP_SetNcbiMessageHook(FHTTP_NcbiMessageHook hook)
Set a message hook procedure for messages originating from NCBI via HTTP.
FDestroy destroy
destroys handle, can be NULL
EHTTP_HeaderParse
The extended version HTTP_CreateConnectorEx() is able to track the HTTP response chain and also chang...
void * handle
data handle of the connector
SMetaConnector * meta
back link to original meta
EHTTP_HeaderParse(* FHTTP_ParseHeader)(const char *http_header, void *user_data, int server_error)
EIO_Status HTTP_CreateTunnelEx(const SConnNetInfo *net_info, THTTP_Flags flags, const void *init_data, size_t init_size, void *user_data, FHTTP_Adjust adjust, SOCK *sock)
Create a tunnel to "net_info->host:net_info->port" via an HTTP proxy server located at "net_info->htt...
const STimeout g_NcbiDefConnTimeout
DEF_CONN_TIMEOUT as STimeout.
Definition: ncbi_connutil.c:54
@ fHTTP_DropUnread
Each microsession drops unread data.
@ fHTTP_AdjustOnRedirect
Call adjust routine for redirects, too.
@ fHTTP_NoUpread
Do not use SOCK_SetReadOnWrite()
@ fHTTP_NoAutoRetry
No auto-retries allowed.
@ fHTTP_AutoReconnect
See HTTP_CreateConnectorEx()
@ fHTTP_KeepHeader
Keep HTTP header (see limitations)
@ fHTTP_NoAutomagicSID
Do not add NCBI SID automagically.
@ fHTTP_SuppressMessages
Most annoying ones reduced to traces.
@ fHTTP_UnsafeRedirects
Any redirect will be honored.
@ fHTTP_Flushable
Connector will really flush on Flush()
@ fHTTP_PushAuth
HTTP/1.1 pushes out auth if present.
@ fHTTP_WriteThru
HTTP/1.1 writes through (chunked)
@ eHTTP_HeaderSuccess
Parse succeeded, retain server status.
@ eHTTP_HeaderError
Parse failed, treat as a server error.
@ eHTTP_HeaderContinue
Parse succeeded, continue with body.
@ eHTTP_HeaderComplete
Parse succeeded, no more processing.
@ fHCC_UrlEncodeOutput
Obsolete, may not work, do not use!
@ fHCC_UrlDecodeInput
Obsolete, may not work, do not use!
@ fHCC_UrlEncodeArgs
NB: Error-prone semantics, do not use!
int SOCK_isipEx(const char *host, int fullquad)
Check whether the given string represents a valid IPv4 address.
Definition: ncbi_socket.c:8684
int SOCK_IsSecure(SOCK sock)
Check whether a socket is using SSL (Secure Socket Layer).
Definition: ncbi_socket.c:8468
EIO_Status SOCK_SetTimeout(SOCK sock, EIO_Event event, const STimeout *timeout)
Specify timeout for the connection I/O (see SOCK_[Read|Write|Close]()).
Definition: ncbi_socket.c:7198
#define SOCK_Destroy(s)
Definition: ncbi_socket.h:875
EIO_Status SOCK_Close(SOCK sock)
Close the SOCK handle, and destroy all relevant internal data.
Definition: ncbi_socket.c:6855
EIO_Status SOCK_Read(SOCK sock, void *buf, size_t size, size_t *n_read, EIO_ReadMethod how)
Read/peek up to "size" bytes from "sock" to a buffer pointed to by "buf".
Definition: ncbi_socket.c:7274
EIO_Status SOCK_Status(SOCK sock, EIO_Event direction)
Return low-level socket I/O status of *last* socket operation.
Definition: ncbi_socket.c:7463
unsigned int SOCK_gethostbyname(const char *hostname)
Same as SOCK_gethostbynameEx(,<current API data logging>)
Definition: ncbi_socket.c:8794
EIO_Status SOCK_Pushback(SOCK sock, const void *data, size_t size)
Push the specified data back to the socket's input queue (in the socket's internal read buffer).
Definition: ncbi_socket.c:7443
EIO_Status SOCK_Wait(SOCK sock, EIO_Event event, const STimeout *timeout)
Block on the socket until either the specified "event" is available or "timeout" expires (if "timeout...
Definition: ncbi_socket.c:6957
EIO_Status SOCK_Abort(SOCK sock)
If there is outstanding connection or output data pending, cancel it.
Definition: ncbi_socket.c:7561
EIO_Status SOCK_Write(SOCK sock, const void *data, size_t size, size_t *n_written, EIO_WriteMethod how)
Write "size" bytes of "data" to "sock".
Definition: ncbi_socket.c:7490
const STimeout * SOCK_GetTimeout(SOCK sock, EIO_Event event)
Get the connection's i/o timeout (or NULL, if the timeout is infinite).
Definition: ncbi_socket.c:7234
const char * SOCK_gethostbyaddr(unsigned int addr, char *name, size_t namelen)
Same as SOCK_gethostbyaddrEx(,,<current API data logging>)
Definition: ncbi_socket.c:8817
unsigned int TSOCK_Flags
bitwise "OR" of ESOCK_Flags
Definition: ncbi_socket.h:503
@ fSOCK_LogDefault
Definition: ncbi_socket.h:491
@ fSOCK_KeepAlive
keep socket alive (if supported by OS)
Definition: ncbi_socket.h:492
@ fSOCK_Secure
subsumes CloseOnExec regardless of Keep
Definition: ncbi_socket.h:497
@ fSOCK_ReadOnWrite
Definition: ncbi_socket.h:500
@ fSOCK_LogOn
Definition: ncbi_socket.h:490
ELOG_Level
Log severity level.
Definition: ncbi_core.h:292
TReqMethod req_method
unsigned EBURLScheme
unsigned external
char http_proxy_host[255+1]
EBFWMode firewall
int ConnNetInfo_ExtendUserHeader(SConnNetInfo *net_info, const char *header)
char http_proxy_user[63+1]
unsigned short http_proxy_port
#define kInfiniteTimeout
Definition: ncbi_types.h:82
#define NCBI_BIGCOUNT_FORMAT_SPEC_HEX
Definition: ncbi_types.h:166
int ConnNetInfo_OverrideUserHeader(SConnNetInfo *net_info, const char *header)
const STimeout * timeout
unsigned short port
int ConnNetInfo_ParseURL(SConnNetInfo *net_info, const char *url)
unsigned http_push_auth
int ConnNetInfo_SetArgs(SConnNetInfo *net_info, const char *args)
int ConnNetInfo_AppendUserHeader(SConnNetInfo *net_info, const char *header)
SConnNetInfo * ConnNetInfo_Clone(const SConnNetInfo *net_info)
LOG CORE_GetLOG(void)
Get the log handle that is to be used by the core internals (CORE LOG).
Definition: ncbi_util.c:138
char user[63+1]
EIO_Status
I/O status.
Definition: ncbi_core.h:132
unsigned int usec
microseconds (modulo 1,000,000)
Definition: ncbi_types.h:78
char http_proxy_pass[63+1]
const char * ConnNetInfo_GetArgs(const SConnNetInfo *net_info)
EReqMethod
const char svc[1]
char * ConnNetInfo_URL(const SConnNetInfo *net_info)
const char * IO_StatusStr(EIO_Status status)
Get the text form of an enum status value.
Definition: ncbi_core.c:56
const char * CORE_GetAppName(void)
Obtain current application name (toolkit dependent).
Definition: ncbi_util.c:716
EIO_ReadMethod
I/O read method.
Definition: ncbi_core.h:88
int ConnNetInfo_SetUserHeader(SConnNetInfo *net_info, const char *header)
void ConnNetInfo_DeleteUserHeader(SConnNetInfo *net_info, const char *header)
void ConnNetInfo_Log(const SConnNetInfo *net_info, ELOG_Level sev, LOG log)
EBURLScheme scheme
const char * http_user_header
unsigned short max_try
char * CORE_GetNcbiRequestID(ENcbiRequestID reqid)
Obtain current NCBI request ID (if known, per thread).
Definition: ncbi_util.c:728
EIO_Status URL_ConnectEx(const char *host, unsigned short port, const char *path, const char *args, TReqMethod req_method, size_t content_length, const STimeout *o_timeout, const STimeout *rw_timeout, const char *user_header, NCBI_CRED cred, TSOCK_Flags flags, SOCK *sock)
unsigned int sec
seconds
Definition: ncbi_types.h:77
NCBI_CRED credentials
char pass[63+1]
EIO_Status SOCK_StripToPattern(SOCK sock, const void *pattern, size_t pattern_size, BUF *discard, size_t *n_discarded)
unsigned http_version
EIO_Event
I/O event (or direction).
Definition: ncbi_core.h:118
void URL_Encode(const void *src_buf, size_t src_size, size_t *src_read, void *dst_buf, size_t dst_size, size_t *dst_written)
#define NCBI_BIGCOUNT_FORMAT_SPEC
Definition: ncbi_types.h:165
char host[255+1]
int ConnNetInfo_Boolean(const char *str)
#define kDefaultTimeout
Definition: ncbi_types.h:81
char path[4095+1]
EBDebugPrintout debug_printout
int URL_DecodeEx(const void *src_buf, size_t src_size, size_t *src_read, void *dst_buf, size_t dst_size, size_t *dst_written, const char *allow_symbols)
const char * http_referer
void ConnNetInfo_Destroy(SConnNetInfo *net_info)
int ConnNetInfo_SetFrag(SConnNetInfo *net_info, const char *frag)
unsigned http_proxy_only
uint64_t TNCBI_BigCount
Big unsigned integer for file size and position.
Definition: ncbi_types.h:164
@ eLOG_Critical
Definition: ncbi_core.h:298
@ eLOG_Error
Definition: ncbi_core.h:297
@ eLOG_Note
Definition: ncbi_core.h:294
@ eLOG_Warning
Definition: ncbi_core.h:296
@ eLOG_Trace
Definition: ncbi_core.h:293
@ eURL_Unspec
@ eURL_Http
@ eURL_Https
@ eIO_Timeout
timeout expired before any I/O succeeded
Definition: ncbi_core.h:134
@ eIO_NotSupported
operation is not supported or is not available
Definition: ncbi_core.h:138
@ eIO_Success
everything is fine, no error occurred
Definition: ncbi_core.h:133
@ eIO_Unknown
unknown I/O error (likely fatal but can retry)
Definition: ncbi_core.h:139
@ eIO_InvalidArg
bad argument / parameter value(s) supplied
Definition: ncbi_core.h:137
@ eReqMethod_Any
@ eReqMethod_v1
@ eReqMethod_Put
@ eReqMethod_Delete
@ eReqMethod_Get
@ eReqMethod_Connect
@ eReqMethod_Head
@ eReqMethod_Post
@ eDebugPrintout_Some
@ eDebugPrintout_Data
@ eIO_ReadPlain
read readily available data only, wait if none
Definition: ncbi_core.h:90
@ eIO_ReadPeek
do eIO_ReadPlain but leave data in input queue
Definition: ncbi_core.h:89
@ eIO_ReadPersist
read exactly as much as requested, w/waits
Definition: ncbi_core.h:91
@ eIO_WritePlain
write as much as possible, report back how much
Definition: ncbi_core.h:101
@ eIO_Write
write
Definition: ncbi_core.h:121
@ eIO_Close
also serves as an error indicator in SOCK_Poll
Definition: ncbi_core.h:123
@ eIO_Read
read
Definition: ncbi_core.h:120
@ eNcbiRequestID_SID
NCBI Session ID.
Definition: ncbi_util.h:435
@ eNcbiRequestID_HitID
NCBI Hit ID.
Definition: ncbi_util.h:434
char * buf
if(yy_accept[yy_current_state])
yy_size_t n
int len
static void text(MDB_val *v)
Definition: mdb_dump.c:62
const struct ncbi::grid::netcache::search::fields::SIZE size
char * strncpy0(char *s1, const char *s2, size_t n)
Copy not more than "n" characters from string "s2" into "s1", and return the result,...
#define strdup
Definition: ncbi_ansi_ext.h:70
#define strncasecmp
#define strcasecmp
#define strndup
Definition: ncbi_ansi_ext.h:89
#define verify(expr)
Definition: ncbi_assert.h:51
#define NCBI_EXTERNAL
Definition: ncbi_comm.h:57
#define HTTP_NCBI_SID
Definition: ncbi_comm.h:43
#define HTTP_NCBI_PHID
Definition: ncbi_comm.h:44
#define HTTP_NCBI_MESSAGE
Definition: ncbi_comm.h:42
SConnNetInfo * ConnNetInfo_CloneInternal(const SConnNetInfo *info)
const char * ConnNetInfo_GetValueInternal(const char *service, const char *param, char *value, size_t value_size, const char *def_value)
SConnNetInfo * ConnNetInfo_CreateInternal(const char *service)
#define CONN_PORT_HTTP
Definition: ncbi_connutil.h:93
#define CONN_HOST_LEN
Definition: ncbi_connutil.h:99
#define CONN_PORT_HTTPS
Definition: ncbi_connutil.h:94
#define CONN_USER_LEN
Definition: ncbi_connutil.h:97
#define CONN_PASS_LEN
Definition: ncbi_connutil.h:98
static int x_IsValidChallenge(const char *text, size_t len)
static EIO_Status x_ReadChunkTail(SHttpConnector *uuu)
static EHTTP_Redirect x_Redirect(SHttpConnector *uuu, const SRetry *retry)
static int x_FixupUserHeader(SConnNetInfo *net_info, int has_ref, int *has_sid)
static int x_SamePort(unsigned short port1, EBURLScheme scheme1, unsigned short port2, EBURLScheme scheme2)
static const char kHttpHostTag[]
EHTTP_Redirect
@ eHTTP_RedirectInvalid
@ eHTTP_RedirectUnsafe
@ eHTTP_RedirectOK
@ eHTTP_RedirectError
@ eHTTP_RedirectTunnel
static int x_UnsafeRedirectOK(SHttpConnector *uuu)
static EIO_Status s_Connect(SHttpConnector *uuu, const STimeout *timeout, EExtractMode extract)
static int x_Pushback(SOCK sock, BUF buf)
@ eCS_ReadBody
@ eCS_Eom
@ eCS_ReadHeader
@ eCS_WriteRequest
@ eCS_FlushRequest
@ eCS_DoneBody
@ eCS_Discard
@ eCS_NotInitiated
static EIO_Status s_Adjust(SHttpConnector *uuu, const SRetry *retry, EIO_Status status, EExtractMode extract)
static void s_DestroyHttpConnector(SHttpConnector *uuu)
unsigned EBSwitch
static char * s_VT_Descr(CONNECTOR connector)
unsigned EBConnState
static EIO_Status s_Read(SHttpConnector *uuu, void *buf, size_t size, size_t *n_read)
static EIO_Status s_VT_Write(CONNECTOR connector, const void *buf, size_t size, size_t *n_written, const STimeout *timeout)
@ eRetry_Redirect303
@ eRetry_Authenticate
@ eRetry_Unused
@ eRetry_ProxyAuthenticate
@ eRetry_None
@ eRetry_Redirect
static EIO_Status s_VT_Read(CONNECTOR connector, void *buf, size_t size, size_t *n_read, const STimeout *timeout)
static char * x_HostPort(size_t reserve, const char *host, unsigned short xport)
EExtractMode
@ eEM_Wait
@ eEM_Flush
@ eEM_Read
@ eEM_Drop
static const char * s_VT_GetType(CONNECTOR connector)
static EIO_Status s_ReadData(SHttpConnector *uuu, void *buf, size_t size, size_t *n_read, EIO_ReadMethod how)
static int x_IsValidParam(const char *param, size_t paramlen)
static EIO_Status s_ReadHeader(SHttpConnector *uuu, SRetry *retry, EExtractMode extract)
static int x_RedirectOK(EBURLScheme scheme_to, const char *host_to, unsigned short port_to, EBURLScheme scheme_from, const char *host_from, unsigned short port_from)
static int x_SameScheme(EBURLScheme scheme1, EBURLScheme scheme2)
static EIO_Status s_VT_Close(CONNECTOR connector, const STimeout *timeout)
@ eHTTP_AuthMissing
@ eHTTP_AuthIllegal
@ eHTTP_AuthUnsafe
@ eHTTP_AuthNotSupp
@ eHTTP_AuthOK
@ eHTTP_AuthError
@ eHTTP_AuthConnect
@ eHTTP_AuthDone
static int s_TunnelAdjust(SConnNetInfo *net_info, void *data, unsigned int arg)
static int s_CallAdjust(SHttpConnector *uuu, unsigned int arg)
static EIO_Status s_VT_Flush(CONNECTOR connector, const STimeout *timeout)
static int x_ErrorHeaderOnly(SHttpConnector *uuu)
static EIO_Status s_VT_Open(CONNECTOR connector, const STimeout *timeout)
static EIO_Status s_ConnectAndSend(SHttpConnector *uuu, const STimeout *timeout, EExtractMode extract)
static int x_SameHost(const char *host1, const char *host2)
#define HTTP_SOAK_READ_SIZE
unsigned EBCanConnect
static CONNECTOR s_CreateConnector(const SConnNetInfo *net_info, const char *user_header, THTTP_Flags flags, FHTTP_ParseHeader parse_header, void *user_data, FHTTP_Adjust adjust, FHTTP_Cleanup cleanup)
static void x_SetRequestIDs(SConnNetInfo *net_info)
static size_t x_PushbackBuf(void *data, const void *buf, size_t size)
static int x_IsWriteThru(const SHttpConnector *uuu)
static void s_OpenHttpConnector(SHttpConnector *uuu, const STimeout *timeout)
static EIO_Status s_Disconnect(SHttpConnector *uuu, const STimeout *timeout, EExtractMode extract)
static EIO_Status x_ReadChunkHead(SHttpConnector *uuu, int first)
static int s_MessageIssued
static FHTTP_NcbiMessageHook s_MessageHook
@ fCC_Once
@ fCC_None
@ fCC_Unlimited
static size_t x_WriteBuf(void *data, const void *buf, size_t size)
static void s_Setup(CONNECTOR connector)
static void s_DropConnection(SHttpConnector *uuu, enum EConnState state)
static EIO_Status s_CreateHttpConnector(const SConnNetInfo *net_info, const char *user_header, int tunnel, THTTP_Flags flags, void *user_data, FHTTP_Adjust adjust, SHttpConnector **http)
static unsigned short x_PortForScheme(unsigned port, EBURLScheme scheme)
static int x_SetHttpHostTag(SConnNetInfo *net_info)
static int x_SameOrDefPort(unsigned short port1, EBURLScheme scheme1, unsigned short port2, EBURLScheme scheme2)
static EIO_Status s_VT_Status(CONNECTOR connector, EIO_Event dir)
static EIO_Status s_VT_Wait(CONNECTOR connector, EIO_Event event, const STimeout *timeout)
static EIO_Status s_PreRead(SHttpConnector *uuu, const STimeout *timeout, EExtractMode extract)
static void x_Close(SHttpConnector *uuu)
static EHTTP_Auth x_Authenticate(SHttpConnector *uuu, ERetry auth, int retry)
static void s_Destroy(CONNECTOR connector)
static const STimeout kZeroTimeout
#define CORE_DATAF_X(subcode, level, data, size, fmt_args)
Definition: ncbi_priv.h:198
#define CORE_LOGF_X(subcode, level, fmt_args)
Definition: ncbi_priv.h:150
#define CORE_LOGF_ERRNO_X(subcode, level, error, fmt_args)
Definition: ncbi_priv.h:166
#define CORE_UNLOCK
Definition: ncbi_priv.h:273
#define CORE_LOCK_READ
Definition: ncbi_priv.h:271
const char * tag
#define NCBI_CXX_TOOLKIT
Definition: ncbiconf_msvc.h:16
int isspace(Uchar c)
Definition: ncbictype.hpp:69
static char tmp[2048]
Definition: utf8.c:42
#define memmove(a, b, c)
char * strerror(int n)
Definition: pcregrep.c:835
static const char * prefix[]
Definition: pcregrep.c:405
#define BASE64_Decode
Definition: ncbi_base64.h:42
#define BASE64_Encode
Definition: ncbi_base64.h:41
#define assert(x)
Definition: srv_diag.hpp:58
static const char * str(char *buf, int n)
Definition: stats.c:84
Connector specification.
TNCBI_BigCount expected
unsigned short http_code
EBConnState conn_state
unsigned char unused[3]
const STimeout * o_timeout
unsigned char minor_fault
EBCanConnect can_connect
unsigned short major_fault
TNCBI_BigCount received
FHTTP_Cleanup cleanup
SConnNetInfo * net_info
FHTTP_ParseHeader parse_header
const STimeout * w_timeout
Standard set of connector methods to handle a connection (corresponding connectors are also in here),...
const char * data
Timeout structure.
Definition: ncbi_types.h:76
void free(voidpf ptr)
voidp malloc(uInt size)
Modified on Sat Dec 02 09:22:00 2023 by modify_doxy.py rev. 669887