NCBI C++ ToolKit
message_handler.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: message_handler.cpp 86689 2019-06-06 17:41:53Z gouriano $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Pavel Ivanov
27  *
28  */
29 
30 #include "nc_pch.hpp"
31 
32 #include <corelib/ncbireg.hpp>
33 #include <corelib/ncbifile.hpp>
34 #include <corelib/request_ctx.hpp>
35 #include <corelib/ncbi_bswap.hpp>
36 #include <util/md5.hpp>
37 
38 #include "netcached.hpp"
39 #include "message_handler.hpp"
40 #include "netcache_version.hpp"
41 #include "nc_stat.hpp"
42 #include "peer_control.hpp"
43 #include "distribution_conf.hpp"
44 #include "periodic_sync.hpp"
45 #include "active_handler.hpp"
46 #include "nc_storage.hpp"
47 #include "nc_storage_blob.hpp"
48 #include "logging.hpp"
49 
50 
51 
53 
54 #if 0
55 #define LOG_CURRENT_FUNCTION SRV_LOG(Warning, "this: " << (void*)this);
56 #else
57 #define LOG_CURRENT_FUNCTION
58 #endif
59 
60 // when 1, server always broadcasts 'blob update' notifications.
61 // This will create empty blob stub on COPY_UPD if needed
62 // This takes time though, and I am not sure this is required.
63 // Another problem is that it overloads communication channels -
64 // NC fails much more often in CNCPeerControl::x_ReserveBGConn (too many connections)
65 #define USE_ALWAYS_COPY_UPD 0
66 
67 /// Definition of all NetCache commands
68 ///
69 /// General format of a "NetCache" command is as follows:
70 ///
71 /// CMD param1 param2 ...
72 ///
73 /// Format of "ICache" command is as follows:
74 ///
75 /// IC(cache) CMD param1 param2 ...
76 ///
77 /// Here "IC" is two letters that appear in command literally. "cache" is name
78 /// of the cache where blob is stored; it's mentioned in parameter list as
79 /// first parameter with type eNSPA_ICPrefix. Every command parameter can be
80 /// given as just value or as name=value pair. String parameter values can be
81 /// enclosed in double quotes. If parameter is declared with the flag
82 /// eNSPA_Optional then it can be skipped from the command, in this case its
83 /// default value will be used (if any). If parameter flag eNSPA_Optchain has
84 /// the same meaning except if it's not provided then all following parameters
85 /// marked as eNSPA_Optional will be assumed not provided too.
86 ///
87 /// If command needs some binary data along with it then it's sent split in
88 /// chunks each having 4-byte integer prefix containing the length of the chunk.
89 /// When all data is sent special chunk length 0xFFFFFFFF should be sent
90 /// at the end. Successful response of each command is sent as one line
91 /// starting with "OK:" and then space-separated parameters that need to be
92 /// returned. If response should contain binary data then initial response
93 /// line should have "SIZE=nnn" with the size of binary data to follow.
94 /// Unsuccessful responses to commands always start with "ERR:" and then error
95 /// explanation follows.
96 ///
97 /// Descriptions of commands have
98 /// - command name as it comes from client;
99 /// - structure containing
100 /// * state function processing this command;
101 /// * command name as it appears in statistics;
102 /// * flags controlling command behavior;
103 /// * type of access to blob if needed;
104 /// * type of proxy command that should be executed if command will need to
105 /// be proxied to other servers;
106 /// - set of structures explaining command parameters. Each structure has
107 /// * parameter name which can be used by client if it passes parameters in
108 /// name=value form. Also name is used to distinguish parameters in
109 /// x_AssignCmdParams();
110 /// * type of parameter - parser makes additional checks to see if given
111 /// value is applicable for necessary parameter type;
112 /// * parameter flags.
114  // "Are you alive?" command. This is old and deprecated command but it's
115  // executed a lot in old ICache clients. All that they need in response is
116  // "OK:".
117  { "A?",
119  "A?", fNoCmdFlags, eNCNone, eProxyNone} },
120  // Requests version of the server.
121  { "VERSION",
123  "VERSION", fNoCmdFlags, eNCNone, eProxyNone},
124  {
125  // Client IP for application requesting the info.
126  { "ip", eNSPT_Str, fNSPA_Optional },
127  // Session ID for application requesting the info.
128  { "sid", eNSPT_Str, eNSPA_Optional },
129  // request Hit ID
130  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
131  } },
132  // Requests some "health" information about the server.
133  { "HEALTH",
135  "HEALTH", fNoCmdFlags, eNCNone, eProxyNone},
136  {
137  // Client IP for application requesting the info.
138  { "ip", eNSPT_Str, fNSPA_Optional },
139  // Session ID for application requesting the info.
140  { "sid", eNSPT_Str, eNSPA_Optional },
141  // request Hit ID
142  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
143  } },
144  // Check if blob exists. Command for "ICache" clients.
145  { "HASB",
147  "IC_HASB",
149  eNCRead,
150  eProxyHasBlob},
151  // Name of cache for blob.
152  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
153  // Blob's key.
154  { "key", eNSPT_Str, eNSPA_Required },
155  // Blob's version.
156  { "version", eNSPT_Int, eNSPA_Required },
157  // Blob's subkey.
158  { "subkey", eNSPT_Str, eNSPA_Required },
159  // Quorum to use for this operation.
160  { "qrum", eNSPT_Int, eNSPA_Optional },
161  // Client IP for application requesting the info.
162  { "ip", eNSPT_Str, fNSPA_Optional },
163  // Session ID for application requesting the info.
164  { "sid", eNSPT_Str, eNSPA_Optional },
165  // Password for blob access.
166  { "pass", eNSPT_Str, eNSPA_Optional },
167  // request Hit ID
168  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
169  } },
170  // Read blob contents. Command for "ICache" clients.
171  { "READ",
173  "IC_READ",
175  eNCReadData,
176  eProxyRead},
177  // Name of cache for blob.
178  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
179  // Blob's key.
180  { "key", eNSPT_Str, eNSPA_Required },
181  // Blob's version.
182  { "version", eNSPT_Int, eNSPA_Required },
183  // Blob's subkey.
184  { "subkey", eNSPT_Str, eNSPA_Required },
185  // Quorum to use for this operation.
186  { "qrum", eNSPT_Int, eNSPA_Optional },
187  // Client IP for application requesting the info.
188  { "ip", eNSPT_Str, fNSPA_Optional },
189  // Session ID for application requesting the info.
190  { "sid", eNSPT_Str, eNSPA_Optional },
191  // Password for blob access.
192  { "pass", eNSPT_Str, eNSPA_Optional },
193  // Max age of blob (returned blob should be younger)
194  { "age", eNSPT_Int, eNSPA_Optional },
195  // request Hit ID
196  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
197  } },
198  // Write blob contents. Command for "ICache" clients.
199  { "STOR",
201  "IC_STOR",
203  eNCCreate,
204  eProxyWrite},
205  // Name of cache for blob.
206  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
207  // Time-to-live for the blob, 0 means default from server settings.
208  { "ttl", eNSPT_Int, eNSPA_Required },
209  // Blob's key.
210  { "key", eNSPT_Str, eNSPA_Required },
211  // Blob's version.
212  { "version", eNSPT_Int, eNSPA_Required },
213  // Blob's subkey.
214  { "subkey", eNSPT_Str, eNSPA_Required },
215  // 1 if client wants confirmation after blob has been written
216  // (by default it just assumes that everything is written and moves
217  // further).
218  { "confirm", eNSPT_Int, eNSPA_Optional },
219  // Quorum to use for this operation.
220  { "qrum", eNSPT_Int, eNSPA_Optional },
221  // Client IP for application requesting the info.
222  { "ip", eNSPT_Str, fNSPA_Optional },
223  // Session ID for application requesting the info.
224  { "sid", eNSPT_Str, eNSPA_Optional },
225  // Password for blob access.
226  { "pass", eNSPT_Str, eNSPA_Optional },
227  // request Hit ID
228  { "ncbi_phid", eNSPT_Str, eNSPA_Optional },
229  // see ENCUserFlags, added in v6.11.0 (CXX-8737)
230  { "flags", eNSPT_Int, eNSPA_Optional }
231  } },
232  // Write blob contents. Old and deprecated command which probably is not
233  // used by modern ICache clients anymore. It has the size of the blob right
234  // in the command (so client should know it beforehand) and it doesn't use
235  // "EOF" marker at the end of blob data.
236  { "STRS",
238  "IC_STRS",
240  eNCCreate,
241  eProxyWrite},
242  // Name of cache for blob.
243  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
244  // Time-to-live for the blob, 0 means default from server settings.
245  { "ttl", eNSPT_Int, eNSPA_Required },
246  // Size of the blob to be written.
247  { "size", eNSPT_Int, eNSPA_Required },
248  // Blob's key.
249  { "key", eNSPT_Str, eNSPA_Required },
250  // Blob's version.
251  { "version", eNSPT_Int, eNSPA_Required },
252  // Blob's subkey.
253  { "subkey", eNSPT_Str, eNSPA_Required },
254  // Quorum to use for this operation.
255  { "qrum", eNSPT_Int, eNSPA_Optional },
256  // Client IP for application requesting the info.
257  { "ip", eNSPT_Str, fNSPA_Optional },
258  // Session ID for application requesting the info.
259  { "sid", eNSPT_Str, eNSPA_Optional },
260  // Password for blob access.
261  { "pass", eNSPT_Str, eNSPA_Optional },
262  // request Hit ID
263  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
264  } },
265  // Read all or part of contents of the "last" version of the blob.
266  // In response to the command NC sends blob contents, blob version it has
267  // and flag showing if this version can be considered "valid", i.e. if it's
268  // not expired yet.
269  { "READLAST",
271  "IC_READLAST",
273  eNCReadData,
275  // Name of cache for blob.
276  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
277  // Blob's key.
278  { "key", eNSPT_Str, eNSPA_Required },
279  // Blob's subkey.
280  { "subkey", eNSPT_Str, eNSPA_Required },
281  // Starting position of the data that needs to be sent.
282  { "start", eNSPT_Int, eNSPA_Optional },
283  // Size of the data that needs to be sent.
284  { "size", eNSPT_Int, eNSPA_Optional },
285  // Quorum to use for this operation.
286  { "qrum", eNSPT_Int, eNSPA_Optional },
287  // Client IP for application requesting the info.
288  { "ip", eNSPT_Str, fNSPA_Optional },
289  // Session ID for application requesting the info.
290  { "sid", eNSPT_Str, eNSPA_Optional },
291  // Password for blob access.
292  { "pass", eNSPT_Str, eNSPA_Optional },
293  // Max age of blob (returned blob should be younger)
294  { "age", eNSPT_Int, eNSPA_Optional },
295  // request Hit ID
296  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
297  } },
298  // Mark the given blob version as "valid" and do that only if this version
299  // is still current and wasn't rewritten with another version.
300  { "SETVALID",
302  "IC_SETVALID",
304  eNCRead,
306  // Name of cache for blob.
307  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
308  // Blob's key.
309  { "key", eNSPT_Str, eNSPA_Required },
310  // Blob's version.
311  { "version", eNSPT_Int, eNSPA_Required },
312  // Blob's subkey.
313  { "subkey", eNSPT_Str, eNSPA_Required },
314  // Quorum to use for this operation.
315  { "qrum", eNSPT_Int, eNSPA_Optional },
316  // Client IP for application requesting the info.
317  { "ip", eNSPT_Str, fNSPA_Optional },
318  // Session ID for application requesting the info.
319  { "sid", eNSPT_Str, eNSPA_Optional },
320  // Password for blob access.
321  { "pass", eNSPT_Str, eNSPA_Optional },
322  // request Hit ID
323  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
324  } },
325  // Write blob contents. Command is issued only by other servers while
326  // mirroring just written blobs or processing quorum requirements,
327  // i.e. writing to other servers before answering to client that blob is
328  // written.
329  { "COPY_PUT",
331  "COPY_PUT",
333  | fCopyLogEvent,
335  // Name of cache for blob (for NC-generated blob keys this will be
336  // empty).
337  { { "cache", eNSPT_Str, eNSPA_Required },
338  // Blob's key.
339  { "key", eNSPT_Str, eNSPA_Required },
340  // Blob's subkey (for NC-generated blob keys this will be empty).
341  { "subkey", eNSPT_Str, eNSPA_Required },
342  // Blob's version (for NC-generated blob keys this will be equal to 0).
343  { "version", eNSPT_Int, eNSPA_Required },
344  // MD5 checksum of blob's password.
345  { "md5_pass",eNSPT_Str, eNSPA_Required },
346  // Creation time of the blob (microseconds since epoch).
347  { "cr_time", eNSPT_Int, eNSPA_Required },
348  // Time-to-live for the blob.
349  { "ttl", eNSPT_Int, eNSPA_Required },
350  // Dead-time for the blob (can be greater than expiration time).
351  { "dead", eNSPT_Int, eNSPA_Required },
352  // Expiration time for the blob
353  { "exp", eNSPT_Int, eNSPA_Required },
354  // Blob size.
355  { "size", eNSPT_Int, eNSPA_Required },
356  // Time-to-live for blob's version.
357  { "ver_ttl", eNSPT_Int, eNSPA_Required },
358  // Blob's version expiration time.
359  { "ver_dead",eNSPT_Int, eNSPA_Required },
360  // Server_id of the server where blob was created.
361  { "cr_srv", eNSPT_Int, eNSPA_Required },
362  // Id of the blob on the server where it was created.
363  { "cr_id", eNSPT_Int, eNSPA_Required },
364  // Record number of the event of blob creation in synchronization
365  // logs of the server where blob was created.
366  { "log_rec", eNSPT_Int, eNSPA_Required },
367  // Version of the command. Field exists for protocol backwards
368  // compatibility with previous versions of NC. In current NC this
369  // version is always 1.
370  { "cmd_ver", eNSPT_Int, eNSPA_Optional, "0" },
371  // Client IP for application that requested writing the blob.
372  // Parameter is not empty only if command is issued as part of
373  // quorum-related functionality, i.e. before client received
374  // confirmation of blob writing.
375  { "ip", eNSPT_Str, fNSPA_Optional },
376  // Session ID for application that requested writing the blob.
377  // Parameter is not empty only if command is issued as part of
378  // quorum-related functionality, i.e. before client received
379  // confirmation of blob writing.
380  { "sid", eNSPT_Str, eNSPA_Optional } } },
381  // Prolong blob lifetime. Command is issued only by other servers while
382  // mirroring prolonged blobs.
383  { "COPY_PROLONG",
385  "COPY_PROLONG",
388  // Name of cache for blob (for NC-generated blob keys this will be
389  // empty).
390  { { "cache", eNSPT_Str, eNSPA_Required },
391  // Blob's key.
392  { "key", eNSPT_Str, eNSPA_Required },
393  // Blob's subkey (for NC-generated blob keys this will be empty).
394  { "subkey", eNSPT_Str, eNSPA_Required },
395  // Creation time of the blob (microseconds since epoch).
396  { "cr_time", eNSPT_Int, eNSPA_Required },
397  // Server_id of the server where blob was created.
398  { "cr_srv", eNSPT_Int, eNSPA_Required },
399  // Id of the blob on the server where it was created.
400  { "cr_id", eNSPT_Int, eNSPA_Required },
401  // Dead-time for the blob (can be greater than expiration time).
402  { "dead", eNSPT_Int, eNSPA_Required },
403  // Expiration time for the blob
404  { "exp", eNSPT_Int, eNSPA_Required },
405  // Blob's version expiration time.
406  { "ver_dead",eNSPT_Int, eNSPA_Required },
407  // Time of creation of initial record in synchronization log about
408  // this operation.
409  { "log_time",eNSPT_Int, fNSPA_Optional },
410  // Server that first made the blob's life prolongation.
411  { "log_srv", eNSPT_Int, eNSPA_Optional },
412  // Record number of the initial record in synchronization log about
413  // this operation.
414  { "log_rec", eNSPT_Int, eNSPA_Optional } } },
415  // Write blob contents. Command for "NetCache" clients.
416  { "PUT3",
418  "PUT3",
420  eNCCreate,
421  eProxyWrite},
422  // Time-to-live for the blob. If not given or 0 then default TTL
423  // is used.
424  { { "ttl", eNSPT_Int, eNSPA_Optional },
425  // Key of the blob. If it's not given or empty then new key will be
426  // generated.
427  { "key", eNSPT_NCID, eNSPA_Optional },
428  // Quorum to use for this operation.
429  { "qrum", eNSPT_Int, eNSPA_Optional },
430  // Client IP for application sending the command.
431  { "ip", eNSPT_Str, fNSPA_Optional },
432  // Session ID for application sending the command.
433  { "sid", eNSPT_Str, eNSPA_Optional },
434  // Password for blob access.
435  { "pass", eNSPT_Str, eNSPA_Optional },
436  // request Hit ID
437  { "ncbi_phid", eNSPT_Str, eNSPA_Optional },
438  // see ENCUserFlags, added in v6.11.0 (CXX-8737)
439  { "flags", eNSPT_Int, eNSPA_Optional }
440  } },
441  // Read blob contents. Command for "NetCache" clients.
442  { "GET2",
444  "GET2",
446  eNCReadData,
447  eProxyRead},
448  // Key of the blob.
449  { { "key", eNSPT_NCID, eNSPA_Required },
450  // Not used and not implemented parameter. Exists just for backwards
451  // compatibility with old clients.
452  { "NW", eNSPT_Id, eNSPA_Obsolete | fNSPA_Match },
453  // Quorum to use for this operation.
454  { "qrum", eNSPT_Int, eNSPA_Optional },
455  // Client IP for application sending the command.
456  { "ip", eNSPT_Str, fNSPA_Optional },
457  // Session ID for application sending the command.
458  { "sid", eNSPT_Str, eNSPA_Optional },
459  // Password for blob access.
460  { "pass", eNSPT_Str, eNSPA_Optional },
461  // Max age of blob (returned blob should be younger)
462  { "age", eNSPT_Int, eNSPA_Optional },
463  // request Hit ID
464  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
465  } },
466  // Check if blob exists. Command for "NetCache" clients.
467  { "HASB",
469  "HASB",
471  eNCRead,
472  eProxyHasBlob},
473  // Key of the blob.
474  { { "key", eNSPT_NCID, eNSPA_Required },
475  // Quorum to use for this operation.
476  { "qrum", eNSPT_Int, eNSPA_Optional },
477  // Client IP for application sending the command.
478  { "ip", eNSPT_Str, fNSPA_Optional },
479  // Session ID for application sending the command.
480  { "sid", eNSPT_Str, eNSPA_Optional },
481  // Password for blob access.
482  { "pass", eNSPT_Str, eNSPA_Optional },
483  // request Hit ID
484  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
485  } },
486  // Delete blob. Command for "NetCache" clients.
487  // If the blob doesn't exist command is still considered successful.
488  { "RMV2",
490  "RMV2",
492  eNCCreate,
493  eProxyRemove},
494  // Key of the blob.
495  { { "key", eNSPT_NCID, eNSPA_Required },
496  // Quorum to use for this operation.
497  { "qrum", eNSPT_Int, eNSPA_Optional },
498  // Client IP for application sending the command.
499  { "ip", eNSPT_Str, fNSPA_Optional },
500  // Session ID for application sending the command.
501  { "sid", eNSPT_Str, eNSPA_Optional },
502  // Password for blob access.
503  { "pass", eNSPT_Str, eNSPA_Optional },
504  // request Hit ID
505  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
506  } },
507  // Get size of the blob. Command for "NetCache" clients.
508  { "GSIZ",
510  "GetSIZe",
512  eNCRead,
513  eProxyGetSize},
514  // Key of the blob.
515  { { "key", eNSPT_NCID, eNSPA_Required },
516  // Quorum to use for this operation.
517  { "qrum", eNSPT_Int, eNSPA_Optional },
518  // Client IP for application sending the command.
519  { "ip", eNSPT_Str, fNSPA_Optional },
520  // Session ID for application sending the command.
521  { "sid", eNSPT_Str, eNSPA_Optional },
522  // Password for blob access.
523  { "pass", eNSPT_Str, eNSPA_Optional },
524  // request Hit ID
525  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
526  } },
527  // Delete blob. Command for "ICache" clients.
528  // If the blob doesn't exist command is still considered successful.
529  { "REMO",
531  "IC_REMOve",
533  eNCCreate,
534  eProxyRemove},
535  // Name of cache for blob.
536  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
537  // Blob's key.
538  { "key", eNSPT_Str, eNSPA_Required },
539  // Blob's version.
540  { "version", eNSPT_Int, eNSPA_Required },
541  // Blob's subkey.
542  { "subkey", eNSPT_Str, eNSPA_Required },
543  // Quorum to use for this operation.
544  { "qrum", eNSPT_Int, eNSPA_Optional },
545  // Client IP for application sending the command.
546  { "ip", eNSPT_Str, fNSPA_Optional },
547  // Session ID for application sending the command.
548  { "sid", eNSPT_Str, eNSPA_Optional },
549  // Password for blob access.
550  { "pass", eNSPT_Str, eNSPA_Optional },
551  // request Hit ID
552  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
553  } },
554  // Get size of the blob. Command for "ICache" clients.
555  { "GSIZ",
557  "IC_GetSIZe",
559  eNCRead,
560  eProxyGetSize},
561  // Name of cache for blob.
562  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
563  // Blob's key.
564  { "key", eNSPT_Str, eNSPA_Required },
565  // Blob's version.
566  { "version", eNSPT_Int, eNSPA_Required },
567  // Blob's subkey.
568  { "subkey", eNSPT_Str, eNSPA_Required },
569  // Quorum to use for this operation.
570  { "qrum", eNSPT_Int, eNSPA_Optional },
571  // Client IP for application sending the command.
572  { "ip", eNSPT_Str, fNSPA_Optional },
573  // Session ID for application sending the command.
574  { "sid", eNSPT_Str, eNSPA_Optional },
575  // Password for blob access.
576  { "pass", eNSPT_Str, eNSPA_Optional },
577  // request Hit ID
578  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
579  } },
580  // Read part of the blob contents. Command for "NetCache" clients.
581  { "GETPART",
583  "GETPART",
585  eNCReadData,
586  eProxyRead},
587  // Key of the blob.
588  { { "key", eNSPT_NCID, eNSPA_Required },
589  // Starting position of the data that needs to be sent.
590  { "start", eNSPT_Int, eNSPA_Required },
591  // Size of the data that needs to be sent.
592  { "size", eNSPT_Int, eNSPA_Required },
593  // Quorum to use for this operation.
594  { "qrum", eNSPT_Int, eNSPA_Optional },
595  // Client IP for application requesting the info.
596  { "ip", eNSPT_Str, fNSPA_Optional },
597  // Session ID for application requesting the info.
598  { "sid", eNSPT_Str, eNSPA_Optional },
599  // Password for blob access.
600  { "pass", eNSPT_Str, eNSPA_Optional },
601  // Max age of blob (returned blob should be younger)
602  { "age", eNSPT_Int, eNSPA_Optional },
603  // request Hit ID
604  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
605  } },
606  // Read part of the blob contents. Command for "ICache" clients.
607  { "READPART",
609  "IC_READPART",
611  eNCReadData,
612  eProxyRead},
613  // Name of cache for blob.
614  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
615  // Blob's key.
616  { "key", eNSPT_Str, eNSPA_Required },
617  // Blob's version.
618  { "version", eNSPT_Int, eNSPA_Required },
619  // Blob's subkey.
620  { "subkey", eNSPT_Str, eNSPA_Required },
621  // Starting position of the data that needs to be sent.
622  { "start", eNSPT_Int, eNSPA_Required },
623  // Size of the data that needs to be sent.
624  { "size", eNSPT_Int, eNSPA_Required },
625  // Quorum to use for this operation.
626  { "qrum", eNSPT_Int, eNSPA_Optional },
627  // Client IP for application requesting the info.
628  { "ip", eNSPT_Str, fNSPA_Optional },
629  // Session ID for application requesting the info.
630  { "sid", eNSPT_Str, eNSPA_Optional },
631  // Password for blob access.
632  { "pass", eNSPT_Str, eNSPA_Optional },
633  // Max age of blob (returned blob should be younger)
634  { "age", eNSPT_Int, eNSPA_Optional },
635  // request Hit ID
636  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
637  } },
638  // Get meta information about the blob. This command is sent only by other
639  // NC servers. And it's used to determine which server has the latest
640  // version of the blob. Thus response to this command is a line containing
641  // enough information to compare the blob's creation time with the same
642  // blob on other servers.
643  { "PROXY_META",
645  "PROXY_META",
649  // Name of cache for blob (for NC-generated blob keys this will be
650  // empty).
651  { { "cache", eNSPT_Str, eNSPA_Required },
652  // Blob's key.
653  { "key", eNSPT_Str, eNSPA_Required },
654  // Blob's subkey (for NC-generated blob keys this will be empty).
655  { "subkey", eNSPT_Str, eNSPA_Required },
656  // Client IP for application on behalf of which the info is requested.
657  { "ip", eNSPT_Str, eNSPA_Required },
658  // Session ID for application on behalf of which the info is requested.
659  { "sid", eNSPT_Str, eNSPA_Required } } },
660 
661  // another NC server notifies this one that the blob was updated there
662  // the blob data will come later (eg, via COPY_PUT)
663  { "COPY_UPD",
665  "UPD",
669 #if USE_ALWAYS_COPY_UPD
671 #else
672  eNCRead
673 #endif
674  , eProxyNone },
675  // Name of cache for blob (for NC-generated blob keys this will be
676  // empty).
677  { { "cache", eNSPT_Str, eNSPA_Required },
678  // Blob's key.
679  { "key", eNSPT_Str, eNSPA_Required },
680  // Blob's subkey (for NC-generated blob keys this will be empty).
681  { "subkey", eNSPT_Str, eNSPA_Required },
682  // time of the update
683  { "cr_time", eNSPT_Int, eNSPA_Required },
684  // Server_id, where the update was made
685  { "cr_srv", eNSPT_Int, eNSPA_Required }
686  }
687  },
688  // Write the blob contents. This command is sent only by other NC servers
689  // in response to client's PUT3 (or similar) command when the server where
690  // client have sent initial command cannot execute it locally for any
691  // reason (slot is not processed by that server or initial database caching
692  // is not completed yet).
693  { "PROXY_PUT",
695  "PROXY_PUT",
697  eNCCreate,
698  eProxyWrite},
699  // Name of cache for blob (for NC-generated blob keys this will be
700  // empty).
701  { { "cache", eNSPT_Str, eNSPA_Required },
702  // Blob's key.
703  { "key", eNSPT_Str, eNSPA_Required },
704  // Blob's subkey (for NC-generated blob keys this will be empty).
705  { "subkey", eNSPT_Str, eNSPA_Required },
706  // Blob's version (for NC-generated blob keys this will be equal to 0).
707  { "version", eNSPT_Int, eNSPA_Required },
708  // Time-to-live for the blob.
709  { "ttl", eNSPT_Int, eNSPA_Required },
710  // Quorum to use for this operation.
711  { "qrum", eNSPT_Int, eNSPA_Required },
712  // Client IP for application requesting the info.
713  { "ip", eNSPT_Str, eNSPA_Required },
714  // Session ID for application requesting the info.
715  { "sid", eNSPT_Str, eNSPA_Required },
716  // see ENCUserFlags, added in v6.11.0 (CXX-8737)
717  { "flags", eNSPT_Int, eNSPA_Optional },
718  // Password for blob access.
719  { "pass", eNSPT_Str, eNSPA_Optional }
720  } },
721  // Read all or a part of the blob contents. This command is sent only
722  // by other NC servers in response to client's GET2 (or similar) command
723  // when the server where client have sent initial command cannot execute it
724  // locally for any reason (slot is not processed by that server, or initial
725  // database caching is not completed yet, or it was determined that this
726  // server has latest version of the blob).
727  { "PROXY_GET",
729  "PROXY_GET",
731  eNCReadData,
732  eProxyRead},
733  // Name of cache for blob (for NC-generated blob keys this will be
734  // empty).
735  { { "cache", eNSPT_Str, eNSPA_Required },
736  // Blob's key.
737  { "key", eNSPT_Str, eNSPA_Required },
738  // Blob's subkey (for NC-generated blob keys this will be empty).
739  { "subkey", eNSPT_Str, eNSPA_Required },
740  // Blob's version (for NC-generated blob keys this will be equal to 0).
741  { "version", eNSPT_Int, eNSPA_Required },
742  // Starting position of the data that needs to be sent.
743  { "start", eNSPT_Int, eNSPA_Required },
744  // Size of the data that needs to be sent.
745  { "size", eNSPT_Int, eNSPA_Required },
746  // Quorum to use for this operation.
747  { "qrum", eNSPT_Int, eNSPA_Required },
748  // Value of the flag "search_on_read" to use with this command, i.e.
749  // whether this server should search the blob if it's not found
750  // locally.
751  { "srch", eNSPT_Int, eNSPA_Required },
752  // Flag whether local execution of this command should be forced
753  // no matter what, i.e. if this flag is set in cases when normal GET2
754  // command would have been proxied to other servers an error should
755  // be returned.
756  { "local", eNSPT_Int, eNSPA_Required },
757  // Client IP for application requesting the info.
758  { "ip", eNSPT_Str, eNSPA_Required },
759  // Session ID for application requesting the info.
760  { "sid", eNSPT_Str, eNSPA_Required },
761  // Password for blob access.
762  { "pass", eNSPT_Str, eNSPA_Optional },
763  // Max age of blob (returned blob should be younger)
764  { "age", eNSPT_Int, eNSPA_Optional }
765  } },
766  // Check if the blob exists. This command is sent only by other NC servers
767  // in response to client's HASB command when the server where
768  // client have sent initial command cannot execute it locally for any
769  // reason (slot is not processed by that server or initial database caching
770  // is not completed yet).
771  { "PROXY_HASB",
773  "PROXY_HASB",
775  eNCRead,
776  eProxyHasBlob},
777  // Name of cache for blob (for NC-generated blob keys this will be
778  // empty).
779  { { "cache", eNSPT_Str, eNSPA_Required },
780  // Blob's key.
781  { "key", eNSPT_Str, eNSPA_Required },
782  // Blob's subkey (for NC-generated blob keys this will be empty).
783  { "subkey", eNSPT_Str, eNSPA_Required },
784  // Quorum to use for this operation.
785  { "qrum", eNSPT_Int, eNSPA_Required },
786  // Client IP for application requesting the info.
787  { "ip", eNSPT_Str, eNSPA_Required },
788  // Session ID for application requesting the info.
789  { "sid", eNSPT_Str, eNSPA_Required },
790  // Password for blob access.
791  { "pass", eNSPT_Str, eNSPA_Optional } } },
792  // Get size of the blob. This command is sent only
793  // by other NC servers in response to client's GSIZ command
794  // when the server where client have sent initial command cannot execute it
795  // locally for any reason (slot is not processed by that server, or initial
796  // database caching is not completed yet, or it was determined that this
797  // server has latest version of the blob).
798  { "PROXY_GSIZ",
800  "PROXY_GetSIZe",
802  eNCRead,
803  eProxyGetSize},
804  // Name of cache for blob (for NC-generated blob keys this will be
805  // empty).
806  { { "cache", eNSPT_Str, eNSPA_Required },
807  // Blob's key.
808  { "key", eNSPT_Str, eNSPA_Required },
809  // Blob's subkey (for NC-generated blob keys this will be empty).
810  { "subkey", eNSPT_Str, eNSPA_Required },
811  // Blob's version (for NC-generated blob keys this will be equal to 0).
812  { "version", eNSPT_Int, eNSPA_Required },
813  // Quorum to use for this operation.
814  { "qrum", eNSPT_Int, eNSPA_Required },
815  // Value of the flag "search_on_read" to use with this command, i.e.
816  // whether this server should search the blob if it's not found
817  // locally.
818  { "srch", eNSPT_Int, eNSPA_Required },
819  // Flag whether local execution of this command should be forced
820  // no matter what, i.e. if this flag is set in cases when normal GET2
821  // command would have been proxied to other servers an error should
822  // be returned.
823  { "local", eNSPT_Int, eNSPA_Required },
824  // Client IP for application requesting the info.
825  { "ip", eNSPT_Str, eNSPA_Required },
826  // Session ID for application requesting the info.
827  { "sid", eNSPT_Str, eNSPA_Required },
828  // Password for blob access.
829  { "pass", eNSPT_Str, eNSPA_Optional } } },
830  // Read all or a part of contents of the "last version" of the blob.
831  // This command is sent only by other NC servers in response to client's
832  // READLAST command when the server where client have sent initial command
833  // cannot execute it locally for any reason (slot is not processed by that
834  // server, or initial database caching is not completed yet, or it was
835  // determined that this server has latest version of the blob).
836  { "PROXY_READLAST",
838  "PROXY_READLAST",
840  eNCReadData,
842  // Name of cache for blob (for NC-generated blob keys this will be
843  // empty).
844  { { "cache", eNSPT_Str, eNSPA_Required },
845  // Blob's key.
846  { "key", eNSPT_Str, eNSPA_Required },
847  // Blob's subkey (for NC-generated blob keys this will be empty).
848  { "subkey", eNSPT_Str, eNSPA_Required },
849  // Starting position of the data that needs to be sent.
850  { "start", eNSPT_Int, eNSPA_Required },
851  // Size of the data that needs to be sent.
852  { "size", eNSPT_Int, eNSPA_Required },
853  // Quorum to use for this operation.
854  { "qrum", eNSPT_Int, eNSPA_Required },
855  // Value of the flag "search_on_read" to use with this command, i.e.
856  // whether this server should search the blob if it's not found
857  // locally.
858  { "srch", eNSPT_Int, eNSPA_Required },
859  // Flag whether local execution of this command should be forced
860  // no matter what, i.e. if this flag is set in cases when normal
861  // READLAST command would have been proxied to other servers an error
862  // should be returned.
863  { "local", eNSPT_Int, eNSPA_Required },
864  // Client IP for application requesting the info.
865  { "ip", eNSPT_Str, eNSPA_Required },
866  // Session ID for application requesting the info.
867  { "sid", eNSPT_Str, eNSPA_Required },
868  // Password for blob access.
869  { "pass", eNSPT_Str, eNSPA_Optional },
870  // Max age of blob (returned blob should be younger)
871  { "age", eNSPT_Int, eNSPA_Optional } } },
872  // Mark the "current version" of the blob as "valid".
873  // This command is sent only by other NC servers in response to client's
874  // SETVALID command when the server where client have sent initial command
875  // cannot execute it locally for any reason (slot is not processed by that
876  // server, or initial database caching is not completed yet).
877  { "PROXY_SETVALID",
879  "PROXY_SETVALID",
881  eNCRead,
883  // Name of cache for blob (for NC-generated blob keys this will be
884  // empty).
885  { { "cache", eNSPT_Str, eNSPA_Required },
886  // Blob's key.
887  { "key", eNSPT_Str, eNSPA_Required },
888  // Blob's subkey (for NC-generated blob keys this will be empty).
889  { "subkey", eNSPT_Str, eNSPA_Required },
890  // Blob's version (for NC-generated blob keys this will be equal to 0).
891  { "version", eNSPT_Int, eNSPA_Required },
892  // Client IP for application requesting the info.
893  { "ip", eNSPT_Str, eNSPA_Required },
894  // Session ID for application requesting the info.
895  { "sid", eNSPT_Str, eNSPA_Required },
896  // Password for blob access.
897  { "pass", eNSPT_Str, eNSPA_Optional } } },
898  // Remove the blob. This command is sent only by other NC servers in response
899  // to client's RMV2 (or similar) command when the server where client have
900  // sent initial command cannot execute it locally for any reason (slot is not
901  // processed by that server, or initial database caching is not completed yet).
902  { "PROXY_RMV",
904  "PROXY_ReMoVe",
906  eNCCreate,
907  eProxyRemove},
908  // Name of cache for blob (for NC-generated blob keys this will be
909  // empty).
910  { { "cache", eNSPT_Str, eNSPA_Required },
911  // Blob's key.
912  { "key", eNSPT_Str, eNSPA_Required },
913  // Blob's subkey (for NC-generated blob keys this will be empty).
914  { "subkey", eNSPT_Str, eNSPA_Required },
915  // Blob's version (for NC-generated blob keys this will be equal to 0).
916  { "version", eNSPT_Int, eNSPA_Required },
917  // Quorum to use for this operation.
918  { "qrum", eNSPT_Int, eNSPA_Required },
919  // Client IP for application requesting the info.
920  { "ip", eNSPT_Str, eNSPA_Required },
921  // Session ID for application requesting the info.
922  { "sid", eNSPT_Str, eNSPA_Required },
923  // Password for blob access.
924  { "pass", eNSPT_Str, eNSPA_Optional }
925  } },
926  // Remove the blob. This command is sent by other NC servers in certain scenarios only
927  { "COPY_RMV",
929  "COPY_ReMoVe",
932  eNCRead,
933  eProxyRemove},
934  // Name of cache for blob (for NC-generated blob keys this will be
935  // empty).
936  { { "cache", eNSPT_Str, eNSPA_Required },
937  // Blob's key.
938  { "key", eNSPT_Str, eNSPA_Required },
939  // Blob's subkey (for NC-generated blob keys this will be empty).
940  { "subkey", eNSPT_Str, eNSPA_Required },
941  // time of the update
942  { "cr_time", eNSPT_Int, eNSPA_Required },
943  // Server_id, requestor
944  { "cr_srv", eNSPT_Int, eNSPA_Required }
945  }
946  },
947  // Read meta information about the blob. This command is sent only by other
948  // NC servers in response to client's GETMETA command when the server where
949  // client have sent initial command cannot execute it locally for any reason
950  // (slot is not processed by that server, or initial database caching
951  // is not completed yet, or it was determined that this server has latest
952  // version of the blob).
953  { "PROXY_GETMETA",
955  "PROXY_GETMETA",
957  eNCRead,
958  eProxyGetMeta},
959  // Name of cache for blob (for NC-generated blob keys this will be
960  // empty).
961  { { "cache", eNSPT_Str, eNSPA_Required },
962  // Blob's key.
963  { "key", eNSPT_Str, eNSPA_Required },
964  // Blob's subkey (for NC-generated blob keys this will be empty).
965  { "subkey", eNSPT_Str, eNSPA_Required },
966  // Quorum to use for this operation.
967  { "qrum", eNSPT_Int, eNSPA_Required },
968  // Flag whether local execution of this command should be forced
969  // no matter what, i.e. if this flag is set in cases when normal
970  // GETMETA command would have been proxied to other servers an error
971  // should be returned.
972  { "local", eNSPT_Int, eNSPA_Required },
973  // Client IP for application requesting the info.
974  { "ip", eNSPT_Str, eNSPA_Required },
975  // Session ID for application requesting the info.
976  { "sid", eNSPT_Str, eNSPA_Required },
977  // HTTP flag: read input in NC format, write output in HTTP one
978  { "http", eNSPT_Int, eNSPA_Optional }
979  } },
980  // Start periodic synchronization session. Command is sent only by other
981  // NC servers when CNCActiveSyncControl in them decides to start
982  // synchronization. Response to this command contains list of events from
983  // sync logs of this server which need to be synchronized. Or if this
984  // server understands that synchronization using blob lists is needed then
985  // first line of response will contain ALL_BLOBS word and then full list
986  // of blobs in this slot will be sent.
987  { "SYNC_START",
989  "SYNC_START",
990  // this command does not need space, but, to do sync, we WILL need space
993  , eNCNone, eProxyNone},
994  // Server id of the server starting synchronization.
995  { { "srv_id", eNSPT_Int, eNSPA_Required },
996  // Slot to start synchronization on.
997  { "slot", eNSPT_Int, eNSPA_Required },
998  // Last synchronized record number (in sync log) of _that_ server
999  // as _that_ server thinks.
1000  { "rec_my", eNSPT_Int, eNSPA_Required },
1001  // Last synchronized record number (in sync log) of _this_ server
1002  // as _that_ server thinks.
1003  { "rec_your",eNSPT_Int, eNSPA_Required } } },
1004  // Get full list of blobs for the slot. Command is sent only by other NC
1005  // servers when that server decides that synchronization using blob lists
1006  // is needed. Command can be sent only after successful execution of
1007  // SYNC_START command.
1008  { "SYNC_BLIST",
1010  "SYNC_BLIST",
1012  // Server id of the server managing the synchronization.
1013  { { "srv_id", eNSPT_Int, eNSPA_Required },
1014  // Slot that synchronization is started on.
1015  { "slot", eNSPT_Int, eNSPA_Required } } },
1016  // Write blob contents. This command is sent only by other NC servers
1017  // during synchronization session if some blob was written on that server
1018  // and the same data didn't make it to this server yet.
1019  { "SYNC_PUT",
1021  "SYNC_PUT",
1025  // Server id of the server managing the synchronization.
1026  { { "srv_id", eNSPT_Int, eNSPA_Required },
1027  // Slot that synchronization is started on.
1028  { "slot", eNSPT_Int, eNSPA_Required },
1029  // Name of cache for blob (for NC-generated blob keys this will be
1030  // empty).
1031  { "cache", eNSPT_Str, eNSPA_Required },
1032  // Blob's key.
1033  { "key", eNSPT_Str, eNSPA_Required },
1034  // Blob's subkey (for NC-generated blob keys this will be empty).
1035  { "subkey", eNSPT_Str, eNSPA_Required },
1036  // Blob's version (for NC-generated blob keys this will be equal to 0).
1037  { "version", eNSPT_Int, eNSPA_Required },
1038  // MD5 checksum of blob's password.
1039  { "md5_pass",eNSPT_Str, eNSPA_Required },
1040  // Creation time of the blob (microseconds since epoch).
1041  { "cr_time", eNSPT_Int, eNSPA_Required },
1042  // Time-to-live for the blob.
1043  { "ttl", eNSPT_Int, eNSPA_Required },
1044  // Dead-time for the blob (can be greater than expiration time).
1045  { "dead", eNSPT_Int, eNSPA_Required },
1046  // Expiration time for the blob
1047  { "exp", eNSPT_Int, eNSPA_Required },
1048  // Blob size.
1049  { "size", eNSPT_Int, eNSPA_Required },
1050  // Time-to-live for blob's version.
1051  { "ver_ttl", eNSPT_Int, eNSPA_Required },
1052  // Blob's version expiration time.
1053  { "ver_dead",eNSPT_Int, eNSPA_Required },
1054  // Server_id of the server where blob was created.
1055  { "cr_srv", eNSPT_Int, eNSPA_Required },
1056  // Id of the blob on the server where it was created.
1057  { "cr_id", eNSPT_Int, eNSPA_Required },
1058  // Record number of the event of blob creation in synchronization
1059  // logs of the server where blob was created.
1060  { "log_rec", eNSPT_Int, eNSPA_Required },
1061  // Version of the command. Field exists for protocol backwards
1062  // compatibility with previous versions of NC. In current NC this
1063  // version is always 1.
1064  { "cmd_ver", eNSPT_Int, eNSPA_Optional, "0" } } },
1065  // Prolong the blob's life. This command is sent only by other NC servers
1066  // during synchronization session if some blob was prolonged on that server
1067  // and the same prolongation didn't happen on this server yet.
1068  { "SYNC_PROLONG",
1070  "SYNC_PROLONG",
1071  eSyncBlobCmd,
1072  eNCRead, eProxyNone},
1073  // Server id of the server managing the synchronization.
1074  { { "srv_id", eNSPT_Int, eNSPA_Required },
1075  // Slot that synchronization is started on.
1076  { "slot", eNSPT_Int, eNSPA_Required },
1077  // Name of cache for blob (for NC-generated blob keys this will be
1078  // empty).
1079  { "cache", eNSPT_Str, eNSPA_Required },
1080  // Blob's key.
1081  { "key", eNSPT_Str, eNSPA_Required },
1082  // Blob's subkey (for NC-generated blob keys this will be empty).
1083  { "subkey", eNSPT_Str, eNSPA_Required },
1084  // Creation time of the blob (microseconds since epoch).
1085  { "cr_time", eNSPT_Int, eNSPA_Required },
1086  // Server_id of the server where blob was created.
1087  { "cr_srv", eNSPT_Int, eNSPA_Required },
1088  // Id of the blob on the server where it was created.
1089  { "cr_id", eNSPT_Int, eNSPA_Required },
1090  // Dead-time for the blob (can be greater than expiration time).
1091  { "dead", eNSPT_Int, eNSPA_Required },
1092  // Expiration time for the blob
1093  { "exp", eNSPT_Int, eNSPA_Required },
1094  // Blob's version expiration time.
1095  { "ver_dead",eNSPT_Int, eNSPA_Required },
1096  // Time of creation of initial record in synchronization log about
1097  // this operation.
1098  { "log_time",eNSPT_Int, fNSPA_Optional },
1099  // Server that first made the blob's life prolongation.
1100  { "log_srv", eNSPT_Int, eNSPA_Optional },
1101  // Record number of the initial record in synchronization log about
1102  // this operation.
1103  { "log_rec", eNSPT_Int, eNSPA_Optional } } },
1104  // Read blob contents. This command is sent only by other NC servers
1105  // during synchronization session if some blob was written on this server
1106  // and the same data didn't make it to that server yet.
1107  { "SYNC_GET",
1109  "SYNC_GET",
1110  eSyncBlobCmd,
1112  // Server id of the server managing the synchronization.
1113  { { "srv_id", eNSPT_Int, eNSPA_Required },
1114  // Slot that synchronization is started on.
1115  { "slot", eNSPT_Int, eNSPA_Required },
1116  // Name of cache for blob (for NC-generated blob keys this will be
1117  // empty).
1118  { "cache", eNSPT_Str, eNSPA_Required },
1119  // Blob's key.
1120  { "key", eNSPT_Str, eNSPA_Required },
1121  // Blob's subkey (for NC-generated blob keys this will be empty).
1122  { "subkey", eNSPT_Str, eNSPA_Required },
1123  // Time of creation of initial record in synchronization log about
1124  // this operation.
1125  { "log_time",eNSPT_Int, eNSPA_Required },
1126  // Creation time of the blob (microseconds since epoch).
1127  { "cr_time", eNSPT_Int, eNSPA_Required },
1128  // Server_id of the server where blob was created.
1129  { "cr_srv", eNSPT_Int, eNSPA_Required },
1130  // Id of the blob on the server where it was created.
1131  { "cr_id", eNSPT_Int, eNSPA_Required } } },
1132  // Get information necessary to prolong the blob's life. This command
1133  // is sent only by other NC servers during synchronization session if some
1134  // blob was prolonged on this server and the same prolongation didn't
1135  // happen on that server yet.
1136  { "SYNC_PROINFO",
1138  "SYNC_PROINFO",
1139  eSyncBlobCmd,
1140  eNCRead, eProxyNone},
1141  // Server id of the server managing the synchronization.
1142  { { "srv_id", eNSPT_Int, eNSPA_Required },
1143  // Slot that synchronization is started on.
1144  { "slot", eNSPT_Int, eNSPA_Required },
1145  // Name of cache for blob (for NC-generated blob keys this will be
1146  // empty).
1147  { "cache", eNSPT_Str, eNSPA_Required },
1148  // Blob's key.
1149  { "key", eNSPT_Str, eNSPA_Required },
1150  // Blob's subkey (for NC-generated blob keys this will be empty).
1151  { "subkey", eNSPT_Str, eNSPA_Required } } },
1152  // "Commit" the synchronization session. This command is sent only by other
1153  // NC servers at the end of synchronization session when all necessary
1154  // commands have been executed successfully.
1155  { "SYNC_COMMIT",
1157  "SYNC_COMMIT",
1159  // Server id of the server managing the synchronization.
1160  { { "srv_id", eNSPT_Int, eNSPA_Required },
1161  // Slot that synchronization is started on.
1162  { "slot", eNSPT_Int, eNSPA_Required },
1163  // Last synchronized record number (in sync log) of _that_ server.
1164  { "rec_my", eNSPT_Int, eNSPA_Required },
1165  // Last synchronized record number (in sync log) of _this_ server.
1166  { "rec_your",eNSPT_Int, eNSPA_Required } } },
1167  // "Cancel" the synchronization session. This command is sent only by other
1168  // NC servers at the end of synchronization session when either this server
1169  // requested or that server decided that synchronization should be aborted
1170  // despite the successful execution of all commands. Reason for
1171  // cancellation could be some server going to shutdown, or requirement to
1172  // clean sync logs (and synchronization already executes for too long).
1173  // The cancellation doesn't cancel any commands already executed in this
1174  // synchronization session. It exists only to quickly mark this
1175  // synchronization as no longer executing so that NC could start
1176  // synchronization with some other server.
1177  { "SYNC_CANCEL",
1179  "SYNC_CANCEL",
1181  // Server id of the server managing the synchronization.
1182  { { "srv_id", eNSPT_Int, eNSPA_Required },
1183  // Slot that synchronization is started on.
1184  { "slot", eNSPT_Int, eNSPA_Required } } },
1185  // Get meta information about the blob. Command for "NetCache" clients.
1186  { "GETMETA",
1188  "GETMETA",
1190  eNCRead,
1191  eProxyGetMeta},
1192  // Key of the blob
1193  { { "key", eNSPT_NCID, eNSPA_Required },
1194  // Flag forcing local execution of the command (without forwarding
1195  // to other servers and without searching for blob on them).
1196  { "local", eNSPT_Int, eNSPA_Optional },
1197  // Quorum to use for this operation.
1198  { "qrum", eNSPT_Int, eNSPA_Optional },
1199  // Client IP for application sending the command.
1200  { "ip", eNSPT_Str, fNSPA_Optional },
1201  // Session ID for application sending the command.
1202  { "sid", eNSPT_Str, eNSPA_Optional },
1203  // request Hit ID
1204  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1205  } },
1206  // Get meta information about the blob. Command for "ICache" clients.
1207  { "GETMETA",
1209  "IC_GETMETA",
1211  eNCRead,
1212  eProxyGetMeta},
1213  // Name of cache for blob.
1214  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
1215  // Blob's key.
1216  { "key", eNSPT_Str, eNSPA_Required },
1217  // Blob's version.
1218  { "version", eNSPT_Int, eNSPA_Required },
1219  // Blob's subkey.
1220  { "subkey", eNSPT_Str, eNSPA_Required },
1221  // Flag forcing local execution of the command (without forwarding
1222  // to other servers and without searching for blob on them).
1223  { "local", eNSPT_Int, eNSPA_Optional },
1224  // Quorum to use for this operation.
1225  { "qrum", eNSPT_Int, eNSPA_Optional },
1226  // Client IP for application sending the command.
1227  { "ip", eNSPT_Str, fNSPA_Optional },
1228  // Session ID for application sending the command.
1229  { "sid", eNSPT_Str, eNSPA_Optional },
1230  // request Hit ID
1231  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1232  } },
1233  // Prolong blob's life for a specific number of seconds from current time.
1234  // Command provides a minimum time this blob should be still available for.
1235  // If blob's expiration time was already later than that this command is
1236  // a no-op.
1237  { "PROLONG",
1239  "PROLONG",
1241  eNCRead,
1242  eProxyProlong},
1243  // Name of cache for blob (for NC-generated blob keys this will be
1244  // empty).
1245  { { "cache", eNSPT_Str, eNSPA_Required },
1246  // Blob's key.
1247  { "key", eNSPT_Str, eNSPA_Required },
1248  // Blob's subkey (for NC-generated blob keys this will be empty).
1249  { "subkey", eNSPT_Str, eNSPA_Required },
1250  // Period of time for the blob to be available.
1251  { "ttl", eNSPT_Int, eNSPA_Required },
1252  // Quorum to use for this operation.
1253  { "qrum", eNSPT_Int, eNSPA_Optional },
1254  // Client IP for application sending the command.
1255  { "ip", eNSPT_Str, fNSPA_Optional },
1256  // Session ID for application sending the command.
1257  { "sid", eNSPT_Str, eNSPA_Optional },
1258  // Password for blob access.
1259  { "pass", eNSPT_Str, eNSPA_Optional },
1260  // request Hit ID
1261  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1262  } },
1263  // Prolong blob's life for a specific number of seconds from current time.
1264  // This command is sent only by other NC servers in response to client's
1265  // PROLONG command when the server where client have sent initial command
1266  // cannot execute it locally for any reason (slot is not processed by that
1267  // server, or initial database caching is not completed yet, or it was
1268  // determined that this server has latest version of the blob).
1269  { "PROXY_PROLONG",
1271  "PROXY_PROLONG",
1273  eNCRead,
1274  eProxyProlong},
1275  // Name of cache for blob (for NC-generated blob keys this will be
1276  // empty).
1277  { { "cache", eNSPT_Str, eNSPA_Required },
1278  // Blob's key.
1279  { "key", eNSPT_Str, eNSPA_Required },
1280  // Blob's subkey (for NC-generated blob keys this will be empty).
1281  { "subkey", eNSPT_Str, eNSPA_Required },
1282  // Period of time for the blob to be available.
1283  { "ttl", eNSPT_Int, eNSPA_Required },
1284  // Quorum to use for this operation.
1285  { "qrum", eNSPT_Int, eNSPA_Required },
1286  // Flag whether local execution of this command should be forced
1287  // no matter what, i.e. if this flag is set in cases when normal GET2
1288  // command would have been proxied to other servers an error should
1289  // be returned.
1290  { "local", eNSPT_Int, eNSPA_Required },
1291  // Value of the flag "search_on_read" to use with this command, i.e.
1292  // whether this server should search the blob if it's not found
1293  // locally.
1294  { "srch", eNSPT_Int, eNSPA_Required },
1295  // Client IP for application sending the command.
1296  { "ip", eNSPT_Str, eNSPA_Required },
1297  // Session ID for application sending the command.
1298  { "sid", eNSPT_Str, eNSPA_Required },
1299  // Password for blob access.
1300  { "pass", eNSPT_Str, eNSPA_Optional } } },
1301  /*{ "BLOBSLIST",
1302  {&CNCMessageHandler::x_DoCmd_GetBlobsList,
1303  "BLOBSLIST",
1304  fNeedsStorageCache + fNeedsAdminClient} },*/
1305  // Write blob contents. Deprecated command used now only by old clients.
1306  // This command is the same as PUT3 except it uses connection closing as
1307  // legitimate EOF marker for blob's data.
1308  { "PUT2",
1310  "PUT2",
1312  eNCCreate,
1313  eProxyWrite},
1314  // Time-to-live for the blob.
1315  { { "ttl", eNSPT_Int, eNSPA_Optional },
1316  // Key of the blob (if skipped or empty then new one will be created).
1317  { "key", eNSPT_NCID, eNSPA_Optional } } },
1318  // Shutdown the server
1319  { "SHUTDOWN",
1321  "SHUTDOWN",
1323  // Client IP for application sending the command.
1324  { { "ip", eNSPT_Str, fNSPA_Optional },
1325  // Session ID for application sending the command.
1326  { "sid", eNSPT_Str, eNSPA_Optional },
1327  // drain: wait until all BLOBs are expired, then shutdown
1328  { "drain", eNSPT_Int, eNSPA_Optional },
1329  // reset: shutdown and leave database guard on disk (CXX-10401)
1330  { "reset", eNSPT_Int, eNSPA_Optional },
1331  // request Hit ID
1332  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1333  } },
1334  // Get server statistics.
1335  { "GETSTAT",
1336  {&CNCMessageHandler::x_DoCmd_GetStat, "GETSTAT"},
1337  // Flag showing whether current (value is 0) or previous (value is 1)
1338  // statistics period should be shown.
1339  { { "prev", eNSPT_Int, fNSPA_Optional, "0" },
1340  // Type of statistics period to show. See top of nc_stat.cpp for
1341  // list of all possible period types.
1342  { "type", eNSPT_Str, eNSPA_Optional, "life" },
1343  // Client IP for application sending the command.
1344  { "ip", eNSPT_Str, fNSPA_Optional },
1345  // Session ID for application sending the command.
1346  { "sid", eNSPT_Str, eNSPA_Optional },
1347  // request Hit ID
1348  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1349  } },
1350  // Read full ini-file used by NetCache for configuration.
1351  { "GETCONF",
1354  {
1355  // Netcached.ini section name
1356  { "section", eNSPT_Str, eNSPA_Optional },
1357  // when section name is "netcache", setup for this port
1358  { "port", eNSPT_Str, eNSPA_Optional },
1359  // when section name is "netcache", setup for this cache
1360  { "cache", eNSPT_Str, eNSPA_Optional },
1361  // Client IP for application sending the command.
1362  { "ip", eNSPT_Str, fNSPA_Optional },
1363  // Session ID for application sending the command.
1364  { "sid", eNSPT_Str, eNSPA_Optional },
1365  // request Hit ID
1366  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1367  } },
1368  // Get state information, added in v6.8.6
1369  { "INFO",
1372  {
1373  // Netcached.ini section name
1374  { "section", eNSPT_Str, fNSPA_Required },
1375  // when section name is "netcache", setup for this port
1376  { "port", eNSPT_Str, eNSPA_Optional },
1377  // when section name is "netcache", setup for this cache
1378  { "cache", eNSPT_Str, eNSPA_Optional },
1379  // Client IP for application sending the command.
1380  { "ip", eNSPT_Str, fNSPA_Optional },
1381  // Session ID for application sending the command.
1382  { "sid", eNSPT_Str, eNSPA_Optional },
1383  // request Hit ID
1384  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1385  } },
1386  // Acknowledge alert, added in v6.8.6
1387  { "ACKALERT",
1390  {
1391  // Alert name
1392  { "alert", eNSPT_Str, fNSPA_Required },
1393  // User name
1394  { "user", eNSPT_Str, fNSPA_Required },
1395  // Client IP for application sending the command.
1396  { "ip", eNSPT_Str, fNSPA_Optional },
1397  // Session ID for application sending the command.
1398  { "sid", eNSPT_Str, eNSPA_Optional },
1399  // request Hit ID
1400  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1401  } },
1402  // Re-read ini-file used by NetCache for configuration.
1403  // Find changes and reconfigure
1404  // Only few changes are supported
1405  { "RECONF",
1407  "RECONF",
1409  {
1410  // Netcached.ini section name
1411  { "section", eNSPT_Str, eNSPA_Required },
1412  // Client IP for application sending the command.
1413  { "ip", eNSPT_Str, fNSPA_Optional },
1414  // Session ID for application sending the command.
1415  { "sid", eNSPT_Str, eNSPA_Optional },
1416  // request Hit ID
1417  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1418  } },
1419 
1420  { "PURGE",
1422  "PURGE",
1424  {
1425  // Cache name.
1426  { "cache", eNSPT_Str, eNSPA_Required },
1427  // Client IP for application sending the command.
1428  { "ip", eNSPT_Str, fNSPA_Optional },
1429  // Session ID for application sending the command.
1430  { "sid", eNSPT_Str, eNSPA_Optional },
1431  // request Hit ID
1432  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1433  } },
1434  { "COPY_PURGE",
1436  "COPY_PURGE", fNoCmdFlags, eNCNone, eProxyNone },
1437  // Cache name.
1438  { { "cache", eNSPT_Str, eNSPA_Required },
1439  // forget blobs created earlier than cr_time
1440  { "cr_time", eNSPT_Int, eNSPA_Required } } },
1441 
1442  // Added in 6.11.7, CXX-8948
1443  { "PURGE2",
1445  "PURGE",
1447  // Cache name.
1448  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
1449  // Blob's key.
1450  { "key", eNSPT_Str, eNSPA_Required },
1451  // Client IP for application sending the command.
1452  { "ip", eNSPT_Str, fNSPA_Optional },
1453  // Session ID for application sending the command.
1454  { "sid", eNSPT_Str, eNSPA_Optional },
1455  // request Hit ID
1456  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1457  } },
1458  // Added in 6.11.7, CXX-8948
1459  { "COPY_PURGE2",
1461  "COPY_PURGE", fNoCmdFlags, eNCNone, eProxyNone },
1462  // Cache name.
1463  { { "cache", eNSPT_Str, eNSPA_Required },
1464  // Blob's key.
1465  { "key", eNSPT_Str, eNSPA_Required },
1466  // forget blobs created earlier than cr_time
1467  { "cr_time", eNSPT_Int, eNSPA_Required } } },
1468 
1469  // Get list of blobs by mask: "cache,key,*"
1470  // Added in 6.9.0, CXX-6246
1471  { "BLIST",
1473  "BLIST",
1477  // Name of cache for blob.
1478  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
1479  // Blob's key.
1480  { "key", eNSPT_Str, eNSPA_Required },
1481  // Blob's subkey.
1482  { "subkey", eNSPT_Str, eNSPA_Optional },
1483  { "local", eNSPT_Int, eNSPA_Optional },
1484  // Client IP for application sending the command.
1485  { "ip", eNSPT_Str, fNSPA_Optional },
1486  // Session ID for application sending the command.
1487  { "sid", eNSPT_Str, eNSPA_Optional },
1488  // request Hit ID
1489  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1490  } },
1491 // added in v6.11.0 (CXX-8737)
1492  { "BLIST2",
1494  "BLIST2",
1498  // Name of cache for blob.
1499  { { "cache", eNSPT_Id, eNSPA_ICPrefix },
1500  // Blob's key.
1501  { "key", eNSPT_Str, eNSPA_Optional },
1502  // Blob's subkey.
1503  { "subkey", eNSPT_Str, eNSPA_Optional },
1504  { "local", eNSPT_Int, eNSPA_Optional },
1505 
1506  // Created more than N seconds ago
1507  { "fcr_ago_ge", eNSPT_Int, eNSPA_Optional },
1508  // Created less than N seconds ago
1509  { "fcr_ago_lt", eNSPT_Int, eNSPA_Optional },
1510  // Created more than N seconds since epoch
1511  { "fcr_epoch_ge", eNSPT_Int, eNSPA_Optional },
1512  // Created less than N seconds since epoch
1513  { "fcr_epoch_lt", eNSPT_Int, eNSPA_Optional },
1514  // Blob will expire in more than N seconds from now
1515  { "fexp_now_ge", eNSPT_Int, eNSPA_Optional },
1516  // Blob will expire in less than N seconds from now
1517  { "fexp_now_lt", eNSPT_Int, eNSPA_Optional },
1518  // Blob will expire in more than N seconds since epoch
1519  { "fexp_epoch_ge", eNSPT_Int, eNSPA_Optional },
1520  // Blob will expire in less than N seconds since epoch
1521  { "fexp_epoch_lt", eNSPT_Int, eNSPA_Optional },
1522  // Version will expire in more than N seconds from now
1523  { "fvexp_now_ge", eNSPT_Int, eNSPA_Optional },
1524  // Version will expire in less than N seconds from now
1525  { "fvexp_now_lt", eNSPT_Int, eNSPA_Optional },
1526  // Version will expire in more than N seconds since epoch
1527  { "fvexp_epoch_ge", eNSPT_Int, eNSPA_Optional },
1528  // Version will expire in less than N seconds since epoch
1529  { "fvexp_epoch_lt", eNSPT_Int, eNSPA_Optional },
1530  // Server_id of the server where blob was created.
1531  { "fcr_srv", eNSPT_Int, eNSPA_Optional },
1532  // blob bigger than this size
1533  { "fsize_ge", eNSPT_Int, eNSPA_Optional },
1534  // blob smaller than this size
1535  { "fsize_lt", eNSPT_Int, eNSPA_Optional },
1536  // Client IP for application sending the command.
1537  { "ip", eNSPT_Str, fNSPA_Optional },
1538  // Session ID for application sending the command.
1539  { "sid", eNSPT_Str, eNSPA_Optional },
1540  // request Hit ID
1541  { "ncbi_phid", eNSPT_Str, eNSPA_Optional }
1542  } },
1543  { "PROXY_BLIST",
1545  "PROXY_BLIST",
1547  eNCNone, eProxyNone},
1548  // Name of cache for blob.
1549  { { "cache", eNSPT_Str, eNSPA_Required },
1550  // Blob's key.
1551  { "key", eNSPT_Str, eNSPA_Required },
1552  // Blob's subkey.
1553  { "subkey", eNSPT_Str, eNSPA_Required },
1554  { "local", eNSPT_Int, eNSPA_Required }
1555  } },
1556 // added in v6.11.0 (CXX-8737)
1557  { "PROXY_BLIST2",
1559  "PROXY_BLIST2",
1561  eNCNone, eProxyNone},
1562  // Name of cache for blob.
1563  { { "cache", eNSPT_Str, eNSPA_Required },
1564  // Blob's key.
1565  { "key", eNSPT_Str, eNSPA_Required },
1566  // Blob's subkey.
1567  { "subkey", eNSPT_Str, eNSPA_Required },
1568  { "local", eNSPT_Int, eNSPA_Required },
1569  // Created more than N seconds ago
1570  { "fcr_ago_ge", eNSPT_Int, eNSPA_Optional },
1571  // Created less than N seconds ago
1572  { "fcr_ago_lt", eNSPT_Int, eNSPA_Optional },
1573  // Created more than N seconds since epoch
1574  { "fcr_epoch_ge", eNSPT_Int, eNSPA_Optional },
1575  // Created less than N seconds since epoch
1576  { "fcr_epoch_lt", eNSPT_Int, eNSPA_Optional },
1577  // Will expire in more than N seconds from now
1578  { "fexp_now_ge", eNSPT_Int, eNSPA_Optional },
1579  // Will expire in less than N seconds from now
1580  { "fexp_now_lt", eNSPT_Int, eNSPA_Optional },
1581  // Will expire in more than N seconds since epoch
1582  { "fexp_epoch_ge", eNSPT_Int, eNSPA_Optional },
1583  // Will expire in less than N seconds since epoch
1584  { "fexp_epoch_lt", eNSPT_Int, eNSPA_Optional },
1585  // Will expire in more than N seconds from now
1586  { "fvexp_now_ge", eNSPT_Int, eNSPA_Optional },
1587  // Will expire in less than N seconds from now
1588  { "fvexp_now_lt", eNSPT_Int, eNSPA_Optional },
1589  // Will expire in more than N seconds since epoch
1590  { "fvexp_epoch_ge", eNSPT_Int, eNSPA_Optional },
1591  // Will expire in less than N seconds since epoch
1592  { "fvexp_epoch_lt", eNSPT_Int, eNSPA_Optional },
1593  // Server_id of the server where blob was created.
1594  { "fcr_srv", eNSPT_Int, eNSPA_Optional },
1595  // blob bigger than this size
1596  { "fsize_ge", eNSPT_Int, eNSPA_Optional },
1597  // blob smaller than this size
1598  { "fsize_lt", eNSPT_Int, eNSPA_Optional }
1599  } },
1600 
1601 // HTTP commands
1602  { "DELETE",
1604  "DELETE",
1606  eNCCreate,
1607  eProxyRemove}
1608  },
1609  { "GET",
1611  "GET",
1613  eNCReadData,
1614  eProxyRead}
1615  },
1616  { "HEAD",
1618  "HEAD",
1620  eNCRead,
1621  eProxyGetMeta}
1622  },
1623  { "POST",
1625  "POST",
1627  eNCCreate,
1628  eProxyWrite}
1629  },
1630  { "PUT",
1632  "PUT",
1634  eNCCreate,
1635  eProxyWrite}
1636  },
1637 
1638 
1639  // All commands below are not implemented and mostly old ones not needed
1640  // anymore. One exception is RECONF - it would be nice to have it but it
1641  // needs some thinking on how to implement it.
1642 
1643  { "REINIT", {&CNCMessageHandler::x_DoCmd_NotImplemented, "REINIT"} },
1644  { "REINIT", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_REINIT"},
1645  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1646 // { "RECONF", {&CNCMessageHandler::x_DoCmd_NotImplemented, "RECONF"} },
1647  { "LOG", {&CNCMessageHandler::x_DoCmd_NotImplemented, "LOG"} },
1648  { "STAT", {&CNCMessageHandler::x_DoCmd_NotImplemented, "STAT"} },
1649  { "MONI", {&CNCMessageHandler::x_DoCmd_NotImplemented, "MONITOR"} },
1650  { "DROPSTAT", {&CNCMessageHandler::x_DoCmd_NotImplemented, "DROPSTAT"} },
1651  { "GBOW", {&CNCMessageHandler::x_DoCmd_NotImplemented, "GBOW"} },
1652  { "ISLK", {&CNCMessageHandler::x_DoCmd_NotImplemented, "ISLK"} },
1653  { "SMR", {&CNCMessageHandler::x_DoCmd_NotImplemented, "SMR"} },
1654  { "SMU", {&CNCMessageHandler::x_DoCmd_NotImplemented, "SMU"} },
1655  { "OK", {&CNCMessageHandler::x_DoCmd_NotImplemented, "OK"} },
1656 // will be used by HTTP
1657 // { "GET", {&CNCMessageHandler::x_DoCmd_NotImplemented, "GET"} },
1658 // { "PUT", {&CNCMessageHandler::x_DoCmd_NotImplemented, "PUT"} },
1659  { "REMOVE", {&CNCMessageHandler::x_DoCmd_NotImplemented, "REMOVE"} },
1660  { "STSP", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_STSP"},
1661  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1662  { "GTSP", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_GTSP"},
1663  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1664  { "SVRP", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_SVRP"},
1665  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1666  { "GVRP", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_GVRP"},
1667  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1668  { "PRG1", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_PRG1"},
1669  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1670  { "REMK", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_REMK"},
1671  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1672  { "GBLW", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_GBLW"},
1673  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1674  { "ISOP", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_ISOP"},
1675  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1676  { "GACT", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_GACT"},
1677  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1678  { "GTOU", {&CNCMessageHandler::x_DoCmd_NotImplemented, "IC_GTOU"},
1679  { { "cache", eNSPT_Id, eNSPA_ICPrefix } } },
1680  { NULL }
1681 };
1682 
1683 // List of arguments that can be in client authentication line.
1685  { "client", eNSPT_Str, eNSPA_Optional, "Unknown client" },
1686  { "params", eNSPT_Str, eNSPA_Ellipsis },
1687  { NULL }
1688 };
1689 
1690 
1691 
1692 
1693 /////////////////////////////////////////////////////////////////////////////
1694 // CNCMessageHandler implementation
1695 
1696 inline void
1698 {
1700  m_Flags = fNoCmdFlags | keep;
1701 }
1702 
1703 inline void
1705 {
1706  m_Flags |= flag;
1707 }
1708 
1709 inline void
1711 {
1712  m_Flags &= ~flag;
1713 }
1714 
1715 inline bool
1717 {
1718  return (m_Flags & flag) != 0;
1719 }
1720 
1721 inline bool
1723 {
1724  return (m_UserFlags & flag) != 0;
1725 }
1726 
1727 inline bool
1729 {
1730  return cmd_status < 300;
1731 }
1732 
1733 //static Uint8 s_CntHndls = 0;
1734 
1736  : m_Flags(0),
1737  m_Parser(s_CommandMap),
1738  m_CmdProcessor(NULL),
1739  m_BlobAccess(NULL),
1740  m_write_event(NULL),
1741  m_ChunkLen(0),
1742  m_SrvsIndex(0),
1743  m_ActiveHub(NULL)
1744 {
1746 #if __NC_TASKS_MONITOR
1747  m_TaskName = "CNCMessageHandler";
1748 #endif
1749  m_CopyBlobInfo = new SNCBlobVerData(nullptr);
1752  m_CmdLog.reserve(32);
1753 
1755 
1756  //Uint8 cnt = AtomicAdd(s_CntHndls, 1);
1757  //INFO("CNCMessageHandler, cnt=" << cnt);
1758  m_HttpMode = eNoHttp;
1759 }
1760 
1762 {
1764  delete m_LatestBlobSum;
1765  delete m_CopyBlobInfo;
1766 
1767  //Uint8 cnt = AtomicSub(s_CntHndls, 1);
1768  //INFO("~CNCMessageHandler, cnt=" << cnt);
1769 }
1770 
1771 CNCMessageHandler::State
1773 {
1775  m_PrevCache.clear();
1777  m_CntCmds = 0;
1778 
1779  string host;
1780  Uint2 port = 0;
1781  GetPeerAddress(host, port);
1782  m_ClientParams["peer"] = host;
1783  m_ClientParams["pport"] = NStr::UIntToString(port);
1786 
1788 }
1789 
1790 CNCMessageHandler::State
1792 {
1794  if (GetDiagCtx()->GetRequestStatus() == eStatus_OK) {
1795  if (HasError() || !CanHaveMoreRead())
1797  else if (CTaskServer::IsInShutdown())
1799  else
1801  }
1802  int status = GetDiagCtx()->GetRequestStatus();
1805  }
1807  GetDiagCtx()->SetRequestStatus(status);
1809 }
1810 
1811 CNCMessageHandler::State
1813 {
1815  //CNCStat::AddClosedConnection(conn_span, GetDiagCtx()->GetRequestStatus(), m_CntCmds);
1818 }
1819 
1820 CNCMessageHandler::State
1822 {
1824  CSrvDiagMsg().PrintExtra().PrintParam("cmds_cnt", m_CntCmds);
1825  CloseSocket();
1826  Terminate();
1827  return NULL;
1828 }
1829 
1830 CNCMessageHandler::State
1832 {
1834 check_again:
1836  if (status == eNCHubError || status == eNCHubSuccess) {
1838  SRV_LOG(Warning, "Error executing command on peer "
1839  << m_ActiveHub->GetFullPeerName() << ", peer says: "
1840  << m_LastPeerError);
1841  m_ActiveHub->Release();
1842  m_ActiveHub = NULL;
1844  }
1845  else if (status != eNCHubCmdInProgress) {
1846  SRV_FATAL("Unexpected client status: " << status);
1847  }
1848  if (NeedEarlyClose())
1850 
1852  if (!active->GotClientResponse())
1853  return NULL;
1854  // Intentionally re-reading the status because it could change since our
1855  // previous read at the beginning of the function.
1857  goto check_again;
1858 
1859  WriteText(active->GetCmdResponse()).WriteText("\n");
1860  Flush();
1861  if (NeedEarlyClose())
1863 
1865 }
1866 
1867 CNCMessageHandler::State
1869 {
1874  else
1877  }
1878 
1879  CTempString auth_line;
1880  if (!ReadLine(&auth_line)) {
1881  if (!HasError() && CanHaveMoreRead())
1882  return NULL;
1883  if (IsReadDataAvailable()) {
1886  }
1887  else {
1890  }
1891  }
1892 
1893  m_HttpMode = eNoHttp;
1894  size_t auth_size = auth_line.size();
1895  if (auth_size > 8 && auth_line[auth_size-8] == 'H') {
1896  if (NStr::strncmp(auth_line.data() + auth_size - 8, "HTTP/1.", 7) == 0) {
1897  if (auth_line[auth_size-1] == '0') {
1898  m_HttpMode = eHttp10;
1899  } else if (auth_line[auth_size-1] == '1') {
1900  m_HttpMode = eHttp11;
1901  }
1902  }
1903  }
1904 
1905  if (!x_IsHttpMode()) {
1906  TNSProtoParams params;
1907  try {
1908  m_Parser.ParseArguments(auth_line, s_AuthArgs, &params);
1909  }
1910  catch (CNSProtoParserException& ex) {
1911  SRV_LOG(Error, "Error authenticating client: '"
1912  << auth_line << "': " << ex);
1913  params["client"] = auth_line;
1915  }
1916  ITERATE(TNSProtoParams, it, params) {
1917  m_ClientParams[it->first] = it->second;
1918  }
1919  CSrvDiagMsg diag_msg;
1920  diag_msg.PrintExtra();
1921  ITERATE(TNSProtoParams, it, params) {
1922  diag_msg.PrintParam(it->first, it->second);
1923  }
1924  diag_msg.Flush();
1925  } else {
1926  m_PosponedCmd = auth_line;
1927 // m_ClientParams["cache"] = "http";
1928 // m_ClientParams["cache"] = "";
1929  }
1930 
1932  if (m_AppSetup->disable) {
1933  SRV_LOG(Warning, "Disabled client is being disconnected ('"
1934  << auth_line << "').");
1937  }
1938  else {
1940  }
1941 }
1942 
1943 void
1945 {
1947  CTempString blob_key, blob_subkey;
1949  m_BlobVersion = 0;
1950  m_NCBlobKey.Clear();
1951  m_BlobPass.clear();
1952  m_RawBlobPass.clear();
1953  m_KeyVersion = 1;
1954  m_BlobTTL = 0;
1955  m_StartPos = 0;
1956  m_Size = Uint8(-1);
1957  m_Slot = 0;
1958  m_OrigRecNo = 0;
1959  m_OrigSrvId = 0;
1960  m_OrigTime = 0;
1961  m_Quorum = 1;
1962  m_CmdVersion = 0;
1963  m_ForceLocal = false;
1964  m_AgeMax = m_AgeCur = 0;
1965  m_SlotsDone.clear();
1966  m_CmdParams.clear();
1967  bool quorum_was_set = false;
1968  bool search_was_set = false;
1969 
1971  delete m_CopyBlobInfo;
1972  m_CopyBlobInfo = new SNCBlobVerData(nullptr);
1973  new (m_BlobFilter) SNCBlobFilter();
1974 
1975  CTempString cache_name;
1976 
1977  if (!x_IsHttpMode()) {
1978  m_HttpMode = eNoHttp;
1980  const CTempString& key = it->first;
1981  CTempString& val = it->second;
1982 
1983  switch (key[0]) {
1984  case 'a':
1985  if (key == "age") {
1987  }
1988  break;
1989  case 'c':
1990  switch (key[1]) {
1991  case 'a':
1992  if (key == "cache") {
1993  cache_name = val;
1994  }
1995  break;
1996  case 'm':
1997  if (key == "cmd_ver") {
1999  }
2000  break;
2001  case 'o':
2002  if (key == "confirm") {
2003  if (val == "1")
2005  else
2007  }
2008  break;
2009  case 'r':
2010  if (key == "cr_time") {
2012  }
2013  else if (key == "cr_id") {
2015  }
2016  else if (key == "cr_srv") {
2018  }
2019  break;
2020  }
2021  break;
2022  case 'd':
2023  if (key == "dead") {
2025  }
2026  break;
2027  case 'e':
2028  if (key == "exp") {
2030  }
2031  break;
2032  case 'f':
2033  if (key == "flags") {
2035  } else if (key == "fcr_ago_ge") {
2037  } else if (key == "fcr_ago_lt") {
2039  } else if (key == "fcr_epoch_ge") {
2041  } else if (key == "fcr_epoch_lt") {
2043  } else if (key == "fexp_now_ge") {
2045  } else if (key == "fexp_now_lt") {
2047  } else if (key == "fexp_epoch_ge") {
2049  } else if (key == "fexp_epoch_lt") {
2051  } else if (key == "fvexp_now_ge") {
2053  } else if (key == "fvexp_now_lt") {
2055  } else if (key == "fvexp_epoch_ge") {
2057  } else if (key == "fvexp_epoch_lt") {
2059  } else if (key == "fcr_srv") {
2061  } else if (key == "fsize_ge") {
2063  } else if (key == "fsize_lt") {
2065  }
2066  break;
2067  case 'h':
2068  if (key == "http") {
2070  }
2071  break;
2072  case 'i':
2073  if (key == "ip") {
2074  if (!val.empty())
2076  // Erase parameter to not print it in request-start, it will be
2077  // printed as a part of standard log header.
2078  m_ParsedCmd.params.erase(it);
2079  }
2080  break;
2081  case 'k':
2082  if (key == "key") {
2083  blob_key = val;
2084  }
2085  break;
2086  case 'l':
2087  if (key == "log_rec") {
2089  }
2090  else if (key == "log_srv") {
2092  }
2093  else if (key == "log_time") {
2095  }
2096  else if (key == "local") {
2097  m_ForceLocal = val == "1";
2098  }
2099  break;
2100  case 'm':
2101  if (key == "md5_pass") {
2102  m_BlobPass = val;
2103  }
2104  break;
2105  case 'n':
2106  if (key == "ncbi_phid") {
2107  if (!val.empty()) {
2108  GetDiagCtx()->SetHitID(val);
2109  }
2110  }
2111  break;
2112  case 'p':
2113  if (key == "pass") {
2114  m_RawBlobPass = val;
2115  CMD5 md5;
2116  md5.Update(val.data(), val.size());
2117  unsigned char digest[16];
2118  md5.Finalize(digest);
2119  m_BlobPass.assign((char*)digest, 16);
2120  // Erase parameter to not expose passwords via logs.
2121  m_ParsedCmd.params.erase(it);
2122  }
2123  else if (key == "prev") {
2124  m_StatPrev = val == "1";
2125  }
2126  break;
2127  case 'q':
2128  if (key == "qrum") {
2130  quorum_was_set = true;
2131  }
2132  break;
2133  case 'r':
2134  if (key == "rec_my") {
2136  }
2137  else if (key == "rec_your") {
2139  }
2140  break;
2141  case 's':
2142  switch (key[1]) {
2143  case 'i':
2144  if (key == "sid") {
2145  if (!val.empty())
2147  // Erase parameter to not print it in request-start,
2148  // it will be printed as a part of standard log header.
2149  m_ParsedCmd.params.erase(it);
2150  }
2151  else if (key == "size") {
2153  }
2154  break;
2155  case 'l':
2156  if (key == "slot") {
2158  }
2159  break;
2160  case 'r':
2161  if (key == "srv_id") {
2163  }
2164  else if (key == "srch") {
2165  m_SearchOnRead = val != "0";
2166  search_was_set = true;
2167  }
2168  break;
2169  case 't':
2170  if (key == "start") {
2172  }
2173  break;
2174  case 'u':
2175  if (key == "subkey") {
2176  blob_subkey = val;
2177  }
2178  break;
2179  }
2180  break;
2181  case 't':
2182  if (key == "ttl") {
2184  }
2185  else if (key == "type") {
2186  m_StatType = val;
2187  }
2188  break;
2189  case 'v':
2190  if (key == "version") {
2192  }
2193  else if (key == "ver_ttl") {
2195  }
2196  else if (key == "ver_dead") {
2198  }
2199  break;
2200  default:
2201  break;
2202  }
2203  }
2204  } else {
2205  m_ClientParams["client"] = "";
2206  cache_name = m_ClientParams["cache"];
2207  if (m_ClientParams.find("key") != m_ClientParams.end()) {
2208  blob_key = m_ClientParams["key"];
2209  }
2210  // parse HTTP header
2211  CTempString cmd_line;
2212  map<string,string> headers;
2213  while (ReadLine(&cmd_line)) {
2214  if (cmd_line.empty()) {
2215  break;
2216  }
2217  const string content_length("Content-Length:");
2218  const string user_agent("User-Agent:");
2219  const string content_range("Range:");
2220  size_t max_pos = cmd_line.size();
2221 
2222  if (NStr::StartsWith(cmd_line, content_length)) {
2223  size_t pos = content_length.size();
2225  CTempString(cmd_line.data() + pos, max_pos - pos),
2227  }
2228  else if (NStr::StartsWith(cmd_line, content_range)) {
2229  size_t pos = content_range.size();
2230  const char* begin = cmd_line.data() + pos;
2231  while (!isdigit(*begin) && pos < max_pos) {
2232  ++pos; ++begin;
2233  }
2234  m_StartPos = 0;
2235  while (isdigit(*begin) && pos < max_pos) {
2236  m_StartPos = m_StartPos * 10 + (*begin - '0');
2237  ++pos; ++begin;
2238  }
2239  while (!isdigit(*begin) && pos < max_pos) {
2240  ++pos; ++begin;
2241  }
2242  m_Size = 0;
2243  while (isdigit(*begin) && pos < max_pos) {
2244  m_Size = m_Size * 10 + (*begin - '0');
2245  ++pos; ++begin;
2246  }
2247  // byte pos are inclusive
2248  m_Size = (m_Size >= m_StartPos) ? (m_Size - m_StartPos + 1) : 0;
2249  }
2250  else if (NStr::StartsWith(cmd_line, user_agent)) {
2251  list<CTempString> arr;
2252  size_t pos = user_agent.size();
2253  NStr::Split( CTempString(cmd_line.data()+pos, max_pos-pos), " ", arr, NStr::fSplit_Tokenize);
2254  if (!arr.empty()) {
2255  m_ClientParams["client"] = NStr::URLEncode( arr.front());
2256  }
2257  }
2258  else {
2259  string key, value;
2260  NStr::SplitInTwo(cmd_line, ":", key, value);
2262  key = NStr::ToUpper(key);
2264  headers[key] = value;
2265  }
2266  }
2267  list<CTempString> arr;
2268  if (headers.find("NCBI-SID") != headers.end()) {
2269  NStr::Split( headers.at("NCBI-SID"), ", \t", arr, NStr::fSplit_Tokenize);
2270  if (!arr.empty()) {
2271  m_CmdParams["sid"] = arr.back();
2272  }
2273  }
2274  arr.clear();
2275  if (headers.find("NCBI-PHID") != headers.end()) {
2276  NStr::Split( headers.at("NCBI-PHID"), ", \t", arr, NStr::fSplit_Tokenize);
2277  if (!arr.empty()) {
2278  m_CmdParams["ncbi_phid"] = arr.back();
2279  }
2280  }
2281  arr.clear();
2282  string client_ip = g_GetClientIP( headers);
2283  if (!client_ip.empty()) {
2284  NStr::Split( client_ip, ", \t", arr, NStr::fSplit_Tokenize);
2285  }
2286  if (!arr.empty()) {
2287  m_CmdParams["ip"] = arr.front();
2288  } else {
2289  m_CmdParams["ip"] = m_ClientParams["peer"];
2290  }
2291  }
2292 
2293  m_NCBlobKey.Assign(cache_name, blob_key, blob_subkey);
2294  if (cache_name.empty()) {
2296  m_ClientParams.erase("cache");
2297  }
2298  else if (cache_name == m_PrevCache) {
2300  }
2301  else {
2302  m_PrevCache = cache_name;
2303  m_ClientParams["cache"] = cache_name;
2305  }
2306  if (!quorum_was_set)
2308  if (m_ForceLocal)
2309  m_Quorum = 1;
2310  if (!search_was_set)
2312 }
2313 
2314 void
2316 {
2318  diag_msg.StartRequest();
2319  diag_msg.PrintParam("_type", "cmd");
2320  diag_msg.PrintParam("cmd", m_ParsedCmd.command->cmd);
2321  diag_msg.PrintParam("client", m_ClientParams["client"]);
2322  diag_msg.PrintParam("conn", m_ConnReqId);
2323  diag_msg.PrintParam("phost", m_ClientParams["peer"]);
2325  diag_msg.PrintParam(it->first, it->second);
2326  }
2328  diag_msg.PrintParam(it->first, it->second);
2329  }
2330  if (!m_BlobPass.empty()) {
2332  }
2333  diag_msg.PrintParam("ncbi_role", CNCServer::GetHostRole());
2334  diag_msg.PrintParam("ncbi_location", CNCServer::GetHostLocation());
2335 }
2336 
2337 CNCMessageHandler::State
2339 {
2342  m_CmdLog.clear();
2344  CSrvDiagMsg diag_msg;
2345  x_PrintRequestStart(diag_msg);
2346 
2347  if (NeedToClose()) {
2348  diag_msg.Flush();
2349  x_ResetFlags();
2351  }
2352  if (HasError() || !CanHaveMoreRead()) {
2353  diag_msg.Flush();
2355  x_ResetFlags();
2357  }
2358 
2360  && m_ClientParams["client"] != CNCServer::GetAdminClient()
2361  && m_ClientParams["client"] != kNCPeerClientName)
2362  {
2363  string msg;
2364  msg += "command: " + string(m_ParsedCmd.command->cmd);
2365  msg += ", peer: " + m_ClientParams["peer"];
2366  msg += ", client: " + m_ClientParams["client"];
2368  diag_msg.Flush();
2370  if (!x_IsHttpMode()) {
2371  x_ResetFlags();
2372  }
2374  }
2375 
2378  {
2379  diag_msg.Flush();
2381  if (!x_IsHttpMode()) {
2382  x_ResetFlags();
2383  }
2385  }
2386 
2387  if (m_AppSetup->disable) {
2388  diag_msg.Flush();
2389  // We'll be here only if generally work for the client is enabled but
2390  // for current particular cache it is disabled.
2392  if (!x_IsHttpMode()) {
2393  x_ResetFlags();
2394  }
2396  }
2397 
2400  m_SrvId, m_Slot,
2402  m_SyncId);
2403  if (start_res == eNetworkError) {
2404  diag_msg.Flush();
2406  if (!x_IsHttpMode()) {
2407  x_ResetFlags();
2408  }
2410  }
2411  else if (start_res == eServerBusy || CTaskServer::IsInSoftShutdown()) {
2412  diag_msg.Flush();
2413  x_ReportOK("OK:SIZE=0, NEED_ABORT1\n");
2415  // Old NC servers (those which used CNetCacheAPI instead of
2416  // CNCActiveHandler) always started to write blob data in SYNC_PUT
2417  // even when we responded to them NEED_ABORT. To avoid breaking the protocol
2418  // we need to read from them those fake blob writes.
2419  bool needs_fake = x_IsFlagSet(fReadExactBlobSize) && m_CmdVersion == 0;
2420  if (start_res == eServerBusy)
2421  x_ResetFlags();
2422  if (needs_fake)
2425  }
2426  }
2427 
2432  }
2433 
2435  diag_msg.Flush();
2436  // if we do not need blob access
2437  return m_CmdProcessor;
2438  }
2439 
2440  if (((m_BlobPass.empty() && m_AppSetup->pass_policy == eNCOnlyWithPass)
2441  || (!m_BlobPass.empty() && m_AppSetup->pass_policy == eNCOnlyWithoutPass))
2443  {
2444  diag_msg.Flush();
2446  // why substr?
2450  }
2451 
2453  string raw_key;
2454 //to test
2455 // bool old_ver = (CSrvTime::Current().CurSecs() % 2) != 0;
2456  bool old_ver = !x_IsHttpMode();
2457 
2460  raw_key, m_BlobSlot, m_TimeBucket, old_ver ? 1 : 3);
2461  m_NCBlobKey.Assign(raw_key);
2462  diag_msg.PrintParam("key", m_NCBlobKey.RawKey());
2463  diag_msg.PrintParam("gen_key", "1");
2464  }
2465  else if (!m_NCBlobKey.IsValid() && !x_IsFlagSet(fNeedsBlobList)) {
2466  diag_msg.Flush();
2468  SRV_LOG(Warning, "Invalid blob key format: " << m_NCBlobKey.RawKey());
2470  }
2471  else if (m_NCBlobKey.IsICacheKey()) {
2473  } else {
2475  }
2476 
2477  m_BlobSize = 0;
2478  if (m_Slot == 0) {
2479  diag_msg.PrintParam("slot", m_BlobSlot);
2480  }
2481 
2485  && !m_ForceLocal
2487  {
2488  diag_msg.PrintParam("proxy", "1");
2489  diag_msg.Flush();
2492  }
2493 
2494  if (!x_IsFlagSet(fNeedsBlobAccess)) {
2495  diag_msg.Flush();
2496  // if we do not need blob access
2497  return m_CmdProcessor;
2498  }
2499 
2500  // no blob access before caching is done
2502  {
2503  diag_msg.Flush();
2505  if (!x_IsHttpMode()) {
2506  x_ResetFlags();
2507  }
2509  }
2510 
2511  diag_msg.Flush();
2512 
2515  {
2516  m_Quorum = 0;
2517  }
2524  {
2528  }
2531  }
2532 
2533  x_LogCmdEvent("RequestMetaInfo");
2539 }
2540 
2541 CNCMessageHandler::State
2543 {
2548  else
2551  }
2552 
2553  CTempString cmd_line;
2554  if (x_IsHttpMode() && !m_PosponedCmd.empty()) {
2555  cmd_line = m_PosponedCmd;
2556  } else if (!ReadLine(&cmd_line)) {
2557  if (!HasError() && CanHaveMoreRead())
2558  return NULL;
2559  if (IsReadDataAvailable())
2562  }
2563 
2564  if (!x_IsHttpMode()) {
2565  try {
2566  m_ParsedCmd = m_Parser.ParseCommand(cmd_line);
2567  }
2568  catch (CNSProtoParserException& ex) {
2569  SRV_LOG(Warning, "Error parsing command: " << ex);
2572  }
2573  } else {
2574  m_ClientParams.erase("key");
2575  list<CTempString> arr;
2576  ncbi_NStr_Split(cmd_line, " ", arr);
2577  bool good = false;
2578  if (arr.size() >= 3) {
2579  CTempString arr_cmd(arr.front());
2580  CTempString arr_uri(*(++arr.begin()));
2581  CTempString arr_key;
2582  if (arr_cmd == "DELETE" ||
2583  arr_cmd == "GET" ||
2584  arr_cmd == "HEAD" ||
2585  arr_cmd == "POST" ||
2586  arr_cmd == "PUT") {
2587  {
2588  // eg, "/" "/service"
2589  list<CTempString> uri_parts;
2590  ncbi_NStr_Split(arr_uri, "/", uri_parts);
2591  if (uri_parts.size() > 0) {
2592  if (arr_cmd != "POST") {
2593  arr_key = uri_parts.back();
2594  m_ClientParams["key"] = arr_key;
2595  uri_parts.pop_back();
2596  }
2597  string service;
2598  if (!uri_parts.empty()) {
2599  service = NStr::Join(uri_parts, "/");
2600  }
2601  m_ClientParams["service"] = service;
2602  }
2603  }
2604  m_ClientParams["cache"].clear();
2605 
2606  try {
2607  m_ParsedCmd = m_Parser.ParseCommand(cmd_line);
2608  good = true;
2609  }
2610  catch (CNSProtoParserException& ) {
2612  }
2613  } else {
2615  SRV_LOG(Error, "Unrecognized command: " << cmd_line);
2616  }
2617  } else {
2618  SRV_LOG(Error, "Error parsing command: " << cmd_line);
2620  }
2621  m_PosponedCmd.clear();
2622  if (!good) {
2625  }
2626  }
2627 
2628  const SCommandExtra& cmd_extra = m_ParsedCmd.command->extra;
2629  m_CmdProcessor = cmd_extra.processor;
2630  m_Flags = cmd_extra.cmd_flags;
2631  CreateNewDiagCtx();
2632  try {
2634  }
2635  catch (CStringException& ex) {
2636  ReleaseDiagCtx();
2637  SRV_LOG(Warning, "Error while parsing command '" << cmd_line
2638  << "': " << ex);
2641  }
2643 }
2644 
2645 void
2647 {
2650  m_SrvsIndex = 0;
2651  Uint4 main_srv_ip = 0;
2652  if (!m_NCBlobKey.IsICacheKey()) {
2654  }
2655  m_ThisServerIsMain = false;
2656  if (main_srv_ip != 0) {
2657  // Note: this check for "main" server for blob assumes that for each
2658  // blob slot only one NetCache instance processing it works on each
2659  // server. It will give false positive results if there are several
2660  // NC instances on the same server processing the same slot. But there's
2661  // no much sense in such setup, so it's pretty safe assumption.
2662  if (Uint4(CNCDistributionConf::GetSelfID() >> 32) == main_srv_ip
2663  // if NeedStopWrite, I cannot be sure that blob on this server is most recent
2664  // and need to ask other servers
2666  // make sure this slot is served here
2668  ) {
2669  //m_ThisServerIsMain = true;
2672  }
2673  if (!m_ThisServerIsMain) {
2675  for (size_t i = 0; i < m_CheckSrvs.size(); ++i) {
2676  if (Uint4(m_CheckSrvs[i] >> 32) == main_srv_ip) {
2677  Uint8 srv_id = m_CheckSrvs[i];
2678  m_CheckSrvs.erase(m_CheckSrvs.begin() + i);
2679  m_CheckSrvs.insert(m_CheckSrvs.begin(), srv_id);
2680  break;
2681  }
2682  }
2683  }
2684  }
2685 }
2686 
2687 void
2688 CNCMessageHandler::x_JournalBlobPutResult(int status, const string& blob_key, Uint2 blob_slot)
2689 {
2690  if (CNCDistributionConf::CountServersForSlot(blob_slot) != 0) {
2691  if (status == eStatus_PrematureClose && status == eStatus_CmdTimeout) {
2692  CNCBlobAccessor::PutFailed(blob_key);
2693  } else {
2695  }
2696  }
2697 }
2698 
2699 void
2701 {
2702  GetDiagCtx()->SetRequestStatus(sts);
2703  x_ReportError( GetMessageByStatus(sts), eol);
2704 }
2705 
2706 void
2707 CNCMessageHandler::x_ReportError( const string& sts, bool eol /*= true*/)
2708 {
2709 // in HTTP mode we leave it to x_CleanCmdResources always
2710  if (!x_IsHttpMode()) {
2711  if (!x_IsFlagSet(fNoReplyOnFinish)) {
2713  WriteText(sts);
2714  if (eol) {
2715  WriteText("\n");
2716  }
2717  }
2718  }
2719 }
2720 
2723 {
2724  if (!x_IsHttpMode()) {
2725  if (!x_IsFlagSet(fNoReplyOnFinish)) {
2727  WriteText(sts);
2728  }
2729  }
2730  return *this;
2731 }
2732 
2733 void
2735 {
2736  CSrvTime cmd_now = CSrvTime::Current();
2737  CSrvTime cmd_len = cmd_now;
2738  cmd_len -= m_CmdPrevTime;
2739  CSrvTime cmd_time = cmd_now;
2740  cmd_time -= m_CmdStartTime;
2741  m_CmdPrevTime = cmd_now;
2742  m_CmdLog.push_back( evt + ": " + NStr::NumericToString(cmd_len.AsUSec()) + "us ("
2743  + NStr::NumericToString(cmd_time.AsUSec()) + "us)");
2744 }
2745 
2747 {
2748  for( const string& l : m_CmdLog) {
2749  SRV_LOG(Warning, l);
2750  }
2751 }
2752 
2753 CNCMessageHandler::State
2755 {
2757  if (!m_BlobAccess->IsMetaInfoReady())
2758  return NULL;
2759  if (NeedEarlyClose())
2761 
2765  }
2766 
2767 // send COPY_UPD notification early, before blob data is received
2768 // another option is to send it in x_FinishReadingBlob,
2769 // after m_BlobAccess->Finalize(), before CNCPeerControl::MirrorWrite
2770 // sending the notification here we assume the risk that the blob will not be received correctly
2771 // and the notification will create confusion only (it will be corrected by periodic sync).
2772  if (
2773  (m_Flags & eProxyBlobWrite) == eProxyBlobWrite // request from clients only
2776  // if the blob exists here, it can exist on mirrors as well
2777  && (m_BlobAccess->IsBlobExists()
2778  // if blob does not exist here, but it is created on another server, I should notify them
2780 #endif
2781  ) {
2783  }
2784 
2785  if (!x_IsFlagSet(fUsesPeerSearch)) {
2786  x_LogCmdEvent("CmdProcessor");
2787  return m_CmdProcessor;
2788  }
2789 
2790  // All commands that have fUsesPeerSearch will operate on m_LatestExist and
2791  // m_LatestBlobSum, so we need to fill it here even if in next "if" we'll go
2792  // almost directly to m_CmdProcessor.
2793  bool is_exist = m_BlobAccess->IsBlobExists();
2794  bool is_valid = m_BlobAccess->IsValid();
2795  m_LatestExist = is_exist
2800  } else {
2801  m_LatestSrvId = is_valid ?
2804  }
2805  if (m_LatestSrvId != 0) {
2806  if (m_LatestExist) {
2813  }
2815  || m_ForceLocal
2816  || (m_Quorum == 1 && (m_LatestExist || !m_SearchOnRead))
2818  {
2820  }
2821  }
2822 
2824  if (m_ThisServerIsMain
2827  {
2828  if (!is_valid) {
2829  m_CheckSrvs.clear();
2830  m_SrvsIndex = 0;
2831  }
2833  }
2834  if (!is_valid) {
2835  Uint8 srv_id = m_BlobAccess->GetValidServer();
2836  if (srv_id) {
2837  for (size_t i = 0; i < m_CheckSrvs.size(); ++i) {
2838  if (m_CheckSrvs[i] == srv_id) {
2839  m_CheckSrvs.erase(m_CheckSrvs.begin() + i);
2840  m_CheckSrvs.insert(m_CheckSrvs.begin(), srv_id);
2841  break;
2842  }
2843  }
2844  }
2845  }
2846 
2847  if (m_LatestExist && m_Quorum != 0) {
2848  --m_Quorum;
2849  }
2850  x_LogCmdEvent("ReadMetaNextPeer");
2852 }
2853 
2854 CNCMessageHandler::State
2856 {
2860  if (!x_IsHttpMode()) {
2861  if (m_AgeMax != 0 && m_BlobAccess->IsBlobExists()) {
2862  WriteText(", AGE=").WriteNumber(m_AgeCur);
2864  }
2865  WriteText("\n");
2866  }
2868 }
2869 
2870 void
2872 {
2874 
2875  CSrvTime cur_srv_time = CSrvTime::Current();
2876  Uint8 cur_time = cur_srv_time.AsUSec();
2877  int now = int(cur_srv_time.Sec());
2878  int new_expire = now + add_time;
2879  if (new_expire < now) {
2880  return;
2881  }
2882  int old_expire = m_BlobAccess->GetCurBlobExpire();
2883  if (!CNCServer::IsDebugMode() &&
2884  new_expire - old_expire < (int)m_AppSetup->ttl_unit) {
2885  return;
2886  }
2887 
2888  if (m_AppSetup->lifespan_ttl > 0) {
2890  int retire = (int)(created + m_AppSetup->lifespan_ttl);
2891  if (retire > created) {
2892  new_expire = min(new_expire,retire);
2893  }
2894  }
2895 
2896  m_BlobAccess->SetCurBlobExpire(new_expire);
2898  SNCSyncEvent* event = new SNCSyncEvent();
2899  event->blob_size = m_BlobAccess->GetCurBlobSize();
2900  event->event_type = eSyncProlong;
2901  event->key = m_NCBlobKey;
2902  event->orig_server = CNCDistributionConf::GetSelfID();
2903  event->orig_time = cur_time;
2906  event->orig_rec_no, cur_time, m_BlobAccess);
2907  }
2908 }
2909 
2910 void
2912 {
2914  CSrvTime cur_srv_time = CSrvTime::Current();
2915  Uint8 cur_time = cur_srv_time.AsUSec();
2916  int new_expire = int(cur_srv_time.Sec()) + m_BlobAccess->GetCurVersionTTL();
2917  int old_expire = m_BlobAccess->GetCurVerExpire();
2918  if (!CNCServer::IsDebugMode() && new_expire - old_expire < m_AppSetup->ttl_unit)
2919  return;
2920 
2921  m_BlobAccess->SetCurVerExpire(new_expire);
2923  SNCSyncEvent* event = new SNCSyncEvent();
2924  event->blob_size = m_BlobAccess->GetCurBlobSize();
2925  event->event_type = eSyncProlong;
2926  event->key = m_NCBlobKey;
2927  event->orig_server = CNCDistributionConf::GetSelfID();
2928  event->orig_time = cur_time;
2931  event->orig_rec_no, cur_time, m_BlobAccess);
2932  }
2933 }
2934 
2935 void
2937 {
2939  int cmd_status = GetDiagCtx()->GetRequestStatus();
2940  bool print_size = false;
2941  Uint8 written_size = 0;
2942  ENCAccessType access_type = eNCCopyCreate;
2943  if (m_write_event) {
2944  delete m_write_event;
2945  m_write_event = NULL;
2946  }
2947  if (m_BlobAccess) {
2948  access_type = m_BlobAccess->GetAccessType();
2950  print_size = true;
2951  if (access_type == eNCRead || access_type == eNCReadData)
2953  else
2955  if (access_type == eNCReadData)
2957  else if (access_type == eNCCreate || access_type == eNCCopyCreate)
2959  }
2960  else if (access_type == eNCCreate) {
2961  written_size = m_BlobAccess->GetNewBlobSize();
2962  }
2963  m_BlobAccess->Release();
2964  m_BlobAccess = NULL;
2965  }
2966  else if (m_ActiveHub) {
2967  switch (m_ParsedCmd.command->extra.proxy_cmd) {
2968  case eProxyRead:
2969  case eProxyReadLast:
2970  print_size = true;
2973  break;
2974  case eProxyWrite:
2975  print_size = true;
2978  break;
2979  default:
2980  break;
2981  }
2983  }
2984 
2986  if (x_IsCmdSucceeded(cmd_status) || x_IsFlagSet(fSyncCmdSuccessful)) {
2988  } else {
2990  }
2991  }
2992  if (!x_IsFlagSet(fNoReplyOnFinish)) {
2993  if (!x_IsHttpMode()) {
2994  if (x_IsCmdSucceeded(cmd_status)) {
2995  WriteText("OK:\n");
2996  } else {
2997  WriteText(GetMessageByStatus(EHTTPStatus(cmd_status))).WriteText("\n");
2998  }
2999  } else {
3001  }
3002  }
3003  Flush();
3004 
3005  if (m_ActiveHub) {
3006  m_ActiveHub->Release();
3007  m_ActiveHub = NULL;
3008  }
3009  m_CheckSrvs.clear();
3010  m_SrvsIndex = 0;
3011  m_ChunkLen = 0;
3012  m_LastPeerError.clear();
3013 
3014  CSrvTime cmd_len = CSrvTime::Current();
3015  cmd_len -= m_CmdStartTime;
3016  Uint8 len_usec = cmd_len.AsUSec();
3017  if (IsLongCommand(len_usec)) {
3018  x_LogCmdLog();
3019  }
3020 
3021  if (print_size && (x_IsCmdSucceeded(cmd_status) || m_BlobSize != 0))
3022  CSrvDiagMsg().PrintExtra().PrintParam("blob_size", m_BlobSize);
3024 
3025 // CSrvTime cmd_len = CSrvTime::Current();
3026 // cmd_len -= m_CmdStartTime;
3027 // Uint8 len_usec = cmd_len.AsUSec();
3028  CNCStat::CmdFinished(m_ParsedCmd.command->cmd, len_usec, cmd_status);
3029  if (m_Flags & fComesFromClient) {
3030  if (access_type == eNCCreate) {
3031  if (print_size && x_IsCmdSucceeded(cmd_status))
3033  else
3034  CNCStat::ClientBlobRollback(written_size);
3035  }
3036  else if (access_type == eNCReadData) {
3037  if (print_size)
3039  }
3040  }
3041  ++m_CntCmds;
3042 
3045 
3046  m_SendBuff.reset();
3047  ReleaseDiagCtx();
3048 
3049  m_PosponedCmd.clear();
3050 }
3051 
3052 CNCMessageHandler::State
3054 {
3056  int status = GetDiagCtx()->GetRequestStatus();
3059  }
3062 
3065  SetRunnable();
3066  return NULL;
3067 }
3068 
3069 CNCMessageHandler::State
3071 {
3073  // Flushing the initial response line that client should receive before it
3074  // will start writing blob data.
3075  Flush();
3076  m_BlobSize = 0;
3077  if (NeedEarlyClose())
3079  else
3081 }
3082 
3083 CNCMessageHandler::State
3085 {
3087  x_LogCmdEvent("FinishReadingBlob");
3088  bool fail = false, keep_conn = !x_IsFlagSet(fNoReplyOnFinish);
3089  string errmsg;
3091  fail = true;
3093  SRV_LOG(Error, "Wrong data for blob size " << m_Size
3094  << " (received " << m_BlobSize << " bytes)");
3095  }
3097  fail = true;
3099  SRV_LOG(Error, "Blob size exceeds the allowed maximum of "
3101  << " (received " << m_BlobSize << " bytes)");
3102  }
3103  if (fail) {
3104  if (keep_conn) {
3106  } else {
3108  }
3109  }
3110 
3111  if (!x_IsCmdSucceeded(GetDiagCtx()->GetRequestStatus()))
3113  if (NeedEarlyClose())
3115 
3116  // Fill all new event data but not add it to CNCSyncLog until we execute
3117  // m_BlobAccess->Finalize().
3118  SNCSyncEvent* write_event = new SNCSyncEvent();
3119  write_event->blob_size = m_BlobSize;
3120  write_event->event_type = eSyncWrite;
3121  write_event->key = m_NCBlobKey;
3122  if (x_IsFlagSet(fCopyLogEvent)) {
3123  write_event->orig_time = m_BlobAccess->GetNewBlobCreateTime();
3124  write_event->orig_server = m_BlobAccess->GetNewCreateServer();
3125  write_event->orig_rec_no = m_OrigRecNo;
3126  }
3127  else {
3128  CSrvTime cur_srv_time = CSrvTime::Current();
3129  Uint8 cur_time = cur_srv_time.AsUSec();
3130  int cur_secs = int(cur_srv_time.Sec());
3131  m_BlobAccess->SetBlobCreateTime(cur_time);
3135  } else {
3136  if (m_BlobAccess->GetNewBlobExpire() == 0) {
3138  }
3140  }
3144  write_event->orig_time = cur_time;
3145  }
3146 
3147  x_LogCmdEvent("Finalize");
3149  if (m_BlobAccess->HasError()) {
3150  delete write_event;
3154 
3155  x_ReportError("ERR:Error while reading blob");
3157  }
3158 
3159  m_MirrorsDone.clear();
3160  if (!x_IsFlagSet(fCopyLogEvent)) {
3162 #if 0
3163  if (m_BlobAccess->IsCurBlobExpired()) {
3164  delete write_event;
3167  }
3168 #endif
3169  // If fCopyLogEvent is not set then this blob comes from client and
3170  // thus we need to check quorum value before answering to client.
3171  // If fCopyLogEvent is set then this write comes from other server
3172  // and we don't care about quorum in this case.
3173  if (m_Quorum != 1) {
3174  if (m_Quorum != 0)
3175  --m_Quorum;
3177  m_write_event = write_event;
3178 // probably, this was made intentionally, but now it looks wrong
3179 // let us respect quorum setting always
3180 // if (!m_ThisServerIsMain || !m_AppSetup->fast_on_main)
3182  }
3186  } else {
3188  SRV_LOG(Warning, "Received blob is too big and will not be mirrored:"
3189  << " blob key:" << m_NCBlobKey.RawKey()
3190  << " blob size: " << m_BlobAccess->GetNewBlobSize()
3191  << " max allowed: " << CNCDistributionConf::GetMaxBlobSizeSync());
3192  }
3195  } else {
3197  if (srvId != 0) {
3198  m_CheckSrvs.push_back(srvId);
3199  m_SrvsIndex = 0;
3201  }
3202  }
3203  delete write_event;
3204  }
3205  }
3206  else if (m_OrigRecNo != 0) {
3207  CNCSyncLog::AddEvent(m_BlobSlot, write_event);
3208  }
3209  else {
3210  // m_OrigRecNo can be 0 if blob comes from another server as a result
3211  // of synchronization by blob lists. In this case there's no event to
3212  // link to and thus we don't need to add event to our sync log.
3213  delete write_event;
3214  }
3215 
3217 }
3218 
3219 CNCMessageHandler::State
3221 {
3223  SRV_LOG(Warning, "Error executing command on peer "
3224  << m_ActiveHub->GetFullPeerName() << ", peer says: "
3225  << m_ActiveHub->GetErrMsg());
3228 }
3229 
3230 CNCMessageHandler::State
3232 {
3234  if (x_IsHttpMode()) {
3236  }
3237 
3238  Uint4 sig = 0;
3239  bool has_sig = ReadNumber(&sig);
3240  if (NeedEarlyClose())
3242  if (!has_sig)
3243  return NULL;
3244 
3245  x_LogCmdEvent("ReadBlobSignature");
3246  if (sig == 0x04030201) {
3249  }
3250  if (sig == 0x01020304) {
3253  }
3254 
3256  SRV_LOG(Error, "Cannot determine the byte order. Got: "
3257  << NStr::UIntToString(sig, 0, 16));
3259 }
3260 
3261 CNCMessageHandler::State
3263 {
3265  if (m_ActiveHub) {
3266  if (ProxyHadError()) {
3267  // If we were proxying blob data from client to another server and some
3268  // error occurred protocol doesn't allow us to anything else but
3269  // close both connections - to client and to other NC server.
3270  CSrvSocketTask* active_sock = m_ActiveHub->GetHandler()->GetSocket();
3271  if (active_sock && active_sock->HasError())
3273  else
3275  }
3277  if (status == eNCHubError)
3279  else if (status != eNCHubCmdInProgress) {
3280  SRV_FATAL("Unexpected client status: " << status);
3281  }
3282 
3283  if (m_ChunkLen != 0) {
3286  }
3287  }
3288 
3289  bool has_chunklen = true;
3291  // Workaround for old STRS
3292  m_ChunkLen = 0xFFFFFFFF;
3293  }
3294  else if (x_IsHttpMode()) {
3295  m_ChunkLen = m_Size;
3296  has_chunklen = m_ChunkLen != 0;
3297  }
3298  else {
3299  has_chunklen = ReadNumber(&m_ChunkLen);
3300  if (!has_chunklen && !CanHaveMoreRead() && x_IsFlagSet(fCursedPUT2Cmd)) {
3302  }
3303  }
3304 
3305  if (NeedEarlyClose())
3307  if (!has_chunklen)
3308  return NULL;
3309 
3311  m_ChunkLen = CByteSwap::GetInt4((const unsigned char*)&m_ChunkLen);
3312  if (m_ChunkLen == 0xFFFFFFFF) {
3313  if (m_ActiveHub) {
3314  // Data transfer is finished, CNCActiveHandler will wait for response
3315  // from other NC server and wake us up.
3318  }
3320  }
3321 
3322  if (!m_BlobAccess && !m_ActiveHub) {
3323  // We can be here only when expecting fake start of blob writing from old
3324  // NC server, but for some reason we got non-EOF chunk length.
3326  SRV_LOG(Critical, "Received non-EOF chunk len from peer "
3327  << m_SrvId << ": " << m_ChunkLen);
3329  }
3330 
3333  RunAfter(1);
3334  return NULL;
3335  }
3336 // this is potentially dangerous, because
3337 // if we close connection here, in the middle of transmission,
3338 // client might have no chance (most likely) to receive our error message
3339 // and might want to retry thinking it was bad luck
3342  SRV_LOG(Error, "Too much data for blob size " << m_Size
3343  << " (received at least "
3344  << (m_BlobSize + m_ChunkLen) << " bytes)");
3346  }
3349  if (x_IsHttpMode()) {
3351  }
3352  SRV_LOG(Error, "Blob size exceeds the allowed maximum of "
3354  << " (received " << m_BlobSize
3355  << ", next chunk " << m_ChunkLen << " bytes)");
3356  // I am not going to read it anyway
3358  Flush();
3359  RunAfter(1);
3360  return NULL;
3361 // return &CNCMessageHandler::x_CloseCmdAndConn;
3362  }
3363  }
3364 
3365  if (m_ActiveHub) {
3366  CSrvSocketTask* active_sock = m_ActiveHub->GetHandler()->GetSocket();
3367  active_sock->WriteData(&m_ChunkLen, sizeof(m_ChunkLen));
3369  StartProxyTo(active_sock, m_ChunkLen);
3370  if (IsProxyInProgress())
3371  return NULL;
3372  else
3374  }
3375 
3377 }
3378 
3379 CNCMessageHandler::State
3381 {
3383 // there are too many of them
3384 // x_LogCmdEvent("ReadBlobChunk");
3385  while (m_ChunkLen != 0) {
3386  Uint4 read_len = Uint4(m_BlobAccess->GetWriteMemSize());
3387  if (m_BlobAccess->HasError()) {
3390  x_ReportError("ERR:Server error");
3391  Flush();
3392  }
3394  SetRunnable();
3395  return NULL;
3396  }
3397  if (read_len == 0)
3398  return NULL;
3399  if (read_len > m_ChunkLen)
3400  read_len = m_ChunkLen;
3401 
3402  Uint4 n_read = Uint4(Read(m_BlobAccess->GetWriteMemPtr(), read_len));
3403  if (n_read != 0) {
3404  if (m_Flags & fComesFromClient)
3405  CNCStat::ClientDataWrite(n_read);
3406  else
3407  CNCStat::PeerDataWrite(n_read);
3408  }
3409  if (NeedEarlyClose())
3411  if (n_read == 0)
3412  return NULL;
3413 
3414  m_BlobAccess->MoveWritePos(n_read);
3415  m_ChunkLen -= n_read;
3416  m_BlobSize += n_read;
3417  }
3419 }
3420 
3421 CNCMessageHandler::State
3423 {
3425  x_LogCmdEvent("WriteBlobData");
3426  while (m_Size != 0) {
3429 
3430  Uint4 want_read = m_BlobAccess->GetReadMemSize();
3431  if (m_BlobAccess->HasError()) {
3434  }
3435  if (m_Size != Uint8(-1) && m_Size < want_read)
3436  want_read = Uint4(m_Size);
3437 
3438  Uint4 n_written = Uint4(Write(m_BlobAccess->GetReadMemPtr(), want_read));
3439 // x_LogCmdEvent("Write");
3440  if (n_written != 0) {
3441  if (m_Flags & fComesFromClient)
3442  CNCStat::ClientDataRead(n_written);
3443  else
3444  CNCStat::PeerDataRead(n_written);
3445  m_BlobAccess->MoveReadPos(n_written);
3446  if (m_Size != Uint8(-1))
3447  m_Size -= n_written;
3448  }
3449  if (NeedEarlyClose())
3451  if (n_written == 0)
3452  return NULL;
3453  }
3455 }
3456 
3457 CNCMessageHandler::State
3459 {
3461  while (m_SendPos != m_SendBuff->size()) {
3462  size_t n_written = Write(m_SendBuff->data() + m_SendPos,
3463  m_SendBuff->size() - m_SendPos);
3464 
3465  if (NeedEarlyClose())
3467  if (n_written == 0)
3468  return NULL;
3469 
3470  m_SendPos += n_written;
3471  }
3472  if (strcmp(m_ParsedCmd.command->cmd, "SYNC_START") == 0) {
3474  }
3476  if (proxy_cmd == eProxyGetBList2) {
3477  Flush();
3479  }
3481 }
3482 
3483 CNCMessageHandler::State
3485 {
3487  WriteText("PURGE:\n");
3490 }
3491 
3492 CNCMessageHandler::State
3494 {
3496  if (NeedEarlyClose())
3498  x_LogCmdEvent("ProxyToNextPeer");
3499  if (m_SrvsIndex < m_CheckSrvs.size()) {
3500  Uint8 srv_id = m_CheckSrvs[m_SrvsIndex++];
3501  if (m_ActiveHub) {
3502  SRV_FATAL("Previous client not released");
3503  }
3504  m_ActiveHub = CNCActiveClientHub::Create(srv_id, this);
3506  }
3507 
3508  // Either there's no servers to execute this command on or all servers were
3509  // tried and some error was the result from all of them.
3510  SRV_LOG(Warning, "Got error on all peer servers, LastPeerError is " << m_LastPeerError);
3511  if (m_LastPeerError.empty())
3512  m_LastPeerError = "ERR:Cannot execute command on peer servers";
3516 }
3517 
3518 CNCMessageHandler::State
3520 {
3523  if (status == eNCHubWaitForConn)
3524  return NULL;
3525  x_LogCmdEvent("SendCmdAsProxy");
3527  CNCActiveHandler* pHandler = m_ActiveHub->GetHandler();
3528  if (status == eNCHubError ||
3529  status == eNCHubSuccess ||
3530  (proxy_cmd == eProxyGetBList && !pHandler->GetPeer()->AcceptsBList()) ||
3531  (proxy_cmd == eProxyGetBList2 && !pHandler->GetPeer()->AcceptsBList2()) ||
3532  !pHandler->GetPeer()->AcceptsBlobKey(m_NCBlobKey)
3533  ) {
3535  m_ActiveHub->Release();
3536  m_ActiveHub = NULL;
3538  }
3539  if (status != eNCHubConnReady) {
3540  SRV_FATAL("Unexpected client status: " << status);
3541  }
3542  if (NeedEarlyClose())
3544 
3545 
3546  switch (proxy_cmd) {
3547  case eProxyRead:
3551  m_AgeMax);
3552  break;
3553  case eProxyWrite:
3556  // The only place that needs to go further to a different state.
3558  case eProxyHasBlob:
3560  m_Quorum);
3561  break;
3562  case eProxyGetSize:
3566  break;
3567  case eProxyReadLast:
3571  break;
3572  case eProxySetValid:
3574  m_BlobVersion);
3575  break;
3576  case eProxyRemove:
3579  break;
3580  case eProxyGetMeta:
3581  pHandler->ProxyGetMeta(GetDiagCtx(), m_NCBlobKey,
3583  break;
3584  case eProxyProlong:
3588  break;
3589  case eProxyGetBList:
3590  pHandler->ProxyBList(GetDiagCtx(), m_NCBlobKey, m_ForceLocal, nullptr);
3591  break;
3592  case eProxyGetBList2:
3594  break;
3595  default:
3596  SRV_FATAL("Unsupported command: " << m_ParsedCmd.command->extra.proxy_cmd);
3597  }
3598 
3600 }
3601 
3602 CNCMessageHandler::State
3604 {
3606  if (NeedEarlyClose())
3609  if (status == eNCHubCmdInProgress)
3610  return NULL;
3611 
3612  x_LogCmdEvent("WaitForPeerAnswer");
3613  if (status == eNCHubError) {
3616 
3618  SRV_LOG(Warning, "Error executing command on peer "
3619  << m_ActiveHub->GetFullPeerName() << ", peer says: " << m_LastPeerError);
3620  m_ActiveHub->Release();
3621  m_ActiveHub = NULL;
3623  }
3624  if (status != eNCHubSuccess) {
3625  SRV_FATAL("Unexpected client status: " << status);
3626  }
3628  if (proxy_cmd == eProxyGetBList2) {
3630  }
3631 
3632  const string& err_msg = m_ActiveHub->GetErrMsg();
3633  EHTTPStatus rst = GetStatusByMessage(err_msg, eStatus_OK);
3634  if (rst != eStatus_OK) {
3635  GetDiagCtx()->SetRequestStatus(rst);
3636  }
3637  if (!err_msg.empty()) {
3638  x_ReportError(err_msg);
3639  }
3641 }
3642 
3643 CNCMessageHandler::State
3645 {
3647  if (NeedEarlyClose())
3649  if (m_SrvsIndex >= m_CheckSrvs.size())
3651 
3652  Uint8 srv_id = m_CheckSrvs[m_SrvsIndex++];
3653  if (m_ActiveHub) {
3654  SRV_FATAL("Previous client not released");
3655  }
3656  m_ActiveHub = CNCActiveClientHub::Create(srv_id, this);
3658 }
3659 
3660 CNCMessageHandler::State
3662 {
3665  if (status == eNCHubWaitForConn)
3666  return NULL;
3667  if (status == eNCHubError || status == eNCHubSuccess ||
3670  m_ActiveHub->Release();
3671  m_ActiveHub = NULL;
3673  }
3674  if (status != eNCHubConnReady) {
3675  SRV_FATAL("Unexpected client status: " << status);
3676  }
3677  if (NeedEarlyClose())
3679 
3682 }
3683 
3684 CNCMessageHandler::State
3686 {
3688  if (NeedEarlyClose())
3691  if (status == eNCHubCmdInProgress)
3692  return NULL;
3693  if (status == eNCHubError)
3694  goto results_processed;
3695  if (status != eNCHubSuccess) {
3696  SRV_FATAL("Unexpected client status: " << status);
3697  }
3698 
3701  const SNCBlobSummary* cur_blob_sum;
3702  cur_blob_sum = &handler->GetBlobSummary();
3703  bool cur_exist;
3704  cur_exist = handler->IsBlobExists();
3705  if (!cur_exist && !x_IsFlagSet(fPeerFindExistsOnly))
3706  goto results_processed;
3707 
3708  if (cur_exist && x_IsFlagSet(fPeerFindExistsOnly)) {
3709  m_LatestExist = true;
3711  goto meta_search_finished;
3712  }
3713  if (cur_exist && (!m_LatestExist || m_LatestBlobSum->isOlder(*cur_blob_sum)))
3714  {
3715  m_LatestExist = true;
3717  *m_LatestBlobSum = *cur_blob_sum;
3718  }
3719  if (cur_blob_sum->size > CNCDistributionConf::GetMaxBlobSizeSync()) {
3720  m_Quorum = 1;
3721  }
3722  if (m_Quorum == 1)
3723  goto meta_search_finished;
3724  if (m_Quorum != 0)
3725  --m_Quorum;
3726 
3727 results_processed:
3728 
3730  m_ActiveHub->Release();
3731  m_ActiveHub = NULL;
3733 
3734 meta_search_finished:
3735 
3737  m_ActiveHub->Release();
3738  m_ActiveHub = NULL;
3739  m_CheckSrvs.clear();
3740  m_SrvsIndex = 0;
3742 }
3743 
3744 CNCMessageHandler::State
3746 {
3748  x_LogCmdEvent("ExecuteOnLatestSrvId");
3749  if (m_LatestSrvId == 0) {
3751  }
3752  if (m_LatestExist) {
3753  // if max age specified, check age
3754  if (m_AgeMax != 0 && m_BlobAccess->IsBlobExists()) {
3755  CSrvTime cur_srv_time = CSrvTime::Current();
3756  unsigned int vttl = m_BlobAccess->GetCurVersionTTL();
3757  Uint8 creation = vttl != 0 ?
3758 // see x_ProlongVersionLife
3759  (m_LatestBlobSum->ver_expire - vttl) :
3761  m_AgeCur = Uint8(cur_srv_time.Sec()) - creation;
3762  if (m_AgeCur > m_AgeMax) {
3764  }
3765  }
3766  }
3769  m_LatestExist = false;
3770  return m_CmdProcessor;
3771  }
3772 #if 0
3774 #else
3777 #endif
3778  }
3780  return m_CmdProcessor;
3781  }
3784  return m_CmdProcessor;
3785  } else {
3787  }
3788  }
3789 
3790  CSrvDiagMsg().PrintExtra().PrintParam("proxy", "1");
3791  // Changing parameters that will go to other server: that server have to
3792  // execute command locally, without quorum (quorum equal to 1), and without
3793  // searching on other servers.
3794  m_Quorum = 1;
3795  m_SearchOnRead = false;
3796  m_ForceLocal = true;
3797  m_CheckSrvs.push_back(m_LatestSrvId);
3798  x_LogCmdEvent("ProxyToNextPeer");
3800 }
3801 
3802 CNCMessageHandler::State
3804 {
3806  x_LogCmdEvent("PutToNextPeer");
3807  if (m_SrvsIndex >= m_CheckSrvs.size() || NeedEarlyClose())
3809 
3810  Uint8 srv_id = m_CheckSrvs[m_SrvsIndex++];
3811  if (m_ActiveHub) {
3812  SRV_FATAL("Previous client not released");
3813  }
3814  if (CNCDistributionConf::GetSelfTrustLevel() < CNCPeerControl::Peer(srv_id)->GetTrustLevel()) {
3816  }
3817  m_ActiveHub = CNCActiveClientHub::Create(srv_id, this);
3819 }
3820 
3821 CNCMessageHandler::State
3823 {
3826  if (status == eNCHubWaitForConn)
3827  return NULL;
3828  if (status == eNCHubError || status == eNCHubSuccess ||
3830  m_MirrorsDone.push_back(m_CheckSrvs[m_SrvsIndex-1]);
3832  m_ActiveHub->Release();
3833  m_ActiveHub = NULL;
3835  }
3836  if (status != eNCHubConnReady) {
3837  SRV_FATAL("Unexpected client status: " << status);
3838  }
3839  if (NeedEarlyClose())
3841 
3844 }
3845 
3846 CNCMessageHandler::State
3848 {
3850  x_LogCmdEvent("ReadPutResults");
3851  if (NeedEarlyClose())
3854  if (status == eNCHubCmdInProgress)
3855  return NULL;
3856  if (status == eNCHubError)
3857  goto results_processed;
3858  if (status != eNCHubSuccess) {
3859  SRV_FATAL("Unexpected client status: " << status);
3860  }
3861 
3862  m_MirrorsDone.push_back(m_CheckSrvs[m_SrvsIndex-1]);
3863  if (m_Quorum == 1) {
3865  if (m_write_event) {
3867  m_write_event = NULL;
3868  }
3871  m_MirrorsDone);
3872  }
3874  }
3875  if (m_Quorum != 0)
3876  --m_Quorum;
3877 
3878 results_processed:
3879  m_ActiveHub->Release();
3880  m_ActiveHub = NULL;
3882 }
3883 
3884 CNCMessageHandler::State
3886 {
3888  if (m_SrvsIndex >= m_CheckSrvs.size() || NeedEarlyClose())
3890 
3891  Uint8 srv_id = m_CheckSrvs[m_SrvsIndex++];
3892  if (m_ActiveHub) {
3893  SRV_FATAL("Previous client not released");
3894  }
3895  if (!m_NCBlobKey.RawKey().empty() && !CNCPeerControl::Peer(srv_id)->AcceptsPurge2()) {
3897  }
3898  m_ActiveHub = CNCActiveClientHub::Create(srv_id, this);
3900 }
3901 
3902 CNCMessageHandler::State
3904 {
3907  if (status == eNCHubWaitForConn)
3908  return NULL;
3909  if (status == eNCHubError || status == eNCHubSuccess) {
3911  m_ActiveHub->Release();
3912  m_ActiveHub = NULL;
3914  }
3915  if (status != eNCHubConnReady) {
3916  SRV_FATAL("Unexpected client status: " << status);
3917  }
3918  if (NeedEarlyClose())
3920 
3923 }
3924 
3925 CNCMessageHandler::State
3927 {
3929  if (NeedEarlyClose())
3932  if (status == eNCHubCmdInProgress)
3933  return NULL;
3934  if (status == eNCHubError)
3935  goto results_processed;
3936  if (status != eNCHubSuccess) {
3937  SRV_FATAL("Unexpected client status: " << status);
3938  }
3939 
3940 results_processed:
3941  m_ActiveHub->Release();
3942  m_ActiveHub = NULL;
3944 }
3945 
3946 
3947 inline unsigned int
3949 {
3952 }
3953 
3954 CNCMessageHandler::State
3956 {
3958  const char* health_coeff = "1";
3960  health_coeff = "0 (does not accept writes)";
3961  } else if (!CNCServer::IsCachingComplete()) {
3962  health_coeff = "0.1 (caching not finished)";
3963  } else if (CNCBlobStorage::IsDraining()) {
3964  health_coeff = "0.2 (draining)";
3965  } else if (!CNCServer::IsInitiallySynced()) {
3966  health_coeff = "0.5 (initial sync not finished)";
3967  } else if (CNCPeerControl::HasPeerInThrottle()) {
3968  health_coeff = "0.8 (some peers unaccessible)";
3969  }
3970  x_ReportOK("OK:HEALTH_COEFF=").WriteText(health_coeff).WriteText("\n");
3971  WriteText("OK:UP_TIME=").WriteNumber(CNCServer::GetUpTime()).WriteText("\n");
3972  WriteText("OK:CACHING_COMPLETE=").WriteText(CNCServer::IsCachingComplete()? "yes": "no").WriteText("\n");
3973  WriteText("OK:INITIALLY_SYNCED=").WriteText(CNCServer::IsInitiallySynced()? "yes": "no").WriteText("\n");
3974  Int8 free_space = CNCBlobStorage::GetDiskFree();
3975  Int8 allowed_size = CNCBlobStorage::GetAllowedDBSize(free_space);
3976  WriteText("OK:DISK_FREE=").WriteNumber(free_space).WriteText("\n");
3977  WriteText("OK:DISK_LIMIT=").WriteNumber(allowed_size).WriteText("\n");
3978  WriteText("OK:DISK_USED=").WriteNumber(CNCBlobStorage::GetDBSize()).WriteText("\n");
3979  WriteText("OK:DISK_LIMIT_ALERT=").WriteText(CNCBlobStorage::IsDBSizeAlert()? "yes": "no").WriteText("\n");
3980  WriteText("OK:N_DB_FILES=").WriteNumber(CNCBlobStorage::GetNDBFiles()).WriteText("\n");
3981  WriteText("OK:COPY_QUEUE_SIZE=").WriteNumber(CNCPeerControl::GetMirrorQueueSize()).WriteText("\n");
3982 
3983  const TNCPeerList& peers = CNCDistributionConf::GetPeers();
3984  ITERATE(TNCPeerList, it_peer, peers) {
3985 // WriteText("OK:QUEUE_SIZE_").WriteNumber(it_peer->first).WriteText("=").WriteNumber(CNCPeerControl::GetMirrorQueueSize(it_peer->first)).WriteText("\n");
3986  WriteText("OK:QUEUE_SIZE_").WriteText(it_peer->second).WriteText("=").WriteNumber(CNCPeerControl::GetMirrorQueueSize(it_peer->first)).WriteText("\n");
3987  }
3988  WriteText("OK:SYNC_LOG_SIZE=").WriteNumber(CNCSyncLog::GetLogSize()).WriteText("\n");
3989 
3990  WriteText("OK:END\n");
3992 }
3993 
3994 CNCMessageHandler::State
3996 {
3999  if (param.find("drain") != param.end() && param["drain"] != "0") {
4001  } else if (param.find("reset") != param.end() && param["reset"] != "0") {
4004  } else {
4006  }
4008 }
4009 
4010 CNCMessageHandler::State
4012 {
4014  x_ReportOK("OK:server_version=" NETCACHED_VERSION
4015  "&storage_version=" NETCACHED_STORAGE_VERSION
4016  "&protocol_version=" NETCACHED_PROTOCOL_VERSION
4017  "&build_date=" + NStr::URLEncode(NETCACHED_BUILD_DATE) +
4019  .WriteText("\n");
4021 }
4022 
4023 CNCMessageHandler::State
4025 {
4027  TNSProtoParams& params = m_ParsedCmd.params;
4028  if (params.find("section") != params.end()) {
4029  string section(params["section"]);
4030  WriteText("{\"").WriteText(section).WriteText("\": {\n\"section\": \"");
4031  WriteText(section).WriteText("\"");
4032  if (section == "task_server") {
4033  CTaskServer::WriteSetup(*this);
4034  } else if (section == "netcache") {
4036  if (params.find("port") != params.end()) {
4037  client["port"] = params["port"];
4038  }
4039  if (params.find("cache") != params.end()) {
4040  client["cache"] = params["cache"];
4041  }
4042  Flush();
4043  m_SendBuff.reset(new TNCBufferType());
4045  m_SendBuff->WriteText("\n}}\nOK:END\n");
4047  m_SendPos = 0;
4049  } else if (section == "storage") {
4051  } else if (section == "mirror") {
4053  } else if (section == "env") {
4054  CNCServer::WriteEnvInfo(*this);
4057  } else if (section == "stat") {
4058  CSrvRef<CNCStat> stat = CNCStat::GetStat("1min", false);
4059  if (stat) {
4060  stat->PrintState(*this);
4061  }
4063 #ifdef _DEBUG
4064  } else if (section == "syncstat") {
4066 #endif
4067 #if __NC_TASKS_MONITOR
4068  } else if (section == "tasks") {
4069  CSrvTask::PrintState(*this);
4070 #endif
4071  } else if (section == "allalerts") {
4072  CNCAlerts::Report(*this, true);
4073  } else if (section == "alerts") {
4074  CNCAlerts::Report(*this, false);
4075  } else if (section == "sync") {
4076  CTempString mask = params.find("port") != params.end() ? params.at("port") : CTempString(kEmptyStr);
4077  Flush();
4078  m_SendBuff.reset(new TNCBufferType());
4080  m_SendBuff->WriteText("\n}}\nOK:END\n");
4082  m_SendPos = 0;
4084  } else if (section == "db") {
4085  CTempString mask = params.find("port") != params.end() ? params.at("port") : CTempString(kEmptyStr);
4086  Flush();
4087  m_SendBuff.reset(new TNCBufferType());
4089  m_SendBuff->WriteText("\n}}\nOK:END\n");
4091  m_SendPos = 0;
4093  } else if (section == "blobs") {
4095  } else if (section == "blist") {
4096  CTempString mask = params.find("port") != params.end() ? params.at("port") : CTempString(kEmptyStr);
4097  Flush();
4098  m_SendBuff.reset(new TNCBufferType());
4100  m_SendBuff->WriteText("\n}}\nOK:END\n");
4102  m_SendPos = 0;
4104  } else {
4105  WriteText(",\n\"error\": \"Unknown section name, valid names: ");
4106 #if __NC_TASKS_MONITOR
4107  WriteText("task_server, netcache, storage, mirror, alerts, allalerts, env, stat, sync, tasks, blobs, db\"");
4108 #else
4109  WriteText("task_server, netcache, storage, mirror, alerts, allalerts, env, stat, sync, blobs, db\"");
4110 #endif
4111  }
4112  WriteText("\n}}");
4113  } else {
4116  string conf = CNcbiOstrstreamToString(str);
4117  WriteText(conf);
4118  }
4119  x_ReportOK("\nOK:END\n");
4121 }
4122 
4123 CNCMessageHandler::State
4125 {
4128  TNSProtoParams& params = m_ParsedCmd.params;
4129  if (params.find("alert") != params.end()) {
4130  string alert(params["alert"]);
4131  if (params.find("user") != params.end()) {
4132  string user(params["user"]);
4133  res = CNCAlerts::Acknowledge(alert, user);
4134  }
4135  }
4136  if (res == CNCAlerts::eNotFound) {
4137  x_ReportError("ERR:Not found");
4138  } else {
4139  x_ReportOK("OK:END\n");
4140  }
4142 }
4143 
4144 CNCMessageHandler::State
4146 {
4149  x_ReportError("ERR:Initial sync not finished");
4152  }
4153  TNSProtoParams& params = m_ParsedCmd.params;
4154  string err_message("ERR:Unknown section name");
4155  bool result = false;
4156  if (params.find("section") != params.end()) {
4157  string section(params["section"]);
4158  if (section != "task_server" &&
4159  section != "mirror" &&
4160  section != "storage") {
4161  err_message += ": " + section;
4162  goto done;
4163  }
4164  CNcbiRegistry* new_reg = NULL;
4165  if (!CTaskServer::ReadConfiguration(new_reg))
4166  {
4167  err_message = "ERR:Failed to load registry";
4168  goto done;
4169  }
4170  if (section == "task_server") {
4171  result = CTaskServer::ReConfig(*new_reg, err_message);
4172  } else if (section == "mirror") {
4173  result = CNCDistributionConf::ReConfig(*new_reg, err_message);
4174  if (result) {
4176  }
4177  } else if (section == "storage") {
4178  result = CNCBlobStorage::ReConfig(*new_reg, err_message);
4179  }
4180  delete new_reg;
4181  }
4182 done:
4183  if (result) {
4184  x_ReportOK("OK:\n");
4185  } else {
4186  x_ReportError(err_message);
4187  }
4189 }
4190 
4191 CNCMessageHandler::State
4193 {
4196  if (!stat) {
4197  x_ReportError("ERR:Unknown statistics type: " + m_StatType);
4199  }
4200  else {
4201  stat->PrintToSocket(this);
4202  x_ReportOK("OK:END\n");
4203  }
4205 }
4206 
4207 CNCMessageHandler::State
4209 {
4213  }
4214  if (!m_BlobAccess->IsBlobExists()) {
4216  if (x_IsHttpMode()) {
4217  // if blob does not exist, verify that it is POST, not PUT
4218  if (!x_IsFlagSet(fCanGenerateKey)) {
4219  sts = eStatus_NotFound;
4220  }