NCBI C++ ToolKit
time_series_stat.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: time_series_stat.cpp 101453 2023-12-14 20:19:17Z satskyse $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Authors: Sergey Satskiy
27  *
28  * File Description:
29  * PSG server request time series statistics
30  *
31  */
32 #include <ncbi_pch.hpp>
33 #include <corelib/ncbistd.hpp>
34 
35 #include "time_series_stat.hpp"
36 
37 
39  m_Accumulated(0), m_AccumulatedCount(0),
40  m_TotalValues(0.0),
41  m_MaxValue(0.0),
42  m_Loop(false),
43  m_TotalMinutesCollected(1),
44  m_CurrentIndex(0)
45 {
46  Reset();
47 }
48 
49 
51 {
52  // Adding happens every 5 seconds and goes to the accumulated values
55 }
56 
57 
59 {
60  // Rotate should:
61  // - calculate avarage
62  // - store it in the current index cell
63  // - rotate the current index
64 
65  size_t current_index = m_CurrentIndex.load();
66  m_Values[current_index] = double(m_Accumulated) / double(m_AccumulatedCount);
67  m_TotalValues += m_Values[current_index];
68  if (m_Values[current_index] > m_MaxValue) {
69  m_MaxValue = m_Values[current_index];
70  }
71 
72  m_Accumulated = 0;
74 
75  size_t new_current_index = m_CurrentIndex.load();
76  if (new_current_index == kSeriesIntervals - 1) {
77  new_current_index = 0;
78  } else {
79  ++new_current_index;
80  }
81 
82  m_Values[new_current_index] = 0;
83 
84  m_CurrentIndex.store(new_current_index);
86  if (new_current_index == 0) {
87  m_Loop = true;
88  }
89 }
90 
91 
93 {
94  for (size_t k = 0; k < kSeriesIntervals; ++k)
95  m_Values[k] = 0.0;
96  m_TotalValues = 0.0;
97  m_MaxValue = 0.0;
98 
99  m_Accumulated = 0;
100  m_AccumulatedCount = 0;
101 
102  m_CurrentIndex.store(0);
103  m_TotalMinutesCollected.store(1);
104  m_Loop = false;
105 }
106 
107 
108 CJsonNode
109 CMomentousCounterSeries::Serialize(const vector<pair<int, int>> & time_series,
110  bool loop, size_t current_index) const
111 {
113 
114  ret.SetByKey("AverageValues",
115  x_SerializeOneSeries(time_series, loop, current_index));
116  return ret;
117 }
118 
119 
120 CJsonNode CMomentousCounterSeries::x_SerializeOneSeries(const vector<pair<int, int>> & time_series,
121  bool loop,
122  size_t current_index) const
123 {
125 
126  if (current_index == 0 && loop == false) {
127  // There is no data collected yet
128  return ret;
129  }
130 
131  CJsonNode output_series(CJsonNode::NewArrayNode());
132 
133  // Index in the array where the data are collected
134  size_t raw_index;
135  if (current_index == 0) {
136  raw_index = kSeriesIntervals - 1;
137  loop = false; // to avoid going the second time over
138  } else {
139  raw_index = current_index - 1;
140  }
141 
142  size_t current_accumulated_mins = 0;
143  double current_accumulated_vals = 0;
144  size_t output_data_index = 0;
145  double total_processed_vals = 0.0;
146 
147  // The current index in the 'time_series', i.e. a pair of
148  // <mins to accumulate>:<last sequential data index>
149  // It is guaranteed they are both > 0.
150  size_t range_index = 0;
151  size_t current_mins_to_accumulate = time_series[range_index].first;
152  size_t current_last_seq_index = time_series[range_index].second;
153 
154  for ( ;; ) {
155  double val = m_Values[raw_index];
156 
157  ++current_accumulated_mins;
158  current_accumulated_vals += val;
159  total_processed_vals += val;
160 
161  if (current_accumulated_mins >= current_mins_to_accumulate) {
162  output_series.AppendDouble(double(current_accumulated_vals) /
163  double(current_accumulated_mins));
164  current_accumulated_mins = 0;
165  current_accumulated_vals = 0.0;
166  }
167 
168  ++output_data_index;
169  if (output_data_index > current_last_seq_index) {
170  ++range_index;
171  current_mins_to_accumulate = time_series[range_index].first;
172  current_last_seq_index = time_series[range_index].second;
173  }
174 
175  if (raw_index == 0)
176  break;
177  --raw_index;
178  }
179 
180  if (loop) {
181  raw_index = kSeriesIntervals - 1;
182  while (raw_index > current_index + 1) {
183  double val = m_Values[raw_index];
184  --raw_index;
185 
186  ++current_accumulated_mins;
187  current_accumulated_vals += val;
188  total_processed_vals += val;
189 
190  if (current_accumulated_mins >= current_mins_to_accumulate) {
191  output_series.AppendDouble(double(current_accumulated_vals) /
192  double(current_accumulated_mins));
193  current_accumulated_mins = 0;
194  current_accumulated_vals = 0;
195  }
196 
197  ++output_data_index;
198  if (output_data_index > current_last_seq_index) {
199  ++range_index;
200  current_mins_to_accumulate = time_series[range_index].first;
201  current_last_seq_index = time_series[range_index].second;
202  }
203  }
204  }
205 
206  if (current_accumulated_mins > 0) {
207  output_series.AppendDouble(double(current_accumulated_vals) /
208  double(current_accumulated_mins));
209  }
210 
211  if (loop) {
212  // The current minute and the last minute in case of a loop are not
213  // sent to avoid unreliable data
214  uint64_t rest_mins = m_TotalMinutesCollected.load() - kSeriesIntervals - 2;
215  double rest_vals = m_TotalValues - total_processed_vals - m_Values[current_index];
216 
217  if (rest_mins > 0) {
218  ret.SetDouble("RestAverageValue", rest_vals / double(rest_mins));
219  } else {
220  ret.SetDouble("RestAverageValue", 0.0);
221  }
222  } else {
223  ret.SetDouble("RestAverageValue", 0.0);
224  }
225 
226  ret.SetDouble("Max", m_MaxValue);
227  if (m_TotalMinutesCollected == 1) {
228  // That's the very beginning; the first minute is still accumulating
229  ret.SetDouble("Avg", 0.0);
230  } else {
231  ret.SetDouble("Avg", m_TotalValues / (m_TotalMinutesCollected.load() - 1));
232  }
233  ret.SetByKey("time_series", output_series);
234  return ret;
235 }
236 
237 
239  m_Loop(false),
240  m_TotalMinutesCollected(1),
241  m_CurrentIndex(0)
242 {
243  Reset();
244 }
245 
246 
248 {
249  size_t current_index = m_CurrentIndex.load();
250  ++m_Requests[current_index];
251  ++m_TotalRequests;
252 }
253 
254 
256 {
257  size_t new_current_index = m_CurrentIndex.load();
258  if (new_current_index == kSeriesIntervals - 1) {
259  new_current_index = 0;
260  } else {
261  ++new_current_index;
262  }
263 
264  m_Requests[new_current_index] = 0;
265 
266  m_CurrentIndex.store(new_current_index);
268  if (new_current_index == 0) {
269  m_Loop = true;
270  }
271 }
272 
273 
275 {
276  memset(m_Requests, 0, sizeof(m_Requests));
277  m_TotalRequests = 0;
278 
279  m_CurrentIndex.store(0);
280  m_TotalMinutesCollected.store(1);
281  m_Loop = false;
282 }
283 
284 
285 CJsonNode CProcessorRequestTimeSeries::Serialize(const vector<pair<int, int>> & time_series,
286  bool loop, size_t current_index) const
287 {
289 
290  ret.SetByKey("Requests",
292  time_series, loop, current_index));
293  return ret;
294 }
295 
296 
298  uint64_t grand_total,
299  const vector<pair<int, int>> & time_series,
300  bool loop,
301  size_t current_index) const
302 {
304 
305  if (current_index == 0 && loop == false) {
306  // There is no data collected yet
307  return ret;
308  }
309 
310  CJsonNode output_series(CJsonNode::NewArrayNode());
311 
312  // Needed to calculate max and average reqs/sec
313  uint64_t max_n_req_per_min = 0;
314  uint64_t total_reqs = 0;
315  uint64_t total_mins = 0;
316 
317  // Index in the array where the data are collected
318  size_t raw_index;
319  if (current_index == 0) {
320  raw_index = kSeriesIntervals - 1;
321  loop = false; // to avoid going the second time over
322  } else {
323  raw_index = current_index - 1;
324  }
325 
326  size_t current_accumulated_mins = 0;
327  uint64_t current_accumulated_reqs = 0;
328  size_t output_data_index = 0;
329 
330  // The current index in the 'time_series', i.e. a pair of
331  // <mins to accumulate>:<last sequential data index>
332  // It is guaranteed they are both > 0.
333  size_t range_index = 0;
334  size_t current_mins_to_accumulate = time_series[range_index].first;
335  size_t current_last_seq_index = time_series[range_index].second;
336 
337  for ( ;; ) {
338  uint64_t reqs = values[raw_index];
339 
340  ++total_mins;
341  max_n_req_per_min = max(max_n_req_per_min, reqs);
342  total_reqs += reqs;
343 
344  ++current_accumulated_mins;
345  current_accumulated_reqs += reqs;
346 
347  if (current_accumulated_mins >= current_mins_to_accumulate) {
348  output_series.AppendDouble(double(current_accumulated_reqs) /
349  (double(current_accumulated_mins) * 60.0));
350  current_accumulated_mins = 0;
351  current_accumulated_reqs = 0;
352  }
353 
354  ++output_data_index;
355  if (output_data_index > current_last_seq_index) {
356  ++range_index;
357  current_mins_to_accumulate = time_series[range_index].first;
358  current_last_seq_index = time_series[range_index].second;
359  }
360 
361  if (raw_index == 0)
362  break;
363  --raw_index;
364  }
365 
366  if (loop) {
367  raw_index = kSeriesIntervals - 1;
368  while (raw_index > current_index + 1) {
369  uint64_t reqs = values[raw_index];
370  --raw_index;
371 
372  ++total_mins;
373  max_n_req_per_min = max(max_n_req_per_min, reqs);
374  total_reqs += reqs;
375 
376  ++current_accumulated_mins;
377  current_accumulated_reqs += reqs;
378 
379  if (current_accumulated_mins >= current_mins_to_accumulate) {
380  output_series.AppendDouble(double(current_accumulated_reqs) /
381  (double(current_accumulated_mins) * 60.0));
382  current_accumulated_mins = 0;
383  current_accumulated_reqs = 0;
384  }
385 
386  ++output_data_index;
387  if (output_data_index > current_last_seq_index) {
388  ++range_index;
389  current_mins_to_accumulate = time_series[range_index].first;
390  current_last_seq_index = time_series[range_index].second;
391  }
392  }
393  }
394 
395  if (current_accumulated_mins > 0) {
396  output_series.AppendDouble(double(current_accumulated_reqs) /
397  (double(current_accumulated_mins) * 60.0));
398  }
399 
400  if (loop) {
401  size_t last_minute_index = current_index + 1;
402  if (last_minute_index >= kSeriesIntervals)
403  last_minute_index = 0;
404 
405  // The current minute and the last minute in case of a loop are not
406  // sent to avoid unreliable data
407  uint64_t rest_reqs = grand_total - values[last_minute_index] - values[current_index];
408  uint64_t rest_mins = m_TotalMinutesCollected.load() - kSeriesIntervals - 2;
409 
410  if (rest_mins > 0) {
411  ret.SetDouble("RestAvgReqPerSec", rest_reqs / (rest_mins * 60.0));
412  } else {
413  ret.SetDouble("RestAvgReqPerSec", 0.0);
414  }
415  } else {
416  ret.SetDouble("RestAvgReqPerSec", 0.0);
417  }
418 
419  ret.SetInteger("TotalRequests", total_reqs);
420  ret.SetDouble("MaxReqPerSec", max_n_req_per_min / 60.0);
421  ret.SetDouble("AvgReqPerSec", total_reqs / (total_mins * 60.0));
422  ret.SetByKey("time_series", output_series);
423 
424  // Just in case: grand total includes everything - sent minutes, not sent
425  // minutes in case of the loops and the rest
426  ret.SetInteger("GrandTotalRequests", grand_total);
427  return ret;
428 }
429 
430 
431 
432 
433 // Converts a request status to the counter in the time series
434 // The logic matches the logic in GRID dashboard
437 {
438  if (status == CRequestStatus::e404_NotFound)
439  return eNotFound;
440 
442  return eError;
443 
444  if (status >= CRequestStatus::e400_BadRequest)
445  return eWarning;
446 
447  return eRequest;
448 }
449 
450 
452 {
453  Reset();
454 }
455 
456 
458 {
459  size_t current_index = m_CurrentIndex.load();
460  switch (counter) {
461  case eRequest:
462  ++m_Requests[current_index];
463  ++m_TotalRequests;
464  break;
465  case eError:
466  ++m_Errors[current_index];
467  ++m_TotalErrors;
468  break;
469  case eWarning:
470  ++m_Warnings[current_index];
471  ++m_TotalWarnings;
472  break;
473  case eNotFound:
474  ++m_NotFound[current_index];
475  ++m_TotalNotFound;
476  break;
477  default:
478  break;
479  }
480 }
481 
482 
484 {
485  size_t new_current_index = m_CurrentIndex.load();
486  if (new_current_index == kSeriesIntervals - 1) {
487  new_current_index = 0;
488  } else {
489  ++new_current_index;
490  }
491 
492  m_Requests[new_current_index] = 0;
493  m_Errors[new_current_index] = 0;
494  m_Warnings[new_current_index] = 0;
495  m_NotFound[new_current_index] = 0;
496 
497  m_CurrentIndex.store(new_current_index);
499  if (new_current_index == 0) {
500  m_Loop = true;
501  }
502 }
503 
504 
506 {
507  memset(m_Requests, 0, sizeof(m_Requests));
508  m_TotalRequests = 0;
509  memset(m_Errors, 0, sizeof(m_Errors));
510  m_TotalErrors = 0;
511  memset(m_Warnings, 0, sizeof(m_Warnings));
512  m_TotalWarnings = 0;
513  memset(m_NotFound, 0, sizeof(m_NotFound));
514  m_TotalNotFound = 0;
515 
516  m_CurrentIndex.store(0);
517  m_TotalMinutesCollected.store(1);
518  m_Loop = false;
519 }
520 
521 
522 CJsonNode CRequestTimeSeries::Serialize(const vector<pair<int, int>> & time_series,
523  bool loop, size_t current_index) const
524 {
526 
527  ret.SetByKey("Requests",
529  time_series, loop, current_index));
530  ret.SetByKey("Errors",
532  time_series, loop, current_index));
533  ret.SetByKey("Warnings",
535  time_series, loop, current_index));
536  ret.SetByKey("NotFound",
538  time_series, loop, current_index));
539  return ret;
540 }
541 
#define false
Definition: bool.h:36
JSON node abstraction.
static CJsonNode NewArrayNode()
Create a new JSON array node.
void SetDouble(const string &key, double value)
Set a JSON object element to the specified floating point value.
void SetInteger(const string &key, Int8 value)
Set a JSON object element to the specified integer value.
void SetByKey(const string &key, CJsonNode::TInstance value)
For a JSON object node, insert a new element or update an existing element.
void AppendDouble(double value)
For an array node, add a floating point node at the end of the array.
static CJsonNode NewObjectNode()
Create a new JSON object node.
void Add(uint64_t value)
CJsonNode x_SerializeOneSeries(const vector< pair< int, int >> &time_series, bool loop, size_t current_index) const
CJsonNode Serialize(const vector< pair< int, int >> &time_series, bool loop, size_t current_index) const
double m_Values[kSeriesIntervals]
atomic_uint_fast64_t m_TotalMinutesCollected
atomic_uint_fast64_t m_CurrentIndex
uint64_t m_Requests[kSeriesIntervals]
atomic_uint_fast64_t m_TotalMinutesCollected
atomic_uint_fast64_t m_CurrentIndex
CJsonNode x_SerializeOneSeries(const uint64_t *values, uint64_t grand_total, const vector< pair< int, int >> &time_series, bool loop, size_t current_index) const
CJsonNode Serialize(const vector< pair< int, int >> &time_series, bool loop, size_t current_index) const
static EPSGSCounter RequestStatusToCounter(CRequestStatus::ECode status)
uint64_t m_Warnings[kSeriesIntervals]
uint64_t m_NotFound[kSeriesIntervals]
CJsonNode Serialize(const vector< pair< int, int >> &time_series, bool loop, size_t current_index) const
uint64_t m_Errors[kSeriesIntervals]
char value[7]
Definition: config.c:431
Include a standard set of the NCBI C++ Toolkit most basic headers.
T max(T x_, T y_)
unsigned __int64 uint64_t
Definition: stdint.h:136
static constexpr size_t kSeriesIntervals
Modified on Sat Mar 02 10:58:25 2024 by modify_doxy.py rev. 669887