NCBI C++ ToolKit
parser.cpp
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: parser.cpp 87555 2019-09-11 15:29:02Z grichenk $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Authors: Anatoliy Kuznetsov
27  *
28  * File Description:
29  * Query parser entry point.
30  * Glues together code generated by BISON, lexical tokenizer,
31  * error processing, etc.
32  *
33  */
34 
35 #include <ncbi_pch.hpp>
36 #include <corelib/ncbimtx.hpp>
37 #include <corelib/ncbiapp.hpp>
39 #include <util/resource_pool.hpp>
40 #include <util/error_codes.hpp>
41 
42 #include <stack>
43 
44 
45 #define NCBI_USE_ERRCODE_X Util_QParse
46 
48 
49 /* extern "C" */ static int ncbi_q_error (const char *s);
50 /* extern "C" */ static int ncbi_q_lex (CQueryParseTree::TNode** lvalp, void* param);
51 
53 
54 
55 
56 /// Query tree garbage collection functor.
57 ///
58 /// Finds nodes in the query tree and removes them from the
59 /// memory pool. All lost nodes are deleted by the pool object.
60 ///
61 /// @internal
62 /// @ingroup QParser
63 ///
65 {
66 public:
68 
69 public:
71  : m_Pool(mem_pool)
72  {}
73 
76  {
77  if (delta == 0 || delta == 1) {
78  m_Pool.Forget(&tr);
79  }
80  return eTreeTraverse;
81  }
82 private:
84 };
85 
86 /// Class passed to Lex/Bison (yylex/yyparse) functions.
87 /// Works as a glue to pass all the information
88 /// (both parsed stream and output syntactic tree)
89 ///
90 /// @internal
91 /// @ingroup QParser
92 ///
94 {
95 public:
97  /// Parsing context stack
98  typedef vector<CQueryParseTree::TNode*> TContextStack;
99 
100 public:
101  CQueryParserEnv(const char* query_str, CQueryParseTree& qtree, unsigned line = 0, unsigned linePos = 0)
102  : m_QTree(qtree),
103  m_Query(query_str),
104  m_Ptr(query_str),
105  m_QueryTree(0),
106  m_Verbose(false),
107  m_Case(CQueryParseTree::eCaseInsensitive),
108  m_Line(line),
109  m_LinePos(linePos),
110  m_ParseTolerance(CQueryParseTree::eSyntaxCheck),
111  m_InNode(0),
112 // m_SelectNode(0),
113  m_FromNode(0),
114  m_Functions(0)
115  {
116  m_QueryLen = (unsigned)::strlen(m_Query);
117  }
118 
120  {
121  try {
122  Free();
123  }
124  catch (exception & ex) {
125  ERR_POST_X(1, "Exception in ~CQueryParserEnv()" << ex.what());
126  }
127  catch (...) {
128  ERR_POST_X(2, "Unknown exception in ~CQueryParserEnv()");
129  }
130 
131  }
132 
133  void Free()
134  {
135  if (m_QueryTree) {
136  // Sync the memory pool not to delete the in-tree nodes
139 
140  delete m_QueryTree; m_QueryTree = 0;
141  }
142  }
143 
144  /// Register pointer on function names (no ownership transfer)
146  /// Get list of registered functions
148 
149 
150  const char* GetQueryBuffer() const { return m_Query; }
151  const char* GetBufPtr() const { return m_Ptr; }
152  const char* GetBufEnd() const { return m_Query + m_QueryLen; }
153  void SetBufPtr(const char* ptr) { m_Ptr = ptr; }
154 
155  int GetChar()
156  {
157  int r = *m_Ptr++;
158  if (r == '\n') {
159  ++m_Line; m_LinePos = 0;
160  return r;
161  }
162  ++m_LinePos;
163  return r;
164  }
165 
166  /// Skip number of characters
167  void Skip(int num)
168  {
169  for (int i = 0; i < num; ++i) {
170  GetChar();
171  }
172  }
173 /*
174  void UnGetChar()
175  {
176  --m_Ptr;
177  }
178 */
180  {
181 //cerr << "AttachToPool was:" << m_QueryTree << " new: " << qt << endl;
182 // m_QueryTree = qt;
183  m_NodePool.Put(qt);
184  }
185 
187  {
188 //cerr << "AttachQueryTree was:" << m_QueryTree << " new: " << qt << endl;
189  m_QueryTree = qt;
190  }
191 
193 
194  /// Return query clause.
195  ///
196  /// Caller is responsible for deleteion
197  ///
199  {
200  if (m_QueryTree) {
201  // Sync the memory pool
204 
206  m_QueryTree = 0;
207 
208  return qc;
209  }
210  return 0;
211  }
212 
214  {
215  m_NodePool.Put(qnode);
216  }
217 
219  {
220  return m_NodePool;
221  }
222 
224  {
225  if (qnode) m_NodePool.Forget(qnode);
226  }
227 
229  CQueryParseTree::TNode* qnode2)
230  {
231  if (qnode1) m_NodePool.Forget(qnode1);
232  if (qnode2) m_NodePool.Forget(qnode2);
233  }
234 
235  bool IsVerbose() const { return m_Verbose; }
236  void SetVerbose(bool verbose=true) { m_Verbose = verbose; }
237 
239  void SetCase(CQueryParseTree::ECase case_sense) { m_Case = case_sense; }
240 
242 
243  /// Src line number
244  unsigned GetLine() const { return m_Line; }
245  /// Get position in line
246  unsigned GetLinePos() const { return m_LinePos; }
247 
249  {
250  m_ParseTolerance = syntax_check;
251  }
253  {
254  return m_ParseTolerance;
255  }
257  {
258  if(m_InNode != 0) _ASSERT(in_node == 0);
259  m_InNode = in_node;
260  }
262 
264  {
265  if (node) {
266  m_SelectCtxStack.push_back(node);
267  } else {
268  m_SelectCtxStack.pop_back();
269  }
270  }
272  {
273  if (m_SelectCtxStack.size() == 0) {
274  return 0;
275  }
276  return m_SelectCtxStack[m_SelectCtxStack.size()-1];
277  }
278 
280  {
281  m_FromNode = from_node;
282  }
284 
286  {
287  if (node) {
288  m_CtxStack.push_back(node);
289  } else {
290  m_CtxStack.pop_back();
291  }
292  }
293 
295  {
296  if (m_CtxStack.size() == 0) {
297  return 0;
298  }
299  return m_CtxStack[m_CtxStack.size()-1];
300  }
301 
302 private:
303  CQueryParseTree& m_QTree; ///< Base query tree reference
304  const char* m_Query; ///< Request buffer. (Source for the scanner)
305  unsigned m_QueryLen;///< Query length
306  const char* m_Ptr; ///< Current position in the request buffer
307 
308  /// Query clause tree. This is the result of the statement parsing.
310 
311  /// Memory pool of query clause tree nodes.
312  /// Used to avoid memory leaks when BISON shifts stack and calls
313  /// yyerror (or othewise throws a C++ exception)
315  /// Debug trace when parsing
316  bool m_Verbose;
317  /// Case sensitivity switch
319 
320  unsigned m_Line;
321  unsigned m_LinePos;
322 
324 
325  /// operator "IN" context pointer
327 
328  /// "SELECT" parsing context
329  //CQueryParseTree::TNode* m_SelectNode;
331 
332  /// "FROM" context pointer
334 
335  /// Common parsing context
337 
338  /// Function names
340 
341 };
342 
343 
344 
345 
347 
348 #ifdef IN
349 #undef IN
350 #endif
351 
352 #include "query_parser_bison.tab.c"
353 #include "lexer.inc"
354 
355 /// Called by yyparse on error
356 static
357 int ncbi_q_error (const char *)
358 {
359  _TRACE("Parsing error!!!");
360 // NCBI_THROW(CQueryParseException, eParserError, "Syntax error!");
361  return 1;
362 }
363 
364 
366 
367 
368 void CQueryParseTree::Parse(const char* query_str,
369  ECase case_sense,
370  ESyntaxCheck syntax_check,
371  bool verbose,
372  const TFunctionNames& functions,
373  unsigned line,
374  unsigned linePos)
375 {
376  CQueryParserEnv env(query_str, *this, line, linePos);
377 
378 #ifdef _DEBUG
380  if (app && app->GetEnvironment().Get("DIAG_TRACE") == "1") {
381  yydebug = 1;
382  verbose = true;
383  }
384 #endif
385  env.SetCase(case_sense);
386  env.SetVerbose(verbose);
387  env.SetParserTolerance(syntax_check);
388  if (functions.size()) {
389  env.SetFunctions(&functions);
390  }
391 
392  int res = ncbi_q_parse((void*) &env);
393  if (res != 0) {
394  NCBI_THROW(CQueryParseException, eParserError, "Parsing error.");
395  }
396 
397  CQueryParseTree::TNode* qt = env.GetQueryTree();
398 
399  if (qt) {
400  qt = env.DetachQueryTree();
401  this->SetQueryTree(qt);
402  } else {
403 
404  // Check if we have only one token node created by the tokenizer
405  // means it's just a single word request (perfectly legal)
406 
407  CQueryParserEnv::TNodePool& pool = env.GetPool();
409 
410  // the only node in pool
411  if (vec.size() == 1) {
412  CQueryParseTree::TNode* qcl = vec[0];
413  this->SetQueryTree(qcl);
414  pool.ForgetAll();
415  }
416  }
417  env.Free();
418 }
419 
420 
int yydebug
#define false
Definition: bool.h:36
Query parser exceptions.
Query tree and associated utility methods.
Class passed to Lex/Bison (yylex/yyparse) functions.
Definition: parser.cpp:94
void SetContext(CQueryParseTree::TNode *node)
Definition: parser.cpp:285
void SetIN_Context(CQueryParseTree::TNode *in_node)
Definition: parser.cpp:256
void Skip(int num)
Skip number of characters.
Definition: parser.cpp:167
const CQueryParseTree::TFunctionNames * GetFunctions() const
Get list of registered functions.
Definition: parser.cpp:147
unsigned m_LinePos
Definition: parser.cpp:321
CQueryParseTree::TNode * GetFROM_Context()
Definition: parser.cpp:283
const char * m_Query
Request buffer. (Source for the scanner)
Definition: parser.cpp:304
TNodePool m_NodePool
Memory pool of query clause tree nodes.
Definition: parser.cpp:314
CResourcePool< CQueryParseTree::TNode, CFastMutex > TNodePool
Definition: parser.cpp:96
void SetFunctions(const CQueryParseTree::TFunctionNames *func)
Register pointer on function names (no ownership transfer)
Definition: parser.cpp:145
CQueryParseTree::TNode * GetQueryTree()
Definition: parser.cpp:192
void AttachToPool(CQueryParseTree::TNode *qt)
Definition: parser.cpp:179
CQueryParseTree::TNode * m_QueryTree
Query clause tree. This is the result of the statement parsing.
Definition: parser.cpp:309
CQueryParseTree & m_QTree
Base query tree reference.
Definition: parser.cpp:303
CQueryParseTree::ECase m_Case
Case sensitivity switch.
Definition: parser.cpp:318
const char * GetBufEnd() const
Definition: parser.cpp:152
CQueryParseTree::ECase GetCase() const
Definition: parser.cpp:238
void SetSELECT_Context(CQueryParseTree::TNode *node)
Definition: parser.cpp:263
unsigned m_Line
Definition: parser.cpp:320
void ForgetPoolNode(CQueryParseTree::TNode *qnode)
Definition: parser.cpp:223
CQueryParseTree & QTree()
Definition: parser.cpp:241
const char * GetBufPtr() const
Definition: parser.cpp:151
unsigned m_QueryLen
Query length.
Definition: parser.cpp:305
unsigned GetLinePos() const
Get position in line.
Definition: parser.cpp:246
void SetFROM_Context(CQueryParseTree::TNode *from_node)
Definition: parser.cpp:279
CQueryParseTree::TNode * m_FromNode
"FROM" context pointer
Definition: parser.cpp:333
void AttachQueryTree(CQueryParseTree::TNode *qt)
Definition: parser.cpp:186
void AddNodeToPool(CQueryParseTree::TNode *qnode)
Definition: parser.cpp:213
CQueryParseTree::TNode * GetIN_Context()
Definition: parser.cpp:261
bool IsVerbose() const
Definition: parser.cpp:235
TContextStack m_CtxStack
Common parsing context.
Definition: parser.cpp:336
void SetBufPtr(const char *ptr)
Definition: parser.cpp:153
void SetVerbose(bool verbose=true)
Definition: parser.cpp:236
void SetParserTolerance(CQueryParseTree::ESyntaxCheck syntax_check)
Definition: parser.cpp:248
CQueryParseTree::TNode * GetContext()
Definition: parser.cpp:294
CQueryParserEnv(const char *query_str, CQueryParseTree &qtree, unsigned line=0, unsigned linePos=0)
Definition: parser.cpp:101
TNodePool & GetPool()
Definition: parser.cpp:218
CQueryParseTree::TNode * GetSELECT_Context()
Definition: parser.cpp:271
CQueryParseTree::TNode * DetachQueryTree()
Return query clause.
Definition: parser.cpp:198
bool m_Verbose
Debug trace when parsing.
Definition: parser.cpp:316
CQueryParseTree::ESyntaxCheck GetParserTolerance() const
Definition: parser.cpp:252
vector< CQueryParseTree::TNode * > TContextStack
Parsing context stack.
Definition: parser.cpp:98
TContextStack m_SelectCtxStack
"SELECT" parsing context
Definition: parser.cpp:330
void ForgetPoolNodes(CQueryParseTree::TNode *qnode1, CQueryParseTree::TNode *qnode2)
Definition: parser.cpp:228
CQueryParseTree::TNode * m_InNode
operator "IN" context pointer
Definition: parser.cpp:326
const CQueryParseTree::TFunctionNames * m_Functions
Function names.
Definition: parser.cpp:339
const char * GetQueryBuffer() const
Definition: parser.cpp:150
void SetCase(CQueryParseTree::ECase case_sense)
Definition: parser.cpp:239
unsigned GetLine() const
Src line number.
Definition: parser.cpp:244
const char * m_Ptr
Current position in the request buffer.
Definition: parser.cpp:306
CQueryParseTree::ESyntaxCheck m_ParseTolerance
Definition: parser.cpp:323
Query tree garbage collection functor.
CQueryTreeCleanPoolFunc(TNodePool &mem_pool)
Definition: parser.cpp:70
CResourcePool< CBDB_Query::TQueryClause > & m_Pool
TNodePool & m_Pool
Definition: parser.cpp:83
ETreeTraverseCode operator()(CQueryParseTree::TNode &tr, int delta)
Definition: parser.cpp:75
CResourcePool< CQueryParseTree::TNode, CFastMutex > TNodePool
Definition: parser.cpp:67
definition of a Culling tree
Definition: ncbi_tree.hpp:100
const CNcbiEnvironment & GetEnvironment(void) const
Get the application's cached environment.
static CNcbiApplicationGuard InstanceGuard(void)
Singleton method.
Definition: ncbiapp.cpp:133
#define _TRACE(message)
Definition: ncbidbg.hpp:122
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
Definition: ncbidiag.hpp:550
const string & Get(const string &name, bool *found=NULL) const
Get environment value by name.
Definition: ncbienv.cpp:109
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
Definition: ncbiexpt.hpp:704
void Parse(const char *query_str, ECase case_sense=eCaseInsensitive, ESyntaxCheck syntax_check=eSyntaxCheck, bool verbose=false, const TFunctionNames &functions=TFunctionNames(0), unsigned line=0, unsigned linePos=0)
Query parser front-end function.
Definition: parser.cpp:368
void SetQueryTree(TNode *qtree)
Replace current query tree with the new one.
ECase
Case sensitive parsing.
vector< string > TFunctionNames
List of keywords recognised as functions.
ESyntaxCheck
Level of tolerance to syntax errors and problems.
void ForgetAll()
Makes pool to forget all objects.
Value * Forget(Value *v)
Makes the pool to forget the object.
void Put(Value *v)
Put object into the pool.
TPoolList & GetFreeList()
Get internal list of free objects.
#define END_NCBI_SCOPE
End previously defined NCBI scope.
Definition: ncbistl.hpp:103
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Definition: ncbistl.hpp:100
Fun TreeDepthFirstTraverse(TTreeNode &tree_node, Fun func)
Depth-first tree traversal algorithm.
Definition: ncbi_tree.hpp:504
ETreeTraverseCode
Tree traverse code returned by the traverse predicate function.
Definition: ncbi_tree.hpp:51
@ eTreeTraverse
Keep traversal.
Definition: ncbi_tree.hpp:52
Definition of all error codes used in util (xutil.lib).
int i
Defines the CNcbiApplication and CAppException classes for creating NCBI applications.
Multi-threading – mutexes; rw-locks; semaphore.
Int4 delta(size_t dimension_, const Int4 *score_)
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
true_type verbose
Definition: processing.cpp:890
Query string parsing components.
int ncbi_q_parse()
#define _ASSERT
static HENV env
Definition: transaction2.c:38
static int ncbi_q_lex(CQueryParseTree::TNode **lvalp, void *param)
static int ncbi_q_error(const char *s)
Called by yyparse on error.
Definition: parser.cpp:357
USING_NCBI_SCOPE
Definition: parser.cpp:47
Modified on Fri Jan 05 07:25:15 2024 by modify_doxy.py rev. 669887