Skip to content

Commit 9df19e0

Browse files
committedMay 12, 2016
Add QgsSQLStatement class to parse SELECT statements
1 parent 12a630f commit 9df19e0

File tree

9 files changed

+3054
-3
lines changed

9 files changed

+3054
-3
lines changed
 

‎python/core/core.sip

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@
127127
%Include qgssnapper.sip
128128
%Include qgssnappingutils.sip
129129
%Include qgsspatialindex.sip
130+
%Include qgssqlstatement.sip
130131
%Include qgsstatisticalsummary.sip
131132
%Include qgsstringstatisticalsummary.sip
132133
%Include qgsstringutils.sip

‎python/core/qgssqlstatement.sip

Lines changed: 602 additions & 0 deletions
Large diffs are not rendered by default.

‎src/core/CMakeLists.txt

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ SET(QGIS_CORE_SRCS
194194
qgsspatialindex.cpp
195195
qgssqlexpressioncompiler.cpp
196196
qgssqliteexpressioncompiler.cpp
197+
qgssqlstatement.cpp
197198
qgsstatisticalsummary.cpp
198199
qgsstringstatisticalsummary.cpp
199200
qgsstringutils.cpp
@@ -427,11 +428,11 @@ IF (QT_MOBILITY_LOCATION_FOUND OR Qt5Positioning_FOUND)
427428
)
428429
ENDIF (QT_MOBILITY_LOCATION_FOUND OR Qt5Positioning_FOUND)
429430

430-
ADD_FLEX_FILES(QGIS_CORE_SRCS qgsexpressionlexer.ll)
431-
ADD_BISON_FILES(QGIS_CORE_SRCS qgsexpressionparser.yy)
431+
ADD_FLEX_FILES(QGIS_CORE_SRCS qgsexpressionlexer.ll qgssqlstatementlexer.ll)
432+
ADD_BISON_FILES(QGIS_CORE_SRCS qgsexpressionparser.yy qgssqlstatementparser.yy)
432433

433434
IF(NOT MSVC)
434-
SET_SOURCE_FILES_PROPERTIES(qgsexpressionparser.cpp PROPERTIES COMPILE_FLAGS -w)
435+
SET_SOURCE_FILES_PROPERTIES(qgsexpressionparser.cpp qgssqlstatementparser.cpp PROPERTIES COMPILE_FLAGS -w)
435436
ELSE(NOT MSVC)
436437
# -wd4702 unreachable code
437438
SET_SOURCE_FILES_PROPERTIES(

‎src/core/qgssqlstatement.cpp

Lines changed: 691 additions & 0 deletions
Large diffs are not rendered by default.

‎src/core/qgssqlstatement.h

Lines changed: 740 additions & 0 deletions
Large diffs are not rendered by default.

‎src/core/qgssqlstatementlexer.ll

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/***************************************************************************
2+
qgssqlstatementlexer.ll
3+
--------------------
4+
begin : April 2016
5+
copyright : (C) 2011 by Martin Dobias
6+
copyright : (C) 2016 by Even Rouault
7+
email : even.rouault at spatialys.com
8+
***************************************************************************
9+
* *
10+
* This program is free software; you can redistribute it and/or modify *
11+
* it under the terms of the GNU General Public License as published by *
12+
* the Free Software Foundation; either version 2 of the License, or *
13+
* (at your option) any later version. *
14+
* *
15+
***************************************************************************/
16+
17+
%option noyywrap
18+
%option case-insensitive
19+
%option never-interactive
20+
%option nounput
21+
%option prefix="sqlstatement_"
22+
// this makes flex generate lexer with context + init/destroy functions
23+
%option reentrant
24+
// this makes Bison send yylex another argument to use instead of using the global variable yylval
25+
%option bison-bridge
26+
27+
// ensure that lexer will be 8-bit (and not just 7-bit)
28+
%option 8bit
29+
30+
%{
31+
32+
#include "qgssqlstatement.h"
33+
struct sqlstatement_parser_context;
34+
#include "qgssqlstatementparser.hpp"
35+
#include <QLocale>
36+
37+
// if not defined, searches for isatty()
38+
// which doesn't in MSVC compiler
39+
#define YY_NEVER_INTERACTIVE 1
40+
41+
#ifndef YY_NO_UNPUT
42+
#define YY_NO_UNPUT // unused
43+
#endif
44+
45+
#ifdef _MSC_VER
46+
#define YY_NO_UNISTD_H
47+
#endif
48+
49+
#define B_OP(x) yylval->b_op = QgsSQLStatement::x
50+
#define U_OP(x) yylval->u_op = QgsSQLStatement::x
51+
#define TEXT yylval->text = new QString( QString::fromUtf8(yytext) );
52+
#define TEXT_FILTER(filter_fn) yylval->text = new QString( filter_fn( QString::fromUtf8(yytext) ) );
53+
54+
static QString stripText(QString text)
55+
{
56+
// strip single quotes on start,end
57+
text = text.mid( 1, text.length() - 2 );
58+
59+
// make single "single quotes" from double "single quotes"
60+
text.replace( "''", "'" );
61+
62+
// strip \n \' etc.
63+
int index = 0;
64+
while (( index = text.indexOf( '\\', index ) ) != -1 )
65+
{
66+
text.remove( index, 1 ); // delete backslash
67+
QChar chr;
68+
switch ( text[index].toLatin1() ) // evaluate backslashed character
69+
{
70+
case 'n': chr = '\n'; break;
71+
case 't': chr = '\t'; break;
72+
case '\\': chr = '\\'; break;
73+
case '\'': chr = '\''; break;
74+
default: chr = '?'; break;
75+
}
76+
text[index++] = chr; // set new character and push index +1
77+
}
78+
return text;
79+
}
80+
81+
// C locale for correct parsing of numbers even if the system locale is different
82+
static QLocale cLocale("C");
83+
84+
%}
85+
86+
white [ \t\r\n]+
87+
88+
non_ascii [\x80-\xFF]
89+
90+
identifier_first [A-Za-z_]|{non_ascii}
91+
identifier_next [A-Za-z0-9_]|{non_ascii}
92+
identifier {identifier_first}{identifier_next}*
93+
94+
identifier_str_char "\"\""|[^\"]
95+
identifier_quoted "\""{identifier_str_char}*"\""
96+
97+
dig [0-9]
98+
num_int [-]?{dig}+{identifier_first}*
99+
num_float [-]?{dig}*(\.{dig}+([eE][-+]?{dig}+)?|[eE][-+]?{dig}+)
100+
boolean "TRUE"|"FALSE"
101+
102+
str_char ('')|(\\.)|[^'\\]
103+
string "'"{str_char}*"'"
104+
105+
%%
106+
107+
"NOT" { U_OP(uoNot); return NOT; }
108+
"AND" { B_OP(boAnd); return AND; }
109+
"OR" { B_OP(boOr); return OR; }
110+
111+
"=" { B_OP(boEQ); return EQ; }
112+
"!=" { B_OP(boNE); return NE; }
113+
"<=" { B_OP(boLE); return LE; }
114+
">=" { B_OP(boGE); return GE; }
115+
"<>" { B_OP(boNE); return NE; }
116+
"<" { B_OP(boLT); return LT; }
117+
">" { B_OP(boGT); return GT; }
118+
119+
"LIKE" { B_OP(boLike); return LIKE; }
120+
"NOT"{white}"LIKE" { B_OP(boNotLike); return LIKE; }
121+
"ILIKE" { B_OP(boILike); return LIKE; }
122+
"NOT"{white}"ILIKE" { B_OP(boNotILike); return LIKE; }
123+
"IS" { B_OP(boIs); return IS; }
124+
"IS"{white}"NOT" { B_OP(boIsNot); return IS; }
125+
"||" { B_OP(boConcat); return CONCAT; }
126+
127+
"+" { B_OP(boPlus); return PLUS; }
128+
"-" { B_OP(boMinus); return MINUS; }
129+
"*" { B_OP(boMul); return MUL_OR_STAR; }
130+
"//" { B_OP(boIntDiv); return INTDIV; }
131+
"/" { B_OP(boDiv); return DIV; }
132+
"%" { B_OP(boMod); return MOD; }
133+
"^" { B_OP(boPow); return POW; }
134+
135+
"IN" { return IN; }
136+
"BETWEEN" { return BETWEEN; }
137+
138+
"NULL" { return NULLVALUE; }
139+
140+
"SELECT" { return SELECT; }
141+
"ALL" { return ALL; }
142+
"DISTINCT" { return DISTINCT; }
143+
"CAST" { return CAST; }
144+
"AS" { return AS; }
145+
"FROM" { return FROM; }
146+
"JOIN" { return JOIN; }
147+
"ON" { return ON; }
148+
"USING" { return USING; }
149+
"WHERE" { return WHERE; }
150+
"ORDER" { return ORDER; }
151+
"BY" { return BY; }
152+
"ASC" { return ASC; }
153+
"DESC" { return DESC; }
154+
"LEFT" { return LEFT; }
155+
"RIGHT" { return RIGHT; }
156+
"INNER" { return INNER; }
157+
"OUTER" { return OUTER; }
158+
"CROSS" { return CROSS; }
159+
"FULL" { return FULL; }
160+
"NATURAL" { return NATURAL; }
161+
"UNION" { return UNION; }
162+
163+
[().] { return yytext[0]; }
164+
165+
"," { return COMMA; }
166+
167+
{num_float} { yylval->numberFloat = cLocale.toDouble( QString::fromAscii(yytext) ); return NUMBER_FLOAT; }
168+
{num_int} {
169+
bool ok;
170+
yylval->numberInt = cLocale.toInt( QString::fromAscii(yytext), &ok );
171+
if( ok )
172+
return NUMBER_INT;
173+
174+
yylval->numberInt64 = cLocale.toLongLong( QString::fromAscii(yytext), &ok );
175+
if( ok )
176+
return NUMBER_INT64;
177+
178+
yylval->numberFloat = cLocale.toDouble( QString::fromAscii(yytext), &ok );
179+
if( ok )
180+
return NUMBER_FLOAT;
181+
182+
return Unknown_CHARACTER;
183+
}
184+
185+
{boolean} { yylval->boolVal = QString( yytext ).compare( "true", Qt::CaseInsensitive ) == 0; return BOOLEAN; }
186+
187+
{string} { TEXT_FILTER(stripText); return STRING; }
188+
189+
{identifier} { TEXT; return IDENTIFIER; }
190+
191+
{identifier_quoted} { TEXT_FILTER(QgsSQLStatement::stripQuotedIdentifier); return IDENTIFIER; }
192+
193+
{white} /* skip blanks and tabs */
194+
195+
. { return Unknown_CHARACTER; }
196+
197+
198+
%%

‎src/core/qgssqlstatementparser.yy

Lines changed: 633 additions & 0 deletions
Large diffs are not rendered by default.

‎tests/src/python/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ ADD_PYTHON_TEST(PyQgsTabfileProvider test_provider_tabfile.py)
6868
ADD_PYTHON_TEST(PyQgsOGRProvider test_provider_ogr.py)
6969
ADD_PYTHON_TEST(PyQgsSpatialIndex test_qgsspatialindex.py)
7070
ADD_PYTHON_TEST(PyQgsSpatialiteProvider test_provider_spatialite.py)
71+
ADD_PYTHON_TEST(PyQgsSQLStatement test_qgssqlstatement.py)
7172
ADD_PYTHON_TEST(PyQgsStringStatisticalSummary test_qgsstringstatisticalsummary.py)
7273
ADD_PYTHON_TEST(PyQgsSymbolLayerV2 test_qgssymbollayerv2.py)
7374
ADD_PYTHON_TEST(PyQgsArrowSymbolLayer test_qgsarrowsymbollayer.py)
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# -*- coding: utf-8 -*-
2+
"""QGIS Unit tests for QgsSQLStatement.
3+
4+
.. note:: This program is free software; you can redistribute it and/or modify
5+
it under the terms of the GNU General Public License as published by
6+
the Free Software Foundation; either version 2 of the License, or
7+
(at your option) any later version.
8+
"""
9+
__author__ = 'Even Rouault'
10+
__date__ = '4/4/2016'
11+
__copyright__ = 'Copyright 2016, The QGIS Project'
12+
# This will get replaced with a git SHA1 when you do a git archive
13+
__revision__ = '$Format:%H$'
14+
15+
from qgis.testing import unittest
16+
from qgis.core import QgsSQLStatement
17+
18+
19+
class TestQgsSQLStatementCustomFunctions(unittest.TestCase):
20+
21+
def checkNominal(self, statement, expected_dump=None):
22+
exp = QgsSQLStatement(statement)
23+
self.assertEqual(exp.hasParserError(), False)
24+
self.assertEqual(exp.parserErrorString(), "")
25+
if expected_dump is None:
26+
expected_dump = statement
27+
self.assertEqual(exp.dump(), expected_dump)
28+
self.assertEqual(exp.dump(), exp.rootNode().clone().dump())
29+
30+
def testNominalSimple(self):
31+
statement = "SELECT a FROM t"
32+
self.checkNominal(statement)
33+
exp = QgsSQLStatement(statement)
34+
statement_node = exp.rootNode()
35+
self.assertEqual(statement_node.nodeType(), QgsSQLStatement.ntSelect)
36+
tables = statement_node.tables()
37+
self.assertEqual(len(tables), 1)
38+
table = tables[0]
39+
self.assertEqual(table.nodeType(), QgsSQLStatement.ntTableDef)
40+
self.assertEqual(table.name(), 't')
41+
self.assertEqual(table.alias(), '')
42+
columns = statement_node.columns()
43+
self.assertEqual(len(columns), 1)
44+
column = columns[0]
45+
self.assertEqual(column.nodeType(), QgsSQLStatement.ntSelectedColumn)
46+
column_ref = column.column()
47+
self.assertEqual(column.alias(), '')
48+
self.assertEqual(column_ref.nodeType(), QgsSQLStatement.ntColumnRef)
49+
self.assertEqual(column_ref.name(), 'a')
50+
self.assertEqual(column_ref.tableName(), '')
51+
52+
def testNominalSelectDistinct(self):
53+
statement = "SELECT DISTINCT a FROM t"
54+
self.checkNominal(statement)
55+
56+
def testNominalColumns(self):
57+
statement = "SELECT null, 1234567890123456789, a, b b_alias, 'literal', CAST(1 AS varchar), "
58+
statement += "\"1c\", *, \"*\", a.*, foo(), bar(baz, baw), t.c AS \"1quoted\", "
59+
statement += "COUNT(*), COUNT(*) a, COUNT(DISTINCT x), COUNT(DISTINCT x) AS a FROM t"
60+
expected_dump = "SELECT NULL, 1234567890123456789, a, b AS b_alias, 'literal', CAST(1 AS varchar), "
61+
expected_dump += "\"1c\", *, \"*\", a.*, foo(), bar(baz, baw), t.c AS \"1quoted\", "
62+
expected_dump += "COUNT(*), COUNT(*) AS a, COUNT(DISTINCT x), COUNT(DISTINCT x) AS a FROM t"
63+
self.checkNominal(statement, expected_dump)
64+
65+
def testNominalFrom(self):
66+
statement = "SELECT a FROM t1, t2 at2, t3 AS at3, \"1quoted\", t4 AS \"2quoted\""
67+
expected_dump = "SELECT a FROM t1, t2 AS at2, t3 AS at3, \"1quoted\", t4 AS \"2quoted\""
68+
self.checkNominal(statement, expected_dump)
69+
70+
def testNominalWhere(self):
71+
statement = "SELECT a FROM t WHERE 1.5 <= 'a' OR TRUE OR FALSE OR a IS NULL AND b IS NOT NULL " + \
72+
"OR NOT d OR 1 + (2 - 3) * 4 / 5 ^ 6 <> 0 OR a IN (1, 2) OR b NOT IN (5) " + \
73+
"OR x BETWEEN 5 AND 6 OR x NOT BETWEEN 5 AND 6 OR c = d OR c > d OR c < d OR c >= d OR c <= d"
74+
self.checkNominal(statement)
75+
76+
def checkJoinType(self, joinType):
77+
statement = "SELECT a FROM t " + joinType + " j1 ON TRUE"
78+
self.checkNominal(statement)
79+
80+
def testJoinTypes(self):
81+
self.checkJoinType('JOIN')
82+
self.checkJoinType('LEFT JOIN')
83+
self.checkJoinType('LEFT OUTER JOIN')
84+
self.checkJoinType('RIGHT JOIN')
85+
self.checkJoinType('RIGHT OUTER JOIN')
86+
self.checkJoinType('CROSS JOIN')
87+
self.checkJoinType('FULL JOIN')
88+
self.checkJoinType('INNER JOIN')
89+
90+
def testJoin(self):
91+
statement = "SELECT a FROM t JOIN j1 ON TRUE JOIN j2 USING (a) JOIN j3 USING (\"1a\", b)"
92+
self.checkNominal(statement)
93+
94+
def testNominalOrderBy(self):
95+
statement = "SELECT a FROM t ORDER BY a, b ASC, c DESC"
96+
expected_dump = "SELECT a FROM t ORDER BY a, b, c DESC"
97+
self.checkNominal(statement, expected_dump)
98+
99+
def testNominalFull(self):
100+
statement = \
101+
"SELECT a FROM t JOIN j1 ON cond1 JOIN j2 ON cond2 WHERE TRUE ORDER BY c"
102+
self.checkNominal(statement)
103+
104+
def checkError(self, statement):
105+
exp = QgsSQLStatement(statement)
106+
self.assertEqual(exp.hasParserError(), True)
107+
self.assertNotEqual(exp.parserErrorString(), '')
108+
self.assertEqual(exp.dump(), "(no root)")
109+
self.assertEqual(exp.rootNode(), None)
110+
111+
def testError(self):
112+
self.checkError("1")
113+
self.checkError("SELECT")
114+
self.checkError("SELECT a")
115+
self.checkError("SELECT a, FROM b")
116+
self.checkError("SELECT 1a FROM b")
117+
self.checkError("SELECT a AS FROM b")
118+
self.checkError("SELECT a,. FROM b")
119+
self.checkError("SELECT f(*) FROM b")
120+
self.checkError("SELECT f(*) a FROM b")
121+
self.checkError("SELECT .")
122+
self.checkError("SELECT a FROM")
123+
self.checkError("SELECT a FROM b WHERE")
124+
self.checkError("SELECT a FROM b WHERE .")
125+
self.checkError("SELECT a FROM b,")
126+
self.checkError("SELECT a FROM b,.")
127+
self.checkError("SELECT a FROM b JOIN")
128+
self.checkError("SELECT a FROM b JOIN c")
129+
self.checkError("SELECT a FROM b JOIN c ON")
130+
self.checkError("SELECT a FROM b JOIN c USING")
131+
self.checkError("SELECT a FROM b JOIN c ON d JOIN")
132+
self.checkError("SELECT a FROM b ORDER BY")
133+
self.checkError("SELECT a FROM b JOIN c ON d ORDER BY e unexpected")
134+
135+
def testBasicValidationCheck(self):
136+
exp = QgsSQLStatement("error")
137+
(b, errorMsg) = exp.doBasicValidationChecks()
138+
self.assertFalse(b)
139+
self.assertEqual(errorMsg, 'No root node')
140+
141+
exp = QgsSQLStatement("SELECT c FROM t")
142+
(b, errorMsg) = exp.doBasicValidationChecks()
143+
self.assertTrue(b)
144+
self.assertEqual(errorMsg, '')
145+
146+
exp = QgsSQLStatement("SELECT t.c FROM t ORDER BY t.c")
147+
(b, errorMsg) = exp.doBasicValidationChecks()
148+
self.assertTrue(b)
149+
self.assertEqual(errorMsg, '')
150+
151+
exp = QgsSQLStatement("SELECT t.c FROM t t_alias")
152+
(b, errorMsg) = exp.doBasicValidationChecks()
153+
self.assertFalse(b)
154+
self.assertEqual(
155+
errorMsg, 'Table t is referenced by column c, but not selected in FROM / JOIN.')
156+
157+
exp = QgsSQLStatement(
158+
"SELECT CAST(1 + foo(t_unknown.a) AS varchar) FROM t")
159+
(b, errorMsg) = exp.doBasicValidationChecks()
160+
self.assertFalse(b)
161+
self.assertEqual(
162+
errorMsg, 'Table t_unknown is referenced by column a, but not selected in FROM / JOIN.')
163+
164+
exp = QgsSQLStatement("SELECT c FROM t WHERE t_unknown.a = 1")
165+
(b, errorMsg) = exp.doBasicValidationChecks()
166+
self.assertFalse(b)
167+
self.assertEqual(
168+
errorMsg, 'Table t_unknown is referenced by column a, but not selected in FROM / JOIN.')
169+
170+
exp = QgsSQLStatement(
171+
"SELECT c FROM t JOIN t2 ON t.c1 = t2.c2 AND t3.c3 IS NOT NULL")
172+
(b, errorMsg) = exp.doBasicValidationChecks()
173+
self.assertFalse(b)
174+
self.assertEqual(
175+
errorMsg, 'Table t3 is referenced by column c3, but not selected in FROM / JOIN.')
176+
177+
exp = QgsSQLStatement("SELECT c FROM t ORDER BY t_unknown.c")
178+
(b, errorMsg) = exp.doBasicValidationChecks()
179+
self.assertFalse(b)
180+
self.assertEqual(
181+
errorMsg, 'Table t_unknown is referenced by column c, but not selected in FROM / JOIN.')
182+
183+
if __name__ == "__main__":
184+
unittest.main()

0 commit comments

Comments
 (0)
Please sign in to comment.