4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
12 ** An tokenizer for SQL
14 ** This file contains C code that splits an SQL input string up into
15 ** individual tokens and sends those tokens one-by-one over to the
16 ** parser for analysis.
25 #include "wine/unicode.h"
30 ** All the keywords of the SQL language are stored as in a hash
31 ** table composed of instances of the following structure.
33 typedef struct Keyword Keyword;
35 const WCHAR *zName; /* The keyword name */
36 int tokenType; /* The token value for this keyword */
39 #define MAX_TOKEN_LEN 11
41 static const WCHAR ABORT_W[] = { 'A','B','O','R','T',0 };
42 static const WCHAR AFTER_W[] = { 'A','F','T','E','R',0 };
43 static const WCHAR ALTER_W[] = { 'A','L','T','E','R',0 };
44 static const WCHAR ALL_W[] = { 'A','L','L',0 };
45 static const WCHAR AND_W[] = { 'A','N','D',0 };
46 static const WCHAR AS_W[] = { 'A','S',0 };
47 static const WCHAR ASC_W[] = { 'A','S','C',0 };
48 static const WCHAR BEFORE_W[] = { 'B','E','F','O','R','E',0 };
49 static const WCHAR BEGIN_W[] = { 'B','E','G','I','N','W',0 };
50 static const WCHAR BETWEEN_W[] = { 'B','E','T','W','E','E','N',0 };
51 static const WCHAR BY_W[] = { 'B','Y',0 };
52 static const WCHAR CASCADE_W[] = { 'C','A','S','C','A','D','E',0 };
53 static const WCHAR CASE_W[] = { 'C','A','S','E',0 };
54 static const WCHAR CHAR_W[] = { 'C','H','A','R',0 };
55 static const WCHAR CHARACTER_W[] = { 'C','H','A','R','A','C','T','E','R',0 };
56 static const WCHAR CHECK_W[] = { 'C','H','E','C','K',0 };
57 static const WCHAR CLUSTER_W[] = { 'C','L','U','S','T','E','R',0 };
58 static const WCHAR COLLATE_W[] = { 'C','O','L','L','A','T','E',0 };
59 static const WCHAR COMMIT_W[] = { 'C','O','M','M','I','T',0 };
60 static const WCHAR CONFLICT_W[] = { 'C','O','N','F','L','I','C','T',0 };
61 static const WCHAR CONSTRAINT_W[] = { 'C','O','N','S','T','R','A','I','N','T',0 };
62 static const WCHAR COPY_W[] = { 'C','O','P','Y',0 };
63 static const WCHAR CREATE_W[] = { 'C','R','E','A','T','E',0 };
64 static const WCHAR CROSS_W[] = { 'C','R','O','S','S',0 };
65 static const WCHAR DEFAULT_W[] = { 'D','E','F','A','U','L','T',0 };
66 static const WCHAR DEFERRED_W[] = { 'D','E','F','E','R','R','E','D',0 };
67 static const WCHAR DEFERRABLE_W[] = { 'D','E','F','E','R','R','A','B','L','E',0 };
68 static const WCHAR DELETE_W[] = { 'D','E','L','E','T','E',0 };
69 static const WCHAR DELIMITERS_W[] = { 'D','E','L','I','M','I','T','E','R','S',0 };
70 static const WCHAR DESC_W[] = { 'D','E','S','C',0 };
71 static const WCHAR DISTINCT_W[] = { 'D','I','S','T','I','N','C','T',0 };
72 static const WCHAR DROP_W[] = { 'D','R','O','P',0 };
73 static const WCHAR END_W[] = { 'E','N','D',0 };
74 static const WCHAR EACH_W[] = { 'E','A','C','H',0 };
75 static const WCHAR ELSE_W[] = { 'E','L','S','E',0 };
76 static const WCHAR EXCEPT_W[] = { 'E','X','C','E','P','T',0 };
77 static const WCHAR EXPLAIN_W[] = { 'E','X','P','L','A','I','N',0 };
78 static const WCHAR FAIL_W[] = { 'F','A','I','L',0 };
79 static const WCHAR FOR_W[] = { 'F','O','R',0 };
80 static const WCHAR FOREIGN_W[] = { 'F','O','R','E','I','G','N',0 };
81 static const WCHAR FREE_W[] = { 'F','R','E','E',0 };
82 static const WCHAR FROM_W[] = { 'F','R','O','M',0 };
83 static const WCHAR FULL_W[] = { 'F','U','L','L',0 };
84 static const WCHAR GLOB_W[] = { 'G','L','O','B',0 };
85 static const WCHAR GROUP_W[] = { 'G','R','O','U','P',0 };
86 static const WCHAR HAVING_W[] = { 'H','A','V','I','N','G',0 };
87 static const WCHAR HOLD_W[] = { 'H','O','L','D',0 };
88 static const WCHAR IGNORE_W[] = { 'I','G','N','O','R','E',0 };
89 static const WCHAR IMMEDIATE_W[] = { 'I','M','M','E','D','I','A','T','E',0 };
90 static const WCHAR IN_W[] = { 'I','N',0 };
91 static const WCHAR INDEX_W[] = { 'I','N','D','E','X',0 };
92 static const WCHAR INITIALLY_W[] = { 'I','N','I','T','I','A','L','L','Y',0 };
93 static const WCHAR INNER_W[] = { 'I','N','N','E','R',0 };
94 static const WCHAR INSERT_W[] = { 'I','N','S','E','R','T',0 };
95 static const WCHAR INSTEAD_W[] = { 'I','N','S','T','E','A','D',0 };
96 static const WCHAR INT_W[] = { 'I','N','T',0 };
97 static const WCHAR INTEGER_W[] = { 'I','N','T','E','G','E','R',0 };
98 static const WCHAR INTERSECT_W[] = { 'I','N','T','E','R','S','E','C','T',0 };
99 static const WCHAR INTO_W[] = { 'I','N','T','O',0 };
100 static const WCHAR IS_W[] = { 'I','S',0 };
101 static const WCHAR ISNULL_W[] = { 'I','S','N','U','L','L',0 };
102 static const WCHAR JOIN_W[] = { 'J','O','I','N',0 };
103 static const WCHAR KEY_W[] = { 'K','E','Y',0 };
104 static const WCHAR LEFT_W[] = { 'L','E','F','T',0 };
105 static const WCHAR LIKE_W[] = { 'L','I','K','E',0 };
106 static const WCHAR LIMIT_W[] = { 'L','I','M','I','T',0 };
107 static const WCHAR LOCALIZABLE_W[] = { 'L','O','C','A','L','I','Z','A','B','L','E',0 };
108 static const WCHAR LONG_W[] = { 'L','O','N','G',0 };
109 static const WCHAR LONGCHAR_W[] = { 'L','O','N','G','C','H','A','R',0 };
110 static const WCHAR MATCH_W[] = { 'M','A','T','C','H',0 };
111 static const WCHAR NATURAL_W[] = { 'N','A','T','U','R','A','L',0 };
112 static const WCHAR NOT_W[] = { 'N','O','T',0 };
113 static const WCHAR NOTNULL_W[] = { 'N','O','T','N','U','L','L',0 };
114 static const WCHAR NULL_W[] = { 'N','U','L','L',0 };
115 static const WCHAR OBJECT_W[] = { 'O','B','J','E','C','T',0 };
116 static const WCHAR OF_W[] = { 'O','F',0 };
117 static const WCHAR OFFSET_W[] = { 'O','F','F','S','E','T',0 };
118 static const WCHAR ON_W[] = { 'O','N',0 };
119 static const WCHAR OR_W[] = { 'O','R',0 };
120 static const WCHAR ORDER_W[] = { 'O','R','D','E','R',0 };
121 static const WCHAR OUTER_W[] = { 'O','U','T','E','R',0 };
122 static const WCHAR PRAGMA_W[] = { 'P','R','A','G','M','A',0 };
123 static const WCHAR PRIMARY_W[] = { 'P','R','I','M','A','R','Y',0 };
124 static const WCHAR RAISE_W[] = { 'R','A','I','S','E',0 };
125 static const WCHAR REFERENCES_W[] = { 'R','E','F','E','R','E','N','C','E','S',0 };
126 static const WCHAR REPLACE_W[] = { 'R','E','P','L','A','C','E',0 };
127 static const WCHAR RESTRICT_W[] = { 'R','E','S','T','R','I','C','T',0 };
128 static const WCHAR RIGHT_W[] = { 'R','I','G','H','T',0 };
129 static const WCHAR ROLLBACK_W[] = { 'R','O','L','L','B','A','C','K',0 };
130 static const WCHAR ROW_W[] = { 'R','O','W',0 };
131 static const WCHAR SELECT_W[] = { 'S','E','L','E','C','T',0 };
132 static const WCHAR SET_W[] = { 'S','E','T',0 };
133 static const WCHAR SHORT_W[] = { 'S','H','O','R','T',0 };
134 static const WCHAR STATEMENT_W[] = { 'S','T','A','T','E','M','E','N','T',0 };
135 static const WCHAR TABLE_W[] = { 'T','A','B','L','E',0 };
136 static const WCHAR TEMP_W[] = { 'T','E','M','P',0 };
137 static const WCHAR TEMPORARY_W[] = { 'T','E','M','P','O','R','A','R','Y',0 };
138 static const WCHAR THEN_W[] = { 'T','H','E','N',0 };
139 static const WCHAR TRANSACTION_W[] = { 'T','R','A','N','S','A','C','T','I','O','N',0 };
140 static const WCHAR TRIGGER_W[] = { 'T','R','I','G','G','E','R',0 };
141 static const WCHAR UNION_W[] = { 'U','N','I','O','N',0 };
142 static const WCHAR UNIQUE_W[] = { 'U','N','I','Q','U','E',0 };
143 static const WCHAR UPDATE_W[] = { 'U','P','D','A','T','E',0 };
144 static const WCHAR USING_W[] = { 'U','S','I','N','G',0 };
145 static const WCHAR VACUUM_W[] = { 'V','A','C','U','U','M',0 };
146 static const WCHAR VALUES_W[] = { 'V','A','L','U','E','S',0 };
147 static const WCHAR VIEW_W[] = { 'V','I','E','W',0 };
148 static const WCHAR WHEN_W[] = { 'W','H','E','N',0 };
149 static const WCHAR WHERE_W[] = { 'W','H','E','R','E',0 };
152 ** These are the keywords
154 static const Keyword aKeywordTable[] = {
155 { ABORT_W, TK_ABORT },
156 { AFTER_W, TK_AFTER },
157 { ALTER_W, TK_ALTER },
162 { BEFORE_W, TK_BEFORE },
163 { BEGIN_W, TK_BEGIN },
164 { BETWEEN_W, TK_BETWEEN },
166 { CASCADE_W, TK_CASCADE },
169 { CHARACTER_W, TK_CHAR },
170 { CHECK_W, TK_CHECK },
171 { CLUSTER_W, TK_CLUSTER },
172 { COLLATE_W, TK_COLLATE },
173 { COMMIT_W, TK_COMMIT },
174 { CONFLICT_W, TK_CONFLICT },
175 { CONSTRAINT_W, TK_CONSTRAINT },
177 { CREATE_W, TK_CREATE },
178 { CROSS_W, TK_JOIN_KW },
179 { DEFAULT_W, TK_DEFAULT },
180 { DEFERRED_W, TK_DEFERRED },
181 { DEFERRABLE_W, TK_DEFERRABLE },
182 { DELETE_W, TK_DELETE },
183 { DELIMITERS_W, TK_DELIMITERS },
185 { DISTINCT_W, TK_DISTINCT },
190 { EXCEPT_W, TK_EXCEPT },
191 { EXPLAIN_W, TK_EXPLAIN },
194 { FOREIGN_W, TK_FOREIGN },
196 { FULL_W, TK_JOIN_KW },
198 { GROUP_W, TK_GROUP },
199 { HAVING_W, TK_HAVING },
201 { IGNORE_W, TK_IGNORE },
202 { IMMEDIATE_W, TK_IMMEDIATE },
204 { INDEX_W, TK_INDEX },
205 { INITIALLY_W, TK_INITIALLY },
206 { INNER_W, TK_JOIN_KW },
207 { INSERT_W, TK_INSERT },
208 { INSTEAD_W, TK_INSTEAD },
210 { INTEGER_W, TK_INT },
211 { INTERSECT_W, TK_INTERSECT },
214 { ISNULL_W, TK_ISNULL },
217 { LEFT_W, TK_JOIN_KW },
219 { LIMIT_W, TK_LIMIT },
220 { LOCALIZABLE_W, TK_LOCALIZABLE },
222 { LONGCHAR_W, TK_LONGCHAR },
223 { MATCH_W, TK_MATCH },
224 { NATURAL_W, TK_JOIN_KW },
226 { NOTNULL_W, TK_NOTNULL },
228 { OBJECT_W, TK_OBJECT },
230 { OFFSET_W, TK_OFFSET },
233 { ORDER_W, TK_ORDER },
234 { OUTER_W, TK_JOIN_KW },
235 { PRAGMA_W, TK_PRAGMA },
236 { PRIMARY_W, TK_PRIMARY },
237 { RAISE_W, TK_RAISE },
238 { REFERENCES_W, TK_REFERENCES },
239 { REPLACE_W, TK_REPLACE },
240 { RESTRICT_W, TK_RESTRICT },
241 { RIGHT_W, TK_JOIN_KW },
242 { ROLLBACK_W, TK_ROLLBACK },
244 { SELECT_W, TK_SELECT },
246 { SHORT_W, TK_SHORT },
247 { STATEMENT_W, TK_STATEMENT },
248 { TABLE_W, TK_TABLE },
249 { TEMPORARY_W, TK_TEMPORARY },
251 { TRANSACTION_W, TK_TRANSACTION },
252 { TRIGGER_W, TK_TRIGGER },
253 { UNION_W, TK_UNION },
254 { UNIQUE_W, TK_UNIQUE },
255 { UPDATE_W, TK_UPDATE },
256 { USING_W, TK_USING },
257 { VACUUM_W, TK_VACUUM },
258 { VALUES_W, TK_VALUES },
261 { WHERE_W, TK_WHERE },
264 #define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )
267 ** Comparison function for binary search.
269 static int compKeyword(const void *m1, const void *m2){
270 const Keyword *k1 = m1, *k2 = m2;
272 return strcmpiW( k1->zName, k2->zName );
276 ** This function looks up an identifier to determine if it is a
277 ** keyword. If it is a keyword, the token code of that keyword is
278 ** returned. If the input is not a keyword, TK_ID is returned.
280 static int sqliteKeywordCode(const WCHAR *z, int n){
281 WCHAR str[MAX_TOKEN_LEN+1];
284 if( n>MAX_TOKEN_LEN )
287 memcpy( str, z, n*sizeof (WCHAR) );
291 r = bsearch( &key, aKeywordTable, KEYWORD_COUNT, sizeof (Keyword), compKeyword );
299 ** If X is a character that can be used in an identifier then
300 ** isIdChar[X] will be 1. Otherwise isIdChar[X] will be 0.
302 ** In this implementation, an identifier can be a string of
303 ** alphabetic characters, digits, and "_" plus any character
304 ** with the high-order bit set. The latter rule means that
305 ** any sequence of UTF-8 characters or characters taken from
306 ** an extended ISO8859 character set can form an identifier.
308 static const char isIdChar[] = {
309 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
310 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
311 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
312 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
313 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
314 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
315 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
316 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
317 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
318 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8x */
319 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9x */
320 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ax */
321 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Bx */
322 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Cx */
323 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Dx */
324 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ex */
325 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Fx */
330 ** Return the length of the token that begins at z[0]. Return
331 ** -1 if the token is (or might be) incomplete. Store the token
332 ** type in *tokenType before returning.
334 int sqliteGetToken(const WCHAR *z, int *tokenType){
337 case ' ': case '\t': case '\n': case '\f': case '\r': {
338 for(i=1; isspace(z[i]); i++){}
339 *tokenType = TK_SPACE;
343 if( z[1]==0 ) return -1;
345 for(i=2; z[i] && z[i]!='\n'; i++){}
346 *tokenType = TK_COMMENT;
349 *tokenType = TK_MINUS;
353 if( z[1]=='+' && z[2]==')' ){
354 *tokenType = TK_ORACLE_OUTER_JOIN;
366 *tokenType = TK_SEMI;
370 *tokenType = TK_PLUS;
374 *tokenType = TK_STAR;
378 if( z[1]!='*' || z[2]==0 ){
379 *tokenType = TK_SLASH;
382 for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
384 *tokenType = TK_COMMENT;
393 return 1 + (z[1]=='=');
399 }else if( z[1]=='>' ){
402 }else if( z[1]=='<' ){
403 *tokenType = TK_LSHIFT;
414 }else if( z[1]=='>' ){
415 *tokenType = TK_RSHIFT;
424 *tokenType = TK_ILLEGAL;
433 *tokenType = TK_BITOR;
436 *tokenType = TK_CONCAT;
441 *tokenType = TK_WILDCARD;
445 *tokenType = TK_COMMA;
449 *tokenType = TK_BITAND;
453 *tokenType = TK_BITNOT;
456 case '`': case '\'': case '"': {
471 *tokenType = TK_STRING;
475 if( !isdigit(z[1]) ){
479 /* Fall thru into the next case */
481 case '0': case '1': case '2': case '3': case '4':
482 case '5': case '6': case '7': case '8': case '9': {
483 *tokenType = TK_INTEGER;
484 for(i=1; isdigit(z[i]); i++){}
487 while( isdigit(z[i]) ){ i++; }
488 *tokenType = TK_FLOAT;
490 if( (z[i]=='e' || z[i]=='E') &&
492 || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
496 while( isdigit(z[i]) ){ i++; }
497 *tokenType = TK_FLOAT;
498 }else if( z[0]=='.' ){
499 *tokenType = TK_FLOAT;
504 for(i=1; z[i] && z[i-1]!=']'; i++){}
512 for(i=1; isIdChar[z[i]]; i++){}
513 *tokenType = sqliteKeywordCode(z, i);
517 *tokenType = TK_ILLEGAL;