4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
12 ** An tokenizer for SQL
14 ** This file contains C code that splits an SQL input string up into
15 ** individual tokens and sends those tokens one-by-one over to the
16 ** parser for analysis.
22 #include "wine/debug.h"
27 WINE_DEFAULT_DEBUG_CHANNEL(msi);
30 ** All the keywords of the SQL language are stored as in a hash
31 ** table composed of instances of the following structure.
33 typedef struct Keyword Keyword;
35 const char *zName; /* The keyword name */
36 int tokenType; /* The token value for this keyword */
40 ** These are the keywords
42 static const Keyword aKeywordTable[] = {
43 { "ABORT", TK_ABORT },
44 { "AFTER", TK_AFTER },
49 { "BEFORE", TK_BEFORE },
50 { "BEGIN", TK_BEGIN },
51 { "BETWEEN", TK_BETWEEN },
53 { "CASCADE", TK_CASCADE },
55 { "CHECK", TK_CHECK },
56 { "CLUSTER", TK_CLUSTER },
57 { "COLLATE", TK_COLLATE },
58 { "COMMIT", TK_COMMIT },
59 { "CONFLICT", TK_CONFLICT },
60 { "CONSTRAINT", TK_CONSTRAINT },
62 { "CREATE", TK_CREATE },
63 { "CROSS", TK_JOIN_KW },
64 { "DEFAULT", TK_DEFAULT },
65 { "DEFERRED", TK_DEFERRED },
66 { "DEFERRABLE", TK_DEFERRABLE },
67 { "DELETE", TK_DELETE },
68 { "DELIMITERS", TK_DELIMITERS },
70 { "DISTINCT", TK_DISTINCT },
75 { "EXCEPT", TK_EXCEPT },
76 { "EXPLAIN", TK_EXPLAIN },
79 { "FOREIGN", TK_FOREIGN },
81 { "FULL", TK_JOIN_KW },
83 { "GROUP", TK_GROUP },
84 { "HAVING", TK_HAVING },
85 { "IGNORE", TK_IGNORE },
86 { "IMMEDIATE", TK_IMMEDIATE },
88 { "INDEX", TK_INDEX },
89 { "INITIALLY", TK_INITIALLY },
90 { "INNER", TK_JOIN_KW },
91 { "INSERT", TK_INSERT },
92 { "INSTEAD", TK_INSTEAD },
93 { "INTERSECT", TK_INTERSECT },
96 { "ISNULL", TK_ISNULL },
99 { "LEFT", TK_JOIN_KW },
101 { "LIMIT", TK_LIMIT },
102 { "MATCH", TK_MATCH },
103 { "NATURAL", TK_JOIN_KW },
105 { "NOTNULL", TK_NOTNULL },
108 { "OFFSET", TK_OFFSET },
111 { "ORDER", TK_ORDER },
112 { "OUTER", TK_JOIN_KW },
113 { "PRAGMA", TK_PRAGMA },
114 { "PRIMARY", TK_PRIMARY },
115 { "RAISE", TK_RAISE },
116 { "REFERENCES", TK_REFERENCES },
117 { "REPLACE", TK_REPLACE },
118 { "RESTRICT", TK_RESTRICT },
119 { "RIGHT", TK_JOIN_KW },
120 { "ROLLBACK", TK_ROLLBACK },
122 { "SELECT", TK_SELECT },
124 { "STATEMENT", TK_STATEMENT },
125 { "TABLE", TK_TABLE },
127 { "TEMPORARY", TK_TEMP },
129 { "TRANSACTION", TK_TRANSACTION },
130 { "TRIGGER", TK_TRIGGER },
131 { "UNION", TK_UNION },
132 { "UNIQUE", TK_UNIQUE },
133 { "UPDATE", TK_UPDATE },
134 { "USING", TK_USING },
135 { "VACUUM", TK_VACUUM },
136 { "VALUES", TK_VALUES },
139 { "WHERE", TK_WHERE },
142 #define KEYWORD_COUNT ( sizeof aKeywordTable/sizeof (Keyword) )
145 ** This function looks up an identifier to determine if it is a
146 ** keyword. If it is a keyword, the token code of that keyword is
147 ** returned. If the input is not a keyword, TK_ID is returned.
149 int sqliteKeywordCode(const WCHAR *z, int n){
153 len = WideCharToMultiByte( CP_ACP, 0, z, n, buffer, sizeof buffer, NULL, NULL );
154 for(i=0; i<KEYWORD_COUNT; i++)
156 if(memcmp(buffer, aKeywordTable[i].zName, len))
158 if(strlen(aKeywordTable[i].zName) == len )
159 return aKeywordTable[i].tokenType;
166 ** If X is a character that can be used in an identifier then
167 ** isIdChar[X] will be 1. Otherwise isIdChar[X] will be 0.
169 ** In this implementation, an identifier can be a string of
170 ** alphabetic characters, digits, and "_" plus any character
171 ** with the high-order bit set. The latter rule means that
172 ** any sequence of UTF-8 characters or characters taken from
173 ** an extended ISO8859 character set can form an identifier.
175 static const char isIdChar[] = {
176 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
177 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
178 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
179 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
180 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
181 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
182 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
183 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
184 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
185 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8x */
186 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9x */
187 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ax */
188 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Bx */
189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Cx */
190 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Dx */
191 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ex */
192 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Fx */
197 ** Return the length of the token that begins at z[0]. Return
198 ** -1 if the token is (or might be) incomplete. Store the token
199 ** type in *tokenType before returning.
201 int sqliteGetToken(const WCHAR *z, int *tokenType){
204 case ' ': case '\t': case '\n': case '\f': case '\r': {
205 for(i=1; isspace(z[i]); i++){}
206 *tokenType = TK_SPACE;
210 if( z[1]==0 ) return -1;
212 for(i=2; z[i] && z[i]!='\n'; i++){}
213 *tokenType = TK_COMMENT;
216 *tokenType = TK_MINUS;
220 if( z[1]=='+' && z[2]==')' ){
221 *tokenType = TK_ORACLE_OUTER_JOIN;
233 *tokenType = TK_SEMI;
237 *tokenType = TK_PLUS;
241 *tokenType = TK_STAR;
245 if( z[1]!='*' || z[2]==0 ){
246 *tokenType = TK_SLASH;
249 for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
251 *tokenType = TK_COMMENT;
260 return 1 + (z[1]=='=');
266 }else if( z[1]=='>' ){
269 }else if( z[1]=='<' ){
270 *tokenType = TK_LSHIFT;
281 }else if( z[1]=='>' ){
282 *tokenType = TK_RSHIFT;
291 *tokenType = TK_ILLEGAL;
300 *tokenType = TK_BITOR;
303 *tokenType = TK_CONCAT;
308 *tokenType = TK_COMMA;
312 *tokenType = TK_BITAND;
316 *tokenType = TK_BITNOT;
319 case '`': case '\'': case '"': {
331 *tokenType = TK_STRING;
335 if( !isdigit(z[1]) ){
339 /* Fall thru into the next case */
341 case '0': case '1': case '2': case '3': case '4':
342 case '5': case '6': case '7': case '8': case '9': {
343 *tokenType = TK_INTEGER;
344 for(i=1; isdigit(z[i]); i++){}
347 while( isdigit(z[i]) ){ i++; }
348 *tokenType = TK_FLOAT;
350 if( (z[i]=='e' || z[i]=='E') &&
352 || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
356 while( isdigit(z[i]) ){ i++; }
357 *tokenType = TK_FLOAT;
358 }else if( z[0]=='.' ){
359 *tokenType = TK_FLOAT;
364 for(i=1; z[i] && z[i-1]!=']'; i++){}
372 for(i=1; isIdChar[z[i]]; i++){}
373 *tokenType = sqliteKeywordCode(z, i);
377 *tokenType = TK_ILLEGAL;