git.oblomov.eu Git - wine/blob - tools/wmc/mcl.c

   1 /*
   2  * Wine Message Compiler lexical scanner
   3  *
   4  * Copyright 2000 Bertho A. Stultiens (BS)
   5  *
   6  */
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <ctype.h>
  10 #include <assert.h>
  11 #include <string.h>
  12
  13 #include "config.h"
  14
  15 #include "utils.h"
  16 #include "wmc.h"
  17 #include "lang.h"
  18
  19 #include "y.tab.h"
  20
  21 /*
  22  * Keywords are case insenitive. All normal input is treated as
  23  * being in codepage iso-8859-1 for ascii input files (unicode
  24  * page 0) and as equivalent unicode if unicode input is selected.
  25  * All normal input, which is not part of a message text, is
  26  * enforced to be unicode page 0. Otherwise an error will be
  27  * generated. The normal file data should only be ASCII because
  28  * that is the basic definition of the grammar.
  29  *
  30  * Byteorder or unicode input is determined automatically by
  31  * reading the first 8 bytes and checking them against unicode
  32  * page 0 byteorder (hibyte must be 0).
  33  * -- FIXME --
  34  * Alternatively, the input is checked against a special byte
  35  * sequence to identify the file.
  36  * -- FIXME --
  37  *
  38  *
  39  * Keywords:
  40  *      Codepages
  41  *      Facility
  42  *      FacilityNames
  43  *      LanguageNames
  44  *      MessageId
  45  *      MessageIdTypedef
  46  *      Severity
  47  *      SeverityNames
  48  *      SymbolicName
  49  *
  50  * Default added identifiers for classes:
  51  * SeverityNames:
  52  *      Success         = 0x0
  53  *      Informational   = 0x1
  54  *      Warning         = 0x2
  55  *      Error           = 0x3
  56  * FacilityNames:
  57  *      System          = 0x0FF
  58  *      Application     = 0xFFF
  59  *
  60  * The 'Codepages' keyword is a wmc extension.
  61  */
  62
  63 static WCHAR ustr_application[] = { 'A', 'p', 'p', 'l', 'i', 'c', 'a', 't', 'i', 'o', 'n', 0 };
  64 static WCHAR ustr_codepages[]           = { 'C', 'o', 'd', 'e', 'p', 'a', 'g', 'e', 's', 0 };
  65 static WCHAR ustr_english[]             = { 'E', 'n', 'g', 'l', 'i', 's', 'h', 0 };
  66 static WCHAR ustr_error[]               = { 'E', 'r', 'r', 'o', 'r', 0 };
  67 static WCHAR ustr_facility[]            = { 'F', 'a', 'c', 'i', 'l', 'i', 't', 'y', 0 };
  68 static WCHAR ustr_facilitynames[]       = { 'F', 'a', 'c', 'i', 'l', 'i', 't', 'y', 'N', 'a', 'm', 'e', 's', 0 };
  69 static WCHAR ustr_informational[]       = { 'I', 'n', 'f', 'o', 'r', 'm', 'a', 't', 'i', 'o', 'n', 'a', 'l', 0 };
  70 static WCHAR ustr_language[]            = { 'L', 'a', 'n', 'g', 'u', 'a', 'g', 'e', 0};
  71 static WCHAR ustr_languagenames[]       = { 'L', 'a', 'n', 'g', 'u', 'a', 'g', 'e', 'N', 'a', 'm', 'e', 's', 0};
  72 static WCHAR ustr_messageid[]           = { 'M', 'e', 's', 's', 'a', 'g', 'e', 'I', 'd', 0 };
  73 static WCHAR ustr_messageidtypedef[]    = { 'M', 'e', 's', 's', 'a', 'g', 'e', 'I', 'd', 'T', 'y', 'p', 'e', 'd', 'e', 'f', 0 };
  74 static WCHAR ustr_outputbase[]  = { 'O', 'u', 't', 'p', 'u', 't', 'B', 'a', 's', 'e', 0 };
  75 static WCHAR ustr_severity[]            = { 'S', 'e', 'v', 'e', 'r', 'i', 't', 'y', 0 };
  76 static WCHAR ustr_severitynames[]       = { 'S', 'e', 'v', 'e', 'r', 'i', 't', 'y', 'N', 'a', 'm', 'e', 's', 0 };
  77 static WCHAR ustr_success[]             = { 'S', 'u', 'c', 'c', 'e', 's', 's', 0 };
  78 static WCHAR ustr_symbolicname[]        = { 'S', 'y', 'm', 'b', 'o', 'l', 'i', 'c', 'N', 'a', 'm', 'e', 0 };
  79 static WCHAR ustr_system[]              = { 'S', 'y', 's', 't', 'e', 'm', 0 };
  80 static WCHAR ustr_warning[]             = { 'W', 'a', 'r', 'n', 'i', 'n', 'g', 0 };
  81 static WCHAR ustr_msg00001[]            = { 'm', 's', 'g', '0', '0', '0', '0', '1', 0 };
  82 /*
  83  * This table is to beat any form of "expression building" to check for
  84  * correct filename characters. It is also used for ident checks.
  85  * FIXME: use it more consistently.
  86  */
  87
  88 #define CH_SHORTNAME    0x01
  89 #define CH_LONGNAME     0x02
  90 #define CH_IDENT        0x04
  91 #define CH_NUMBER       0x08
  92 /*#define CH_WILDCARD   0x10*/
  93 /*#define CH_DOT        0x20*/
  94 #define CH_PUNCT        0x40
  95 #define CH_INVALID      0x80
  96
  97 static const char char_table[256] = {
  98         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x00 - 0x07 */
  99         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x08 - 0x0F */
 100         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x10 - 0x17 */
 101         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x18 - 0x1F */
 102         0x80, 0x03, 0x80, 0x03, 0x03, 0x03, 0x03, 0x03, /* 0x20 - 0x27 " !"#$%&'" */
 103         0x43, 0x43, 0x10, 0x80, 0x03, 0x03, 0x22, 0x80, /* 0x28 - 0x2F "()*+,-./" */
 104         0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, /* 0x30 - 0x37 "01234567" */
 105         0x0b, 0x0b, 0xc0, 0x80, 0x80, 0x80, 0x80, 0x10, /* 0x38 - 0x3F "89:;<=>?" */
 106         0x03, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x40 - 0x47 "@ABCDEFG" */
 107         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x48 - 0x4F "HIJKLMNO" */
 108         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x50 - 0x57 "PQRSTUVW" */
 109         0x07, 0x07, 0x07, 0x80, 0x80, 0x80, 0x80, 0x07, /* 0x58 - 0x5F "XYZ[\]^_" */
 110         0x03, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x60 - 0x67 "`abcdefg" */
 111         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x68 - 0x6F "hijklmno" */
 112         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x70 - 0x77 "pqrstuvw" */
 113         0x07, 0x07, 0x07, 0x03, 0x80, 0x03, 0x03, 0x80, /* 0x78 - 0x7F "xyz{|}~ " */
 114         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x80 - 0x87 */
 115         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x88 - 0x8F */
 116         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x90 - 0x97 */
 117         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x98 - 0x9F */
 118         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xA0 - 0xA7 */
 119         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xA8 - 0xAF */
 120         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xB0 - 0xB7 */
 121         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xB8 - 0xBF */
 122         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xC0 - 0xC7 */
 123         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xC8 - 0xCF */
 124         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xD0 - 0xD7 */
 125         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xD8 - 0xDF */
 126         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xE0 - 0xE7 */
 127         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xE8 - 0xEF */
 128         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xF0 - 0xF7 */
 129         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x80, /* 0xF8 - 0xFF */
 130 };
 131
 132 static int isisochar(int ch)
 133 {
 134         return !(ch & (~0xff));
 135 }
 136
 137 static int codepage;
 138 static const union cptable *codepage_def;
 139
 140 void set_codepage(int cp)
 141 {
 142         codepage = cp;
 143         codepage_def = find_codepage(codepage);
 144         if(!codepage_def)
 145                 xyyerror("Codepage %d not found; cannot process", codepage);
 146 }
 147
 148 /*
 149  * Input functions
 150  */
 151 static int nungetstack = 0;
 152 static int allocungetstack = 0;
 153 static char *ungetstack = NULL;
 154 static int ninputbuffer = 0;
 155 static WCHAR *inputbuffer = NULL;
 156 static char *xlatebuffer = NULL;
 157
 158 #define INPUTBUFFER_SIZE        2048    /* Must be larger than 4 and approx. large enough to hold a line */
 159
 160 /*
 161  * Fill the input buffer with *one* line of input.
 162  * The line is '\n' terminated so that scanning
 163  * messages with translation works as expected
 164  * (otherwise we cannot pre-translate because the
 165  * language is first known one line before the
 166  * actual message).
 167  */
 168 static int fill_inputbuffer(void)
 169 {
 170         int n;
 171         static char err_fatalread[] = "Fatal: reading input failed";
 172         static int endian = -1;
 173
 174         if(!inputbuffer)
 175         {
 176                 inputbuffer = xmalloc(INPUTBUFFER_SIZE);
 177                 xlatebuffer = xmalloc(INPUTBUFFER_SIZE);
 178         }
 179
 180 try_again:
 181         if(!unicodein)
 182         {
 183                 char *cptr;
 184                 cptr = fgets(xlatebuffer, INPUTBUFFER_SIZE, yyin);
 185                 if(!cptr && ferror(yyin))
 186                         xyyerror(err_fatalread);
 187                 else if(!cptr)
 188                         return 0;
 189                 assert(codepage_def != NULL);
 190                 n = cp_mbstowcs(codepage_def, 0, xlatebuffer, strlen(xlatebuffer)+1, inputbuffer, INPUTBUFFER_SIZE);
 191                 if(n < 0)
 192                         internal_error(__FILE__, __LINE__, "Could not translate to unicode (%d)", n);
 193                 if(n <= 1)
 194                         goto try_again; /* Should not hapen */
 195                 n--;    /* Strip added conversion '\0' from input length */
 196                 /*
 197                  * FIXME:
 198                  * Detect UTF-8 in the first time we read some bytes by
 199                  * checking the special sequence "FE..." or something like
 200                  * that. I need to check www.unicode.org for details.
 201                  */
 202         }
 203         else
 204         {
 205                 if(endian == -1)
 206                 {
 207                         n = fread(inputbuffer, 1, 8, yyin);
 208                         if(n != 8)
 209                         {
 210                                 if(!n && ferror(yyin))
 211                                         xyyerror(err_fatalread);
 212                                 else
 213                                         xyyerror("Fatal: file to short to determine byteorder (should never happen)");
 214                         }
 215                         if(isisochar(inputbuffer[0]) &&
 216                                 isisochar(inputbuffer[1]) &&
 217                                 isisochar(inputbuffer[2]) &&
 218                                 isisochar(inputbuffer[3]))
 219                         {
 220 #ifdef WORDS_BIGENDIAN
 221                                 endian = WMC_BO_BIG;
 222 #else
 223                                 endian = WMC_BO_LITTLE;
 224 #endif
 225                         }
 226                         else if(isisochar(BYTESWAP_WORD(inputbuffer[0])) &&
 227                                 isisochar(BYTESWAP_WORD(inputbuffer[1])) &&
 228                                 isisochar(BYTESWAP_WORD(inputbuffer[2])) &&
 229                                 isisochar(BYTESWAP_WORD(inputbuffer[3])))
 230                         {
 231 #ifdef WORDS_BIGENDIAN
 232                                 endian = WMC_BO_LITTLE;
 233 #else
 234                                 endian = WMC_BO_BIG;
 235 #endif
 236                         }
 237                         else
 238                                 xyyerror("Fatal: cannot determine file's byteorder");
 239                         /* FIXME:
 240                          * Determine the file-endian with the leader-bytes
 241                          * "FF FE..."; can't remember the exact sequence.
 242                          */
 243                         n /= 2;
 244 #ifdef WORDS_BIGENDIAN
 245                         if(endian == WMC_BO_LITTLE)
 246 #else
 247                         if(endian == WMC_BO_BIG)
 248 #endif
 249                         {
 250                                 inputbuffer[0] = BYTESWAP_WORD(inputbuffer[0]);
 251                                 inputbuffer[1] = BYTESWAP_WORD(inputbuffer[1]);
 252                                 inputbuffer[2] = BYTESWAP_WORD(inputbuffer[2]);
 253                                 inputbuffer[3] = BYTESWAP_WORD(inputbuffer[3]);
 254                         }
 255
 256                 }
 257                 else
 258                 {
 259                         int i;
 260                         n = 0;
 261                         for(i = 0; i < INPUTBUFFER_SIZE; i++)
 262                         {
 263                                 int t;
 264                                 t = fread(&inputbuffer[i], 2, 1, yyin);
 265                                 if(!t && ferror(yyin))
 266                                         xyyerror(err_fatalread);
 267                                 else if(!t && n)
 268                                         break;
 269                                 n++;
 270 #ifdef WORDS_BIGENDIAN
 271                                 if(endian == WMC_BO_LITTLE)
 272 #else
 273                                 if(endian == WMC_BO_BIG)
 274 #endif
 275                                 {
 276                                         if((inputbuffer[i] = BYTESWAP_WORD(inputbuffer[i])) == '\n')
 277                                                 break;
 278                                 }
 279                                 else
 280                                 {
 281                                         if(inputbuffer[i] == '\n')
 282                                                 break;
 283                                 }
 284                         }
 285                 }
 286
 287         }
 288
 289         if(!n)
 290         {
 291                 yywarning("Re-read line (input was or converted to zilch)");
 292                 goto try_again; /* Should not happen, but could be due to stdin reading and a signal */
 293         }
 294
 295         ninputbuffer += n;
 296         return 1;
 297 }
 298
 299 static int get_unichar(void)
 300 {
 301         static WCHAR *b = NULL;
 302         char_number++;
 303
 304         if(nungetstack)
 305                 return ungetstack[--nungetstack];
 306
 307         if(!ninputbuffer)
 308         {
 309                 if(!fill_inputbuffer())
 310                         return EOF;
 311                 b = inputbuffer;
 312         }
 313
 314         ninputbuffer--;
 315         return (int)(*b++ & 0xffff);
 316 }
 317
 318 static void unget_unichar(int ch)
 319 {
 320         if(ch == EOF)
 321                 return;
 322
 323         char_number--;
 324
 325         if(nungetstack == allocungetstack)
 326         {
 327                 allocungetstack += 32;
 328                 ungetstack = xrealloc(ungetstack, allocungetstack * sizeof(*ungetstack));
 329         }
 330
 331         ungetstack[nungetstack++] = (WCHAR)ch;
 332 }
 333
 334
 335 /*
 336  * Normal character stack.
 337  * Used for number scanning.
 338  */
 339 static int ncharstack = 0;
 340 static int alloccharstack = 0;
 341 static char *charstack = NULL;
 342
 343 static void empty_char_stack(void)
 344 {
 345         ncharstack = 0;
 346 }
 347
 348 static void push_char(int ch)
 349 {
 350         if(ncharstack == alloccharstack)
 351         {
 352                 alloccharstack += 32;
 353                 charstack = xrealloc(charstack, alloccharstack * sizeof(*charstack));
 354         }
 355         charstack[ncharstack++] = (char)ch;
 356 }
 357
 358 static int tos_char_stack(void)
 359 {
 360         if(!ncharstack)
 361                 return 0;
 362         else
 363                 return (int)(charstack[ncharstack-1] & 0xff);
 364 }
 365
 366 static char *get_char_stack(void)
 367 {
 368         return charstack;
 369 }
 370
 371 /*
 372  * Unicode character stack.
 373  * Used for general scanner.
 374  */
 375 static int nunicharstack = 0;
 376 static int allocunicharstack = 0;
 377 static WCHAR *unicharstack = NULL;
 378
 379 static void empty_unichar_stack(void)
 380 {
 381         nunicharstack = 0;
 382 }
 383
 384 static void push_unichar(int ch)
 385 {
 386         if(nunicharstack == allocunicharstack)
 387         {
 388                 allocunicharstack += 128;
 389                 unicharstack = xrealloc(unicharstack, allocunicharstack * sizeof(*unicharstack));
 390         }
 391         unicharstack[nunicharstack++] = (WCHAR)ch;
 392 }
 393
 394 #if 0
 395 static int tos_unichar_stack(void)
 396 {
 397         if(!nunicharstack)
 398                 return 0;
 399         else
 400                 return (int)(unicharstack[nunicharstack-1] & 0xffff);
 401 }
 402 #endif
 403
 404 static WCHAR *get_unichar_stack(void)
 405 {
 406         return unicharstack;
 407 }
 408
 409 /*
 410  * Number scanner
 411  *
 412  * state |      ch         | next state
 413  * ------+-----------------+--------------------------
 414  *   0   | [0]             | 1
 415  *   0   | [1-9]           | 4
 416  *   0   | .               | error (should never occur)
 417  *   1   | [xX]            | 2
 418  *   1   | [0-7]           | 3
 419  *   1   | [89a-wyzA-WYZ_] | error invalid digit
 420  *   1   | .               | return 0
 421  *   2   | [0-9a-fA-F]     | 2
 422  *   2   | [g-zG-Z_]       | error invalid hex digit
 423  *   2   | .               | return (hex-number) if TOS != [xX] else error
 424  *   3   | [0-7]           | 3
 425  *   3   | [89a-zA-Z_]     | error invalid octal digit
 426  *   3   | .               | return (octal-number)
 427  *   4   | [0-9]           | 4
 428  *   4   | [a-zA-Z_]       | error invalid decimal digit
 429  *   4   | .               | return (decimal-number)
 430  *
 431  * All non-identifier characters [^a-zA-Z_0-9] terminate the scan
 432  * and return the value. This is not entirely correct, but close
 433  * enough (should check punctuators as trailing context, but the
 434  * char_table is not adapted to that and it is questionable whether
 435  * it is worth the trouble).
 436  * All non-iso-8859-1 characters are an error.
 437  */
 438 static int scan_number(int ch)
 439 {
 440         int state = 0;
 441         int base = 10;
 442         empty_char_stack();
 443
 444         while(1)
 445         {
 446                 if(!isisochar(ch))
 447                         xyyerror("Invalid digit");
 448
 449                 switch(state)
 450                 {
 451                 case 0:
 452                         if(isdigit(ch))
 453                         {
 454                                 push_char(ch);
 455                                 if(ch == '0')
 456                                         state = 1;
 457                                 else
 458                                         state = 4;
 459                         }
 460                         else
 461                                 internal_error(__FILE__, __LINE__, "Non-digit in first number-scanner state");
 462                         break;
 463                 case 1:
 464                         if(ch == 'x' || ch == 'X')
 465                         {
 466                                 push_char(ch);
 467                                 state = 2;
 468                         }
 469                         else if(ch >= '0' && ch <= '7')
 470                         {
 471                                 push_char(ch);
 472                                 state = 3;
 473                         }
 474                         else if(isalpha(ch) || ch == '_')
 475                                 xyyerror("Invalid number digit");
 476                         else
 477                         {
 478                                 unget_unichar(ch);
 479                                 yylval.num = 0;
 480                                 return tNUMBER;
 481                         }
 482                         break;
 483                 case 2:
 484                         if(isxdigit(ch))
 485                                 push_char(ch);
 486                         else if(isalpha(ch) || ch == '_' || !isxdigit(tos_char_stack()))
 487                                 xyyerror("Invalid hex digit");
 488                         else
 489                         {
 490                                 base = 16;
 491                                 goto finish;
 492                         }
 493                         break;
 494                 case 3:
 495                         if(ch >= '0' && ch <= '7')
 496                                 push_char(ch);
 497                         else if(isalnum(ch) || ch == '_')
 498                                 xyyerror("Invalid octal digit");
 499                         else
 500                         {
 501                                 base = 8;
 502                                 goto finish;
 503                         }
 504                         break;
 505                 case 4:
 506                         if(isdigit(ch))
 507                                 push_char(ch);
 508                         else if(isalnum(ch) || ch == '_')
 509                                 xyyerror("Invalid decimal digit");
 510                         else
 511                         {
 512                                 base = 10;
 513                                 goto finish;
 514                         }
 515                         break;
 516                 default:
 517                         internal_error(__FILE__, __LINE__, "Invalid state in number-scanner");
 518                 }
 519                 ch = get_unichar();
 520         }
 521 finish:
 522         unget_unichar(ch);
 523         push_char(0);
 524         yylval.num = strtoul(get_char_stack(), NULL, base);
 525         return tNUMBER;
 526 }
 527
 528 static void newline(void)
 529 {
 530         line_number++;
 531         char_number = 1;
 532 }
 533
 534 static int unisort(const void *p1, const void *p2)
 535 {
 536         return unistricmp(((token_t *)p1)->name, ((token_t *)p2)->name);
 537 }
 538
 539 static token_t *tokentable = NULL;
 540 static int ntokentable = 0;
 541
 542 token_t *lookup_token(const WCHAR *s)
 543 {
 544         token_t tok;
 545
 546         tok.name = s;
 547         return (token_t *)bsearch(&tok, tokentable, ntokentable, sizeof(*tokentable), unisort);
 548 }
 549
 550 void add_token(tok_e type, const WCHAR *name, int tok, int cp, const WCHAR *alias, int fix)
 551 {
 552         ntokentable++;
 553         tokentable = xrealloc(tokentable, ntokentable * sizeof(*tokentable));
 554         tokentable[ntokentable-1].type = type;
 555         tokentable[ntokentable-1].name = name;
 556         tokentable[ntokentable-1].token = tok;
 557         tokentable[ntokentable-1].codepage = cp;
 558         tokentable[ntokentable-1].alias = alias;
 559         tokentable[ntokentable-1].fixed = fix;
 560         qsort(tokentable, ntokentable, sizeof(*tokentable), unisort);
 561 }
 562
 563 void get_tokentable(token_t **tab, int *len)
 564 {
 565         assert(tab != NULL);
 566         assert(len != NULL);
 567         *tab = tokentable;
 568         *len = ntokentable;
 569 }
 570
 571 /*
 572  * The scanner
 573  *
 574  */
 575 int yylex(void)
 576 {
 577         static WCHAR ustr_dot1[] = { '.', '\n', 0 };
 578         static WCHAR ustr_dot2[] = { '.', '\r', '\n', 0 };
 579         static int isinit = 0;
 580         int ch;
 581
 582         if(!isinit)
 583         {
 584                 isinit++;
 585                 set_codepage(WMC_DEFAULT_CODEPAGE);
 586                 add_token(tok_keyword,  ustr_codepages,         tCODEPAGE,      0, NULL, 0);
 587                 add_token(tok_keyword,  ustr_facility,          tFACILITY,      0, NULL, 1);
 588                 add_token(tok_keyword,  ustr_facilitynames,     tFACNAMES,      0, NULL, 1);
 589                 add_token(tok_keyword,  ustr_language,          tLANGUAGE,      0, NULL, 1);
 590                 add_token(tok_keyword,  ustr_languagenames,     tLANNAMES,      0, NULL, 1);
 591                 add_token(tok_keyword,  ustr_messageid,         tMSGID,         0, NULL, 1);
 592                 add_token(tok_keyword,  ustr_messageidtypedef,  tTYPEDEF,       0, NULL, 1);
 593                 add_token(tok_keyword,  ustr_outputbase,        tBASE,          0, NULL, 1);
 594                 add_token(tok_keyword,  ustr_severity,          tSEVERITY,      0, NULL, 1);
 595                 add_token(tok_keyword,  ustr_severitynames,     tSEVNAMES,      0, NULL, 1);
 596                 add_token(tok_keyword,  ustr_symbolicname,      tSYMNAME,       0, NULL, 1);
 597                 add_token(tok_severity, ustr_error,             0x03,           0, NULL, 0);
 598                 add_token(tok_severity, ustr_warning,           0x02,           0, NULL, 0);
 599                 add_token(tok_severity, ustr_informational,     0x01,           0, NULL, 0);
 600                 add_token(tok_severity, ustr_success,           0x00,           0, NULL, 0);
 601                 add_token(tok_facility, ustr_application,       0xFFF,          0, NULL, 0);
 602                 add_token(tok_facility, ustr_system,            0x0FF,          0, NULL, 0);
 603                 add_token(tok_language, ustr_english,           0x409,          437, ustr_msg00001, 0);
 604         }
 605
 606         empty_unichar_stack();
 607
 608         while(1)
 609         {
 610                 if(want_line)
 611                 {
 612                         while((ch = get_unichar()) != '\n')
 613                         {
 614                                 if(ch == EOF)
 615                                         xyyerror("Unexpected EOF");
 616                                 push_unichar(ch);
 617                         }
 618                         newline();
 619                         push_unichar(ch);
 620                         push_unichar(0);
 621                         if(!unistrcmp(ustr_dot1, get_unichar_stack()) || !unistrcmp(ustr_dot2, get_unichar_stack()))
 622                         {
 623                                 want_line = 0;
 624                                 /* Reset the codepage to our default after each message */
 625                                 set_codepage(WMC_DEFAULT_CODEPAGE);
 626                                 return tMSGEND;
 627                         }
 628                         yylval.str = xunistrdup(get_unichar_stack());
 629                         return tLINE;
 630                 }
 631
 632                 ch = get_unichar();
 633
 634                 if(ch == EOF)
 635                         return EOF;
 636
 637                 if(ch == '\n')
 638                 {
 639                         newline();
 640                         if(want_nl)
 641                         {
 642                                 want_nl = 0;
 643                                 return tNL;
 644                         }
 645                         continue;
 646                 }
 647
 648                 if(isisochar(ch))
 649                 {
 650                         if(want_file)
 651                         {
 652                                 int n = 0;
 653                                 while(n < 8 && isisochar(ch))
 654                                 {
 655                                         int t = char_table[ch];
 656                                         if((t & CH_PUNCT) || !(t & CH_SHORTNAME))
 657                                                 break;
 658
 659                                         push_unichar(ch);
 660                                         n++;
 661                                         ch = get_unichar();
 662                                 }
 663                                 unget_unichar(ch);
 664                                 push_unichar(0);
 665                                 want_file = 0;
 666                                 yylval.str = xunistrdup(get_unichar_stack());
 667                                 return tFILE;
 668                         }
 669
 670                         if(char_table[ch] & CH_IDENT)
 671                         {
 672                                 token_t *tok;
 673                                 while(isisochar(ch) && (char_table[ch] & (CH_IDENT|CH_NUMBER)))
 674                                 {
 675                                         push_unichar(ch);
 676                                         ch = get_unichar();
 677                                 }
 678                                 unget_unichar(ch);
 679                                 push_unichar(0);
 680                                 if(!(tok = lookup_token(get_unichar_stack())))
 681                                 {
 682                                         yylval.str = xunistrdup(get_unichar_stack());
 683                                         return tIDENT;
 684                                 }
 685                                 switch(tok->type)
 686                                 {
 687                                 case tok_keyword:
 688                                         return tok->token;
 689
 690                                 case tok_language:
 691                                         codepage = tok->codepage;
 692                                         /* Fall through */
 693                                 case tok_severity:
 694                                 case tok_facility:
 695                                         yylval.tok = tok;
 696                                         return tTOKEN;
 697
 698                                 default:
 699                                         internal_error(__FILE__, __LINE__, "Invalid token type encountered");
 700                                 }
 701                         }
 702
 703                         if(isspace(ch)) /* Ignore space */
 704                                 continue;
 705
 706                         if(isdigit(ch))
 707                                 return scan_number(ch);
 708                 }
 709
 710                 switch(ch)
 711                 {
 712                 case ':':
 713                 case '=':
 714                 case '+':
 715                 case '(':
 716                 case ')':
 717                         return ch;
 718                 case ';':
 719                         while(ch != '\n' && ch != EOF)
 720                         {
 721                                 push_unichar(ch);
 722                                 ch = get_unichar();
 723                         }
 724                         newline();
 725                         push_unichar(ch);       /* Include the newline */
 726                         push_unichar(0);
 727                         yylval.str = xunistrdup(get_unichar_stack());
 728                         return tCOMMENT;
 729                 default:
 730                         xyyerror("Invalid character '%c' (0x%04x)", isisochar(ch) && isprint(ch) ? ch : '.', ch);
 731                 }
 732         }
 733 }
 734