git.oblomov.eu Git - wine/blob - libs/wpp/ppl.l

   1 /* -*-C-*-
   2  * Wrc preprocessor lexical analysis
   3  *
   4  * Copyright 1999-2000  Bertho A. Stultiens (BS)
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  *
  20  * History:
  21  * 24-Apr-2000 BS       - Started from scratch to restructure everything
  22  *                        and reintegrate the source into the wine-tree.
  23  * 04-Jan-2000 BS       - Added comments about the lexicographical
  24  *                        grammar to give some insight in the complexity.
  25  * 28-Dec-1999 BS       - Eliminated backing-up of the flexer by running
  26  *                        `flex -b' on the source. This results in some
  27  *                        weirdo extra rules, but a much faster scanner.
  28  * 23-Dec-1999 BS       - Started this file
  29  *
  30  *-------------------------------------------------------------------------
  31  * The preprocessor's lexographical grammar (approximately):
  32  *
  33  * pp           := {ws} # {ws} if {ws} {expr} {ws} \n
  34  *              |  {ws} # {ws} ifdef {ws} {id} {ws} \n
  35  *              |  {ws} # {ws} ifndef {ws} {id} {ws} \n
  36  *              |  {ws} # {ws} elif {ws} {expr} {ws} \n
  37  *              |  {ws} # {ws} else {ws} \n
  38  *              |  {ws} # {ws} endif {ws} \n
  39  *              |  {ws} # {ws} include {ws} < {anytext} > \n
  40  *              |  {ws} # {ws} include {ws} " {anytext} " \n
  41  *              |  {ws} # {ws} define {ws} {anytext} \n
  42  *              |  {ws} # {ws} define( {arglist} ) {ws} {expansion} \n
  43  *              |  {ws} # {ws} pragma {ws} {anytext} \n
  44  *              |  {ws} # {ws} ident {ws} {anytext} \n
  45  *              |  {ws} # {ws} error {ws} {anytext} \n
  46  *              |  {ws} # {ws} warning {ws} {anytext} \n
  47  *              |  {ws} # {ws} line {ws} " {anytext} " {number} \n
  48  *              |  {ws} # {ws} {number} " {anytext} " {number} [ {number} [{number}] ] \n
  49  *              |  {ws} # {ws} \n
  50  *
  51  * ws           := [ \t\r\f\v]*
  52  *
  53  * expr         := {expr} [+-*%^/|&] {expr}
  54  *              |  {expr} {logor|logand} {expr}
  55  *              |  [!~+-] {expr}
  56  *              |  {expr} ? {expr} : {expr}
  57  *
  58  * logor        := ||
  59  *
  60  * logand       := &&
  61  *
  62  * id           := [a-zA-Z_][a-zA-Z0-9_]*
  63  *
  64  * anytext      := [^\n]*       (see note)
  65  *
  66  * arglist      :=
  67  *              |  {id}
  68  *              |  {arglist} , {id}
  69  *              |  {arglist} , {id} ...
  70  *
  71  * expansion    := {id}
  72  *              |  # {id}
  73  *              |  {anytext}
  74  *              |  {anytext} ## {anytext}
  75  *
  76  * number       := [0-9]+
  77  *
  78  * Note: "anytext" is not always "[^\n]*". This is because the
  79  *       trailing context must be considered as well.
  80  *
  81  * The only certain assumption for the preprocessor to make is that
  82  * directives start at the beginning of the line, followed by a '#'
  83  * and end with a newline.
  84  * Any directive may be suffixed with a line-continuation. Also
  85  * classical comment / *...* / (note: no comments within comments,
  86  * therefore spaces) is considered to be a line-continuation
  87  * (according to gcc and egcs AFAIK, ANSI is a bit vague).
  88  * Comments have not been added to the above grammar for simplicity
  89  * reasons. However, it is allowed to enter comment anywhere within
  90  * the directives as long as they do not interfere with the context.
  91  * All comments are considered to be deletable whitespace (both
  92  * classical form "/ *...* /" and C++ form "//...\n").
  93  *
  94  * All recursive scans, except for macro-expansion, are done by the
  95  * parser, whereas the simple state transitions of non-recursive
  96  * directives are done in the scanner. This results in the many
  97  * exclusive start-conditions of the scanner.
  98  *
  99  * Macro expansions are slightly more difficult because they have to
 100  * prescan the arguments. Parameter substitution is literal if the
 101  * substitution is # or ## (either side). This enables new identifiers
 102  * to be created (see 'info cpp' node Macro|Pitfalls|Prescan for more
 103  * information).
 104  *
 105  * FIXME: Variable macro parameters is recognized, but not yet
 106  * expanded. I have to reread the ANSI standard on the subject (yes,
 107  * ANSI defines it).
 108  *
 109  * The following special defines are supported:
 110  * __FILE__     -> "thissource.c"
 111  * __LINE__     -> 123
 112  * __DATE__     -> "May  1 2000"
 113  * __TIME__     -> "23:59:59"
 114  * These macros expand, as expected, into their ANSI defined values.
 115  *
 116  * The same include prevention is implemented as gcc and egcs does.
 117  * This results in faster processing because we do not read the text
 118  * at all. Some wine-sources attempt to include the same file 4 or 5
 119  * times. This strategy also saves a lot blank output-lines, which in
 120  * its turn improves the real resource scanner/parser.
 121  *
 122  */
 123
 124 /*
 125  * Special flex options and exclusive scanner start-conditions
 126  */
 127 %option stack
 128 %option 8bit never-interactive
 129 %option noinput nounput
 130 %option prefix="ppy_"
 131
 132 %x pp_pp
 133 %x pp_eol
 134 %x pp_inc
 135 %x pp_dqs
 136 %x pp_sqs
 137 %x pp_iqs
 138 %x pp_comment
 139 %x pp_def
 140 %x pp_define
 141 %x pp_macro
 142 %x pp_mbody
 143 %x pp_macign
 144 %x pp_macscan
 145 %x pp_macexp
 146 %x pp_if
 147 %x pp_ifd
 148 %x pp_endif
 149 %x pp_line
 150 %x pp_defined
 151 %x pp_ignore
 152 %x RCINCL
 153
 154 ws      [ \v\f\t\r]
 155 cident  [a-zA-Z_][0-9a-zA-Z_]*
 156 ul      [uUlL]|[uUlL][lL]|[lL][uU]|[lL][lL][uU]|[uU][lL][lL]|[lL][uU][lL]
 157
 158 %{
 159 #include "config.h"
 160 #include "wine/port.h"
 161 #include <stdio.h>
 162 #include <stdlib.h>
 163 #include <string.h>
 164 #include <ctype.h>
 165 #include <assert.h>
 166 #include <errno.h>
 167 #include <limits.h>
 168
 169 #ifndef LLONG_MAX
 170 # define LLONG_MAX  ((long long)0x7fffffff << 32 | 0xffffffff)
 171 # define LLONG_MIN  (-LLONG_MAX - 1)
 172 #endif
 173 #ifndef ULLONG_MAX
 174 # define ULLONG_MAX ((long long)0xffffffff << 32 | 0xffffffff)
 175 #endif
 176
 177 #ifndef HAVE_UNISTD_H
 178 #define YY_NO_UNISTD_H
 179 #endif
 180
 181 #include "wpp_private.h"
 182 #include "ppy.tab.h"
 183
 184 /*
 185  * Make sure that we are running an appropriate version of flex.
 186  */
 187 #if !defined(YY_FLEX_MAJOR_VERSION) || (1000 * YY_FLEX_MAJOR_VERSION + YY_FLEX_MINOR_VERSION < 2005)
 188 #error Must use flex version 2.5.1 or higher (yy_scan_* routines are required).
 189 #endif
 190
 191 #define YY_READ_BUF_SIZE        65536           /* So we read most of a file at once */
 192
 193 #define yy_current_state()      YY_START
 194 #define yy_pp_state(x)          yy_pop_state(); yy_push_state(x)
 195
 196 /*
 197  * Always update the current character position within a line
 198  */
 199 #define YY_USER_ACTION  pp_status.char_number+=ppy_leng;
 200
 201 /*
 202  * Buffer management for includes and expansions
 203  */
 204 #define MAXBUFFERSTACK  128     /* Nesting more than 128 includes or macro expansion textss is insane */
 205
 206 typedef struct bufferstackentry {
 207         YY_BUFFER_STATE bufferstate;    /* Buffer to switch back to */
 208         pp_entry_t      *define;        /* Points to expanding define or NULL if handling includes */
 209         int             line_number;    /* Line that we were handling */
 210         int             char_number;    /* The current position on that line */
 211         const char      *filename;      /* Filename that we were handling */
 212         int             if_depth;       /* How many #if:s deep to check matching #endif:s */
 213         int             ncontinuations; /* Remember the continuation state */
 214         int             should_pop;     /* Set if we must pop the start-state on EOF */
 215         /* Include management */
 216         include_state_t incl;
 217         char            *include_filename;
 218 } bufferstackentry_t;
 219
 220 #define ALLOCBLOCKSIZE  (1 << 10)       /* Allocate these chunks at a time for string-buffers */
 221
 222 /*
 223  * Macro expansion nesting
 224  * We need the stack to handle expansions while scanning
 225  * a macro's arguments. The TOS must always be the macro
 226  * that receives the current expansion from the scanner.
 227  */
 228 #define MAXMACEXPSTACK  128     /* Nesting more than 128 macro expansions is insane */
 229
 230 typedef struct macexpstackentry {
 231         pp_entry_t      *ppp;           /* This macro we are scanning */
 232         char            **args;         /* With these arguments */
 233         char            **ppargs;       /* Resulting in these preprocessed arguments */
 234         int             *nnls;          /* Number of newlines per argument */
 235         int             nargs;          /* And this many arguments scanned */
 236         int             parentheses;    /* Nesting level of () */
 237         int             curargsize;     /* Current scanning argument's size */
 238         int             curargalloc;    /* Current scanning argument's block allocated */
 239         char            *curarg;        /* Current scanning argument's content */
 240 } macexpstackentry_t;
 241
 242 #define MACROPARENTHESES()      (top_macro()->parentheses)
 243
 244 /*
 245  * Prototypes
 246  */
 247 static void newline(int);
 248 static int make_number(int radix, YYSTYPE *val, const char *str, int len);
 249 static void put_buffer(const char *s, int len);
 250 /* Buffer management */
 251 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop);
 252 static bufferstackentry_t *pop_buffer(void);
 253 /* String functions */
 254 static void new_string(void);
 255 static void add_string(const char *str, int len);
 256 static char *get_string(void);
 257 static void put_string(void);
 258 static int string_start(void);
 259 /* Macro functions */
 260 static void push_macro(pp_entry_t *ppp);
 261 static macexpstackentry_t *top_macro(void);
 262 static macexpstackentry_t *pop_macro(void);
 263 static void free_macro(macexpstackentry_t *mep);
 264 static void add_text_to_macro(const char *text, int len);
 265 static void macro_add_arg(int last);
 266 static void macro_add_expansion(void);
 267 /* Expansion */
 268 static void expand_special(pp_entry_t *ppp);
 269 static void expand_define(pp_entry_t *ppp);
 270 static void expand_macro(macexpstackentry_t *mep);
 271
 272 /*
 273  * Local variables
 274  */
 275 static int ncontinuations;
 276
 277 static int strbuf_idx = 0;
 278 static int strbuf_alloc = 0;
 279 static char *strbuffer = NULL;
 280 static int str_startline;
 281
 282 static macexpstackentry_t *macexpstack[MAXMACEXPSTACK];
 283 static int macexpstackidx = 0;
 284
 285 static bufferstackentry_t bufferstack[MAXBUFFERSTACK];
 286 static int bufferstackidx = 0;
 287
 288 /*
 289  * Global variables
 290  */
 291 include_state_t pp_incl_state =
 292 {
 293     -1,    /* state */
 294     NULL,  /* ppp */
 295     0,     /* ifdepth */
 296     0      /* seen_junk */
 297 };
 298
 299 includelogicentry_t *pp_includelogiclist = NULL;
 300
 301 %}
 302
 303 /*
 304  **************************************************************************
 305  * The scanner starts here
 306  **************************************************************************
 307  */
 308
 309 %%
 310         /*
 311          * Catch line-continuations.
 312          * Note: Gcc keeps the line-continuations in, for example, strings
 313          * intact. However, I prefer to remove them all so that the next
 314          * scanner will not need to reduce the continuation state.
 315          *
 316          * <*>\\\n              newline(0);
 317          */
 318
 319         /*
 320          * Detect the leading # of a preprocessor directive.
 321          */
 322 <INITIAL,pp_ignore>^{ws}*#      pp_incl_state.seen_junk++; yy_push_state(pp_pp);
 323
 324         /*
 325          * Scan for the preprocessor directives
 326          */
 327 <pp_pp>{ws}*include{ws}*        if(yy_top_state() != pp_ignore) {yy_pp_state(pp_inc); return tINCLUDE;} else {yy_pp_state(pp_eol);}
 328 <pp_pp>{ws}*define{ws}*         yy_pp_state(yy_current_state() != pp_ignore ? pp_def : pp_eol);
 329 <pp_pp>{ws}*error{ws}*          yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tERROR;
 330 <pp_pp>{ws}*warning{ws}*        yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tWARNING;
 331 <pp_pp>{ws}*pragma{ws}*         yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tPRAGMA;
 332 <pp_pp>{ws}*ident{ws}*          yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tPPIDENT;
 333 <pp_pp>{ws}*undef{ws}*          if(yy_top_state() != pp_ignore) {yy_pp_state(pp_ifd); return tUNDEF;} else {yy_pp_state(pp_eol);}
 334 <pp_pp>{ws}*ifdef{ws}*          yy_pp_state(pp_ifd);    return tIFDEF;
 335 <pp_pp>{ws}*ifndef{ws}*         pp_incl_state.seen_junk--; yy_pp_state(pp_ifd); return tIFNDEF;
 336 <pp_pp>{ws}*if{ws}*             yy_pp_state(pp_if);     return tIF;
 337 <pp_pp>{ws}*elif{ws}*           yy_pp_state(pp_if);     return tELIF;
 338 <pp_pp>{ws}*else{ws}*           yy_pp_state(pp_endif);  return tELSE;
 339 <pp_pp>{ws}*endif{ws}*          yy_pp_state(pp_endif);  return tENDIF;
 340 <pp_pp>{ws}*line{ws}*           if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tLINE;} else {yy_pp_state(pp_eol);}
 341 <pp_pp>{ws}+                    if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tGCCLINE;} else {yy_pp_state(pp_eol);}
 342 <pp_pp>{ws}*[a-z]+              ppy_error("Invalid preprocessor token '%s'", ppy_text);
 343 <pp_pp>\r?\n                    newline(1); yy_pop_state(); return tNL; /* This could be the null-token */
 344 <pp_pp>\\\r?\n                  newline(0);
 345 <pp_pp>\\\r?                    ppy_error("Preprocessor junk '%s'", ppy_text);
 346 <pp_pp>.                        return *ppy_text;
 347
 348         /*
 349          * Handle #include and #line
 350          */
 351 <pp_line>[0-9]+                 return make_number(10, &ppy_lval, ppy_text, ppy_leng);
 352 <pp_inc>\<                      new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_iqs);
 353 <pp_inc,pp_line>\"              new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 354 <pp_inc,pp_line>{ws}+           ;
 355 <pp_inc,pp_line>\n              newline(1); yy_pop_state(); return tNL;
 356 <pp_inc,pp_line>\\\r?\n         newline(0);
 357 <pp_inc,pp_line>(\\\r?)|(.)     ppy_error(yy_current_state() == pp_inc ? "Trailing junk in #include" : "Trailing junk in #line");
 358
 359         /*
 360          * Ignore all input when a false clause is parsed
 361          */
 362 <pp_ignore>[^#/\\\n]+           ;
 363 <pp_ignore>\n                   newline(1);
 364 <pp_ignore>\\\r?\n              newline(0);
 365 <pp_ignore>(\\\r?)|(.)          ;
 366
 367         /*
 368          * Handle #if and #elif.
 369          * These require conditionals to be evaluated, but we do not
 370          * want to jam the scanner normally when we see these tokens.
 371          * Note: tIDENT is handled below.
 372          */
 373
 374 <pp_if>0[0-7]*{ul}?             return make_number(8, &ppy_lval, ppy_text, ppy_leng);
 375 <pp_if>0[0-7]*[8-9]+{ul}?       ppy_error("Invalid octal digit");
 376 <pp_if>[1-9][0-9]*{ul}?         return make_number(10, &ppy_lval, ppy_text, ppy_leng);
 377 <pp_if>0[xX][0-9a-fA-F]+{ul}?   return make_number(16, &ppy_lval, ppy_text, ppy_leng);
 378 <pp_if>0[xX]                    ppy_error("Invalid hex number");
 379 <pp_if>defined                  yy_push_state(pp_defined); return tDEFINED;
 380 <pp_if>"<<"                     return tLSHIFT;
 381 <pp_if>">>"                     return tRSHIFT;
 382 <pp_if>"&&"                     return tLOGAND;
 383 <pp_if>"||"                     return tLOGOR;
 384 <pp_if>"=="                     return tEQ;
 385 <pp_if>"!="                     return tNE;
 386 <pp_if>"<="                     return tLTE;
 387 <pp_if>">="                     return tGTE;
 388 <pp_if>\n                       newline(1); yy_pop_state(); return tNL;
 389 <pp_if>\\\r?\n                  newline(0);
 390 <pp_if>\\\r?                    ppy_error("Junk in conditional expression");
 391 <pp_if>{ws}+                    ;
 392 <pp_if>\'                       new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 393 <pp_if>\"                       ppy_error("String constants not allowed in conditionals");
 394 <pp_if>.                        return *ppy_text;
 395
 396         /*
 397          * Handle #ifdef, #ifndef and #undef
 398          * to get only an untranslated/unexpanded identifier
 399          */
 400 <pp_ifd>{cident}        ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 401 <pp_ifd>{ws}+           ;
 402 <pp_ifd>\n              newline(1); yy_pop_state(); return tNL;
 403 <pp_ifd>\\\r?\n         newline(0);
 404 <pp_ifd>(\\\r?)|(.)     ppy_error("Identifier expected");
 405
 406         /*
 407          * Handle #else and #endif.
 408          */
 409 <pp_endif>{ws}+         ;
 410 <pp_endif>\n            newline(1); yy_pop_state(); return tNL;
 411 <pp_endif>\\\r?\n       newline(0);
 412 <pp_endif>.             ppy_error("Garbage after #else or #endif.");
 413
 414         /*
 415          * Handle the special 'defined' keyword.
 416          * This is necessary to get the identifier prior to any
 417          * substitutions.
 418          */
 419 <pp_defined>{cident}            yy_pop_state(); ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 420 <pp_defined>{ws}+               ;
 421 <pp_defined>(\()|(\))           return *ppy_text;
 422 <pp_defined>\\\r?\n             newline(0);
 423 <pp_defined>(\\.)|(\n)|(.)      ppy_error("Identifier expected");
 424
 425         /*
 426          * Handle #error, #warning, #pragma and #ident.
 427          * Pass everything literally to the parser, which
 428          * will act appropriately.
 429          * Comments are stripped from the literal text.
 430          */
 431 <pp_eol>[^/\\\n]+               if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
 432 <pp_eol>\/[^/\\\n*]*            if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
 433 <pp_eol>(\\\r?)|(\/[^/*])       if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
 434 <pp_eol>\n                      newline(1); yy_pop_state(); if(yy_current_state() != pp_ignore) { return tNL; }
 435 <pp_eol>\\\r?\n                 newline(0);
 436
 437         /*
 438          * Handle left side of #define
 439          */
 440 <pp_def>{cident}\(              ppy_lval.cptr = pp_xstrdup(ppy_text); if(ppy_lval.cptr) ppy_lval.cptr[ppy_leng-1] = '\0'; yy_pp_state(pp_macro);  return tMACRO;
 441 <pp_def>{cident}                ppy_lval.cptr = pp_xstrdup(ppy_text); yy_pp_state(pp_define); return tDEFINE;
 442 <pp_def>{ws}+                   ;
 443 <pp_def>\\\r?\n                 newline(0);
 444 <pp_def>(\\\r?)|(\n)|(.)        perror("Identifier expected");
 445
 446         /*
 447          * Scan the substitution of a define
 448          */
 449 <pp_define>[^'"/\\\n]+          ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 450 <pp_define>(\\\r?)|(\/[^/*])    ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 451 <pp_define>\\\r?\n{ws}+         newline(0); ppy_lval.cptr = pp_xstrdup(" "); return tLITERAL;
 452 <pp_define>\\\r?\n              newline(0);
 453 <pp_define>\n                   newline(1); yy_pop_state(); return tNL;
 454 <pp_define>\'                   new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 455 <pp_define>\"                   new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 456
 457         /*
 458          * Scan the definition macro arguments
 459          */
 460 <pp_macro>\){ws}*               yy_pp_state(pp_mbody); return tMACROEND;
 461 <pp_macro>{ws}+                 ;
 462 <pp_macro>{cident}              ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 463 <pp_macro>,                     return ',';
 464 <pp_macro>"..."                 return tELIPSIS;
 465 <pp_macro>(\\\r?)|(\n)|(.)|(\.\.?)      ppy_error("Argument identifier expected");
 466 <pp_macro>\\\r?\n               newline(0);
 467
 468         /*
 469          * Scan the substitution of a macro
 470          */
 471 <pp_mbody>[^a-zA-Z0-9'"#/\\\n]+ ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 472 <pp_mbody>{cident}              ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 473 <pp_mbody>\#\#                  return tCONCAT;
 474 <pp_mbody>\#                    return tSTRINGIZE;
 475 <pp_mbody>[0-9][^'"#/\\\n]*     ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 476 <pp_mbody>(\\\r?)|(\/[^/*'"#\\\n]*)     ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 477 <pp_mbody>\\\r?\n{ws}+          newline(0); ppy_lval.cptr = pp_xstrdup(" "); return tLITERAL;
 478 <pp_mbody>\\\r?\n               newline(0);
 479 <pp_mbody>\n                    newline(1); yy_pop_state(); return tNL;
 480 <pp_mbody>\'                    new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 481 <pp_mbody>\"                    new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 482
 483         /*
 484          * Macro expansion text scanning.
 485          * This state is active just after the identifier is scanned
 486          * that triggers an expansion. We *must* delete the leading
 487          * whitespace before we can start scanning for arguments.
 488          *
 489          * If we do not see a '(' as next trailing token, then we have
 490          * a false alarm. We just continue with a nose-bleed...
 491          */
 492 <pp_macign>{ws}*/\(     yy_pp_state(pp_macscan);
 493 <pp_macign>{ws}*\n      {
 494                 if(yy_top_state() != pp_macscan)
 495                         newline(0);
 496         }
 497 <pp_macign>{ws}*\\\r?\n newline(0);
 498 <pp_macign>{ws}+|{ws}*\\\r?|.   {
 499                 macexpstackentry_t *mac = pop_macro();
 500                 yy_pop_state();
 501                 put_buffer(mac->ppp->ident, strlen(mac->ppp->ident));
 502                 put_buffer(ppy_text, ppy_leng);
 503                 free_macro(mac);
 504         }
 505
 506         /*
 507          * Macro expansion argument text scanning.
 508          * This state is active when a macro's arguments are being read for expansion.
 509          */
 510 <pp_macscan>\(  {
 511                 if(++MACROPARENTHESES() > 1)
 512                         add_text_to_macro(ppy_text, ppy_leng);
 513         }
 514 <pp_macscan>\)  {
 515                 if(--MACROPARENTHESES() == 0)
 516                 {
 517                         yy_pop_state();
 518                         macro_add_arg(1);
 519                 }
 520                 else
 521                         add_text_to_macro(ppy_text, ppy_leng);
 522         }
 523 <pp_macscan>,           {
 524                 if(MACROPARENTHESES() > 1)
 525                         add_text_to_macro(ppy_text, ppy_leng);
 526                 else
 527                         macro_add_arg(0);
 528         }
 529 <pp_macscan>\"          new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 530 <pp_macscan>\'          new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 531 <pp_macscan>"/*"        yy_push_state(pp_comment); add_text_to_macro(" ", 1);
 532 <pp_macscan>\n          pp_status.line_number++; pp_status.char_number = 1; add_text_to_macro(ppy_text, ppy_leng);
 533 <pp_macscan>([^/(),\\\n"']+)|(\/[^/*(),\\\n'"]*)|(\\\r?)|(.)    add_text_to_macro(ppy_text, ppy_leng);
 534 <pp_macscan>\\\r?\n     newline(0);
 535
 536         /*
 537          * Comment handling (almost all start-conditions)
 538          */
 539 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_endif,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,RCINCL>"/*" yy_push_state(pp_comment);
 540 <pp_comment>[^*\n]*|"*"+[^*/\n]*        ;
 541 <pp_comment>\n                          newline(0);
 542 <pp_comment>"*"+"/"                     yy_pop_state();
 543
 544         /*
 545          * Remove C++ style comment (almost all start-conditions)
 546          */
 547 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_endif,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,pp_macscan,RCINCL>"//"[^\n]* {
 548                 if(ppy_text[ppy_leng-1] == '\\')
 549                         ppy_warning("C++ style comment ends with an escaped newline (escape ignored)");
 550         }
 551
 552         /*
 553          * Single, double and <> quoted constants
 554          */
 555 <INITIAL,pp_macexp>\"           pp_incl_state.seen_junk++; new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 556 <INITIAL,pp_macexp>\'           pp_incl_state.seen_junk++; new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 557 <pp_dqs>[^"\\\n]+               add_string(ppy_text, ppy_leng);
 558 <pp_dqs>\"                      {
 559                 add_string(ppy_text, ppy_leng);
 560                 yy_pop_state();
 561                 switch(yy_current_state())
 562                 {
 563                 case pp_pp:
 564                 case pp_define:
 565                 case pp_mbody:
 566                 case pp_inc:
 567                 case RCINCL:
 568                         if (yy_current_state()==RCINCL) yy_pop_state();
 569                         ppy_lval.cptr = get_string();
 570                         return tDQSTRING;
 571                 case pp_line:
 572                         ppy_lval.cptr = get_string();
 573                         return tDQSTRING;
 574                 default:
 575                         put_string();
 576                 }
 577         }
 578 <pp_sqs>[^'\\\n]+               add_string(ppy_text, ppy_leng);
 579 <pp_sqs>\'                      {
 580                 add_string(ppy_text, ppy_leng);
 581                 yy_pop_state();
 582                 switch(yy_current_state())
 583                 {
 584                 case pp_if:
 585                 case pp_define:
 586                 case pp_mbody:
 587                         ppy_lval.cptr = get_string();
 588                         return tSQSTRING;
 589                 default:
 590                         put_string();
 591                 }
 592         }
 593 <pp_iqs>[^\>\\\n]+              add_string(ppy_text, ppy_leng);
 594 <pp_iqs>\>                      {
 595                 add_string(ppy_text, ppy_leng);
 596                 yy_pop_state();
 597                 ppy_lval.cptr = get_string();
 598                 return tIQSTRING;
 599         }
 600 <pp_dqs>\\\r?\n         {
 601                 /*
 602                  * This is tricky; we need to remove the line-continuation
 603                  * from preprocessor strings, but OTOH retain them in all
 604                  * other strings. This is because the resource grammar is
 605                  * even more braindead than initially analysed and line-
 606                  * continuations in strings introduce, sigh, newlines in
 607                  * the output. There goes the concept of non-breaking, non-
 608                  * spacing whitespace.
 609                  */
 610                 switch(yy_top_state())
 611                 {
 612                 case pp_pp:
 613                 case pp_define:
 614                 case pp_mbody:
 615                 case pp_inc:
 616                 case pp_line:
 617                         newline(0);
 618                         break;
 619                 default:
 620                         add_string(ppy_text, ppy_leng);
 621                         newline(-1);
 622                 }
 623         }
 624 <pp_iqs,pp_dqs,pp_sqs>\\.       add_string(ppy_text, ppy_leng);
 625 <pp_iqs,pp_dqs,pp_sqs>\n        {
 626                 newline(1);
 627                 add_string(ppy_text, ppy_leng);
 628                 ppy_warning("Newline in string constant encounterd (started line %d)", string_start());
 629         }
 630
 631         /*
 632          * Identifier scanning
 633          */
 634 <INITIAL,pp_if,pp_inc,pp_macexp>{cident}        {
 635                 pp_entry_t *ppp;
 636                 pp_incl_state.seen_junk++;
 637                 if(!(ppp = pplookup(ppy_text)))
 638                 {
 639                         if(yy_current_state() == pp_inc)
 640                                 ppy_error("Expected include filename");
 641
 642                         else if(yy_current_state() == pp_if)
 643                         {
 644                                 ppy_lval.cptr = pp_xstrdup(ppy_text);
 645                                 return tIDENT;
 646                         }
 647                         else {
 648                                 if((yy_current_state()==INITIAL) && (strcasecmp(ppy_text,"RCINCLUDE")==0)){
 649                                         yy_push_state(RCINCL);
 650                                         return tRCINCLUDE;
 651                                 }
 652                                 else put_buffer(ppy_text, ppy_leng);
 653                         }
 654                 }
 655                 else if(!ppp->expanding)
 656                 {
 657                         switch(ppp->type)
 658                         {
 659                         case def_special:
 660                                 expand_special(ppp);
 661                                 break;
 662                         case def_define:
 663                                 expand_define(ppp);
 664                                 break;
 665                         case def_macro:
 666                                 yy_push_state(pp_macign);
 667                                 push_macro(ppp);
 668                                 break;
 669                         default:
 670                                 pp_internal_error(__FILE__, __LINE__, "Invalid define type %d\n", ppp->type);
 671                         }
 672                 }
 673                 else put_buffer(ppy_text, ppy_leng);
 674         }
 675
 676         /*
 677          * Everything else that needs to be passed and
 678          * newline and continuation handling
 679          */
 680 <INITIAL,pp_macexp>[^a-zA-Z_#'"/\\\n \r\t\f\v]+|(\/|\\)[^a-zA-Z_/*'"\\\n \r\t\v\f]*     pp_incl_state.seen_junk++; put_buffer(ppy_text, ppy_leng);
 681 <INITIAL,pp_macexp>{ws}+        put_buffer(ppy_text, ppy_leng);
 682 <INITIAL>\n                     newline(1);
 683 <INITIAL>\\\r?\n                newline(0);
 684 <INITIAL>\\\r?                  pp_incl_state.seen_junk++; put_buffer(ppy_text, ppy_leng);
 685
 686         /*
 687          * Special catcher for macro argmument expansion to prevent
 688          * newlines to propagate to the output or admin.
 689          */
 690 <pp_macexp>(\n)|(.)|(\\\r?(\n|.))       put_buffer(ppy_text, ppy_leng);
 691
 692 <RCINCL>[A-Za-z0-9_\.\\/]+ {
 693                 ppy_lval.cptr=pp_xstrdup(ppy_text);
 694                 yy_pop_state();
 695                 return tRCINCLUDEPATH;
 696         }
 697
 698 <RCINCL>{ws}+ ;
 699
 700 <RCINCL>\"              {
 701                 new_string(); add_string(ppy_text,ppy_leng);yy_push_state(pp_dqs);
 702         }
 703
 704         /*
 705          * This is a 'catch-all' rule to discover errors in the scanner
 706          * in an orderly manner.
 707          */
 708 <*>.            pp_incl_state.seen_junk++; ppy_warning("Unmatched text '%c' (0x%02x); please report\n", isprint(*ppy_text & 0xff) ? *ppy_text : ' ', *ppy_text);
 709
 710 <<EOF>> {
 711                 YY_BUFFER_STATE b = YY_CURRENT_BUFFER;
 712                 bufferstackentry_t *bep = pop_buffer();
 713
 714                 if((!bep && pp_get_if_depth()) || (bep && pp_get_if_depth() != bep->if_depth))
 715                         ppy_warning("Unmatched #if/#endif at end of file");
 716
 717                 if(!bep)
 718                 {
 719                         if(YY_START != INITIAL)
 720                                 ppy_error("Unexpected end of file during preprocessing");
 721                         yyterminate();
 722                 }
 723                 else if(bep->should_pop == 2)
 724                 {
 725                         macexpstackentry_t *mac;
 726                         mac = pop_macro();
 727                         expand_macro(mac);
 728                 }
 729                 ppy__delete_buffer(b);
 730         }
 731
 732 %%
 733 /*
 734  **************************************************************************
 735  * Support functions
 736  **************************************************************************
 737  */
 738
 739 #ifndef ppy_wrap
 740 int ppy_wrap(void)
 741 {
 742         return 1;
 743 }
 744 #endif
 745
 746
 747 /*
 748  *-------------------------------------------------------------------------
 749  * Output newlines or set them as continuations
 750  *
 751  * Input: -1 - Don't count this one, but update local position (see pp_dqs)
 752  *         0 - Line-continuation seen and cache output
 753  *         1 - Newline seen and flush output
 754  *-------------------------------------------------------------------------
 755  */
 756 static void newline(int dowrite)
 757 {
 758         pp_status.line_number++;
 759         pp_status.char_number = 1;
 760
 761         if(dowrite == -1)
 762                 return;
 763
 764         ncontinuations++;
 765         if(dowrite)
 766         {
 767                 for(;ncontinuations; ncontinuations--)
 768                         put_buffer("\n", 1);
 769         }
 770 }
 771
 772
 773 /*
 774  *-------------------------------------------------------------------------
 775  * Make a number out of an any-base and suffixed string
 776  *
 777  * Possible number extensions:
 778  * - ""         int
 779  * - "L"        long int
 780  * - "LL"       long long int
 781  * - "U"        unsigned int
 782  * - "UL"       unsigned long int
 783  * - "ULL"      unsigned long long int
 784  * - "LU"       unsigned long int
 785  * - "LLU"      unsigned long long int
 786  * - "LUL"      invalid
 787  *
 788  * FIXME:
 789  * The sizes of resulting 'int' and 'long' are compiler specific.
 790  * I depend on sizeof(int) > 2 here (although a relatively safe
 791  * assumption).
 792  * Long longs are not yet implemented because this is very compiler
 793  * specific and I don't want to think too much about the problems.
 794  *
 795  *-------------------------------------------------------------------------
 796  */
 797 static int make_number(int radix, YYSTYPE *val, const char *str, int len)
 798 {
 799         int is_l  = 0;
 800         int is_ll = 0;
 801         int is_u  = 0;
 802         char ext[4];
 803         long l;
 804
 805         ext[3] = '\0';
 806         ext[2] = toupper(str[len-1]);
 807         ext[1] = len > 1 ? toupper(str[len-2]) : ' ';
 808         ext[0] = len > 2 ? toupper(str[len-3]) : ' ';
 809
 810         if(!strcmp(ext, "LUL"))
 811         {
 812                 ppy_error("Invalid constant suffix");
 813                 return 0;
 814         }
 815         else if(!strcmp(ext, "LLU") || !strcmp(ext, "ULL"))
 816         {
 817                 is_ll++;
 818                 is_u++;
 819         }
 820         else if(!strcmp(ext+1, "LU") || !strcmp(ext+1, "UL"))
 821         {
 822                 is_l++;
 823                 is_u++;
 824         }
 825         else if(!strcmp(ext+1, "LL"))
 826         {
 827                 is_ll++;
 828         }
 829         else if(!strcmp(ext+2, "L"))
 830         {
 831                 is_l++;
 832         }
 833         else if(!strcmp(ext+2, "U"))
 834         {
 835                 is_u++;
 836         }
 837
 838         if(is_ll)
 839         {
 840 /* Assume as in the declaration of wrc_ull_t and wrc_sll_t */
 841 #ifdef HAVE_LONG_LONG
 842                 if (is_u)
 843                 {
 844                         errno = 0;
 845                         val->ull = strtoull(str, NULL, radix);
 846                         if (val->ull == ULLONG_MAX && errno == ERANGE)
 847                                 ppy_error("integer constant %s is too large\n", str);
 848                         return tULONGLONG;
 849                 }
 850                 else
 851                 {
 852                         errno = 0;
 853                         val->sll = strtoll(str, NULL, radix);
 854                         if ((val->sll == LLONG_MIN || val->sll == LLONG_MAX) && errno == ERANGE)
 855                                 ppy_error("integer constant %s is too large\n", str);
 856                         return tSLONGLONG;
 857                 }
 858 #else
 859                 pp_internal_error(__FILE__, __LINE__, "long long constants not supported on this platform");
 860 #endif
 861         }
 862         else if(is_u && is_l)
 863         {
 864                 errno = 0;
 865                 val->ulong = strtoul(str, NULL, radix);
 866                 if (val->ulong == ULONG_MAX && errno == ERANGE)
 867                         ppy_error("integer constant %s is too large\n", str);
 868                 return tULONG;
 869         }
 870         else if(!is_u && is_l)
 871         {
 872                 errno = 0;
 873                 val->slong = strtol(str, NULL, radix);
 874                 if ((val->slong == LONG_MIN || val->slong == LONG_MAX) && errno == ERANGE)
 875                         ppy_error("integer constant %s is too large\n", str);
 876                 return tSLONG;
 877         }
 878         else if(is_u && !is_l)
 879         {
 880                 unsigned long ul;
 881                 errno = 0;
 882                 ul = strtoul(str, NULL, radix);
 883                 if ((ul == ULONG_MAX && errno == ERANGE) || (ul > UINT_MAX))
 884                         ppy_error("integer constant %s is too large\n", str);
 885                 val->uint = (unsigned int)ul;
 886                 return tUINT;
 887         }
 888
 889         /* Else it must be an int... */
 890         errno = 0;
 891         l = strtol(str, NULL, radix);
 892         if (((l == LONG_MIN || l == LONG_MAX) && errno == ERANGE) ||
 893                 (l > INT_MAX) || (l < INT_MIN))
 894                 ppy_error("integer constant %s is too large\n", str);
 895         val->sint = (int)l;
 896         return tSINT;
 897 }
 898
 899
 900 /*
 901  *-------------------------------------------------------------------------
 902  * Macro and define expansion support
 903  *
 904  * FIXME: Variable macro arguments.
 905  *-------------------------------------------------------------------------
 906  */
 907 static void expand_special(pp_entry_t *ppp)
 908 {
 909         const char *dbgtext = "?";
 910         static char *buf = NULL;
 911         char *new_buf;
 912
 913         assert(ppp->type == def_special);
 914
 915         if(!strcmp(ppp->ident, "__LINE__"))
 916         {
 917                 dbgtext = "def_special(__LINE__)";
 918                 new_buf = pp_xrealloc(buf, 32);
 919                 if(!new_buf)
 920                         return;
 921                 buf = new_buf;
 922                 sprintf(buf, "%d", pp_status.line_number);
 923         }
 924         else if(!strcmp(ppp->ident, "__FILE__"))
 925         {
 926                 dbgtext = "def_special(__FILE__)";
 927                 new_buf = pp_xrealloc(buf, strlen(pp_status.input) + 3);
 928                 if(!new_buf)
 929                         return;
 930                 buf = new_buf;
 931                 sprintf(buf, "\"%s\"", pp_status.input);
 932         }
 933         else
 934                 pp_internal_error(__FILE__, __LINE__, "Special macro '%s' not found...\n", ppp->ident);
 935
 936         if(pp_flex_debug)
 937                 fprintf(stderr, "expand_special(%d): %s:%d: '%s' -> '%s'\n",
 938                         macexpstackidx,
 939                         pp_status.input,
 940                         pp_status.line_number,
 941                         ppp->ident,
 942                         buf ? buf : "");
 943
 944         if(buf && buf[0])
 945         {
 946                 push_buffer(ppp, NULL, NULL, 0);
 947                 yy_scan_string(buf);
 948         }
 949 }
 950
 951 static void expand_define(pp_entry_t *ppp)
 952 {
 953         assert(ppp->type == def_define);
 954
 955         if(pp_flex_debug)
 956                 fprintf(stderr, "expand_define(%d): %s:%d: '%s' -> '%s'\n",
 957                         macexpstackidx,
 958                         pp_status.input,
 959                         pp_status.line_number,
 960                         ppp->ident,
 961                         ppp->subst.text);
 962         if(ppp->subst.text && ppp->subst.text[0])
 963         {
 964                 push_buffer(ppp, NULL, NULL, 0);
 965                 yy_scan_string(ppp->subst.text);
 966         }
 967 }
 968
 969 static int curdef_idx = 0;
 970 static int curdef_alloc = 0;
 971 static char *curdef_text = NULL;
 972
 973 static void add_text(const char *str, int len)
 974 {
 975         int new_alloc;
 976         char *new_text;
 977
 978         if(len == 0)
 979                 return;
 980         if(curdef_idx >= curdef_alloc || curdef_alloc - curdef_idx < len)
 981         {
 982                 new_alloc = curdef_alloc + ((len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1));
 983                 new_text = pp_xrealloc(curdef_text, new_alloc * sizeof(curdef_text[0]));
 984                 if(!new_text)
 985                         return;
 986                 curdef_text = new_text;
 987                 curdef_alloc = new_alloc;
 988                 if(curdef_alloc > 65536)
 989                         ppy_warning("Reallocating macro-expansion buffer larger than 64kB");
 990         }
 991         memcpy(&curdef_text[curdef_idx], str, len);
 992         curdef_idx += len;
 993 }
 994
 995 static mtext_t *add_expand_text(mtext_t *mtp, macexpstackentry_t *mep, int *nnl)
 996 {
 997         char *cptr;
 998         char *exp;
 999         int tag;
1000         int n;
1001
1002         if(mtp == NULL)
1003                 return NULL;
1004
1005         switch(mtp->type)
1006         {
1007         case exp_text:
1008                 if(pp_flex_debug)
1009                         fprintf(stderr, "add_expand_text: exp_text: '%s'\n", mtp->subst.text);
1010                 add_text(mtp->subst.text, strlen(mtp->subst.text));
1011                 break;
1012
1013         case exp_stringize:
1014                 if(pp_flex_debug)
1015                         fprintf(stderr, "add_expand_text: exp_stringize(%d): '%s'\n",
1016                                 mtp->subst.argidx,
1017                                 mep->args[mtp->subst.argidx]);
1018                 cptr = mep->args[mtp->subst.argidx];
1019                 add_text("\"", 1);
1020                 while(*cptr)
1021                 {
1022                         if(*cptr == '"' || *cptr == '\\')
1023                                 add_text("\\", 1);
1024                         add_text(cptr, 1);
1025                         cptr++;
1026                 }
1027                 add_text("\"", 1);
1028                 break;
1029
1030         case exp_concat:
1031                 if(pp_flex_debug)
1032                         fprintf(stderr, "add_expand_text: exp_concat\n");
1033                 /* Remove trailing whitespace from current expansion text */
1034                 while(curdef_idx)
1035                 {
1036                         if(isspace(curdef_text[curdef_idx-1] & 0xff))
1037                                 curdef_idx--;
1038                         else
1039                                 break;
1040                 }
1041                 /* tag current position and recursively expand the next part */
1042                 tag = curdef_idx;
1043                 mtp = add_expand_text(mtp->next, mep, nnl);
1044
1045                 /* Now get rid of the leading space of the expansion */
1046                 cptr = &curdef_text[tag];
1047                 n = curdef_idx - tag;
1048                 while(n)
1049                 {
1050                         if(isspace(*cptr & 0xff))
1051                         {
1052                                 cptr++;
1053                                 n--;
1054                         }
1055                         else
1056                                 break;
1057                 }
1058                 if(cptr != &curdef_text[tag])
1059                 {
1060                         memmove(&curdef_text[tag], cptr, n);
1061                         curdef_idx -= (curdef_idx - tag) - n;
1062                 }
1063                 break;
1064
1065         case exp_subst:
1066                 if((mtp->next && mtp->next->type == exp_concat) || (mtp->prev && mtp->prev->type == exp_concat))
1067                         exp = mep->args[mtp->subst.argidx];
1068                 else
1069                         exp = mep->ppargs[mtp->subst.argidx];
1070                 if(exp)
1071                 {
1072                         add_text(exp, strlen(exp));
1073                         *nnl -= mep->nnls[mtp->subst.argidx];
1074                         cptr = strchr(exp, '\n');
1075                         while(cptr)
1076                         {
1077                                 *cptr = ' ';
1078                                 cptr = strchr(cptr+1, '\n');
1079                         }
1080                         mep->nnls[mtp->subst.argidx] = 0;
1081                 }
1082                 if(pp_flex_debug)
1083                         fprintf(stderr, "add_expand_text: exp_subst(%d): '%s'\n", mtp->subst.argidx, exp);
1084                 break;
1085
1086         default:
1087                 pp_internal_error(__FILE__, __LINE__, "Invalid expansion type (%d) in macro expansion\n", mtp->type);
1088         }
1089         return mtp;
1090 }
1091
1092 static void expand_macro(macexpstackentry_t *mep)
1093 {
1094         mtext_t *mtp;
1095         int n, k;
1096         char *cptr;
1097         int nnl = 0;
1098         pp_entry_t *ppp = mep->ppp;
1099         int nargs = mep->nargs;
1100
1101         assert(ppp->type == def_macro);
1102         assert(ppp->expanding == 0);
1103
1104         if((ppp->nargs >= 0 && nargs != ppp->nargs) || (ppp->nargs < 0 && nargs < -ppp->nargs))
1105         {
1106                 ppy_error("Too %s macro arguments (%d)", nargs < abs(ppp->nargs) ? "few" : "many", nargs);
1107                 return;
1108         }
1109
1110         for(n = 0; n < nargs; n++)
1111                 nnl += mep->nnls[n];
1112
1113         if(pp_flex_debug)
1114                 fprintf(stderr, "expand_macro(%d): %s:%d: '%s'(%d,%d) -> ...\n",
1115                         macexpstackidx,
1116                         pp_status.input,
1117                         pp_status.line_number,
1118                         ppp->ident,
1119                         mep->nargs,
1120                         nnl);
1121
1122         curdef_idx = 0;
1123
1124         for(mtp = ppp->subst.mtext; mtp; mtp = mtp->next)
1125         {
1126                 if(!(mtp = add_expand_text(mtp, mep, &nnl)))
1127                         break;
1128         }
1129
1130         for(n = 0; n < nnl; n++)
1131                 add_text("\n", 1);
1132
1133         /* To make sure there is room and termination (see below) */
1134         add_text(" \0", 2);
1135
1136         /* Strip trailing whitespace from expansion */
1137         for(k = curdef_idx, cptr = &curdef_text[curdef_idx-1]; k > 0; k--, cptr--)
1138         {
1139                 if(!isspace(*cptr & 0xff))
1140                         break;
1141         }
1142
1143         /*
1144          * We must add *one* whitespace to make sure that there
1145          * is a token-separation after the expansion.
1146          */
1147         *(++cptr) = ' ';
1148         *(++cptr) = '\0';
1149         k++;
1150
1151         /* Strip leading whitespace from expansion */
1152         for(n = 0, cptr = curdef_text; n < k; n++, cptr++)
1153         {
1154                 if(!isspace(*cptr & 0xff))
1155                         break;
1156         }
1157
1158         if(k - n > 0)
1159         {
1160                 if(pp_flex_debug)
1161                         fprintf(stderr, "expand_text: '%s'\n", curdef_text + n);
1162                 push_buffer(ppp, NULL, NULL, 0);
1163                 /*yy_scan_bytes(curdef_text + n, k - n);*/
1164                 yy_scan_string(curdef_text + n);
1165         }
1166 }
1167
1168 /*
1169  *-------------------------------------------------------------------------
1170  * String collection routines
1171  *-------------------------------------------------------------------------
1172  */
1173 static void new_string(void)
1174 {
1175 #ifdef DEBUG
1176         if(strbuf_idx)
1177                 ppy_warning("new_string: strbuf_idx != 0");
1178 #endif
1179         strbuf_idx = 0;
1180         str_startline = pp_status.line_number;
1181 }
1182
1183 static void add_string(const char *str, int len)
1184 {
1185         int new_alloc;
1186         char *new_buffer;
1187
1188         if(len == 0)
1189                 return;
1190         if(strbuf_idx >= strbuf_alloc || strbuf_alloc - strbuf_idx < len)
1191         {
1192                 new_alloc = strbuf_alloc + ((len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1));
1193                 new_buffer = pp_xrealloc(strbuffer, new_alloc * sizeof(strbuffer[0]));
1194                 if(!new_buffer)
1195                         return;
1196                 strbuffer = new_buffer;
1197                 strbuf_alloc = new_alloc;
1198                 if(strbuf_alloc > 65536)
1199                         ppy_warning("Reallocating string buffer larger than 64kB");
1200         }
1201         memcpy(&strbuffer[strbuf_idx], str, len);
1202         strbuf_idx += len;
1203 }
1204
1205 static char *get_string(void)
1206 {
1207         char *str = pp_xmalloc(strbuf_idx + 1);
1208         if(!str)
1209                 return NULL;
1210         memcpy(str, strbuffer, strbuf_idx);
1211         str[strbuf_idx] = '\0';
1212 #ifdef DEBUG
1213         strbuf_idx = 0;
1214 #endif
1215         return str;
1216 }
1217
1218 static void put_string(void)
1219 {
1220         put_buffer(strbuffer, strbuf_idx);
1221 #ifdef DEBUG
1222         strbuf_idx = 0;
1223 #endif
1224 }
1225
1226 static int string_start(void)
1227 {
1228         return str_startline;
1229 }
1230
1231
1232 /*
1233  *-------------------------------------------------------------------------
1234  * Buffer management
1235  *-------------------------------------------------------------------------
1236  */
1237 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop)
1238 {
1239         if(ppy_debug)
1240                 printf("push_buffer(%d): %p %p %p %d\n", bufferstackidx, ppp, filename, incname, pop);
1241         if(bufferstackidx >= MAXBUFFERSTACK)
1242                 pp_internal_error(__FILE__, __LINE__, "Buffer stack overflow");
1243
1244         memset(&bufferstack[bufferstackidx], 0, sizeof(bufferstack[0]));
1245         bufferstack[bufferstackidx].bufferstate = YY_CURRENT_BUFFER;
1246         bufferstack[bufferstackidx].define      = ppp;
1247         bufferstack[bufferstackidx].line_number = pp_status.line_number;
1248         bufferstack[bufferstackidx].char_number = pp_status.char_number;
1249         bufferstack[bufferstackidx].if_depth    = pp_get_if_depth();
1250         bufferstack[bufferstackidx].should_pop  = pop;
1251         bufferstack[bufferstackidx].filename    = pp_status.input;
1252         bufferstack[bufferstackidx].ncontinuations      = ncontinuations;
1253         bufferstack[bufferstackidx].incl                = pp_incl_state;
1254         bufferstack[bufferstackidx].include_filename    = incname;
1255
1256         if(ppp)
1257                 ppp->expanding = 1;
1258         else if(filename)
1259         {
1260                 /* These will track the ppy_error to the correct file and line */
1261                 pp_status.line_number = 1;
1262                 pp_status.char_number = 1;
1263                 pp_status.input  = filename;
1264                 ncontinuations = 0;
1265         }
1266         else if(!pop)
1267                 pp_internal_error(__FILE__, __LINE__, "Pushing buffer without knowing where to go to");
1268         bufferstackidx++;
1269 }
1270
1271 static bufferstackentry_t *pop_buffer(void)
1272 {
1273         if(bufferstackidx < 0)
1274                 pp_internal_error(__FILE__, __LINE__, "Bufferstack underflow?");
1275
1276         if(bufferstackidx == 0)
1277                 return NULL;
1278
1279         bufferstackidx--;
1280
1281         if(bufferstack[bufferstackidx].define)
1282                 bufferstack[bufferstackidx].define->expanding = 0;
1283         else
1284         {
1285                 if(!bufferstack[bufferstackidx].should_pop)
1286                 {
1287                         fclose(ppy_in);
1288                         fprintf(ppy_out, "# %d \"%s\" 2\n", bufferstack[bufferstackidx].line_number, bufferstack[bufferstackidx].filename);
1289
1290                         /* We have EOF, check the include logic */
1291                         if(pp_incl_state.state == 2 && !pp_incl_state.seen_junk && pp_incl_state.ppp)
1292                         {
1293                                 pp_entry_t *ppp = pplookup(pp_incl_state.ppp);
1294                                 if(ppp)
1295                                 {
1296                                         includelogicentry_t *iep = pp_xmalloc(sizeof(includelogicentry_t));
1297                                         if(!iep)
1298                                                 return NULL;
1299
1300                                         iep->ppp = ppp;
1301                                         ppp->iep = iep;
1302                                         iep->filename = bufferstack[bufferstackidx].include_filename;
1303                                         iep->prev = NULL;
1304                                         iep->next = pp_includelogiclist;
1305                                         if(iep->next)
1306                                                 iep->next->prev = iep;
1307                                         pp_includelogiclist = iep;
1308                                         if(pp_status.debug)
1309                                                 fprintf(stderr, "pop_buffer: %s:%d: includelogic added, include_ppp='%s', file='%s'\n", bufferstack[bufferstackidx].filename, bufferstack[bufferstackidx].line_number, pp_incl_state.ppp, iep->filename);
1310                                 }
1311                                 else
1312                                         free(bufferstack[bufferstackidx].include_filename);
1313                         }
1314                         free(pp_incl_state.ppp);
1315                         pp_incl_state   = bufferstack[bufferstackidx].incl;
1316
1317                 }
1318                 pp_status.line_number = bufferstack[bufferstackidx].line_number;
1319                 pp_status.char_number = bufferstack[bufferstackidx].char_number;
1320                 pp_status.input  = bufferstack[bufferstackidx].filename;
1321                 ncontinuations = bufferstack[bufferstackidx].ncontinuations;
1322         }
1323
1324         if(ppy_debug)
1325                 printf("pop_buffer(%d): %p %p (%d, %d, %d) %p %d\n",
1326                         bufferstackidx,
1327                         bufferstack[bufferstackidx].bufferstate,
1328                         bufferstack[bufferstackidx].define,
1329                         bufferstack[bufferstackidx].line_number,
1330                         bufferstack[bufferstackidx].char_number,
1331                         bufferstack[bufferstackidx].if_depth,
1332                         bufferstack[bufferstackidx].filename,
1333                         bufferstack[bufferstackidx].should_pop);
1334
1335         ppy__switch_to_buffer(bufferstack[bufferstackidx].bufferstate);
1336
1337         if(bufferstack[bufferstackidx].should_pop)
1338         {
1339                 if(yy_current_state() == pp_macexp)
1340                         macro_add_expansion();
1341                 else
1342                         pp_internal_error(__FILE__, __LINE__, "Pop buffer and state without macro expansion state");
1343                 yy_pop_state();
1344         }
1345
1346         return &bufferstack[bufferstackidx];
1347 }
1348
1349
1350 /*
1351  *-------------------------------------------------------------------------
1352  * Macro nestng support
1353  *-------------------------------------------------------------------------
1354  */
1355 static void push_macro(pp_entry_t *ppp)
1356 {
1357         if(macexpstackidx >= MAXMACEXPSTACK)
1358         {
1359                 ppy_error("Too many nested macros");
1360                 return;
1361         }
1362
1363         macexpstack[macexpstackidx] = pp_xmalloc(sizeof(macexpstack[0][0]));
1364         if(!macexpstack[macexpstackidx])
1365                 return;
1366         memset( macexpstack[macexpstackidx], 0, sizeof(macexpstack[0][0]));
1367         macexpstack[macexpstackidx]->ppp = ppp;
1368         macexpstackidx++;
1369 }
1370
1371 static macexpstackentry_t *top_macro(void)
1372 {
1373         return macexpstackidx > 0 ? macexpstack[macexpstackidx-1] : NULL;
1374 }
1375
1376 static macexpstackentry_t *pop_macro(void)
1377 {
1378         if(macexpstackidx <= 0)
1379                 pp_internal_error(__FILE__, __LINE__, "Macro expansion stack underflow\n");
1380         return macexpstack[--macexpstackidx];
1381 }
1382
1383 static void free_macro(macexpstackentry_t *mep)
1384 {
1385         int i;
1386
1387         for(i = 0; i < mep->nargs; i++)
1388                 free(mep->args[i]);
1389         free(mep->args);
1390         free(mep->nnls);
1391         free(mep->curarg);
1392         free(mep);
1393 }
1394
1395 static void add_text_to_macro(const char *text, int len)
1396 {
1397         macexpstackentry_t *mep = top_macro();
1398
1399         assert(mep->ppp->expanding == 0);
1400
1401         if(mep->curargalloc - mep->curargsize <= len+1) /* +1 for '\0' */
1402         {
1403                 char *new_curarg;
1404                 int new_alloc = mep->curargalloc + (ALLOCBLOCKSIZE > len+1) ? ALLOCBLOCKSIZE : len+1;
1405                 new_curarg = pp_xrealloc(mep->curarg, new_alloc * sizeof(mep->curarg[0]));
1406                 if(!new_curarg)
1407                         return;
1408                 mep->curarg = new_curarg;
1409                 mep->curargalloc = new_alloc;
1410         }
1411         memcpy(mep->curarg + mep->curargsize, text, len);
1412         mep->curargsize += len;
1413         mep->curarg[mep->curargsize] = '\0';
1414 }
1415
1416 static void macro_add_arg(int last)
1417 {
1418         int nnl = 0;
1419         char *cptr;
1420         char **new_args, **new_ppargs;
1421         int *new_nnls;
1422         macexpstackentry_t *mep = top_macro();
1423
1424         assert(mep->ppp->expanding == 0);
1425
1426         new_args = pp_xrealloc(mep->args, (mep->nargs+1) * sizeof(mep->args[0]));
1427         if(!new_args)
1428                 return;
1429         mep->args = new_args;
1430
1431         new_ppargs = pp_xrealloc(mep->ppargs, (mep->nargs+1) * sizeof(mep->ppargs[0]));
1432         if(!new_ppargs)
1433                 return;
1434         mep->ppargs = new_ppargs;
1435
1436         new_nnls = pp_xrealloc(mep->nnls, (mep->nargs+1) * sizeof(mep->nnls[0]));
1437         if(!new_nnls)
1438                 return;
1439         mep->nnls = new_nnls;
1440
1441         mep->args[mep->nargs] = pp_xstrdup(mep->curarg ? mep->curarg : "");
1442         if(!mep->args[mep->nargs])
1443                 return;
1444         cptr = mep->args[mep->nargs]-1;
1445         while((cptr = strchr(cptr+1, '\n')))
1446         {
1447                 nnl++;
1448         }
1449         mep->nnls[mep->nargs] = nnl;
1450         mep->nargs++;
1451         free(mep->curarg);
1452         mep->curargalloc = mep->curargsize = 0;
1453         mep->curarg = NULL;
1454
1455         if(pp_flex_debug)
1456                 fprintf(stderr, "macro_add_arg: %s:%d: %d -> '%s'\n",
1457                         pp_status.input,
1458                         pp_status.line_number,
1459                         mep->nargs-1,
1460                         mep->args[mep->nargs-1]);
1461
1462         /* Each macro argument must be expanded to cope with stingize */
1463         if(last || mep->args[mep->nargs-1][0])
1464         {
1465                 yy_push_state(pp_macexp);
1466                 push_buffer(NULL, NULL, NULL, last ? 2 : 1);
1467                 yy_scan_string(mep->args[mep->nargs-1]);
1468                 /*mep->bufferstackidx = bufferstackidx;  But not nested! */
1469         }
1470 }
1471
1472 static void macro_add_expansion(void)
1473 {
1474         macexpstackentry_t *mep = top_macro();
1475
1476         assert(mep->ppp->expanding == 0);
1477
1478         mep->ppargs[mep->nargs-1] = pp_xstrdup(mep->curarg ? mep->curarg : "");
1479         free(mep->curarg);
1480         mep->curargalloc = mep->curargsize = 0;
1481         mep->curarg = NULL;
1482
1483         if(pp_flex_debug)
1484                 fprintf(stderr, "macro_add_expansion: %s:%d: %d -> '%s'\n",
1485                         pp_status.input,
1486                         pp_status.line_number,
1487                         mep->nargs-1,
1488                         mep->ppargs[mep->nargs-1] ? mep->ppargs[mep->nargs-1] : "");
1489 }
1490
1491
1492 /*
1493  *-------------------------------------------------------------------------
1494  * Output management
1495  *-------------------------------------------------------------------------
1496  */
1497 static void put_buffer(const char *s, int len)
1498 {
1499         if(top_macro())
1500                 add_text_to_macro(s, len);
1501         else
1502            fwrite(s, 1, len, ppy_out);
1503 }
1504
1505
1506 /*
1507  *-------------------------------------------------------------------------
1508  * Include management
1509  *-------------------------------------------------------------------------
1510  */
1511 void pp_do_include(char *fname, int type)
1512 {
1513         char *newpath;
1514         int n;
1515         includelogicentry_t *iep;
1516
1517         for(iep = pp_includelogiclist; iep; iep = iep->next)
1518         {
1519                 if(!strcmp(iep->filename, fname))
1520                 {
1521                         /*
1522                          * We are done. The file was included before.
1523                          * If the define was deleted, then this entry would have
1524                          * been deleted too.
1525                          */
1526                         return;
1527                 }
1528         }
1529
1530         n = strlen(fname);
1531
1532         if(n <= 2)
1533         {
1534                 ppy_error("Empty include filename");
1535                 return;
1536         }
1537
1538         /* Undo the effect of the quotation */
1539         fname[n-1] = '\0';
1540
1541         if((ppy_in = pp_open_include(fname+1, type ? pp_status.input : NULL, &newpath)) == NULL)
1542         {
1543                 ppy_error("Unable to open include file %s", fname+1);
1544                 return;
1545         }
1546
1547         fname[n-1] = *fname;    /* Redo the quotes */
1548         push_buffer(NULL, newpath, fname, 0);
1549         pp_incl_state.seen_junk = 0;
1550         pp_incl_state.state = 0;
1551         pp_incl_state.ppp = NULL;
1552
1553         if(pp_status.debug)
1554                 fprintf(stderr, "pp_do_include: %s:%d: include_state=%d, include_ppp='%s', include_ifdepth=%d\n",
1555                         pp_status.input, pp_status.line_number, pp_incl_state.state, pp_incl_state.ppp, pp_incl_state.ifdepth);
1556         ppy__switch_to_buffer(ppy__create_buffer(ppy_in, YY_BUF_SIZE));
1557
1558         fprintf(ppy_out, "# 1 \"%s\" 1%s\n", newpath, type ? "" : " 3");
1559 }
1560
1561 /*
1562  *-------------------------------------------------------------------------
1563  * Push/pop preprocessor ignore state when processing conditionals
1564  * which are false.
1565  *-------------------------------------------------------------------------
1566  */
1567 void pp_push_ignore_state(void)
1568 {
1569         yy_push_state(pp_ignore);
1570 }
1571
1572 void pp_pop_ignore_state(void)
1573 {
1574         yy_pop_state();
1575 }