diff options
Diffstat (limited to 'dimension/tokenize.c')
| -rw-r--r-- | dimension/tokenize.c | 236 |
1 files changed, 153 insertions, 83 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 4e499d5..e872e0c 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -84,42 +84,79 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, int dmnsn_ld_yyparse(const char *filename, void *yyscanner, dmnsn_symbol_table *symtable); -static dmnsn_token_buffer * -dmnsn_include_buffer(int token, dmnsn_token_buffer *prev, - dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, - const char *filename, dmnsn_symbol_table *symtable, - void *yyscanner) +static int +dmnsn_buffer_balanced(dmnsn_token_buffer *tbuffer, bool recursive, + dmnsn_token_type left, dmnsn_token_type right, + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) { - dmnsn_token_buffer *include_buffer - = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); + dmnsn_buffered_token buffered; - /* Buffer the current token */ - dmnsn_buffered_token buffered = { - .type = token, - .lval = *lvalp, - .lloc = *llocp - }; - dmnsn_array_push(include_buffer->buffered, &buffered); + int nesting = -1; + while (1) { + if (recursive) { + buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, + filename, symtable, yyscanner); + } else { + buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc, + filename, symtable, yyscanner); + } - /* Recursive call */ - buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, - filename, symtable, yyscanner); + if (buffered.type == DMNSN_T_EOF) { + dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, + buffered.lloc.first_column, + "syntax error, unexpected end-of-file"); + return 1; + } else if (buffered.type == DMNSN_T_LEX_ERROR) { + return 1; + } + + dmnsn_array_push(tbuffer->buffered, &buffered); + + if (buffered.type == left) { + if (nesting < 0) + nesting = 1; + else + ++nesting; + } else if (buffered.type == right) { + --nesting; + if (nesting == 0) { + break; + } + } + } + + return 0; +} + +static int +dmnsn_buffer_strexp(dmnsn_token_buffer *tbuffer, bool recursive, + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) +{ + dmnsn_buffered_token buffered; + + if (recursive) { + buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, + filename, symtable, yyscanner); + } else { + buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc, + filename, symtable, yyscanner); + } if (buffered.type == DMNSN_T_EOF) { dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, buffered.lloc.first_column, "syntax error, unexpected end-of-file"); - dmnsn_delete_token_buffer(include_buffer); - return NULL; + return 1; } else if (buffered.type == DMNSN_T_LEX_ERROR) { - dmnsn_delete_token_buffer(include_buffer); - return NULL; + return 1; } - /* Buffer the next token */ - dmnsn_array_push(include_buffer->buffered, &buffered); + /* Buffer the first token */ + dmnsn_array_push(tbuffer->buffered, &buffered); bool is_strexp = buffered.type != DMNSN_T_STRING; - if (is_strexp && buffered.type == DMNSN_T_IDENTIFIER) { + if (buffered.type == DMNSN_T_IDENTIFIER) { /* Check if it's a string identifier or a macro */ dmnsn_astnode *inode = dmnsn_find_symbol(symtable, buffered.lval.value); if (!inode || inode->type == DMNSN_AST_STRING) { @@ -129,37 +166,37 @@ dmnsn_include_buffer(int token, dmnsn_token_buffer *prev, if (is_strexp) { /* Grab all the tokens belonging to the string expression */ - int parenlevel = -1; - while (1) { - /* Recursive call - permit other directives inside the condition */ - buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, - filename, symtable, yyscanner); + return dmnsn_buffer_balanced(tbuffer, recursive, + DMNSN_T_LPAREN, DMNSN_T_RPAREN, + filename, symtable, yyscanner); + } - if (buffered.type == DMNSN_T_EOF) { - dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, - buffered.lloc.first_column, - "syntax error, unexpected end-of-file"); - dmnsn_delete_token_buffer(include_buffer); - return NULL; - } else if (buffered.type == DMNSN_T_LEX_ERROR) { - dmnsn_delete_token_buffer(include_buffer); - return NULL; - } + return 0; +} - dmnsn_array_push(include_buffer->buffered, &buffered); - - if (buffered.type == DMNSN_T_LPAREN) { - if (parenlevel < 0) - parenlevel = 1; - else - ++parenlevel; - } else if (buffered.type == DMNSN_T_RPAREN) { - --parenlevel; - if (parenlevel == 0) { - break; - } - } - } +static dmnsn_token_buffer * +dmnsn_include_buffer(int token, dmnsn_token_buffer *prev, + dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) +{ + dmnsn_token_buffer *include_buffer + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); + + /* Buffer the current token */ + dmnsn_buffered_token buffered = { + .type = token, + .lval = *lvalp, + .lloc = *llocp + }; + dmnsn_array_push(include_buffer->buffered, &buffered); + + /* Buffer the following string expression */ + if (dmnsn_buffer_strexp(include_buffer, true, filename, symtable, yyscanner) + != 0) + { + dmnsn_delete_token_buffer(include_buffer); + return NULL; } /* Fake EOF */ @@ -360,36 +397,12 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, dmnsn_array_push(cond_buffer->buffered, &buffered); /* Grab all the tokens belonging to the #if (...) */ - int parenlevel = -1; - while (1) { - /* Recursive call - permit other directives inside the condition */ - buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, - filename, symtable, yyscanner); - - if (buffered.type == DMNSN_T_EOF) { - dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, - buffered.lloc.first_column, - "syntax error, unexpected end-of-file"); - dmnsn_delete_token_buffer(cond_buffer); - return NULL; - } else if (buffered.type == DMNSN_T_LEX_ERROR) { - dmnsn_delete_token_buffer(cond_buffer); - return NULL; - } - - dmnsn_array_push(cond_buffer->buffered, &buffered); - - if (buffered.type == DMNSN_T_LPAREN) { - if (parenlevel < 0) - parenlevel = 1; - else - ++parenlevel; - } else if (buffered.type == DMNSN_T_RPAREN) { - --parenlevel; - if (parenlevel == 0) { - break; - } - } + if (dmnsn_buffer_balanced(cond_buffer, true, DMNSN_T_LPAREN, DMNSN_T_RPAREN, + filename, symtable, yyscanner) + != 0) + { + dmnsn_delete_token_buffer(cond_buffer); + return NULL; } /* Fake EOF */ @@ -586,6 +599,39 @@ dmnsn_version_buffer(int token, dmnsn_token_buffer *prev, return tbuffer; } +static dmnsn_token_buffer * +dmnsn_stream_buffer(int token, dmnsn_token_buffer *prev, + dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) +{ + dmnsn_token_buffer *tbuffer + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); + + /* Buffer the current token */ + dmnsn_buffered_token buffered = { + .type = token, + .lval = *lvalp, + .lloc = *llocp + }; + dmnsn_array_push(tbuffer->buffered, &buffered); + + /* Buffer the following string expression */ + if (dmnsn_buffer_strexp(tbuffer, true, filename, symtable, yyscanner) + != 0) + { + dmnsn_delete_token_buffer(tbuffer); + return NULL; + } + + /* Fake EOF */ + buffered.type = DMNSN_T_EOF; + buffered.lval.value = NULL; + dmnsn_array_push(tbuffer->buffered, &buffered); + + return tbuffer; +} + int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, const char *filename, void *yyscanner); @@ -785,6 +831,30 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, break; } + case DMNSN_T_DEBUG: + case DMNSN_T_ERROR: + case DMNSN_T_WARNING: + { + dmnsn_token_buffer *tb = dmnsn_stream_buffer( + token, tbuffer, lvalp, llocp, filename, symtable, yyscanner + ); + if (!tb) { + return DMNSN_T_LEX_ERROR; + } + + dmnsn_yyset_extra(tb, yyscanner); + if (dmnsn_ld_yyparse(filename, yyscanner, symtable) != 0) { + dmnsn_yyset_extra(tb->prev, yyscanner); + dmnsn_delete_token_buffer(tb); + return DMNSN_T_LEX_ERROR; + } + + /* Restore the previous extra pointer */ + dmnsn_yyset_extra(tb->prev, yyscanner); + dmnsn_delete_token_buffer(tb); + break; + } + default: return token; } |