diff options
Diffstat (limited to 'dimension/tokenize.c')
| -rw-r--r-- | dimension/tokenize.c | 73 |
1 files changed, 63 insertions, 10 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 48d0958..7315a60 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -33,9 +33,12 @@ typedef struct dmnsn_buffered_token { typedef struct dmnsn_token_buffer { dmnsn_token_type type; - /* Indicate that the first token should be returned as-is */ + /* To indicate that the first token should be returned as-is */ #define DMNSN_T_LEX_VERBATIM DMNSN_T_EOF + /* Whether to automatically delete this buffer if we reach its end */ + bool auto_delete; + dmnsn_parse_location lloc; dmnsn_array *buffered; size_t i; @@ -51,14 +54,14 @@ dmnsn_new_token_buffer(dmnsn_parse_location lloc, dmnsn_token_type type, dmnsn_token_buffer *prev, const char *filename) { dmnsn_token_buffer *tbuffer = dmnsn_malloc(sizeof(dmnsn_token_buffer)); - - tbuffer->type = type; - tbuffer->lloc = lloc; - tbuffer->buffered = dmnsn_new_array(sizeof(dmnsn_buffered_token)); - tbuffer->i = 0; - tbuffer->prev = prev; - tbuffer->filename = filename; - tbuffer->ptr = NULL; + tbuffer->type = type; + tbuffer->auto_delete = true; + tbuffer->lloc = lloc; + tbuffer->buffered = dmnsn_new_array(sizeof(dmnsn_buffered_token)); + tbuffer->i = 0; + tbuffer->prev = prev; + tbuffer->filename = filename; + tbuffer->ptr = NULL; return tbuffer; } @@ -91,6 +94,9 @@ dmnsn_buffer_balanced(dmnsn_token_buffer *tbuffer, bool recursive, { dmnsn_buffered_token buffered; + if (tbuffer->prev) + tbuffer->prev->auto_delete = false; + int nesting = -1; while (1) { if (recursive) { @@ -123,6 +129,9 @@ dmnsn_buffer_balanced(dmnsn_token_buffer *tbuffer, bool recursive, } } + if (tbuffer->prev) + tbuffer->prev->auto_delete = true; + return 0; } @@ -133,6 +142,9 @@ dmnsn_buffer_strexp(dmnsn_token_buffer *tbuffer, bool recursive, { dmnsn_buffered_token buffered; + if (tbuffer->prev) + tbuffer->prev->auto_delete = false; + if (recursive) { buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, filename, symtable, yyscanner); @@ -166,6 +178,9 @@ dmnsn_buffer_strexp(dmnsn_token_buffer *tbuffer, bool recursive, filename, symtable, yyscanner); } + if (tbuffer->prev) + tbuffer->prev->auto_delete = true; + return 0; } @@ -285,11 +300,14 @@ dmnsn_declaration_buffer(int token, dmnsn_token_buffer *prev, }; dmnsn_array_push(tbuffer->buffered, &buffered); + if (prev) + prev->auto_delete = false; + /* Grab all the tokens belonging to the #declare/#local, i.e. until the braces balance or we hit a semicolon */ int bracelevel = -1; while (1) { - /* Recursive call - permit other directives inside the declaration */ + /* Recursive call -- permit other directives inside the declaration */ buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, filename, symtable, yyscanner); @@ -319,6 +337,9 @@ dmnsn_declaration_buffer(int token, dmnsn_token_buffer *prev, } } + if (prev) + prev->auto_delete = true; + /* Fake EOF */ buffered.type = DMNSN_T_EOF; buffered.lval.value = NULL; @@ -344,9 +365,13 @@ dmnsn_undef_buffer(int token, dmnsn_token_buffer *prev, }; dmnsn_array_push(tbuffer->buffered, &buffered); + if (prev) + prev->auto_delete = false; /* Recursive call */ buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, filename, symtable, yyscanner); + if (prev) + prev->auto_delete = true; if (buffered.type == DMNSN_T_EOF) { dmnsn_diagnostic(buffered.lloc, "syntax error, unexpected end-of-file"); @@ -425,6 +450,9 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, dmnsn_undef_symbol(symtable, "$cond"); + if (prev) + prev->auto_delete = false; + int nesting = 1, else_seen = 0; while (1) { /* Non-recursive call */ @@ -478,6 +506,9 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, } } + if (prev) + prev->auto_delete = true; + return tbuffer; } @@ -498,6 +529,9 @@ dmnsn_while_buffer(int token, dmnsn_token_buffer *prev, }; dmnsn_array_push(tbuffer->buffered, &buffered); + if (prev) + prev->auto_delete = false; + /* Grab all the tokens belonging to the #while ... #end */ int nesting = 1; while (1) { @@ -536,6 +570,9 @@ dmnsn_while_buffer(int token, dmnsn_token_buffer *prev, } } + if (prev) + prev->auto_delete = true; + return tbuffer; } @@ -556,6 +593,9 @@ dmnsn_version_buffer(int token, dmnsn_token_buffer *prev, }; dmnsn_array_push(tbuffer->buffered, &buffered); + if (prev) + prev->auto_delete = false; + while (buffered.type != DMNSN_T_SEMICOLON) { /* Recursive call */ buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, @@ -573,6 +613,9 @@ dmnsn_version_buffer(int token, dmnsn_token_buffer *prev, dmnsn_array_push(tbuffer->buffered, &buffered); } + if (prev) + prev->auto_delete = true; + /* Fake EOF */ buffered.type = DMNSN_T_EOF; buffered.lval.value = NULL; @@ -666,6 +709,9 @@ dmnsn_declare_macro(int token, dmnsn_token_buffer *prev, dmnsn_assert(mnode->type == DMNSN_AST_MACRO, "#macro has wrong type."); dmnsn_undef_symbol(symtable, "$macro"); + if (prev) + prev->auto_delete = false; + int nesting = 1; while (1) { /* Non-recursive call */ @@ -702,6 +748,9 @@ dmnsn_declare_macro(int token, dmnsn_token_buffer *prev, dmnsn_array_push(tbuffer->buffered, &buffered); } + if (prev) + prev->auto_delete = true; + mnode->ptr = tbuffer; mnode->free_fn = &dmnsn_delete_token_buffer; return true; @@ -772,6 +821,10 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, if (tbuffer->type == DMNSN_T_WHILE) { tbuffer->i = 0; } else { + if (!tbuffer->auto_delete) { + return DMNSN_T_EOF; + } + if (dmnsn_array_size(tbuffer->buffered) == 0 && tbuffer->prev && tbuffer->prev->type == DMNSN_T_WHILE) { |