diff options
Diffstat (limited to 'dimension/tokenize.c')
| -rw-r--r-- | dimension/tokenize.c | 227 |
1 files changed, 209 insertions, 18 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 63435c9..95aaa9b 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -20,7 +20,9 @@ #include "tokenize.h" #include "directives.h" #include "utility.h" +#include <libgen.h> #include <stdbool.h> +#include <string.h> typedef struct dmnsn_buffered_token { int type; @@ -37,10 +39,13 @@ typedef struct dmnsn_token_buffer { unsigned int i; struct dmnsn_token_buffer *prev; + + const char *filename; + void *ptr; } dmnsn_token_buffer; static dmnsn_token_buffer * -dmnsn_new_token_buffer(int type, dmnsn_token_buffer *prev) +dmnsn_new_token_buffer(int type, dmnsn_token_buffer *prev, const char *filename) { dmnsn_token_buffer *tbuffer = malloc(sizeof(dmnsn_token_buffer)); if (!tbuffer) { @@ -51,6 +56,8 @@ dmnsn_new_token_buffer(int type, dmnsn_token_buffer *prev) tbuffer->buffered = dmnsn_new_array(sizeof(dmnsn_buffered_token)); tbuffer->i = 0; tbuffer->prev = prev; + tbuffer->filename = filename; + tbuffer->ptr = NULL; return tbuffer; } @@ -72,18 +79,177 @@ dmnsn_delete_token_buffer(dmnsn_token_buffer *tbuffer) static int dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, - const char *filename, void *yyscanner); + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner); int dmnsn_ld_yyparse(const char *filename, void *yyscanner, dmnsn_symbol_table *symtable); static dmnsn_token_buffer * +dmnsn_include_buffer(int token, dmnsn_token_buffer *prev, + dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) +{ + dmnsn_token_buffer *include_buffer + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); + + /* Buffer the current token */ + dmnsn_buffered_token buffered = { + .type = token, + .lval = *lvalp, + .lloc = *llocp + }; + dmnsn_array_push(include_buffer->buffered, &buffered); + + /* Recursive call */ + buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, + filename, symtable, yyscanner); + + if (buffered.type == DMNSN_T_EOF) { + dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, + buffered.lloc.first_column, + "syntax error, unexpected end-of-file"); + dmnsn_delete_token_buffer(include_buffer); + return NULL; + } else if (buffered.type == DMNSN_T_LEX_ERROR) { + dmnsn_delete_token_buffer(include_buffer); + return NULL; + } + /* Buffer the next token */ + dmnsn_array_push(include_buffer->buffered, &buffered); + + bool is_strexp = buffered.type != DMNSN_T_STRING; + if (is_strexp && buffered.type == DMNSN_T_IDENTIFIER) { + /* Check if it's a string identifier or a macro */ + dmnsn_astnode *inode = dmnsn_find_symbol(symtable, buffered.lval.value); + if (!inode || inode->type == DMNSN_AST_STRING) { + is_strexp = false; + } + } + + if (is_strexp) { + /* Grab all the tokens belonging to the string expression */ + int parenlevel = -1; + while (1) { + /* Recursive call - permit other directives inside the condition */ + buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc, + filename, symtable, yyscanner); + + if (buffered.type == DMNSN_T_EOF) { + dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, + buffered.lloc.first_column, + "syntax error, unexpected end-of-file"); + dmnsn_delete_token_buffer(include_buffer); + return NULL; + } else if (buffered.type == DMNSN_T_LEX_ERROR) { + dmnsn_delete_token_buffer(include_buffer); + return NULL; + } + + dmnsn_array_push(include_buffer->buffered, &buffered); + + if (buffered.type == DMNSN_T_LPAREN) { + if (parenlevel < 0) + parenlevel = 1; + else + ++parenlevel; + } else if (buffered.type == DMNSN_T_RPAREN) { + --parenlevel; + if (parenlevel == 0) { + break; + } + } + } + } + + /* Fake EOF */ + buffered.type = DMNSN_T_EOF; + buffered.lval.value = NULL; + dmnsn_array_push(include_buffer->buffered, &buffered); + + dmnsn_yyset_extra(include_buffer, yyscanner); + if (dmnsn_ld_yyparse(filename, yyscanner, symtable) != 0) { + dmnsn_yyset_extra(include_buffer->prev, yyscanner); + dmnsn_delete_token_buffer(include_buffer); + return NULL; + } + + dmnsn_yyset_extra(include_buffer->prev, yyscanner); + dmnsn_delete_token_buffer(include_buffer); + + dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev, filename); + + dmnsn_astnode *inode = dmnsn_find_symbol(symtable, "__include__"); + if (!inode) { + dmnsn_error(DMNSN_SEVERITY_HIGH, "__include__ unset."); + } + + const char *include = inode->ptr; + if (inode->type != DMNSN_AST_STRING) { + dmnsn_error(DMNSN_SEVERITY_HIGH, "__include__ has wrong type."); + } + + char *filename_copy = strdup(filename); + char *localdir = dirname(filename_copy); + char *local_include = malloc(strlen(localdir) + strlen(include) + 2); + strcpy(local_include, localdir); + strcat(local_include, "/"); + strcat(local_include, include); + free(filename_copy); + dmnsn_undef_symbol(symtable, "__include__"); + + FILE *file = fopen(local_include, "r"); + if (!file) { + dmnsn_diagnostic(llocp->first_filename, llocp->first_line, + llocp->first_column, + "Couldn't open include file '%s'", local_include); + free(local_include); + dmnsn_delete_token_buffer(tbuffer); + return NULL; + } + tbuffer->ptr = file; + + void *buffer = dmnsn_yy_make_buffer(file, yyscanner); + if (!buffer) { + dmnsn_diagnostic(llocp->first_filename, llocp->first_line, + llocp->first_column, + "Couldn't allocate buffer for include file '%s'", + local_include); + fclose(file); + free(local_include); + dmnsn_delete_token_buffer(tbuffer); + return NULL; + } + dmnsn_yy_push_buffer(buffer, yyscanner); + + /* Stuff the filename in the symbol table to persist it */ + dmnsn_astnode *includes = dmnsn_find_symbol(symtable, "__includes__"); + if (!includes) { + dmnsn_declare_symbol(symtable, "__includes__", dmnsn_new_ast_array()); + includes = dmnsn_find_symbol(symtable, "__includes__"); + } + if (includes->type != DMNSN_AST_ARRAY) { + dmnsn_error(DMNSN_SEVERITY_HIGH, "__includes__ has wrong type."); + } + + dmnsn_astnode fnode = dmnsn_new_ast_string(local_include); + free(local_include); + tbuffer->filename = fnode.ptr; + dmnsn_array_push(includes->children, &fnode); + + dmnsn_push_scope(symtable); + + return tbuffer; +} + +static dmnsn_token_buffer * dmnsn_declaration_buffer(int token, dmnsn_token_buffer *prev, dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, const char *filename, dmnsn_symbol_table *symtable, void *yyscanner) { dmnsn_token_buffer *tbuffer - = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev); + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); /* Buffer the current token */ dmnsn_buffered_token buffered = { @@ -144,7 +310,7 @@ dmnsn_undef_buffer(int token, dmnsn_token_buffer *prev, void *yyscanner) { dmnsn_token_buffer *tbuffer - = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev); + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); /* Buffer the current token */ dmnsn_buffered_token buffered = { @@ -186,7 +352,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, void *yyscanner) { dmnsn_token_buffer *cond_buffer - = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev); + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); /* Buffer the current token */ dmnsn_buffered_token buffered = { @@ -204,7 +370,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, filename, symtable, yyscanner); if (buffered.type == DMNSN_T_EOF) { - dmnsn_diagnostic(filename, buffered.lloc.first_line, + dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line, buffered.lloc.first_column, "syntax error, unexpected end-of-file"); dmnsn_delete_token_buffer(cond_buffer); @@ -244,7 +410,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, dmnsn_yyset_extra(cond_buffer->prev, yyscanner); dmnsn_delete_token_buffer(cond_buffer); - dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev); + dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev, filename); dmnsn_astnode *cnode = dmnsn_find_symbol(symtable, "__cond__"); if (!cnode) { @@ -266,7 +432,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev, while (1) { /* Non-recursive call */ buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc, - filename, yyscanner); + filename, symtable, yyscanner); if (buffered.type == DMNSN_T_EOF) { dmnsn_diagnostic(filename, buffered.lloc.first_line, @@ -328,7 +494,7 @@ dmnsn_while_buffer(int token, dmnsn_token_buffer *prev, const char *filename, dmnsn_symbol_table *symtable, void *yyscanner) { - dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev); + dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev, filename); /* Pretend to be an if */ dmnsn_buffered_token buffered = { @@ -343,7 +509,7 @@ dmnsn_while_buffer(int token, dmnsn_token_buffer *prev, while (1) { /* Non-recursive call */ buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc, - filename, yyscanner); + filename, symtable, yyscanner); if (buffered.type == DMNSN_T_EOF) { dmnsn_diagnostic(filename, buffered.lloc.first_line, @@ -388,7 +554,7 @@ dmnsn_version_buffer(int token, dmnsn_token_buffer *prev, void *yyscanner) { dmnsn_token_buffer *tbuffer - = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev); + = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename); /* Buffer the current token */ dmnsn_buffered_token buffered = { @@ -430,11 +596,12 @@ int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, static int dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, - const char *filename, void *yyscanner) + const char *filename, dmnsn_symbol_table *symtable, + void *yyscanner) { dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner); - if (tbuffer) { + if (tbuffer && tbuffer->type != DMNSN_T_INCLUDE) { while (tbuffer && tbuffer->i >= dmnsn_array_size(tbuffer->buffered)) { if (tbuffer->type == DMNSN_T_WHILE) { tbuffer->i = 0; @@ -456,7 +623,7 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, int token; - if (tbuffer) { + if (tbuffer && tbuffer->type != DMNSN_T_INCLUDE) { /* Return buffered tokens */ dmnsn_buffered_token buffered; @@ -472,7 +639,16 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, *llocp = buffered.lloc; ++tbuffer->i; } else { - token = dmnsn_yylex_impl(lvalp, llocp, filename, yyscanner); + const char *real_filename = tbuffer ? tbuffer->filename : filename; + token = dmnsn_yylex_impl(lvalp, llocp, real_filename, yyscanner); + if (token == DMNSN_T_EOF && tbuffer && tbuffer->type == DMNSN_T_INCLUDE) { + dmnsn_yy_pop_buffer(yyscanner); + fclose(tbuffer->ptr); + dmnsn_pop_scope(symtable); + dmnsn_yyset_extra(tbuffer->prev, yyscanner); + dmnsn_delete_token_buffer(tbuffer); + return dmnsn_yylex_wrapper(lvalp, llocp, filename, symtable, yyscanner); + } } return token; @@ -496,7 +672,9 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, */ while (1) { - int token = dmnsn_yylex_wrapper(lvalp, llocp, filename, yyscanner); + int token = dmnsn_yylex_wrapper( + lvalp, llocp, filename, symtable, yyscanner + ); dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner); if (tbuffer && tbuffer->type == DMNSN_T_LEX_VERBATIM && tbuffer->i == 1) { @@ -504,6 +682,19 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, } switch (token) { + case DMNSN_T_INCLUDE: + { + dmnsn_token_buffer *tb = dmnsn_include_buffer( + token, tbuffer, lvalp, llocp, filename, symtable, yyscanner + ); + if (!tb) { + return DMNSN_T_LEX_ERROR; + } + + dmnsn_yyset_extra(tb, yyscanner); + break; + } + case DMNSN_T_DECLARE: case DMNSN_T_LOCAL: { @@ -561,7 +752,7 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, } dmnsn_yyset_extra(tb, yyscanner); - continue; + break; } case DMNSN_T_WHILE: @@ -574,7 +765,7 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp, } dmnsn_yyset_extra(tb, yyscanner); - continue; + break; } case DMNSN_T_VERSION: |