summaryrefslogtreecommitdiffstats
path: root/dimension/tokenize.c
diff options
context:
space:
mode:
Diffstat (limited to 'dimension/tokenize.c')
-rw-r--r--dimension/tokenize.c236
1 files changed, 153 insertions, 83 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 4e499d5..e872e0c 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -84,42 +84,79 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
int dmnsn_ld_yyparse(const char *filename, void *yyscanner,
dmnsn_symbol_table *symtable);
-static dmnsn_token_buffer *
-dmnsn_include_buffer(int token, dmnsn_token_buffer *prev,
- dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
- const char *filename, dmnsn_symbol_table *symtable,
- void *yyscanner)
+static int
+dmnsn_buffer_balanced(dmnsn_token_buffer *tbuffer, bool recursive,
+ dmnsn_token_type left, dmnsn_token_type right,
+ const char *filename, dmnsn_symbol_table *symtable,
+ void *yyscanner)
{
- dmnsn_token_buffer *include_buffer
- = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
+ dmnsn_buffered_token buffered;
- /* Buffer the current token */
- dmnsn_buffered_token buffered = {
- .type = token,
- .lval = *lvalp,
- .lloc = *llocp
- };
- dmnsn_array_push(include_buffer->buffered, &buffered);
+ int nesting = -1;
+ while (1) {
+ if (recursive) {
+ buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
+ filename, symtable, yyscanner);
+ } else {
+ buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc,
+ filename, symtable, yyscanner);
+ }
- /* Recursive call */
- buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
- filename, symtable, yyscanner);
+ if (buffered.type == DMNSN_T_EOF) {
+ dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
+ buffered.lloc.first_column,
+ "syntax error, unexpected end-of-file");
+ return 1;
+ } else if (buffered.type == DMNSN_T_LEX_ERROR) {
+ return 1;
+ }
+
+ dmnsn_array_push(tbuffer->buffered, &buffered);
+
+ if (buffered.type == left) {
+ if (nesting < 0)
+ nesting = 1;
+ else
+ ++nesting;
+ } else if (buffered.type == right) {
+ --nesting;
+ if (nesting == 0) {
+ break;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static int
+dmnsn_buffer_strexp(dmnsn_token_buffer *tbuffer, bool recursive,
+ const char *filename, dmnsn_symbol_table *symtable,
+ void *yyscanner)
+{
+ dmnsn_buffered_token buffered;
+
+ if (recursive) {
+ buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
+ filename, symtable, yyscanner);
+ } else {
+ buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc,
+ filename, symtable, yyscanner);
+ }
if (buffered.type == DMNSN_T_EOF) {
dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
buffered.lloc.first_column,
"syntax error, unexpected end-of-file");
- dmnsn_delete_token_buffer(include_buffer);
- return NULL;
+ return 1;
} else if (buffered.type == DMNSN_T_LEX_ERROR) {
- dmnsn_delete_token_buffer(include_buffer);
- return NULL;
+ return 1;
}
- /* Buffer the next token */
- dmnsn_array_push(include_buffer->buffered, &buffered);
+ /* Buffer the first token */
+ dmnsn_array_push(tbuffer->buffered, &buffered);
bool is_strexp = buffered.type != DMNSN_T_STRING;
- if (is_strexp && buffered.type == DMNSN_T_IDENTIFIER) {
+ if (buffered.type == DMNSN_T_IDENTIFIER) {
/* Check if it's a string identifier or a macro */
dmnsn_astnode *inode = dmnsn_find_symbol(symtable, buffered.lval.value);
if (!inode || inode->type == DMNSN_AST_STRING) {
@@ -129,37 +166,37 @@ dmnsn_include_buffer(int token, dmnsn_token_buffer *prev,
if (is_strexp) {
/* Grab all the tokens belonging to the string expression */
- int parenlevel = -1;
- while (1) {
- /* Recursive call - permit other directives inside the condition */
- buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
- filename, symtable, yyscanner);
+ return dmnsn_buffer_balanced(tbuffer, recursive,
+ DMNSN_T_LPAREN, DMNSN_T_RPAREN,
+ filename, symtable, yyscanner);
+ }
- if (buffered.type == DMNSN_T_EOF) {
- dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
- buffered.lloc.first_column,
- "syntax error, unexpected end-of-file");
- dmnsn_delete_token_buffer(include_buffer);
- return NULL;
- } else if (buffered.type == DMNSN_T_LEX_ERROR) {
- dmnsn_delete_token_buffer(include_buffer);
- return NULL;
- }
+ return 0;
+}
- dmnsn_array_push(include_buffer->buffered, &buffered);
-
- if (buffered.type == DMNSN_T_LPAREN) {
- if (parenlevel < 0)
- parenlevel = 1;
- else
- ++parenlevel;
- } else if (buffered.type == DMNSN_T_RPAREN) {
- --parenlevel;
- if (parenlevel == 0) {
- break;
- }
- }
- }
+static dmnsn_token_buffer *
+dmnsn_include_buffer(int token, dmnsn_token_buffer *prev,
+ dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+ const char *filename, dmnsn_symbol_table *symtable,
+ void *yyscanner)
+{
+ dmnsn_token_buffer *include_buffer
+ = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
+
+ /* Buffer the current token */
+ dmnsn_buffered_token buffered = {
+ .type = token,
+ .lval = *lvalp,
+ .lloc = *llocp
+ };
+ dmnsn_array_push(include_buffer->buffered, &buffered);
+
+ /* Buffer the following string expression */
+ if (dmnsn_buffer_strexp(include_buffer, true, filename, symtable, yyscanner)
+ != 0)
+ {
+ dmnsn_delete_token_buffer(include_buffer);
+ return NULL;
}
/* Fake EOF */
@@ -360,36 +397,12 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev,
dmnsn_array_push(cond_buffer->buffered, &buffered);
/* Grab all the tokens belonging to the #if (...) */
- int parenlevel = -1;
- while (1) {
- /* Recursive call - permit other directives inside the condition */
- buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
- filename, symtable, yyscanner);
-
- if (buffered.type == DMNSN_T_EOF) {
- dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
- buffered.lloc.first_column,
- "syntax error, unexpected end-of-file");
- dmnsn_delete_token_buffer(cond_buffer);
- return NULL;
- } else if (buffered.type == DMNSN_T_LEX_ERROR) {
- dmnsn_delete_token_buffer(cond_buffer);
- return NULL;
- }
-
- dmnsn_array_push(cond_buffer->buffered, &buffered);
-
- if (buffered.type == DMNSN_T_LPAREN) {
- if (parenlevel < 0)
- parenlevel = 1;
- else
- ++parenlevel;
- } else if (buffered.type == DMNSN_T_RPAREN) {
- --parenlevel;
- if (parenlevel == 0) {
- break;
- }
- }
+ if (dmnsn_buffer_balanced(cond_buffer, true, DMNSN_T_LPAREN, DMNSN_T_RPAREN,
+ filename, symtable, yyscanner)
+ != 0)
+ {
+ dmnsn_delete_token_buffer(cond_buffer);
+ return NULL;
}
/* Fake EOF */
@@ -586,6 +599,39 @@ dmnsn_version_buffer(int token, dmnsn_token_buffer *prev,
return tbuffer;
}
+static dmnsn_token_buffer *
+dmnsn_stream_buffer(int token, dmnsn_token_buffer *prev,
+ dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+ const char *filename, dmnsn_symbol_table *symtable,
+ void *yyscanner)
+{
+ dmnsn_token_buffer *tbuffer
+ = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
+
+ /* Buffer the current token */
+ dmnsn_buffered_token buffered = {
+ .type = token,
+ .lval = *lvalp,
+ .lloc = *llocp
+ };
+ dmnsn_array_push(tbuffer->buffered, &buffered);
+
+ /* Buffer the following string expression */
+ if (dmnsn_buffer_strexp(tbuffer, true, filename, symtable, yyscanner)
+ != 0)
+ {
+ dmnsn_delete_token_buffer(tbuffer);
+ return NULL;
+ }
+
+ /* Fake EOF */
+ buffered.type = DMNSN_T_EOF;
+ buffered.lval.value = NULL;
+ dmnsn_array_push(tbuffer->buffered, &buffered);
+
+ return tbuffer;
+}
+
int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
const char *filename, void *yyscanner);
@@ -785,6 +831,30 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
break;
}
+ case DMNSN_T_DEBUG:
+ case DMNSN_T_ERROR:
+ case DMNSN_T_WARNING:
+ {
+ dmnsn_token_buffer *tb = dmnsn_stream_buffer(
+ token, tbuffer, lvalp, llocp, filename, symtable, yyscanner
+ );
+ if (!tb) {
+ return DMNSN_T_LEX_ERROR;
+ }
+
+ dmnsn_yyset_extra(tb, yyscanner);
+ if (dmnsn_ld_yyparse(filename, yyscanner, symtable) != 0) {
+ dmnsn_yyset_extra(tb->prev, yyscanner);
+ dmnsn_delete_token_buffer(tb);
+ return DMNSN_T_LEX_ERROR;
+ }
+
+ /* Restore the previous extra pointer */
+ dmnsn_yyset_extra(tb->prev, yyscanner);
+ dmnsn_delete_token_buffer(tb);
+ break;
+ }
+
default:
return token;
}