Support #include.

1 files changed, 209 insertions, 18 deletions

diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 63435c9..95aaa9b 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -20,7 +20,9 @@
 #include "tokenize.h"
 #include "directives.h"
 #include "utility.h"
+#include <libgen.h>
 #include <stdbool.h>
+#include <string.h>
 
 typedef struct dmnsn_buffered_token {
   int type;
@@ -37,10 +39,13 @@ typedef struct dmnsn_token_buffer {
   unsigned int i;
 
   struct dmnsn_token_buffer *prev;
+
+  const char *filename;
+  void *ptr;
 } dmnsn_token_buffer;
 
 static dmnsn_token_buffer *
-dmnsn_new_token_buffer(int type, dmnsn_token_buffer *prev)
+dmnsn_new_token_buffer(int type, dmnsn_token_buffer *prev, const char *filename)
 {
   dmnsn_token_buffer *tbuffer = malloc(sizeof(dmnsn_token_buffer));
   if (!tbuffer) {
@@ -51,6 +56,8 @@ dmnsn_new_token_buffer(int type, dmnsn_token_buffer *prev)
   tbuffer->buffered = dmnsn_new_array(sizeof(dmnsn_buffered_token));
   tbuffer->i = 0;
   tbuffer->prev = prev;
+  tbuffer->filename = filename;
+  tbuffer->ptr = NULL;
   return tbuffer;
 }
 
@@ -72,18 +79,177 @@ dmnsn_delete_token_buffer(dmnsn_token_buffer *tbuffer)
 
 static int
 dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
-                    const char *filename, void *yyscanner);
+                    const char *filename, dmnsn_symbol_table *symtable,
+                    void *yyscanner);
 int dmnsn_ld_yyparse(const char *filename, void *yyscanner,
                      dmnsn_symbol_table *symtable);
 
 static dmnsn_token_buffer *
+dmnsn_include_buffer(int token, dmnsn_token_buffer *prev,
+                     dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+                     const char *filename, dmnsn_symbol_table *symtable,
+                     void *yyscanner)
+{
+  dmnsn_token_buffer *include_buffer
+    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
+
+  /* Buffer the current token */
+  dmnsn_buffered_token buffered = {
+    .type = token,
+    .lval = *lvalp,
+    .lloc = *llocp
+  };
+  dmnsn_array_push(include_buffer->buffered, &buffered);
+
+  /* Recursive call */
+  buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
+                              filename, symtable, yyscanner);
+
+  if (buffered.type == DMNSN_T_EOF) {
+    dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
+                     buffered.lloc.first_column,
+                     "syntax error, unexpected end-of-file");
+    dmnsn_delete_token_buffer(include_buffer);
+    return NULL;
+  } else if (buffered.type == DMNSN_T_LEX_ERROR) {
+    dmnsn_delete_token_buffer(include_buffer);
+    return NULL;
+  }
+  /* Buffer the next token */
+  dmnsn_array_push(include_buffer->buffered, &buffered);
+
+  bool is_strexp = buffered.type != DMNSN_T_STRING;
+  if (is_strexp && buffered.type == DMNSN_T_IDENTIFIER) {
+    /* Check if it's a string identifier or a macro */
+    dmnsn_astnode *inode = dmnsn_find_symbol(symtable, buffered.lval.value);
+    if (!inode || inode->type == DMNSN_AST_STRING) {
+      is_strexp = false;
+    }
+  }
+
+  if (is_strexp) {
+    /* Grab all the tokens belonging to the string expression  */
+    int parenlevel = -1;
+    while (1) {
+      /* Recursive call - permit other directives inside the condition */
+      buffered.type = dmnsn_yylex(&buffered.lval, &buffered.lloc,
+                                  filename, symtable, yyscanner);
+
+      if (buffered.type == DMNSN_T_EOF) {
+        dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
+                         buffered.lloc.first_column,
+                         "syntax error, unexpected end-of-file");
+        dmnsn_delete_token_buffer(include_buffer);
+        return NULL;
+      } else if (buffered.type == DMNSN_T_LEX_ERROR) {
+        dmnsn_delete_token_buffer(include_buffer);
+        return NULL;
+      }
+
+      dmnsn_array_push(include_buffer->buffered, &buffered);
+
+      if (buffered.type == DMNSN_T_LPAREN) {
+        if (parenlevel < 0)
+          parenlevel = 1;
+        else
+          ++parenlevel;
+      } else if (buffered.type == DMNSN_T_RPAREN) {
+        --parenlevel;
+        if (parenlevel == 0) {
+          break;
+        }
+      }
+    }
+  }
+
+  /* Fake EOF */
+  buffered.type = DMNSN_T_EOF;
+  buffered.lval.value = NULL;
+  dmnsn_array_push(include_buffer->buffered, &buffered);
+
+  dmnsn_yyset_extra(include_buffer, yyscanner);
+  if (dmnsn_ld_yyparse(filename, yyscanner, symtable) != 0) {
+    dmnsn_yyset_extra(include_buffer->prev, yyscanner);
+    dmnsn_delete_token_buffer(include_buffer);
+    return NULL;
+  }
+
+  dmnsn_yyset_extra(include_buffer->prev, yyscanner);
+  dmnsn_delete_token_buffer(include_buffer);
+
+  dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev, filename);
+
+  dmnsn_astnode *inode = dmnsn_find_symbol(symtable, "__include__");
+  if (!inode) {
+    dmnsn_error(DMNSN_SEVERITY_HIGH, "__include__ unset.");
+  }
+
+  const char *include = inode->ptr;
+  if (inode->type != DMNSN_AST_STRING) {
+    dmnsn_error(DMNSN_SEVERITY_HIGH, "__include__ has wrong type.");
+  }
+
+  char *filename_copy = strdup(filename);
+  char *localdir = dirname(filename_copy);
+  char *local_include = malloc(strlen(localdir) + strlen(include) + 2);
+  strcpy(local_include, localdir);
+  strcat(local_include, "/");
+  strcat(local_include, include);
+  free(filename_copy);
+  dmnsn_undef_symbol(symtable, "__include__");
+
+  FILE *file = fopen(local_include, "r");
+  if (!file) {
+    dmnsn_diagnostic(llocp->first_filename, llocp->first_line,
+                     llocp->first_column,
+                     "Couldn't open include file '%s'", local_include);
+    free(local_include);
+    dmnsn_delete_token_buffer(tbuffer);
+    return NULL;
+  }
+  tbuffer->ptr = file;
+
+  void *buffer = dmnsn_yy_make_buffer(file, yyscanner);
+  if (!buffer) {
+    dmnsn_diagnostic(llocp->first_filename, llocp->first_line,
+                     llocp->first_column,
+                     "Couldn't allocate buffer for include file '%s'",
+                     local_include);
+    fclose(file);
+    free(local_include);
+    dmnsn_delete_token_buffer(tbuffer);
+    return NULL;
+  }
+  dmnsn_yy_push_buffer(buffer, yyscanner);
+
+  /* Stuff the filename in the symbol table to persist it */
+  dmnsn_astnode *includes = dmnsn_find_symbol(symtable, "__includes__");
+  if (!includes) {
+    dmnsn_declare_symbol(symtable, "__includes__", dmnsn_new_ast_array());
+    includes = dmnsn_find_symbol(symtable, "__includes__");
+  }
+  if (includes->type != DMNSN_AST_ARRAY) {
+    dmnsn_error(DMNSN_SEVERITY_HIGH, "__includes__ has wrong type.");
+  }
+
+  dmnsn_astnode fnode = dmnsn_new_ast_string(local_include);
+  free(local_include);
+  tbuffer->filename = fnode.ptr;
+  dmnsn_array_push(includes->children, &fnode);
+
+  dmnsn_push_scope(symtable);
+
+  return tbuffer;
+}
+
+static dmnsn_token_buffer *
 dmnsn_declaration_buffer(int token, dmnsn_token_buffer *prev,
                          dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
                          const char *filename, dmnsn_symbol_table *symtable,
                          void *yyscanner)
 {
   dmnsn_token_buffer *tbuffer
-    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev);
+    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
 
   /* Buffer the current token */
   dmnsn_buffered_token buffered = {
@@ -144,7 +310,7 @@ dmnsn_undef_buffer(int token, dmnsn_token_buffer *prev,
                    void *yyscanner)
 {
   dmnsn_token_buffer *tbuffer
-    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev);
+    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
 
   /* Buffer the current token */
   dmnsn_buffered_token buffered = {
@@ -186,7 +352,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev,
                 void *yyscanner)
 {
   dmnsn_token_buffer *cond_buffer
-    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev);
+    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
 
   /* Buffer the current token */
   dmnsn_buffered_token buffered = {
@@ -204,7 +370,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev,
                                 filename, symtable, yyscanner);
 
     if (buffered.type == DMNSN_T_EOF) {
-      dmnsn_diagnostic(filename, buffered.lloc.first_line,
+      dmnsn_diagnostic(buffered.lloc.first_filename, buffered.lloc.first_line,
                        buffered.lloc.first_column,
                        "syntax error, unexpected end-of-file");
       dmnsn_delete_token_buffer(cond_buffer);
@@ -244,7 +410,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev,
   dmnsn_yyset_extra(cond_buffer->prev, yyscanner);
   dmnsn_delete_token_buffer(cond_buffer);
 
-  dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev);
+  dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev, filename);
 
   dmnsn_astnode *cnode = dmnsn_find_symbol(symtable, "__cond__");
   if (!cnode) {
@@ -266,7 +432,7 @@ dmnsn_if_buffer(int token, dmnsn_token_buffer *prev,
   while (1) {
     /* Non-recursive call */
     buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc,
-                                        filename, yyscanner);
+                                        filename, symtable, yyscanner);
 
     if (buffered.type == DMNSN_T_EOF) {
       dmnsn_diagnostic(filename, buffered.lloc.first_line,
@@ -328,7 +494,7 @@ dmnsn_while_buffer(int token, dmnsn_token_buffer *prev,
                    const char *filename, dmnsn_symbol_table *symtable,
                    void *yyscanner)
 {
-  dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev);
+  dmnsn_token_buffer *tbuffer = dmnsn_new_token_buffer(token, prev, filename);
 
   /* Pretend to be an if */
   dmnsn_buffered_token buffered = {
@@ -343,7 +509,7 @@ dmnsn_while_buffer(int token, dmnsn_token_buffer *prev,
   while (1) {
     /* Non-recursive call */
     buffered.type = dmnsn_yylex_wrapper(&buffered.lval, &buffered.lloc,
-                                        filename, yyscanner);
+                                        filename, symtable, yyscanner);
 
     if (buffered.type == DMNSN_T_EOF) {
       dmnsn_diagnostic(filename, buffered.lloc.first_line,
@@ -388,7 +554,7 @@ dmnsn_version_buffer(int token, dmnsn_token_buffer *prev,
                      void *yyscanner)
 {
   dmnsn_token_buffer *tbuffer
-    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev);
+    = dmnsn_new_token_buffer(DMNSN_T_LEX_VERBATIM, prev, filename);
 
   /* Buffer the current token */
   dmnsn_buffered_token buffered = {
@@ -430,11 +596,12 @@ int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
 
 static int
 dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
-                    const char *filename, void *yyscanner)
+                    const char *filename, dmnsn_symbol_table *symtable,
+                    void *yyscanner)
 {
   dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner);
 
-  if (tbuffer) {
+  if (tbuffer && tbuffer->type != DMNSN_T_INCLUDE) {
     while (tbuffer && tbuffer->i >= dmnsn_array_size(tbuffer->buffered)) {
       if (tbuffer->type == DMNSN_T_WHILE) {
         tbuffer->i = 0;
@@ -456,7 +623,7 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
 
   int token;
 
-  if (tbuffer) {
+  if (tbuffer && tbuffer->type != DMNSN_T_INCLUDE) {
     /* Return buffered tokens */
     dmnsn_buffered_token buffered;
 
@@ -472,7 +639,16 @@ dmnsn_yylex_wrapper(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
     *llocp = buffered.lloc;
     ++tbuffer->i;
   } else {
-    token = dmnsn_yylex_impl(lvalp, llocp, filename, yyscanner);
+    const char *real_filename = tbuffer ? tbuffer->filename : filename;
+    token = dmnsn_yylex_impl(lvalp, llocp, real_filename, yyscanner);
+    if (token == DMNSN_T_EOF && tbuffer && tbuffer->type == DMNSN_T_INCLUDE) {
+      dmnsn_yy_pop_buffer(yyscanner);
+      fclose(tbuffer->ptr);
+      dmnsn_pop_scope(symtable);
+      dmnsn_yyset_extra(tbuffer->prev, yyscanner);
+      dmnsn_delete_token_buffer(tbuffer);
+      return dmnsn_yylex_wrapper(lvalp, llocp, filename, symtable, yyscanner);
+    }
   }
 
   return token;
@@ -496,7 +672,9 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
    */
 
   while (1) {
-    int token = dmnsn_yylex_wrapper(lvalp, llocp, filename, yyscanner);
+    int token = dmnsn_yylex_wrapper(
+      lvalp, llocp, filename, symtable, yyscanner
+    );
     dmnsn_token_buffer *tbuffer = dmnsn_yyget_extra(yyscanner);
 
     if (tbuffer && tbuffer->type == DMNSN_T_LEX_VERBATIM && tbuffer->i == 1) {
@@ -504,6 +682,19 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
     }
 
     switch (token) {
+    case DMNSN_T_INCLUDE:
+      {
+        dmnsn_token_buffer *tb = dmnsn_include_buffer(
+          token, tbuffer, lvalp, llocp, filename, symtable, yyscanner
+        );
+        if (!tb) {
+          return DMNSN_T_LEX_ERROR;
+        }
+
+        dmnsn_yyset_extra(tb, yyscanner);
+        break;
+      }
+
     case DMNSN_T_DECLARE:
     case DMNSN_T_LOCAL:
       {
@@ -561,7 +752,7 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
         }
 
         dmnsn_yyset_extra(tb, yyscanner);
-        continue;
+        break;
       }
 
     case DMNSN_T_WHILE:
@@ -574,7 +765,7 @@ dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
         }
 
         dmnsn_yyset_extra(tb, yyscanner);
-        continue;
+        break;
       }
 
     case DMNSN_T_VERSION: