Skip over invalid tokens in non-taken conditionals.

1 files changed, 5 insertions, 11 deletions

diff --git a/dimension/lexer.l b/dimension/lexer.l
index cd02da1..eb5184c 100644
--- a/dimension/lexer.l
+++ b/dimension/lexer.l
@@ -285,17 +285,11 @@ dmnsn_tokenize(FILE *file, const char *filename)
 
   while ((token.type = dmnsn_yylex_impl(&item, &location, filename, scanner))
          != 0) {
-    if (token.type == DMNSN_T_LEX_ERROR) {
-      dmnsn_delete_tokens(tokens);
-      tokens = NULL;
-      break;
-    } else {
-      token.value    = item.value;
-      token.filename = location.first_filename;
-      token.line     = location.first_line;
-      token.col      = location.first_column;
-      dmnsn_array_push(tokens, &token);
-    }
+    token.value    = item.value;
+    token.filename = location.first_filename;
+    token.line     = location.first_line;
+    token.col      = location.first_column;
+    dmnsn_array_push(tokens, &token);
   }
 
   yylex_destroy(scanner);