Begin middle-tier lexer.

1 files changed, 11 insertions, 1 deletions

diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 9ea27ee..801092a 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -21,6 +21,7 @@
 #define TOKENIZE_H
 
 #include "../libdimension/dimension.h"
+#include "parse.h"
 
 #define yytokentype dmnsn_yytokentype
 #define YYSTYPE
@@ -45,7 +46,16 @@ struct dmnsn_token {
   int line, col;
 };
 
-/* For debugging */
+/* Set up the scanner */
+int  dmnsn_yylex_init(void **scannerp);
+void dmnsn_yyset_in(FILE *file, void *scanner);
+int  dmnsn_yylex_destroy(void *scanner);
+
+/* Actual lexer */
+int dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+                const char *filename, void *yyscanner);
+
+/* For debugging - returns an array of raw tokens */
 dmnsn_array *dmnsn_tokenize(FILE *file, const char *filename);
 
 /* Token destruction */