summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTavian Barnes <tavianator@gmail.com>2010-01-29 15:54:04 -0500
committerTavian Barnes <tavianator@gmail.com>2010-01-29 15:54:04 -0500
commite310c2cf9bdfcf86cbd1294567905f537832a488 (patch)
tree15b7f12c5aa23171c14bca8437236d82aa20d6e3
parent15269687f6a9bdccd30dd370875bf18416f60dca (diff)
downloaddimension-e310c2cf9bdfcf86cbd1294567905f537832a488.tar.xz
Begin middle-tier lexer.
-rw-r--r--dimension/Makefile.am1
-rw-r--r--dimension/grammar.y6
-rw-r--r--dimension/lexer.l10
-rw-r--r--dimension/tokenize.c30
-rw-r--r--dimension/tokenize.h12
5 files changed, 48 insertions, 11 deletions
diff --git a/dimension/Makefile.am b/dimension/Makefile.am
index 6f19389..1d18374 100644
--- a/dimension/Makefile.am
+++ b/dimension/Makefile.am
@@ -33,6 +33,7 @@ dimension_SOURCES = grammar.y \
progressbar.h \
realize.c \
realize.h \
+ tokenize.c \
tokenize.h \
utility.c \
utility.h \
diff --git a/dimension/grammar.y b/dimension/grammar.y
index 325efce..b41100f 100644
--- a/dimension/grammar.y
+++ b/dimension/grammar.y
@@ -27,12 +27,6 @@
#define YYSTYPE dmnsn_parse_item
#define YYLTYPE dmnsn_parse_location
-int dmnsn_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, const char *filename,
- void *yyscanner);
-void dmnsn_yylex_init(void **scannerp);
-void dmnsn_yyset_in(FILE *file, void *scanner);
-void dmnsn_yylex_destroy(void *scanner);
-
#define YYLLOC_DEFAULT(Current, Rhs, N) \
do { \
if (N) { \
diff --git a/dimension/lexer.l b/dimension/lexer.l
index 03f91dd..90c5a6b 100644
--- a/dimension/lexer.l
+++ b/dimension/lexer.l
@@ -32,9 +32,10 @@
#include <stdlib.h>
#include <stdio.h>
-#define YY_DECL int yylex(dmnsn_parse_item *lvalp, \
- dmnsn_parse_location *llocp, \
- const char *filename, yyscan_t yyscanner)
+#define YY_DECL int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, \
+ dmnsn_parse_location *llocp, \
+ const char *filename, \
+ yyscan_t yyscanner)
%}
%x DMNSN_BLOCK_COMMENT
@@ -278,7 +279,8 @@ dmnsn_tokenize(FILE *file, const char *filename)
yylex_init(&scanner);
yyset_in(file, scanner);
- while ((token.type = yylex(&item, &location, filename, scanner)) != 0) {
+ while ((token.type = dmnsn_yylex_impl(&item, &location, filename, scanner))
+ != 0) {
if (token.type == 1 || token.type == 2) {
dmnsn_delete_tokens(tokens);
tokens = NULL;
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
new file mode 100644
index 0000000..74caace
--- /dev/null
+++ b/dimension/tokenize.c
@@ -0,0 +1,30 @@
+/*************************************************************************
+ * Copyright (C) 2010 Tavian Barnes <tavianator@gmail.com> *
+ * *
+ * This file is part of Dimension. *
+ * *
+ * Dimension is free software; you can redistribute it and/or modify it *
+ * under the terms of the GNU General Public License as published by the *
+ * Free Software Foundation; either version 3 of the License, or (at *
+ * your option) any later version. *
+ * *
+ * Dimension is distributed in the hope that it will be useful, but *
+ * WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
+ * General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+#include "tokenize.h"
+
+int dmnsn_yylex_impl(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+ const char *filename, void *yyscanner);
+
+int
+dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+ const char *filename, void *yyscanner)
+{
+ return dmnsn_yylex_impl(lvalp, llocp, filename, yyscanner);
+}
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 9ea27ee..801092a 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -21,6 +21,7 @@
#define TOKENIZE_H
#include "../libdimension/dimension.h"
+#include "parse.h"
#define yytokentype dmnsn_yytokentype
#define YYSTYPE
@@ -45,7 +46,16 @@ struct dmnsn_token {
int line, col;
};
-/* For debugging */
+/* Set up the scanner */
+int dmnsn_yylex_init(void **scannerp);
+void dmnsn_yyset_in(FILE *file, void *scanner);
+int dmnsn_yylex_destroy(void *scanner);
+
+/* Actual lexer */
+int dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
+ const char *filename, void *yyscanner);
+
+/* For debugging - returns an array of raw tokens */
dmnsn_array *dmnsn_tokenize(FILE *file, const char *filename);
/* Token destruction */