summaryrefslogtreecommitdiffstats
path: root/dimension/tokenize.l
diff options
context:
space:
mode:
authorTavian Barnes <tavianator@gmail.com>2009-11-23 01:13:59 -0500
committerTavian Barnes <tavianator@gmail.com>2009-11-23 01:13:59 -0500
commit1d8445edd5168359de218ece3452f4da1453a4ee (patch)
tree9914ce2117803d3ed29e61a4de69e5e2f0172547 /dimension/tokenize.l
parent3be0a0b78a3a59d10a53ed08bbff04aa9cafbdad (diff)
downloaddimension-1d8445edd5168359de218ece3452f4da1453a4ee.tar.xz
Calculate column correctly in tokenizing.
Diffstat (limited to 'dimension/tokenize.l')
-rw-r--r--dimension/tokenize.l59
1 files changed, 38 insertions, 21 deletions
diff --git a/dimension/tokenize.l b/dimension/tokenize.l
index 84b0694..222eaa1 100644
--- a/dimension/tokenize.l
+++ b/dimension/tokenize.l
@@ -36,35 +36,45 @@
%%
%{
-#define PUSH_TOKEN(token_type) \
+/* Some helpful macros that set fields of a token correctly, and other stuff */
+
+#define NEW_TOKEN(token_type) \
do { \
token.type = token_type; \
token.filename = filename; \
token.line = yylineno; \
token.col = yycolumn; \
token.value = NULL; \
- PUSH(); \
+ } while (0)
+
+#define CALCULATE_COLUMN() yycolumn += yyleng
+
+#define PUSH() \
+ do { \
+ dmnsn_array_push(tokens, &token); \
+ CALCULATE_COLUMN(); \
+ } while (0)
+
+#define PUSH_TOKEN(token_type) \
+ do { \
+ NEW_TOKEN(token_type); \
+ PUSH(); \
} while (0)
#define PUSH_VALUE_TOKEN(token_type) \
do { \
- token.type = token_type; \
- token.filename = filename; \
- token.line = yylineno; \
- token.col = yycolumn; \
+ NEW_TOKEN(token_type); \
token.value = strdup(yytext); \
PUSH(); \
} while (0)
#define STRING_TOKEN() \
do { \
- token.type = DMNSN_T_STRING; \
- token.filename = filename; \
- token.line = yylineno; \
- token.col = yycolumn; \
+ NEW_TOKEN(DMNSN_T_STRING); \
token.value = malloc(1); \
token.value[0] = '\0'; \
string_length = 0; \
+ CALCULATE_COLUMN(); \
} while (0)
#define STRCAT(str, len) \
@@ -72,10 +82,9 @@
token.value = realloc(token.value, string_length + len + 1); \
strncpy(token.value + string_length, str, len); \
string_length += len; \
+ CALCULATE_COLUMN(); \
} while(0)
-#define PUSH() dmnsn_array_push(tokens, &token)
-
dmnsn_token token;
size_t string_length;
unsigned long wchar;
@@ -85,16 +94,20 @@ unsigned long wchar;
<INITIAL,DMNSN_BLOCK_COMMENT>"/*" {
yy_push_state(DMNSN_BLOCK_COMMENT, yyscanner);
+ CALCULATE_COLUMN();
}
-<DMNSN_BLOCK_COMMENT>"*/" yy_pop_state(yyscanner);
-<DMNSN_BLOCK_COMMENT>[^*/\n]* ;
-<DMNSN_BLOCK_COMMENT>"/" ;
-<DMNSN_BLOCK_COMMENT>"*" ;
+<DMNSN_BLOCK_COMMENT>"*/" CALCULATE_COLUMN(); yy_pop_state(yyscanner);
+<DMNSN_BLOCK_COMMENT>[^*/\n]* CALCULATE_COLUMN();
+<DMNSN_BLOCK_COMMENT>"/" CALCULATE_COLUMN();
+<DMNSN_BLOCK_COMMENT>"*" CALCULATE_COLUMN();
<DMNSN_BLOCK_COMMENT>\n ;
-"//" yy_push_state(DMNSN_LINE_COMMENT, yyscanner);
-<DMNSN_LINE_COMMENT>\n yy_pop_state(yyscanner);
-<DMNSN_LINE_COMMENT>[^\n]+ ;
+"//" {
+ yy_push_state(DMNSN_LINE_COMMENT, yyscanner);
+ CALCULATE_COLUMN();
+}
+<DMNSN_LINE_COMMENT>\n ; yy_pop_state(yyscanner);
+<DMNSN_LINE_COMMENT>[^\n]+ CALCULATE_COLUMN();
(?# Punctuation)
"{" PUSH_TOKEN(DMNSN_T_LBRACE);
@@ -153,7 +166,10 @@ unsigned long wchar;
(?# String escape sequences)
-<DMNSN_STRING>"\\" yy_push_state(DMNSN_STRING_ESCAPE, yyscanner);
+<DMNSN_STRING>"\\" {
+ yy_push_state(DMNSN_STRING_ESCAPE, yyscanner);
+ CALCULATE_COLUMN();
+}
<DMNSN_STRING_ESCAPE>"a" STRCAT("\a", 1); yy_pop_state(yyscanner);
<DMNSN_STRING_ESCAPE>"b" STRCAT("\b", 1); yy_pop_state(yyscanner);
<DMNSN_STRING_ESCAPE>"f" STRCAT("\f", 1); yy_pop_state(yyscanner);
@@ -180,7 +196,8 @@ unsigned long wchar;
}
(?# Ignore whitespace)
-[[:space:]]+ ;
+[\b\r\t\v ]+ CALCULATE_COLUMN();
+\n ;
(?# Fall-through)
. {