summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--dimension/tokenize.l46
-rw-r--r--tests/dimension/punctuation.pov4
2 files changed, 28 insertions, 22 deletions
diff --git a/dimension/tokenize.l b/dimension/tokenize.l
index 5ebd4ed..84b0694 100644
--- a/dimension/tokenize.l
+++ b/dimension/tokenize.l
@@ -17,7 +17,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
-%option reentrant yylineno noyywrap
+%option reentrant stack yylineno noyywrap
%{
#define YY_DECL static int yylex(const char *filename, dmnsn_array *tokens, \
@@ -81,15 +81,19 @@ size_t string_length;
unsigned long wchar;
%}
-"/*" BEGIN(DMNSN_BLOCK_COMMENT);
-"//" BEGIN(DMNSN_LINE_COMMENT);
+(?# Comments)
-<DMNSN_BLOCK_COMMENT>"*/" BEGIN(INITIAL);
-<DMNSN_BLOCK_COMMENT>[^*\n]* ;
+<INITIAL,DMNSN_BLOCK_COMMENT>"/*" {
+ yy_push_state(DMNSN_BLOCK_COMMENT, yyscanner);
+}
+<DMNSN_BLOCK_COMMENT>"*/" yy_pop_state(yyscanner);
+<DMNSN_BLOCK_COMMENT>[^*/\n]* ;
+<DMNSN_BLOCK_COMMENT>"/" ;
<DMNSN_BLOCK_COMMENT>"*" ;
<DMNSN_BLOCK_COMMENT>\n ;
-<DMNSN_LINE_COMMENT>\n BEGIN(INITIAL);
+"//" yy_push_state(DMNSN_LINE_COMMENT, yyscanner);
+<DMNSN_LINE_COMMENT>\n yy_pop_state(yyscanner);
<DMNSN_LINE_COMMENT>[^\n]+ ;
(?# Punctuation)
@@ -143,36 +147,36 @@ unsigned long wchar;
(?# Strings)
-"\"" STRING_TOKEN(); BEGIN(DMNSN_STRING);
+"\"" STRING_TOKEN(); yy_push_state(DMNSN_STRING, yyscanner);
<DMNSN_STRING>[^\\\"\n]* STRCAT(yytext, yyleng);
-<DMNSN_STRING>"\"" PUSH(); BEGIN(INITIAL);
+<DMNSN_STRING>"\"" PUSH(); yy_pop_state(yyscanner);
(?# String escape sequences)
-<DMNSN_STRING>"\\" BEGIN(DMNSN_STRING_ESCAPE);
-<DMNSN_STRING_ESCAPE>"a" STRCAT("\a", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"b" STRCAT("\b", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"f" STRCAT("\f", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"n" STRCAT("\n", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"r" STRCAT("\r", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"t" STRCAT("\t", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"v" STRCAT("\v", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"\\" STRCAT("\\", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"'" STRCAT("'", 1); BEGIN(DMNSN_STRING);
-<DMNSN_STRING_ESCAPE>"\"" STRCAT("\"", 1); BEGIN(DMNSN_STRING);
+<DMNSN_STRING>"\\" yy_push_state(DMNSN_STRING_ESCAPE, yyscanner);
+<DMNSN_STRING_ESCAPE>"a" STRCAT("\a", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"b" STRCAT("\b", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"f" STRCAT("\f", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"n" STRCAT("\n", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"r" STRCAT("\r", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"t" STRCAT("\t", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"v" STRCAT("\v", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"\\" STRCAT("\\", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"'" STRCAT("'", 1); yy_pop_state(yyscanner);
+<DMNSN_STRING_ESCAPE>"\"" STRCAT("\"", 1); yy_pop_state(yyscanner);
<DMNSN_STRING_ESCAPE>"u"[[:digit:]aAbBcCdDeEfF]{4} {
wchar = strtoul(yytext + 1, NULL, 16);
STRCAT("", 2);
token.value[string_length - 2] = wchar/256;
token.value[string_length - 1] = wchar%256;
- BEGIN(DMNSN_STRING);
+ yy_pop_state(yyscanner);
}
<DMNSN_STRING_ESCAPE>. {
dmnsn_diagnostic(filename, yylineno, yycolumn,
"WARNING: unrecognised escape sequence '\\%c'",
(int)*yytext);
STRCAT(yytext, yyleng);
- BEGIN(DMNSN_STRING);
+ yy_pop_state(yyscanner);
}
(?# Ignore whitespace)
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
index 9e8a906..e68646b 100644
--- a/tests/dimension/punctuation.pov
+++ b/tests/dimension/punctuation.pov
@@ -17,6 +17,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
-// Test that we correctly tokenize all simple punctuation marks
+/* Test that we correctly tokenize all simple punctuation marks
+ * // Also make sure we handle nested /* comments */ properly
+ */
{}()[]+-*/,;?:&.|=<>!<= >= !=