summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTavian Barnes <tavianator@gmail.com>2009-10-29 21:50:58 -0400
committerTavian Barnes <tavianator@gmail.com>2009-10-29 21:50:58 -0400
commit7c414dea9bb4c65d6a0e4426a0d6ac172500f4b6 (patch)
tree177a1105a14f8125c3e6e532e589ee00b02b444a
parent6ecfb43aace8231d5fcb1499c286ddedec19577a (diff)
downloaddimension-7c414dea9bb4c65d6a0e4426a0d6ac172500f4b6.tar.xz
Correctly tokenize <=, >= and !=.
-rw-r--r--dimension/tokenize.c54
-rw-r--r--dimension/tokenize.h11
-rw-r--r--tests/dimension/punctuation.pov10
-rwxr-xr-xtests/dimension/punctuation.sh2
4 files changed, 55 insertions, 22 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 278e3cd..1a74d44 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -886,19 +886,52 @@ dmnsn_tokenize(const char *filename, FILE *file)
dmnsn_simple_token(')', DMNSN_T_RPAREN);
dmnsn_simple_token('[', DMNSN_T_LBRACKET);
dmnsn_simple_token(']', DMNSN_T_RBRACKET);
- dmnsn_simple_token('<', DMNSN_T_LT);
- dmnsn_simple_token('>', DMNSN_T_GT);
dmnsn_simple_token('+', DMNSN_T_PLUS);
dmnsn_simple_token('-', DMNSN_T_MINUS);
dmnsn_simple_token('*', DMNSN_T_STAR);
dmnsn_simple_token(',', DMNSN_T_COMMA);
- dmnsn_simple_token('=', DMNSN_T_EQUALS);
dmnsn_simple_token(';', DMNSN_T_SEMICOLON);
dmnsn_simple_token('?', DMNSN_T_QUESTION);
dmnsn_simple_token(':', DMNSN_T_COLON);
dmnsn_simple_token('&', DMNSN_T_AND);
- dmnsn_simple_token('!', DMNSN_T_BANG);
dmnsn_simple_token('|', DMNSN_T_PIPE);
+ dmnsn_simple_token('=', DMNSN_T_EQUALS);
+
+ case '<':
+ if (*(next + 1) == '=') {
+ token.type = DMNSN_T_LESS_EQUAL;
+ ++col;
+ ++next;
+ } else {
+ token.type = DMNSN_T_LESS;
+ }
+ ++col;
+ ++next;
+ break;
+
+ case '>':
+ if (*(next + 1) == '=') {
+ token.type = DMNSN_T_GREATER_EQUAL;
+ ++col;
+ ++next;
+ } else {
+ token.type = DMNSN_T_GREATER;
+ }
+ ++col;
+ ++next;
+ break;
+
+ case '!':
+ if (*(next + 1) == '=') {
+ token.type = DMNSN_T_NOT_EQUAL;
+ ++col;
+ ++next;
+ } else {
+ token.type = DMNSN_T_BANG;
+ }
+ ++col;
+ ++next;
+ break;
/* Possible comment */
case '/':
@@ -1063,27 +1096,32 @@ dmnsn_token_name(dmnsn_token_type token_type)
return str;
/* Punctuation */
+
dmnsn_token_map(DMNSN_T_LBRACE, "{");
dmnsn_token_map(DMNSN_T_RBRACE, "}")
dmnsn_token_map(DMNSN_T_LPAREN, "(");
dmnsn_token_map(DMNSN_T_RPAREN, ")");
dmnsn_token_map(DMNSN_T_LBRACKET, "[");
dmnsn_token_map(DMNSN_T_RBRACKET, "]");
- dmnsn_token_map(DMNSN_T_LT, "<");
- dmnsn_token_map(DMNSN_T_GT, ">");
dmnsn_token_map(DMNSN_T_PLUS, "+");
dmnsn_token_map(DMNSN_T_MINUS, "-");
dmnsn_token_map(DMNSN_T_STAR, "*");
dmnsn_token_map(DMNSN_T_SLASH, "/");
dmnsn_token_map(DMNSN_T_COMMA, ",");
- dmnsn_token_map(DMNSN_T_EQUALS, "=");
dmnsn_token_map(DMNSN_T_SEMICOLON, ";");
dmnsn_token_map(DMNSN_T_QUESTION, "?");
dmnsn_token_map(DMNSN_T_COLON, ":");
dmnsn_token_map(DMNSN_T_AND, "&");
- dmnsn_token_map(DMNSN_T_BANG, "!");
dmnsn_token_map(DMNSN_T_DOT, ".");
dmnsn_token_map(DMNSN_T_PIPE, "|");
+ dmnsn_token_map(DMNSN_T_LESS, "<");
+ dmnsn_token_map(DMNSN_T_GREATER, ">");
+ dmnsn_token_map(DMNSN_T_BANG, "!");
+ dmnsn_token_map(DMNSN_T_EQUALS, "=");
+
+ dmnsn_token_map(DMNSN_T_LESS_EQUAL, "<=");
+ dmnsn_token_map(DMNSN_T_GREATER_EQUAL, ">=");
+ dmnsn_token_map(DMNSN_T_NOT_EQUAL, "!=");
/* Numeric values */
dmnsn_token_map(DMNSN_T_INTEGER, "int");
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 92ae11a..894ee13 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -27,21 +27,24 @@ typedef enum {
DMNSN_T_RPAREN, /* ) */
DMNSN_T_LBRACKET, /* [ */
DMNSN_T_RBRACKET, /* ] */
- DMNSN_T_LT, /* < */
- DMNSN_T_GT, /* > */
DMNSN_T_PLUS, /* + */
DMNSN_T_MINUS, /* - */
DMNSN_T_STAR, /* * */
DMNSN_T_SLASH, /* / */
DMNSN_T_COMMA, /* , */
- DMNSN_T_EQUALS, /* = */
DMNSN_T_SEMICOLON, /* ; */
DMNSN_T_QUESTION, /* ? */
DMNSN_T_COLON, /* : */
DMNSN_T_AND, /* & */
- DMNSN_T_BANG, /* ! */
DMNSN_T_DOT, /* . */
DMNSN_T_PIPE, /* | */
+ DMNSN_T_LESS, /* < */
+ DMNSN_T_GREATER, /* > */
+ DMNSN_T_BANG, /* ! */
+ DMNSN_T_EQUALS, /* = */
+ DMNSN_T_LESS_EQUAL, /* <= */
+ DMNSN_T_GREATER_EQUAL, /* >= */
+ DMNSN_T_NOT_EQUAL, /* != */
/* Numeric values */
DMNSN_T_INTEGER,
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
index a691693..9e8a906 100644
--- a/tests/dimension/punctuation.pov
+++ b/tests/dimension/punctuation.pov
@@ -19,12 +19,4 @@
// Test that we correctly tokenize all simple punctuation marks
-{
- (
- [
- <
- + - * / ,
- >
- ]
- )
-}
+{}()[]+-*/,;?:&.|=<>!<= >= !=
diff --git a/tests/dimension/punctuation.sh b/tests/dimension/punctuation.sh
index 5364670..3765ffa 100755
--- a/tests/dimension/punctuation.sh
+++ b/tests/dimension/punctuation.sh
@@ -20,7 +20,7 @@
#########################################################################
punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
-punctuation_exp='({ \( [ < + - * / , > ] \) })'
+punctuation_exp='({ } \( \) [ ] + - * / , ; ? : & . | = < > ! <= >= !=)'
if [ "$punctuation" != "$punctuation_exp" ]; then
echo "punctuation.pov tokenized as \"$punctuation\"" >&2