From 7c414dea9bb4c65d6a0e4426a0d6ac172500f4b6 Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@gmail.com>
Date: Thu, 29 Oct 2009 21:50:58 -0400
Subject: Correctly tokenize <=, >= and !=.

---
 dimension/tokenize.c            | 54 +++++++++++++++++++++++++++++++++++------
 dimension/tokenize.h            | 11 ++++++---
 tests/dimension/punctuation.pov | 10 +-------
 tests/dimension/punctuation.sh  |  2 +-
 4 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 278e3cd..1a74d44 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -886,19 +886,52 @@ dmnsn_tokenize(const char *filename, FILE *file)
     dmnsn_simple_token(')', DMNSN_T_RPAREN);
     dmnsn_simple_token('[', DMNSN_T_LBRACKET);
     dmnsn_simple_token(']', DMNSN_T_RBRACKET);
-    dmnsn_simple_token('<', DMNSN_T_LT);
-    dmnsn_simple_token('>', DMNSN_T_GT);
     dmnsn_simple_token('+', DMNSN_T_PLUS);
     dmnsn_simple_token('-', DMNSN_T_MINUS);
     dmnsn_simple_token('*', DMNSN_T_STAR);
     dmnsn_simple_token(',', DMNSN_T_COMMA);
-    dmnsn_simple_token('=', DMNSN_T_EQUALS);
     dmnsn_simple_token(';', DMNSN_T_SEMICOLON);
     dmnsn_simple_token('?', DMNSN_T_QUESTION);
     dmnsn_simple_token(':', DMNSN_T_COLON);
     dmnsn_simple_token('&', DMNSN_T_AND);
-    dmnsn_simple_token('!', DMNSN_T_BANG);
     dmnsn_simple_token('|', DMNSN_T_PIPE);
+    dmnsn_simple_token('=', DMNSN_T_EQUALS);
+
+    case '<':
+      if (*(next + 1) == '=') {
+        token.type = DMNSN_T_LESS_EQUAL;
+        ++col;
+        ++next;
+      } else {
+        token.type = DMNSN_T_LESS;
+      }
+      ++col;
+      ++next;
+      break;
+
+    case '>':
+      if (*(next + 1) == '=') {
+        token.type = DMNSN_T_GREATER_EQUAL;
+        ++col;
+        ++next;
+      } else {
+        token.type = DMNSN_T_GREATER;
+      }
+      ++col;
+      ++next;
+      break;
+
+    case '!':
+      if (*(next + 1) == '=') {
+        token.type = DMNSN_T_NOT_EQUAL;
+        ++col;
+        ++next;
+      } else {
+        token.type = DMNSN_T_BANG;
+      }
+      ++col;
+      ++next;
+      break;
 
     /* Possible comment */
     case '/':
@@ -1063,27 +1096,32 @@ dmnsn_token_name(dmnsn_token_type token_type)
     return str;
 
   /* Punctuation */
+
   dmnsn_token_map(DMNSN_T_LBRACE,    "{");
   dmnsn_token_map(DMNSN_T_RBRACE,    "}")
   dmnsn_token_map(DMNSN_T_LPAREN,    "(");
   dmnsn_token_map(DMNSN_T_RPAREN,    ")");
   dmnsn_token_map(DMNSN_T_LBRACKET,  "[");
   dmnsn_token_map(DMNSN_T_RBRACKET,  "]");
-  dmnsn_token_map(DMNSN_T_LT,        "<");
-  dmnsn_token_map(DMNSN_T_GT,        ">");
   dmnsn_token_map(DMNSN_T_PLUS,      "+");
   dmnsn_token_map(DMNSN_T_MINUS,     "-");
   dmnsn_token_map(DMNSN_T_STAR,      "*");
   dmnsn_token_map(DMNSN_T_SLASH,     "/");
   dmnsn_token_map(DMNSN_T_COMMA,     ",");
-  dmnsn_token_map(DMNSN_T_EQUALS,    "=");
   dmnsn_token_map(DMNSN_T_SEMICOLON, ";");
   dmnsn_token_map(DMNSN_T_QUESTION,  "?");
   dmnsn_token_map(DMNSN_T_COLON,     ":");
   dmnsn_token_map(DMNSN_T_AND,       "&");
-  dmnsn_token_map(DMNSN_T_BANG,      "!");
   dmnsn_token_map(DMNSN_T_DOT,       ".");
   dmnsn_token_map(DMNSN_T_PIPE,      "|");
+  dmnsn_token_map(DMNSN_T_LESS,      "<");
+  dmnsn_token_map(DMNSN_T_GREATER,   ">");
+  dmnsn_token_map(DMNSN_T_BANG,      "!");
+  dmnsn_token_map(DMNSN_T_EQUALS,    "=");
+
+  dmnsn_token_map(DMNSN_T_LESS_EQUAL,    "<=");
+  dmnsn_token_map(DMNSN_T_GREATER_EQUAL, ">=");
+  dmnsn_token_map(DMNSN_T_NOT_EQUAL,     "!=");
 
   /* Numeric values */
   dmnsn_token_map(DMNSN_T_INTEGER, "int");
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 92ae11a..894ee13 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -27,21 +27,24 @@ typedef enum {
   DMNSN_T_RPAREN,    /* ) */
   DMNSN_T_LBRACKET,  /* [ */
   DMNSN_T_RBRACKET,  /* ] */
-  DMNSN_T_LT,        /* < */
-  DMNSN_T_GT,        /* > */
   DMNSN_T_PLUS,      /* + */
   DMNSN_T_MINUS,     /* - */
   DMNSN_T_STAR,      /* * */
   DMNSN_T_SLASH,     /* / */
   DMNSN_T_COMMA,     /* , */
-  DMNSN_T_EQUALS,    /* = */
   DMNSN_T_SEMICOLON, /* ; */
   DMNSN_T_QUESTION,  /* ? */
   DMNSN_T_COLON,     /* : */
   DMNSN_T_AND,       /* & */
-  DMNSN_T_BANG,      /* ! */
   DMNSN_T_DOT,       /* . */
   DMNSN_T_PIPE,      /* | */
+  DMNSN_T_LESS,      /* < */
+  DMNSN_T_GREATER,   /* > */
+  DMNSN_T_BANG,      /* ! */
+  DMNSN_T_EQUALS,    /* = */
+  DMNSN_T_LESS_EQUAL,    /* <= */
+  DMNSN_T_GREATER_EQUAL, /* >= */
+  DMNSN_T_NOT_EQUAL,     /* != */
 
   /* Numeric values */
   DMNSN_T_INTEGER,
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
index a691693..9e8a906 100644
--- a/tests/dimension/punctuation.pov
+++ b/tests/dimension/punctuation.pov
@@ -19,12 +19,4 @@
 
 // Test that we correctly tokenize all simple punctuation marks
 
-{
-  (
-    [
-      <
-        + - * / ,
-      >
-    ]
-  )
-}
+{}()[]+-*/,;?:&.|=<>!<= >= !=
diff --git a/tests/dimension/punctuation.sh b/tests/dimension/punctuation.sh
index 5364670..3765ffa 100755
--- a/tests/dimension/punctuation.sh
+++ b/tests/dimension/punctuation.sh
@@ -20,7 +20,7 @@
 #########################################################################
 
 punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
-punctuation_exp='({ \( [ < + - * / , > ] \) })'
+punctuation_exp='({ } \( \) [ ] + - * / , ; ? : & . | = < > ! <= >= !=)'
 
 if [ "$punctuation" != "$punctuation_exp" ]; then
   echo "punctuation.pov tokenized as \"$punctuation\"" >&2
-- 
cgit v1.2.3