Ignore comments in tokenizer.

4 files changed, 87 insertions, 6 deletions

diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 1d53f2f..360a228 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -52,7 +52,7 @@ dmnsn_tokenize(FILE *file)
   dmnsn_token token;
   dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token));
 
-  unsigned int line = 0, col = 0;
+  unsigned int line = 1, col = 0;
   unsigned int i;
 
   while (next - map < size) {
@@ -96,9 +96,39 @@ dmnsn_tokenize(FILE *file)
     dmnsn_simple_token('+', DMNSN_PLUS);
     dmnsn_simple_token('-', DMNSN_MINUS);
     dmnsn_simple_token('*', DMNSN_STAR);
-    dmnsn_simple_token('/', DMNSN_SLASH);
     dmnsn_simple_token(',', DMNSN_COMMA);
 
+    /* Possible comment */
+    case '/':
+      if (*(next + 1) == '/') {
+        /* A '//' comment block */
+        do {
+          ++next;
+        } while (next - map < size && *next != '\n');
+
+        ++line;
+        col = 0;
+        continue;
+      } else if (*(next + 1) == '*') {
+        /* A '/*' comment block */
+        do {
+          ++col;
+          if (*next == '\n') {
+            ++line;
+            col = 0;
+          }
+
+          ++next;
+        } while (next - map < size - 1
+                 && !(*next == '*' && *(next + 1) == '/'));
+        next += 2;
+        continue;
+      } else {
+        /* Just the normal punctuation mark */
+        token.type = DMNSN_SLASH;
+      }
+      break;
+
     /* Numeric values */
     case '.': /* Number begins with a decimal point, as in `.2' */
     case '0':
@@ -145,8 +175,9 @@ dmnsn_tokenize(FILE *file)
 
     default:
       /* Unrecognised character */
-      fprintf(stderr, "Unrecognized character 0x%X in input.\n",
-              (unsigned int)*next);
+      fprintf(stderr,
+              "Unrecognized character 0x%X in input at line %u, column %u.\n",
+              (unsigned int)*next, line, col);
       dmnsn_delete_tokens(tokens);
       munmap(map, size);
       return NULL;
diff --git a/tests/dimension/numeric.pov b/tests/dimension/numeric.pov
index df850d6..e9d0eae 100644
--- a/tests/dimension/numeric.pov
+++ b/tests/dimension/numeric.pov
@@ -1,2 +1,27 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of The Dimension Test Suite.                        *
+ *                                                                       *
+ * The Dimension Test Suite is free software; you can redistribute it    *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the    *
+ * License, or (at your option) any later version.                       *
+ *                                                                       *
+ * The Dimension Test Suite is distributed in the hope that it will be   *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty   *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize numeric values.  Note that the `-' in `-1',
+// for example, is a separate token and not part of the number.
+
+// Integers:
 1 123456789 01234567 0x123456789 -0x01
+
+// Floats:
 .1 0.1 1.0 0.123456789 -0.123456789
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
index db9911f..a691693 100644
--- a/tests/dimension/punctuation.pov
+++ b/tests/dimension/punctuation.pov
@@ -1,3 +1,24 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of The Dimension Test Suite.                        *
+ *                                                                       *
+ * The Dimension Test Suite is free software; you can redistribute it    *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the    *
+ * License, or (at your option) any later version.                       *
+ *                                                                       *
+ * The Dimension Test Suite is distributed in the hope that it will be   *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty   *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize all simple punctuation marks
+
 {
   (
     [
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index edd3bb4..5f126ce 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -19,13 +19,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>. #
 #########################################################################
 
+exitstatus=0
+
 punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
 punctuation_exp='({ \( [ < + - * / , > ] \) })'
 
 if [ "$punctuation" != "$punctuation_exp" ]; then
   echo "punctuation.pov tokenized as \"$punctuation\"" >&2
   echo "                 -- expected \"$punctuation_exp\"" >&2
-  exit 1;
+  exitstatus=1
 fi
 
 numeric=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/numeric.pov)
@@ -34,5 +36,7 @@ numeric_exp='((int "1") (int "123456789") (int "01234567") (int "0x123456789") -
 if [ "$numeric" != "$numeric_exp" ]; then
   echo "numeric.pov tokenized as \"$numeric\"" >&2
   echo "             -- expected \"$numeric_exp\"" >&2
-  exit 1;
+  exitstatus=1
 fi
+
+exit $exitstatus
\ No newline at end of file