Ignore comments in tokenizer.

3 files changed, 52 insertions, 2 deletions

diff --git a/tests/dimension/numeric.pov b/tests/dimension/numeric.pov
index df850d6..e9d0eae 100644
--- a/tests/dimension/numeric.pov
+++ b/tests/dimension/numeric.pov
@@ -1,2 +1,27 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of The Dimension Test Suite.                        *
+ *                                                                       *
+ * The Dimension Test Suite is free software; you can redistribute it    *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the    *
+ * License, or (at your option) any later version.                       *
+ *                                                                       *
+ * The Dimension Test Suite is distributed in the hope that it will be   *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty   *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize numeric values.  Note that the `-' in `-1',
+// for example, is a separate token and not part of the number.
+
+// Integers:
 1 123456789 01234567 0x123456789 -0x01
+
+// Floats:
 .1 0.1 1.0 0.123456789 -0.123456789
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
index db9911f..a691693 100644
--- a/tests/dimension/punctuation.pov
+++ b/tests/dimension/punctuation.pov
@@ -1,3 +1,24 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of The Dimension Test Suite.                        *
+ *                                                                       *
+ * The Dimension Test Suite is free software; you can redistribute it    *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the    *
+ * License, or (at your option) any later version.                       *
+ *                                                                       *
+ * The Dimension Test Suite is distributed in the hope that it will be   *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty   *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize all simple punctuation marks
+
 {
   (
     [
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index edd3bb4..5f126ce 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -19,13 +19,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>. #
 #########################################################################
 
+exitstatus=0
+
 punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
 punctuation_exp='({ \( [ < + - * / , > ] \) })'
 
 if [ "$punctuation" != "$punctuation_exp" ]; then
   echo "punctuation.pov tokenized as \"$punctuation\"" >&2
   echo "                 -- expected \"$punctuation_exp\"" >&2
-  exit 1;
+  exitstatus=1
 fi
 
 numeric=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/numeric.pov)
@@ -34,5 +36,7 @@ numeric_exp='((int "1") (int "123456789") (int "01234567") (int "0x123456789") -
 if [ "$numeric" != "$numeric_exp" ]; then
   echo "numeric.pov tokenized as \"$numeric\"" >&2
   echo "             -- expected \"$numeric_exp\"" >&2
-  exit 1;
+  exitstatus=1
 fi
+
+exit $exitstatus
\ No newline at end of file