Tokenize keywords.

3 files changed, 45 insertions, 2 deletions

diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am
index e9b7f7a..c0ea38e 100644
--- a/tests/dimension/Makefile.am
+++ b/tests/dimension/Makefile.am
@@ -25,4 +25,7 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir)
 tokenizer.sh:
 	cp $(srcdir)/tokenizer.sh .
 
-EXTRA_DIST = tokenizer.sh punctuation.pov numeric.pov
+EXTRA_DIST = tokenizer.sh                                                      \
+             punctuation.pov                                                   \
+             numeric.pov                                                       \
+             labels.pov
diff --git a/tests/dimension/labels.pov b/tests/dimension/labels.pov
new file mode 100644
index 0000000..fd72979
--- /dev/null
+++ b/tests/dimension/labels.pov
@@ -0,0 +1,31 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of The Dimension Test Suite.                        *
+ *                                                                       *
+ * The Dimension Test Suite is free software; you can redistribute it    *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the    *
+ * License, or (at your option) any later version.                       *
+ *                                                                       *
+ * The Dimension Test Suite is distributed in the hope that it will be   *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty   *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize identifiers and keywords
+
+camera {
+}
+
+sphere {
+  color new_identifier
+}
+
+box {
+  colour new_identifier
+}
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index 5f126ce..e23e281 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -39,4 +39,13 @@ if [ "$numeric" != "$numeric_exp" ]; then
   exitstatus=1
 fi
 
-exit $exitstatus
\ No newline at end of file
+labels=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/labels.pov)
+labels_exp='(camera { } sphere { color (identifier "new_identifier") } box { color (identifier "new_identifier") })';
+
+if [ "$labels" != "$labels_exp" ]; then
+  echo "labels.pov tokenized as \"$labels\"" >&2
+  echo "            -- expected \"$labels_exp\"" >&2
+  exitstatus=1
+fi
+
+exit $exitstatus