Tokenize strings and language directives.

1 files changed, 19 insertions, 0 deletions

diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index 6d67da3..f1f2b6b 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -39,6 +39,16 @@ if [ "$numeric" != "$numeric_exp" ]; then
   exitstatus=1
 fi
 
+strings=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/strings.pov)
+strings_exp='((string "This is a string with
+"escape sequences"\"))'
+
+if [ "$strings" != "$strings_exp" ]; then
+  echo "strings.pov tokenized as \"$strings\"" >&2
+  echo "             -- expected \"$strings_exp\"" >&2
+  exitstatus=1
+fi
+
 labels=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/labels.pov)
 labels_exp='(camera { } sphere { color (identifier "new_identifier") } box { color (identifier "new_identifier") })';
 
@@ -48,4 +58,13 @@ if [ "$labels" != "$labels_exp" ]; then
   exitstatus=1
 fi
 
+directives=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/directives.pov)
+directives_exp='(#include (string "punctuation.pov") #declare (identifier "x"))';
+
+if [ "$directives" != "$directives_exp" ]; then
+  echo "directives.pov tokenized as \"$directives\"" >&2
+  echo "                -- expected \"$directives_exp\"" >&2
+  exitstatus=1
+fi
+
 exit $exitstatus