Tokenize keywords.

5 files changed, 89 insertions, 13 deletions

diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 28deda2..8401dc7 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -41,7 +41,7 @@ dmnsn_tokenize_comment(char *map, size_t size,
     ++*line;
     *col = 0;
   } else if (*(*next + 1) == '*') {
-    /* A '/*' comment block */
+    /* A multi-line comment block (like this one) */
     do {
       ++col;
       if (**next == '\n') {
@@ -96,9 +96,10 @@ dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token,
   return 0;
 }
 
+/* Tokenize a keyword or an identifier */
 static int
-dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token,
-                          char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token,
+                     char **next, unsigned int *line, unsigned int *col)
 {
   unsigned int i = 0, alloc = 32;
 
@@ -109,7 +110,7 @@ dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token,
   token->type  = DMNSN_IDENTIFIER;
   token->value = malloc(alloc);
 
-  while (*next - map < size && (isalnum(**next) || **next == '_')) {
+  do {
     if (i + 1 >= alloc) {
       alloc *= 2;
       token->value = realloc(token->value, alloc);
@@ -120,9 +121,28 @@ dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token,
     ++i;
     ++*col;
     ++*next;
-  }
+  }  while (*next - map < size && (isalnum(**next) || **next == '_'));
 
   token->value[i] = '\0';
+
+  /* Now check if we really found a keyword */
+
+#define dmnsn_keyword(str, tp)                                                 \
+  do {                                                                         \
+    if (strcmp(token->value, str) == 0) {                                      \
+      free(token->value);                                                      \
+      token->value = NULL;                                                     \
+      token->type  = tp;                                                       \
+      return 0;                                                                \
+    }                                                                          \
+  } while (0)
+
+  dmnsn_keyword("camera", DMNSN_CAMERA);
+  dmnsn_keyword("color",  DMNSN_COLOR);
+  dmnsn_keyword("colour", DMNSN_COLOR);
+  dmnsn_keyword("sphere", DMNSN_SPHERE);
+  dmnsn_keyword("box",    DMNSN_BOX);
+
   return 0;
 }
 
@@ -178,9 +198,11 @@ dmnsn_tokenize(FILE *file)
       continue;
 
     /* Macro to make basic symbol tokens easier */
-    #define dmnsn_simple_token(c, tp)                                          \
+#define dmnsn_simple_token(c, tp)                                              \
     case c:                                                                    \
       token.type = tp;                                                         \
+      ++col;                                                                   \
+      ++next;                                                                  \
       break
 
     /* Some simple punctuation marks */
@@ -204,6 +226,8 @@ dmnsn_tokenize(FILE *file)
       } else {
         /* Just the normal punctuation mark */
         token.type = DMNSN_SLASH;
+        ++col;
+        ++next;
       }
       break;
 
@@ -227,8 +251,7 @@ dmnsn_tokenize(FILE *file)
       break;
 
     default:
-      if (dmnsn_tokenize_identifier(map, size, &token, &next, &line, &col) != 0)
-      {
+      if (dmnsn_tokenize_label(map, size, &token, &next, &line, &col) != 0) {
         /* Unrecognised character */
         fprintf(stderr,
                 "Unrecognized character 0x%X in input at line %u, column %u.\n",
@@ -238,8 +261,6 @@ dmnsn_tokenize(FILE *file)
     }
 
     dmnsn_array_push(tokens, &token);
-    ++next;
-    ++col;
   }
 
   munmap(map, size);
@@ -301,7 +322,7 @@ dmnsn_token_name(dmnsn_token_type token_type)
 {
   switch (token_type) {
   /* Macro to shorten this huge switch */
-  #define dmnsn_token_map(type, str)                                           \
+#define dmnsn_token_map(type, str)                                             \
   case type:                                                                   \
     return str;
 
@@ -324,6 +345,12 @@ dmnsn_token_name(dmnsn_token_type token_type)
   dmnsn_token_map(DMNSN_INT,   "int");
   dmnsn_token_map(DMNSN_FLOAT, "float");
 
+  /* Keywords */
+  dmnsn_token_map(DMNSN_CAMERA, "camera");
+  dmnsn_token_map(DMNSN_COLOR,  "color");
+  dmnsn_token_map(DMNSN_SPHERE, "sphere");
+  dmnsn_token_map(DMNSN_BOX,    "box");
+
   /* Identifiers */
   dmnsn_token_map(DMNSN_IDENTIFIER, "identifier");
 
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index aead839..2e82f98 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -39,6 +39,12 @@ typedef enum {
   DMNSN_INT,
   DMNSN_FLOAT,
 
+  /* Keywords */
+  DMNSN_CAMERA,
+  DMNSN_COLOR,
+  DMNSN_SPHERE,
+  DMNSN_BOX,
+
   /* Identifiers */
   DMNSN_IDENTIFIER,
 } dmnsn_token_type;
diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am
index e9b7f7a..c0ea38e 100644
--- a/tests/dimension/Makefile.am
+++ b/tests/dimension/Makefile.am
@@ -25,4 +25,7 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir)
 tokenizer.sh:
 	cp $(srcdir)/tokenizer.sh .
 
-EXTRA_DIST = tokenizer.sh punctuation.pov numeric.pov
+EXTRA_DIST = tokenizer.sh                                                      \
+             punctuation.pov                                                   \
+             numeric.pov                                                       \
+             labels.pov
diff --git a/tests/dimension/labels.pov b/tests/dimension/labels.pov
new file mode 100644
index 0000000..fd72979
--- /dev/null
+++ b/tests/dimension/labels.pov
@@ -0,0 +1,31 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com>               *
+ *                                                                       *
+ * This file is part of The Dimension Test Suite.                        *
+ *                                                                       *
+ * The Dimension Test Suite is free software; you can redistribute it    *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the    *
+ * License, or (at your option) any later version.                       *
+ *                                                                       *
+ * The Dimension Test Suite is distributed in the hope that it will be   *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty   *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  *
+ * General Public License for more details.                              *
+ *                                                                       *
+ * You should have received a copy of the GNU General Public License     *
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize identifiers and keywords
+
+camera {
+}
+
+sphere {
+  color new_identifier
+}
+
+box {
+  colour new_identifier
+}
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index 5f126ce..e23e281 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -39,4 +39,13 @@ if [ "$numeric" != "$numeric_exp" ]; then
   exitstatus=1
 fi
 
-exit $exitstatus
\ No newline at end of file
+labels=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/labels.pov)
+labels_exp='(camera { } sphere { color (identifier "new_identifier") } box { color (identifier "new_identifier") })';
+
+if [ "$labels" != "$labels_exp" ]; then
+  echo "labels.pov tokenized as \"$labels\"" >&2
+  echo "            -- expected \"$labels_exp\"" >&2
+  exitstatus=1
+fi
+
+exit $exitstatus