summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTavian Barnes <tavianator@gmail.com>2009-10-28 02:32:06 -0400
committerTavian Barnes <tavianator@gmail.com>2009-10-28 02:32:45 -0400
commit9fe396b0148e02114a4a70273df078e08868141c (patch)
tree4f551487d46e3ee99228d6c06aa609a609da831d
parent61d8844371d8f0d61432ec1ef91381e7d17e897f (diff)
downloaddimension-9fe396b0148e02114a4a70273df078e08868141c.tar.xz
Tokenize keywords.
-rw-r--r--dimension/tokenize.c49
-rw-r--r--dimension/tokenize.h6
-rw-r--r--tests/dimension/Makefile.am5
-rw-r--r--tests/dimension/labels.pov31
-rwxr-xr-xtests/dimension/tokenizer.sh11
5 files changed, 89 insertions, 13 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 28deda2..8401dc7 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -41,7 +41,7 @@ dmnsn_tokenize_comment(char *map, size_t size,
++*line;
*col = 0;
} else if (*(*next + 1) == '*') {
- /* A '/*' comment block */
+ /* A multi-line comment block (like this one) */
do {
++col;
if (**next == '\n') {
@@ -96,9 +96,10 @@ dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token,
return 0;
}
+/* Tokenize a keyword or an identifier */
static int
-dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token,
- char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token,
+ char **next, unsigned int *line, unsigned int *col)
{
unsigned int i = 0, alloc = 32;
@@ -109,7 +110,7 @@ dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token,
token->type = DMNSN_IDENTIFIER;
token->value = malloc(alloc);
- while (*next - map < size && (isalnum(**next) || **next == '_')) {
+ do {
if (i + 1 >= alloc) {
alloc *= 2;
token->value = realloc(token->value, alloc);
@@ -120,9 +121,28 @@ dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token,
++i;
++*col;
++*next;
- }
+ } while (*next - map < size && (isalnum(**next) || **next == '_'));
token->value[i] = '\0';
+
+ /* Now check if we really found a keyword */
+
+#define dmnsn_keyword(str, tp) \
+ do { \
+ if (strcmp(token->value, str) == 0) { \
+ free(token->value); \
+ token->value = NULL; \
+ token->type = tp; \
+ return 0; \
+ } \
+ } while (0)
+
+ dmnsn_keyword("camera", DMNSN_CAMERA);
+ dmnsn_keyword("color", DMNSN_COLOR);
+ dmnsn_keyword("colour", DMNSN_COLOR);
+ dmnsn_keyword("sphere", DMNSN_SPHERE);
+ dmnsn_keyword("box", DMNSN_BOX);
+
return 0;
}
@@ -178,9 +198,11 @@ dmnsn_tokenize(FILE *file)
continue;
/* Macro to make basic symbol tokens easier */
- #define dmnsn_simple_token(c, tp) \
+#define dmnsn_simple_token(c, tp) \
case c: \
token.type = tp; \
+ ++col; \
+ ++next; \
break
/* Some simple punctuation marks */
@@ -204,6 +226,8 @@ dmnsn_tokenize(FILE *file)
} else {
/* Just the normal punctuation mark */
token.type = DMNSN_SLASH;
+ ++col;
+ ++next;
}
break;
@@ -227,8 +251,7 @@ dmnsn_tokenize(FILE *file)
break;
default:
- if (dmnsn_tokenize_identifier(map, size, &token, &next, &line, &col) != 0)
- {
+ if (dmnsn_tokenize_label(map, size, &token, &next, &line, &col) != 0) {
/* Unrecognised character */
fprintf(stderr,
"Unrecognized character 0x%X in input at line %u, column %u.\n",
@@ -238,8 +261,6 @@ dmnsn_tokenize(FILE *file)
}
dmnsn_array_push(tokens, &token);
- ++next;
- ++col;
}
munmap(map, size);
@@ -301,7 +322,7 @@ dmnsn_token_name(dmnsn_token_type token_type)
{
switch (token_type) {
/* Macro to shorten this huge switch */
- #define dmnsn_token_map(type, str) \
+#define dmnsn_token_map(type, str) \
case type: \
return str;
@@ -324,6 +345,12 @@ dmnsn_token_name(dmnsn_token_type token_type)
dmnsn_token_map(DMNSN_INT, "int");
dmnsn_token_map(DMNSN_FLOAT, "float");
+ /* Keywords */
+ dmnsn_token_map(DMNSN_CAMERA, "camera");
+ dmnsn_token_map(DMNSN_COLOR, "color");
+ dmnsn_token_map(DMNSN_SPHERE, "sphere");
+ dmnsn_token_map(DMNSN_BOX, "box");
+
/* Identifiers */
dmnsn_token_map(DMNSN_IDENTIFIER, "identifier");
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index aead839..2e82f98 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -39,6 +39,12 @@ typedef enum {
DMNSN_INT,
DMNSN_FLOAT,
+ /* Keywords */
+ DMNSN_CAMERA,
+ DMNSN_COLOR,
+ DMNSN_SPHERE,
+ DMNSN_BOX,
+
/* Identifiers */
DMNSN_IDENTIFIER,
} dmnsn_token_type;
diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am
index e9b7f7a..c0ea38e 100644
--- a/tests/dimension/Makefile.am
+++ b/tests/dimension/Makefile.am
@@ -25,4 +25,7 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir)
tokenizer.sh:
cp $(srcdir)/tokenizer.sh .
-EXTRA_DIST = tokenizer.sh punctuation.pov numeric.pov
+EXTRA_DIST = tokenizer.sh \
+ punctuation.pov \
+ numeric.pov \
+ labels.pov
diff --git a/tests/dimension/labels.pov b/tests/dimension/labels.pov
new file mode 100644
index 0000000..fd72979
--- /dev/null
+++ b/tests/dimension/labels.pov
@@ -0,0 +1,31 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> *
+ * *
+ * This file is part of The Dimension Test Suite. *
+ * *
+ * The Dimension Test Suite is free software; you can redistribute it *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the *
+ * License, or (at your option) any later version. *
+ * *
+ * The Dimension Test Suite is distributed in the hope that it will be *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
+ * General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize identifiers and keywords
+
+camera {
+}
+
+sphere {
+ color new_identifier
+}
+
+box {
+ colour new_identifier
+}
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index 5f126ce..e23e281 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -39,4 +39,13 @@ if [ "$numeric" != "$numeric_exp" ]; then
exitstatus=1
fi
-exit $exitstatus \ No newline at end of file
+labels=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/labels.pov)
+labels_exp='(camera { } sphere { color (identifier "new_identifier") } box { color (identifier "new_identifier") })';
+
+if [ "$labels" != "$labels_exp" ]; then
+ echo "labels.pov tokenized as \"$labels\"" >&2
+ echo " -- expected \"$labels_exp\"" >&2
+ exitstatus=1
+fi
+
+exit $exitstatus