diff options
| -rw-r--r-- | dimension/tokenize.c | 49 | ||||
| -rw-r--r-- | dimension/tokenize.h | 6 | ||||
| -rw-r--r-- | tests/dimension/Makefile.am | 5 | ||||
| -rw-r--r-- | tests/dimension/labels.pov | 31 | ||||
| -rwxr-xr-x | tests/dimension/tokenizer.sh | 11 |
5 files changed, 89 insertions, 13 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 28deda2..8401dc7 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -41,7 +41,7 @@ dmnsn_tokenize_comment(char *map, size_t size, ++*line; *col = 0; } else if (*(*next + 1) == '*') { - /* A '/*' comment block */ + /* A multi-line comment block (like this one) */ do { ++col; if (**next == '\n') { @@ -96,9 +96,10 @@ dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token, return 0; } +/* Tokenize a keyword or an identifier */ static int -dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token, - char **next, unsigned int *line, unsigned int *col) +dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token, + char **next, unsigned int *line, unsigned int *col) { unsigned int i = 0, alloc = 32; @@ -109,7 +110,7 @@ dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token, token->type = DMNSN_IDENTIFIER; token->value = malloc(alloc); - while (*next - map < size && (isalnum(**next) || **next == '_')) { + do { if (i + 1 >= alloc) { alloc *= 2; token->value = realloc(token->value, alloc); @@ -120,9 +121,28 @@ dmnsn_tokenize_identifier(char *map, size_t size, dmnsn_token *token, ++i; ++*col; ++*next; - } + } while (*next - map < size && (isalnum(**next) || **next == '_')); token->value[i] = '\0'; + + /* Now check if we really found a keyword */ + +#define dmnsn_keyword(str, tp) \ + do { \ + if (strcmp(token->value, str) == 0) { \ + free(token->value); \ + token->value = NULL; \ + token->type = tp; \ + return 0; \ + } \ + } while (0) + + dmnsn_keyword("camera", DMNSN_CAMERA); + dmnsn_keyword("color", DMNSN_COLOR); + dmnsn_keyword("colour", DMNSN_COLOR); + dmnsn_keyword("sphere", DMNSN_SPHERE); + dmnsn_keyword("box", DMNSN_BOX); + return 0; } @@ -178,9 +198,11 @@ dmnsn_tokenize(FILE *file) continue; /* Macro to make basic symbol tokens easier */ - #define dmnsn_simple_token(c, tp) \ +#define dmnsn_simple_token(c, tp) \ case c: \ token.type = tp; \ + ++col; \ + ++next; \ break /* Some simple punctuation marks */ @@ -204,6 +226,8 @@ dmnsn_tokenize(FILE *file) } else { /* Just the normal punctuation mark */ token.type = DMNSN_SLASH; + ++col; + ++next; } break; @@ -227,8 +251,7 @@ dmnsn_tokenize(FILE *file) break; default: - if (dmnsn_tokenize_identifier(map, size, &token, &next, &line, &col) != 0) - { + if (dmnsn_tokenize_label(map, size, &token, &next, &line, &col) != 0) { /* Unrecognised character */ fprintf(stderr, "Unrecognized character 0x%X in input at line %u, column %u.\n", @@ -238,8 +261,6 @@ dmnsn_tokenize(FILE *file) } dmnsn_array_push(tokens, &token); - ++next; - ++col; } munmap(map, size); @@ -301,7 +322,7 @@ dmnsn_token_name(dmnsn_token_type token_type) { switch (token_type) { /* Macro to shorten this huge switch */ - #define dmnsn_token_map(type, str) \ +#define dmnsn_token_map(type, str) \ case type: \ return str; @@ -324,6 +345,12 @@ dmnsn_token_name(dmnsn_token_type token_type) dmnsn_token_map(DMNSN_INT, "int"); dmnsn_token_map(DMNSN_FLOAT, "float"); + /* Keywords */ + dmnsn_token_map(DMNSN_CAMERA, "camera"); + dmnsn_token_map(DMNSN_COLOR, "color"); + dmnsn_token_map(DMNSN_SPHERE, "sphere"); + dmnsn_token_map(DMNSN_BOX, "box"); + /* Identifiers */ dmnsn_token_map(DMNSN_IDENTIFIER, "identifier"); diff --git a/dimension/tokenize.h b/dimension/tokenize.h index aead839..2e82f98 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -39,6 +39,12 @@ typedef enum { DMNSN_INT, DMNSN_FLOAT, + /* Keywords */ + DMNSN_CAMERA, + DMNSN_COLOR, + DMNSN_SPHERE, + DMNSN_BOX, + /* Identifiers */ DMNSN_IDENTIFIER, } dmnsn_token_type; diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am index e9b7f7a..c0ea38e 100644 --- a/tests/dimension/Makefile.am +++ b/tests/dimension/Makefile.am @@ -25,4 +25,7 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir) tokenizer.sh: cp $(srcdir)/tokenizer.sh . -EXTRA_DIST = tokenizer.sh punctuation.pov numeric.pov +EXTRA_DIST = tokenizer.sh \ + punctuation.pov \ + numeric.pov \ + labels.pov diff --git a/tests/dimension/labels.pov b/tests/dimension/labels.pov new file mode 100644 index 0000000..fd72979 --- /dev/null +++ b/tests/dimension/labels.pov @@ -0,0 +1,31 @@ +/************************************************************************* + * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> * + * * + * This file is part of The Dimension Test Suite. * + * * + * The Dimension Test Suite is free software; you can redistribute it * + * and/or modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 3 of the * + * License, or (at your option) any later version. * + * * + * The Dimension Test Suite is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + *************************************************************************/ + +// Test that we correctly tokenize identifiers and keywords + +camera { +} + +sphere { + color new_identifier +} + +box { + colour new_identifier +} diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh index 5f126ce..e23e281 100755 --- a/tests/dimension/tokenizer.sh +++ b/tests/dimension/tokenizer.sh @@ -39,4 +39,13 @@ if [ "$numeric" != "$numeric_exp" ]; then exitstatus=1 fi -exit $exitstatus
\ No newline at end of file +labels=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/labels.pov) +labels_exp='(camera { } sphere { color (identifier "new_identifier") } box { color (identifier "new_identifier") })'; + +if [ "$labels" != "$labels_exp" ]; then + echo "labels.pov tokenized as \"$labels\"" >&2 + echo " -- expected \"$labels_exp\"" >&2 + exitstatus=1 +fi + +exit $exitstatus |