diff options
| -rw-r--r-- | dimension/tokenize.c | 64 | ||||
| -rw-r--r-- | dimension/tokenize.h | 15 | ||||
| -rw-r--r-- | tests/dimension/Makefile.am | 2 | ||||
| -rw-r--r-- | tests/dimension/braces.pov | 8 | ||||
| -rw-r--r-- | tests/dimension/punctuation.pov | 9 | ||||
| -rwxr-xr-x | tests/dimension/tokenizer.sh | 29 |
6 files changed, 93 insertions, 34 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index e1c2fce..7019ad7 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -51,6 +51,9 @@ dmnsn_tokenize(FILE *file) dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token)); while (next - map < size) { + /* Saves us some code repetition in the vast majority of cases */ + token.value = NULL; + switch (*next) { case ' ': case '\n': @@ -59,19 +62,29 @@ dmnsn_tokenize(FILE *file) case '\f': case '\v': /* Skip whitespace */ - break; - - case '{': - token.type = DMNSN_LBRACE; - token.value = NULL; - dmnsn_array_push(tokens, &token); - break; - - case '}': - token.type = DMNSN_RBRACE; - token.value = NULL; - dmnsn_array_push(tokens, &token); - break; + ++next; + continue; + + /* Macro to make basic symbol tokens easier */ + #define dmnsn_simple_token(c, tp) \ + case c: \ + token.type = tp; \ + break + + /* Some simple punctuation marks */ + dmnsn_simple_token('{', DMNSN_LBRACE); + dmnsn_simple_token('}', DMNSN_RBRACE); + dmnsn_simple_token('(', DMNSN_LPAREN); + dmnsn_simple_token(')', DMNSN_RPAREN); + dmnsn_simple_token('[', DMNSN_LBRACKET); + dmnsn_simple_token(']', DMNSN_RBRACKET); + dmnsn_simple_token('<', DMNSN_LT); + dmnsn_simple_token('>', DMNSN_GT); + dmnsn_simple_token('+', DMNSN_PLUS); + dmnsn_simple_token('-', DMNSN_MINUS); + dmnsn_simple_token('*', DMNSN_STAR); + dmnsn_simple_token('/', DMNSN_SLASH); + dmnsn_simple_token(',', DMNSN_COMMA); default: /* Unrecognised character */ @@ -82,6 +95,7 @@ dmnsn_tokenize(FILE *file) return NULL; } + dmnsn_array_push(tokens, &token); ++next; } @@ -138,13 +152,27 @@ static const char * dmnsn_token_name(dmnsn_token_type token_type) { switch (token_type) { - case DMNSN_LBRACE: - return "{"; - - case DMNSN_RBRACE: - return "}"; + /* Macro to shorten this huge switch */ + #define dmnsn_token_map(type, str) \ + case type: \ + return str; + + dmnsn_token_map(DMNSN_LBRACE, "{"); + dmnsn_token_map(DMNSN_RBRACE, "}") + dmnsn_token_map(DMNSN_LPAREN, "\\("); + dmnsn_token_map(DMNSN_RPAREN, "\\)"); + dmnsn_token_map(DMNSN_LBRACKET, "["); + dmnsn_token_map(DMNSN_RBRACKET, "]"); + dmnsn_token_map(DMNSN_LT, "<"); + dmnsn_token_map(DMNSN_GT, ">"); + dmnsn_token_map(DMNSN_PLUS, "+"); + dmnsn_token_map(DMNSN_MINUS, "-"); + dmnsn_token_map(DMNSN_STAR, "*"); + dmnsn_token_map(DMNSN_SLASH, "/"); + dmnsn_token_map(DMNSN_COMMA, ","); default: + printf("Warning: unrecognised token %d.\n", (int)token_type); return "unrecognized-token"; } } diff --git a/dimension/tokenize.h b/dimension/tokenize.h index 3798d1e..e64b7eb 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -20,8 +20,19 @@ #include "../libdimension/dimension.h" typedef enum { - DMNSN_LBRACE, - DMNSN_RBRACE + DMNSN_LBRACE, /* { */ + DMNSN_RBRACE, /* } */ + DMNSN_LPAREN, /* ( */ + DMNSN_RPAREN, /* ) */ + DMNSN_LBRACKET, /* [ */ + DMNSN_RBRACKET, /* ] */ + DMNSN_LT, /* < */ + DMNSN_GT, /* > */ + DMNSN_PLUS, /* + */ + DMNSN_MINUS, /* - */ + DMNSN_STAR, /* * */ + DMNSN_SLASH, /* / */ + DMNSN_COMMA, /* , */ } dmnsn_token_type; typedef struct dmnsn_token dmnsn_token; diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am index f5ec097..7bacab5 100644 --- a/tests/dimension/Makefile.am +++ b/tests/dimension/Makefile.am @@ -25,4 +25,4 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir) tokenizer.sh: cp $(srcdir)/tokenizer.sh . -EXTRA_DIST = tokenizer.sh braces.pov +EXTRA_DIST = tokenizer.sh punctuation.pov diff --git a/tests/dimension/braces.pov b/tests/dimension/braces.pov deleted file mode 100644 index 295a69a..0000000 --- a/tests/dimension/braces.pov +++ /dev/null @@ -1,8 +0,0 @@ -{ - { - } -} -} - } - { -{ diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov new file mode 100644 index 0000000..db9911f --- /dev/null +++ b/tests/dimension/punctuation.pov @@ -0,0 +1,9 @@ +{ + ( + [ + < + + - * / , + > + ] + ) +} diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh index b6de05a..d965741 100755 --- a/tests/dimension/tokenizer.sh +++ b/tests/dimension/tokenizer.sh @@ -1,10 +1,29 @@ #!/bin/sh -braces=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/braces.pov) -braces_exp='({ { } } } } { {)' +######################################################################### +# Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> # +# # +# This file is part of The Dimension Test Suite. # +# # +# The Dimension Test Suite is free software; you can redistribute it # +# and/or modify it under the terms of the GNU General Public License as # +# published by the Free Software Foundation; either version 3 of the # +# License, or (at your option) any later version. # +# # +# The Dimension Test Suite is distributed in the hope that it will be # +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty # +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see <http://www.gnu.org/licenses/>. # +######################################################################### -if [ "$braces" != "$braces_exp" ]; then - echo "braces.pov tokenized as \"$braces\"" >&2 - echo " -- expected \"$braces_exp\"" >&2 +punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov) +punctuation_exp='({ \( [ < + - * / , > ] \) })' + +if [ "$punctuation" != "$punctuation_exp" ]; then + echo "punctuation.pov tokenized as \"$punctuation\"" >&2 + echo " -- expected \"$punctuation_exp\"" >&2 exit 1; fi |