summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTavian Barnes <tavianator@gmail.com>2009-10-27 17:33:44 -0400
committerTavian Barnes <tavianator@gmail.com>2009-10-27 19:35:16 -0400
commit50d844e91589a8f51cade42b732bcd1de889987b (patch)
treede7fd28f212412d397c93e3aee18ebbf1f121d9d
parent9d093988213e7471a5d8ba6042e16c8d2ab5f053 (diff)
downloaddimension-50d844e91589a8f51cade42b732bcd1de889987b.tar.xz
Support more symbols in tokenizer.
-rw-r--r--dimension/tokenize.c64
-rw-r--r--dimension/tokenize.h15
-rw-r--r--tests/dimension/Makefile.am2
-rw-r--r--tests/dimension/braces.pov8
-rw-r--r--tests/dimension/punctuation.pov9
-rwxr-xr-xtests/dimension/tokenizer.sh29
6 files changed, 93 insertions, 34 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index e1c2fce..7019ad7 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -51,6 +51,9 @@ dmnsn_tokenize(FILE *file)
dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token));
while (next - map < size) {
+ /* Saves us some code repetition in the vast majority of cases */
+ token.value = NULL;
+
switch (*next) {
case ' ':
case '\n':
@@ -59,19 +62,29 @@ dmnsn_tokenize(FILE *file)
case '\f':
case '\v':
/* Skip whitespace */
- break;
-
- case '{':
- token.type = DMNSN_LBRACE;
- token.value = NULL;
- dmnsn_array_push(tokens, &token);
- break;
-
- case '}':
- token.type = DMNSN_RBRACE;
- token.value = NULL;
- dmnsn_array_push(tokens, &token);
- break;
+ ++next;
+ continue;
+
+ /* Macro to make basic symbol tokens easier */
+ #define dmnsn_simple_token(c, tp) \
+ case c: \
+ token.type = tp; \
+ break
+
+ /* Some simple punctuation marks */
+ dmnsn_simple_token('{', DMNSN_LBRACE);
+ dmnsn_simple_token('}', DMNSN_RBRACE);
+ dmnsn_simple_token('(', DMNSN_LPAREN);
+ dmnsn_simple_token(')', DMNSN_RPAREN);
+ dmnsn_simple_token('[', DMNSN_LBRACKET);
+ dmnsn_simple_token(']', DMNSN_RBRACKET);
+ dmnsn_simple_token('<', DMNSN_LT);
+ dmnsn_simple_token('>', DMNSN_GT);
+ dmnsn_simple_token('+', DMNSN_PLUS);
+ dmnsn_simple_token('-', DMNSN_MINUS);
+ dmnsn_simple_token('*', DMNSN_STAR);
+ dmnsn_simple_token('/', DMNSN_SLASH);
+ dmnsn_simple_token(',', DMNSN_COMMA);
default:
/* Unrecognised character */
@@ -82,6 +95,7 @@ dmnsn_tokenize(FILE *file)
return NULL;
}
+ dmnsn_array_push(tokens, &token);
++next;
}
@@ -138,13 +152,27 @@ static const char *
dmnsn_token_name(dmnsn_token_type token_type)
{
switch (token_type) {
- case DMNSN_LBRACE:
- return "{";
-
- case DMNSN_RBRACE:
- return "}";
+ /* Macro to shorten this huge switch */
+ #define dmnsn_token_map(type, str) \
+ case type: \
+ return str;
+
+ dmnsn_token_map(DMNSN_LBRACE, "{");
+ dmnsn_token_map(DMNSN_RBRACE, "}")
+ dmnsn_token_map(DMNSN_LPAREN, "\\(");
+ dmnsn_token_map(DMNSN_RPAREN, "\\)");
+ dmnsn_token_map(DMNSN_LBRACKET, "[");
+ dmnsn_token_map(DMNSN_RBRACKET, "]");
+ dmnsn_token_map(DMNSN_LT, "<");
+ dmnsn_token_map(DMNSN_GT, ">");
+ dmnsn_token_map(DMNSN_PLUS, "+");
+ dmnsn_token_map(DMNSN_MINUS, "-");
+ dmnsn_token_map(DMNSN_STAR, "*");
+ dmnsn_token_map(DMNSN_SLASH, "/");
+ dmnsn_token_map(DMNSN_COMMA, ",");
default:
+ printf("Warning: unrecognised token %d.\n", (int)token_type);
return "unrecognized-token";
}
}
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 3798d1e..e64b7eb 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -20,8 +20,19 @@
#include "../libdimension/dimension.h"
typedef enum {
- DMNSN_LBRACE,
- DMNSN_RBRACE
+ DMNSN_LBRACE, /* { */
+ DMNSN_RBRACE, /* } */
+ DMNSN_LPAREN, /* ( */
+ DMNSN_RPAREN, /* ) */
+ DMNSN_LBRACKET, /* [ */
+ DMNSN_RBRACKET, /* ] */
+ DMNSN_LT, /* < */
+ DMNSN_GT, /* > */
+ DMNSN_PLUS, /* + */
+ DMNSN_MINUS, /* - */
+ DMNSN_STAR, /* * */
+ DMNSN_SLASH, /* / */
+ DMNSN_COMMA, /* , */
} dmnsn_token_type;
typedef struct dmnsn_token dmnsn_token;
diff --git a/tests/dimension/Makefile.am b/tests/dimension/Makefile.am
index f5ec097..7bacab5 100644
--- a/tests/dimension/Makefile.am
+++ b/tests/dimension/Makefile.am
@@ -25,4 +25,4 @@ TESTS_ENVIRONMENT = top_builddir=$(top_builddir)
tokenizer.sh:
cp $(srcdir)/tokenizer.sh .
-EXTRA_DIST = tokenizer.sh braces.pov
+EXTRA_DIST = tokenizer.sh punctuation.pov
diff --git a/tests/dimension/braces.pov b/tests/dimension/braces.pov
deleted file mode 100644
index 295a69a..0000000
--- a/tests/dimension/braces.pov
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- {
- }
-}
-}
- }
- {
-{
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
new file mode 100644
index 0000000..db9911f
--- /dev/null
+++ b/tests/dimension/punctuation.pov
@@ -0,0 +1,9 @@
+{
+ (
+ [
+ <
+ + - * / ,
+ >
+ ]
+ )
+}
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index b6de05a..d965741 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -1,10 +1,29 @@
#!/bin/sh
-braces=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/braces.pov)
-braces_exp='({ { } } } } { {)'
+#########################################################################
+# Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> #
+# #
+# This file is part of The Dimension Test Suite. #
+# #
+# The Dimension Test Suite is free software; you can redistribute it #
+# and/or modify it under the terms of the GNU General Public License as #
+# published by the Free Software Foundation; either version 3 of the #
+# License, or (at your option) any later version. #
+# #
+# The Dimension Test Suite is distributed in the hope that it will be #
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty #
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
+# General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+#########################################################################
-if [ "$braces" != "$braces_exp" ]; then
- echo "braces.pov tokenized as \"$braces\"" >&2
- echo " -- expected \"$braces_exp\"" >&2
+punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
+punctuation_exp='({ \( [ < + - * / , > ] \) })'
+
+if [ "$punctuation" != "$punctuation_exp" ]; then
+ echo "punctuation.pov tokenized as \"$punctuation\"" >&2
+ echo " -- expected \"$punctuation_exp\"" >&2
exit 1;
fi