summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTavian Barnes <tavianator@gmail.com>2009-10-27 21:12:08 -0400
committerTavian Barnes <tavianator@gmail.com>2009-10-27 21:12:08 -0400
commit4d47846852dcbccc193368fcc6ef31f872101416 (patch)
tree55f47cefe26ec5b5584c62e3e3259a798f0dcd89
parent33c3e2121e185e619cd993280b23038b4490f4f5 (diff)
downloaddimension-4d47846852dcbccc193368fcc6ef31f872101416.tar.xz
Ignore comments in tokenizer.
-rw-r--r--dimension/tokenize.c39
-rw-r--r--tests/dimension/numeric.pov25
-rw-r--r--tests/dimension/punctuation.pov21
-rwxr-xr-xtests/dimension/tokenizer.sh8
4 files changed, 87 insertions, 6 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 1d53f2f..360a228 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -52,7 +52,7 @@ dmnsn_tokenize(FILE *file)
dmnsn_token token;
dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token));
- unsigned int line = 0, col = 0;
+ unsigned int line = 1, col = 0;
unsigned int i;
while (next - map < size) {
@@ -96,9 +96,39 @@ dmnsn_tokenize(FILE *file)
dmnsn_simple_token('+', DMNSN_PLUS);
dmnsn_simple_token('-', DMNSN_MINUS);
dmnsn_simple_token('*', DMNSN_STAR);
- dmnsn_simple_token('/', DMNSN_SLASH);
dmnsn_simple_token(',', DMNSN_COMMA);
+ /* Possible comment */
+ case '/':
+ if (*(next + 1) == '/') {
+ /* A '//' comment block */
+ do {
+ ++next;
+ } while (next - map < size && *next != '\n');
+
+ ++line;
+ col = 0;
+ continue;
+ } else if (*(next + 1) == '*') {
+ /* A '/*' comment block */
+ do {
+ ++col;
+ if (*next == '\n') {
+ ++line;
+ col = 0;
+ }
+
+ ++next;
+ } while (next - map < size - 1
+ && !(*next == '*' && *(next + 1) == '/'));
+ next += 2;
+ continue;
+ } else {
+ /* Just the normal punctuation mark */
+ token.type = DMNSN_SLASH;
+ }
+ break;
+
/* Numeric values */
case '.': /* Number begins with a decimal point, as in `.2' */
case '0':
@@ -145,8 +175,9 @@ dmnsn_tokenize(FILE *file)
default:
/* Unrecognised character */
- fprintf(stderr, "Unrecognized character 0x%X in input.\n",
- (unsigned int)*next);
+ fprintf(stderr,
+ "Unrecognized character 0x%X in input at line %u, column %u.\n",
+ (unsigned int)*next, line, col);
dmnsn_delete_tokens(tokens);
munmap(map, size);
return NULL;
diff --git a/tests/dimension/numeric.pov b/tests/dimension/numeric.pov
index df850d6..e9d0eae 100644
--- a/tests/dimension/numeric.pov
+++ b/tests/dimension/numeric.pov
@@ -1,2 +1,27 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> *
+ * *
+ * This file is part of The Dimension Test Suite. *
+ * *
+ * The Dimension Test Suite is free software; you can redistribute it *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the *
+ * License, or (at your option) any later version. *
+ * *
+ * The Dimension Test Suite is distributed in the hope that it will be *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
+ * General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize numeric values. Note that the `-' in `-1',
+// for example, is a separate token and not part of the number.
+
+// Integers:
1 123456789 01234567 0x123456789 -0x01
+
+// Floats:
.1 0.1 1.0 0.123456789 -0.123456789
diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov
index db9911f..a691693 100644
--- a/tests/dimension/punctuation.pov
+++ b/tests/dimension/punctuation.pov
@@ -1,3 +1,24 @@
+/*************************************************************************
+ * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> *
+ * *
+ * This file is part of The Dimension Test Suite. *
+ * *
+ * The Dimension Test Suite is free software; you can redistribute it *
+ * and/or modify it under the terms of the GNU General Public License as *
+ * published by the Free Software Foundation; either version 3 of the *
+ * License, or (at your option) any later version. *
+ * *
+ * The Dimension Test Suite is distributed in the hope that it will be *
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty *
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
+ * General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
+// Test that we correctly tokenize all simple punctuation marks
+
{
(
[
diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh
index edd3bb4..5f126ce 100755
--- a/tests/dimension/tokenizer.sh
+++ b/tests/dimension/tokenizer.sh
@@ -19,13 +19,15 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. #
#########################################################################
+exitstatus=0
+
punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov)
punctuation_exp='({ \( [ < + - * / , > ] \) })'
if [ "$punctuation" != "$punctuation_exp" ]; then
echo "punctuation.pov tokenized as \"$punctuation\"" >&2
echo " -- expected \"$punctuation_exp\"" >&2
- exit 1;
+ exitstatus=1
fi
numeric=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/numeric.pov)
@@ -34,5 +36,7 @@ numeric_exp='((int "1") (int "123456789") (int "01234567") (int "0x123456789") -
if [ "$numeric" != "$numeric_exp" ]; then
echo "numeric.pov tokenized as \"$numeric\"" >&2
echo " -- expected \"$numeric_exp\"" >&2
- exit 1;
+ exitstatus=1
fi
+
+exit $exitstatus \ No newline at end of file