diff options
| -rw-r--r-- | dimension/tokenize.c | 39 | ||||
| -rw-r--r-- | tests/dimension/numeric.pov | 25 | ||||
| -rw-r--r-- | tests/dimension/punctuation.pov | 21 | ||||
| -rwxr-xr-x | tests/dimension/tokenizer.sh | 8 |
4 files changed, 87 insertions, 6 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 1d53f2f..360a228 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -52,7 +52,7 @@ dmnsn_tokenize(FILE *file) dmnsn_token token; dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token)); - unsigned int line = 0, col = 0; + unsigned int line = 1, col = 0; unsigned int i; while (next - map < size) { @@ -96,9 +96,39 @@ dmnsn_tokenize(FILE *file) dmnsn_simple_token('+', DMNSN_PLUS); dmnsn_simple_token('-', DMNSN_MINUS); dmnsn_simple_token('*', DMNSN_STAR); - dmnsn_simple_token('/', DMNSN_SLASH); dmnsn_simple_token(',', DMNSN_COMMA); + /* Possible comment */ + case '/': + if (*(next + 1) == '/') { + /* A '//' comment block */ + do { + ++next; + } while (next - map < size && *next != '\n'); + + ++line; + col = 0; + continue; + } else if (*(next + 1) == '*') { + /* A '/*' comment block */ + do { + ++col; + if (*next == '\n') { + ++line; + col = 0; + } + + ++next; + } while (next - map < size - 1 + && !(*next == '*' && *(next + 1) == '/')); + next += 2; + continue; + } else { + /* Just the normal punctuation mark */ + token.type = DMNSN_SLASH; + } + break; + /* Numeric values */ case '.': /* Number begins with a decimal point, as in `.2' */ case '0': @@ -145,8 +175,9 @@ dmnsn_tokenize(FILE *file) default: /* Unrecognised character */ - fprintf(stderr, "Unrecognized character 0x%X in input.\n", - (unsigned int)*next); + fprintf(stderr, + "Unrecognized character 0x%X in input at line %u, column %u.\n", + (unsigned int)*next, line, col); dmnsn_delete_tokens(tokens); munmap(map, size); return NULL; diff --git a/tests/dimension/numeric.pov b/tests/dimension/numeric.pov index df850d6..e9d0eae 100644 --- a/tests/dimension/numeric.pov +++ b/tests/dimension/numeric.pov @@ -1,2 +1,27 @@ +/************************************************************************* + * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> * + * * + * This file is part of The Dimension Test Suite. * + * * + * The Dimension Test Suite is free software; you can redistribute it * + * and/or modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 3 of the * + * License, or (at your option) any later version. * + * * + * The Dimension Test Suite is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + *************************************************************************/ + +// Test that we correctly tokenize numeric values. Note that the `-' in `-1', +// for example, is a separate token and not part of the number. + +// Integers: 1 123456789 01234567 0x123456789 -0x01 + +// Floats: .1 0.1 1.0 0.123456789 -0.123456789 diff --git a/tests/dimension/punctuation.pov b/tests/dimension/punctuation.pov index db9911f..a691693 100644 --- a/tests/dimension/punctuation.pov +++ b/tests/dimension/punctuation.pov @@ -1,3 +1,24 @@ +/************************************************************************* + * Copyright (C) 2009 Tavian Barnes <tavianator@gmail.com> * + * * + * This file is part of The Dimension Test Suite. * + * * + * The Dimension Test Suite is free software; you can redistribute it * + * and/or modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 3 of the * + * License, or (at your option) any later version. * + * * + * The Dimension Test Suite is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + *************************************************************************/ + +// Test that we correctly tokenize all simple punctuation marks + { ( [ diff --git a/tests/dimension/tokenizer.sh b/tests/dimension/tokenizer.sh index edd3bb4..5f126ce 100755 --- a/tests/dimension/tokenizer.sh +++ b/tests/dimension/tokenizer.sh @@ -19,13 +19,15 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # ######################################################################### +exitstatus=0 + punctuation=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/punctuation.pov) punctuation_exp='({ \( [ < + - * / , > ] \) })' if [ "$punctuation" != "$punctuation_exp" ]; then echo "punctuation.pov tokenized as \"$punctuation\"" >&2 echo " -- expected \"$punctuation_exp\"" >&2 - exit 1; + exitstatus=1 fi numeric=$(${top_builddir}/dimension/dimension --tokenize ${srcdir}/numeric.pov) @@ -34,5 +36,7 @@ numeric_exp='((int "1") (int "123456789") (int "01234567") (int "0x123456789") - if [ "$numeric" != "$numeric_exp" ]; then echo "numeric.pov tokenized as \"$numeric\"" >&2 echo " -- expected \"$numeric_exp\"" >&2 - exit 1; + exitstatus=1 fi + +exit $exitstatus
\ No newline at end of file |