summaryrefslogtreecommitdiffstats
path: root/dimension
diff options
context:
space:
mode:
Diffstat (limited to 'dimension')
-rw-r--r--dimension/tokenize.c75
1 files changed, 69 insertions, 6 deletions
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 9eeab45..e34564c 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -220,6 +220,8 @@ dmnsn_tokenize_string(const char *filename,
char *map, size_t size, char **next, dmnsn_token *token)
{
unsigned int i = 0, alloc = 32;
+ char unicode[5] = { 0 }, *end;
+ unsigned long wchar;
if (**next != '"') {
return 1;
@@ -240,18 +242,74 @@ dmnsn_tokenize_string(const char *filename,
++*next;
switch (**next) {
- case '\\':
- token->value[i] = '\\';
+ case 'a':
+ token->value[i] = '\a';
break;
- case '"':
- token->value[i] = '"';
+ case 'b':
+ token->value[i] = '\b';
+ break;
+
+ case 'f':
+ token->value[i] = '\f';
break;
case 'n':
token->value[i] = '\n';
break;
+ case 'r':
+ token->value[i] = '\r';
+ break;
+
+ case 't':
+ token->value[i] = '\t';
+ break;
+
+ case 'u':
+ /* Escaped unicode character */
+ strncpy(unicode, *next + 1, 4);
+ wchar = strtoul(unicode, &end, 16);
+ if (*next - map >= size - 4) {
+ dmnsn_diagnostic(filename, *line, *col,
+ "EOF before end of escape sequence");
+ free(token->value);
+ return 1;
+ }
+ if (end != &unicode[4]) {
+ dmnsn_diagnostic(filename, *line, *col,
+ "WARNING: Invalid unicode character \"\\u%s\"",
+ unicode);
+ } else {
+ token->value[i] = wchar/256;
+ ++i;
+ if (i + 1 >= alloc) {
+ alloc *= 2;
+ token->value = realloc(token->value, alloc);
+ }
+ token->value[i] = wchar%256;
+
+ *col += 4;
+ *next += 4;
+ }
+ break;
+
+ case 'v':
+ token->value[i] = '\v';
+ break;
+
+ case '\\':
+ token->value[i] = '\\';
+ break;
+
+ case '\'':
+ token->value[i] = '\'';
+ break;
+
+ case '"':
+ token->value[i] = '"';
+ break;
+
default:
dmnsn_diagnostic(filename, *line, *col,
"WARNING: unrecognised escape sequence '\\%c'",
@@ -267,10 +325,15 @@ dmnsn_tokenize_string(const char *filename,
++*col;
++*next;
}
- ++*next;
- token->value[i] = '\0';
+ if (**next != '"') {
+ dmnsn_diagnostic(filename, *line, *col, "Non-terminated string");
+ free(token->value);
+ return 1;
+ }
+ ++*next;
+ token->value[i] = '\0';
return 0;
}