summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--dimension/main.c2
-rw-r--r--dimension/tokenize.c118
-rw-r--r--dimension/tokenize.h5
3 files changed, 76 insertions, 49 deletions
diff --git a/dimension/main.c b/dimension/main.c
index 2ccadbf..e15180f 100644
--- a/dimension/main.c
+++ b/dimension/main.c
@@ -98,7 +98,7 @@ main(int argc, char **argv) {
}
/* Tokenize the input file */
- tokens = dmnsn_tokenize(input_file);
+ tokens = dmnsn_tokenize(input, input_file);
if (!tokens) {
dmnsn_error(DMNSN_SEVERITY_HIGH, "Error tokenizing input file.");
}
diff --git a/dimension/tokenize.c b/dimension/tokenize.c
index 72418cc..da190be 100644
--- a/dimension/tokenize.c
+++ b/dimension/tokenize.c
@@ -20,13 +20,29 @@
#include "tokenize.h"
#include <stdlib.h>
#include <string.h>
+#include <stdarg.h>
#include <ctype.h>
#include <sys/mman.h>
#include <unistd.h>
+static void
+dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col,
+ const char *format, ...)
+{
+ va_list ap;
+ va_start(ap, format);
+
+ fprintf(stderr, "%s:%u:%u: ", filename, line, col);
+ vfprintf(stderr, format, ap);
+ fprintf(stderr, "\n");
+
+ va_end(ap);
+}
+
static int
-dmnsn_tokenize_comment(char *map, size_t size,
- char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_comment(const char *filename,
+ unsigned int *line, unsigned int *col,
+ char *map, size_t size, char **next)
{
if (**next != '/')
return 1;
@@ -43,10 +59,10 @@ dmnsn_tokenize_comment(char *map, size_t size,
} else if (*(*next + 1) == '*') {
/* A multi-line comment block (like this one) */
do {
- ++col;
+ ++*col;
if (**next == '\n') {
- ++line;
- col = 0;
+ ++*line;
+ *col = 0;
}
++*next;
@@ -61,8 +77,9 @@ dmnsn_tokenize_comment(char *map, size_t size,
}
static int
-dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token,
- char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_number(const char *filename,
+ unsigned int *line, unsigned int *col,
+ char *map, size_t size, char **next, dmnsn_token *token)
{
char *endf, *endi;
@@ -98,8 +115,9 @@ dmnsn_tokenize_number(char *map, size_t size, dmnsn_token *token,
/* Tokenize a keyword or an identifier */
static int
-dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token,
- char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_label(const char *filename,
+ unsigned int *line, unsigned int *col,
+ char *map, size_t size, char **next, dmnsn_token *token)
{
unsigned int i = 0, alloc = 32;
@@ -148,8 +166,10 @@ dmnsn_tokenize_label(char *map, size_t size, dmnsn_token *token,
/* Tokenize a language directive (#include, #declare, etc.) */
static int
-dmnsn_tokenize_directive(char *map, size_t size, dmnsn_token *token,
- char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_directive(const char *filename,
+ unsigned int *line, unsigned int *col,
+ char *map, size_t size, char **next,
+ dmnsn_token *token)
{
unsigned int i = 0, alloc = 32;
@@ -194,8 +214,9 @@ dmnsn_tokenize_directive(char *map, size_t size, dmnsn_token *token,
/* Tokenize a string */
static int
-dmnsn_tokenize_string(char *map, size_t size, dmnsn_token *token,
- char **next, unsigned int *line, unsigned int *col)
+dmnsn_tokenize_string(const char *filename,
+ unsigned int *line, unsigned int *col,
+ char *map, size_t size, char **next, dmnsn_token *token)
{
unsigned int i = 0, alloc = 32;
@@ -231,10 +252,9 @@ dmnsn_tokenize_string(char *map, size_t size, dmnsn_token *token,
break;
default:
- fprintf(stderr,
- "Warning: unrecognised escape sequence '\\%c'"
- " on line %u, column %u\n",
- (int)**next, *line, *col);
+ dmnsn_diagnostic(filename, *line, *col,
+ "WARNING: unrecognised escape sequence '\\%c'",
+ (int)**next);
token->value[i] = **next;
break;
}
@@ -254,10 +274,10 @@ dmnsn_tokenize_string(char *map, size_t size, dmnsn_token *token,
}
dmnsn_array *
-dmnsn_tokenize(FILE *file)
+dmnsn_tokenize(const char *filename, FILE *file)
{
if (fseeko(file, 0, SEEK_END) != 0) {
- fprintf(stderr, "Couldn't seek on input stream.\n");
+ fprintf(stderr, "Couldn't seek on input stream\n");
return NULL;
}
@@ -265,14 +285,14 @@ dmnsn_tokenize(FILE *file)
int fd = fileno(file);
if (fd == -1) {
- fprintf(stderr, "Couldn't get file descriptor to input stream.\n");
+ fprintf(stderr, "Couldn't get file descriptor to input stream\n");
return NULL;
}
char *map = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0), *next = map;
if (map == MAP_FAILED) {
- fprintf(stderr, "Couldn't mmap() input stream.\n");
+ fprintf(stderr, "Couldn't mmap() input stream\n");
return NULL;
}
@@ -328,7 +348,8 @@ dmnsn_tokenize(FILE *file)
/* Possible comment */
case '/':
- if (dmnsn_tokenize_comment(map, size, &next, &line, &col) == 0) {
+ if (dmnsn_tokenize_comment(filename, &line, &col,
+ map, size, &next) == 0) {
continue;
} else {
/* Just the normal punctuation mark */
@@ -350,41 +371,43 @@ dmnsn_tokenize(FILE *file)
case '7':
case '8':
case '9':
- if (dmnsn_tokenize_number(map, size, &token, &next, &line, &col) != 0) {
- fprintf(stderr, "Invalid numeric value on line %u, column %u.\n",
- line, col);
+ if (dmnsn_tokenize_number(filename, &line, &col,
+ map, size, &next, &token) != 0) {
+ dmnsn_diagnostic(filename, line, col, "Invalid numeric value");
goto bailout;
}
break;
case '#':
/* Language directive */
- if (dmnsn_tokenize_directive(map, size, &token,
- &next, &line, &col) == 0) {
+ if (dmnsn_tokenize_directive(filename, &line, &col,
+ map, size, &next, &token) == 0) {
if (token.type == DMNSN_INCLUDE) {
/* Skip whitespace */
while (next - map < size && isspace(*next) && *next != '\n') {
++next;
}
- if (dmnsn_tokenize_string(map, size, &token,
- &next, &line, &col) != 0) {
- fprintf(stderr,
- "Expected string after #include on line %u, column %u.\n",
- line, col);
+ if (dmnsn_tokenize_string(filename, &line, &col,
+ map, size, &next, &token) != 0) {
+ dmnsn_diagnostic(filename, line, col,
+ "Expected string after #include");
goto bailout;
}
FILE *included = fopen(token.value, "r");
if (!included) {
- fprintf(stderr, "Couldn't include \"%s\" on line %u, column %u.\n",
- token.value, line, col);
+ dmnsn_diagnostic(filename, line, col,
+ "Couldn't open included file \"%s\"",
+ token.value);
goto bailout;
}
- dmnsn_array *included_tokens = dmnsn_tokenize(included);
+ dmnsn_array *included_tokens = dmnsn_tokenize(token.value, included);
if (!included_tokens) {
- fprintf(stderr, "Error tokenizing \"%s\"\n", token.value);
+ dmnsn_diagnostic(filename, line, col,
+ "Error tokenizing included file \"%s\"",
+ token.value);
goto bailout;
}
@@ -393,36 +416,38 @@ dmnsn_tokenize(FILE *file)
dmnsn_array_push(tokens, dmnsn_array_at(included_tokens, i));
}
+ free(token.value);
dmnsn_delete_array(included_tokens);
continue;
}
} else {
- fprintf(stderr, "Invalid directive on line %u, column %u.\n",
- line, col);
+ dmnsn_diagnostic(filename, line, col, "Invalid language directive");
goto bailout;
}
break;
case '"':
- if (dmnsn_tokenize_string(map, size, &token, &next, &line, &col) != 0) {
- fprintf(stderr, "Invalid string on line %u, column %u.\n",
- line, col);
+ if (dmnsn_tokenize_string(filename, &line, &col,
+ map, size, &next, &token) != 0) {
+ dmnsn_diagnostic(filename, line, col, "Invalid string");
goto bailout;
}
break;
default:
- if (dmnsn_tokenize_label(map, size, &token, &next, &line, &col) != 0) {
+ if (dmnsn_tokenize_label(filename, &line, &col,
+ map, size, &next, &token) != 0) {
/* Unrecognised character */
- fprintf(stderr,
- "Unrecognized character '%c' (0x%X) in input at line %u,"
- " column %u.\n",
- (int)*next, (unsigned int)*next, line, col);
+ dmnsn_diagnostic(filename, line, col,
+ "Unrecognized character '%c' (0x%X)",
+ (int)*next, (unsigned int)*next);
goto bailout;
}
break;
}
+ token.filename = malloc(strlen(filename) + 1);
+ strcpy(token.filename, filename);
dmnsn_array_push(tokens, &token);
}
@@ -442,6 +467,7 @@ dmnsn_delete_tokens(dmnsn_array *tokens)
unsigned int i;
for (i = 0; i < dmnsn_array_size(tokens); ++i) {
token = dmnsn_array_at(tokens, i);
+ free(token->filename);
free(token->value);
}
dmnsn_delete_array(tokens);
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index cc8148f..35255c3 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -62,10 +62,11 @@ struct dmnsn_token {
dmnsn_token_type type;
char *value;
- /* Line and column numbers from source code */
+ /* File name, and line and column numbers from source code */
+ char *filename;
unsigned int line, col;
};
-dmnsn_array *dmnsn_tokenize(FILE *file);
+dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file);
void dmnsn_delete_tokens(dmnsn_array *tokens);
void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens);