From 281a8741c0407e3cc504e98dee7a1e10782b46b2 Mon Sep 17 00:00:00 2001 From: tavianator Date: Thu, 29 Oct 2009 12:35:32 -0400 Subject: Refactor tokenizer a bit. --- dimension/Makefile.am | 4 +++- dimension/tokenize.c | 49 +++++++++++++++++++++++-------------------------- dimension/tokenize.h | 10 +++++++++- dimension/utility.c | 36 ++++++++++++++++++++++++++++++++++++ dimension/utility.h | 22 ++++++++++++++++++++++ 5 files changed, 93 insertions(+), 28 deletions(-) create mode 100644 dimension/utility.c create mode 100644 dimension/utility.h diff --git a/dimension/Makefile.am b/dimension/Makefile.am index 099a81b..da8d86d 100644 --- a/dimension/Makefile.am +++ b/dimension/Makefile.am @@ -23,5 +23,7 @@ bin_PROGRAMS = dimension dimension_SOURCES = main.c \ tokenize.c \ - tokenize.h + tokenize.h \ + utility.c \ + utility.h dimension_LDADD = $(top_builddir)/libdimension/libdimension.la diff --git a/dimension/tokenize.c b/dimension/tokenize.c index 4d53bdf..ec8c6ee 100644 --- a/dimension/tokenize.c +++ b/dimension/tokenize.c @@ -18,28 +18,14 @@ *************************************************************************/ #include "tokenize.h" +#include "utility.h" #include /* For strtoul(), etc. */ #include -#include #include /* For isalpha(), etc. */ #include /* For mmap() */ #include /* For dirname() */ #include -static void -dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - - fprintf(stderr, "%s:%u:%u: ", filename, line, col); - vfprintf(stderr, format, ap); - fprintf(stderr, "\n"); - - va_end(ap); -} - static int dmnsn_tokenize_comment(const char *filename, unsigned int *linep, unsigned int *colp, @@ -596,31 +582,42 @@ dmnsn_delete_tokens(dmnsn_array *tokens) dmnsn_delete_array(tokens); } -static const char *dmnsn_token_name(dmnsn_token_type token_type); - static void -dmnsn_print_token(FILE *file, dmnsn_token *token) +dmnsn_print_token(FILE *file, const dmnsn_token *token) { + const char *tname; + if (token->type == DMNSN_T_LPAREN) { + tname = "\\("; + } else if (token->type == DMNSN_T_RPAREN) { + tname = "\\)"; + } else { + tname = dmnsn_token_name(token->type); + } + if (token->value) { - fprintf(file, "(%s \"%s\")", dmnsn_token_name(token->type), token->value); + fprintf(file, "(%s \"%s\")", tname, token->value); } else { - fprintf(file, "%s", dmnsn_token_name(token->type)); + fprintf(file, "%s", tname); } } void -dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens) +dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens) { + dmnsn_token token; unsigned int i; + if (dmnsn_array_size(tokens) == 0) { fprintf(file, "()"); } else { fprintf(file, "("); - dmnsn_print_token(file, dmnsn_array_at(tokens, 0)); + dmnsn_array_get(tokens, 0, &token); + dmnsn_print_token(file, &token); for (i = 1; i < dmnsn_array_size(tokens); ++i) { fprintf(file, " "); - dmnsn_print_token(file, dmnsn_array_at(tokens, i)); + dmnsn_array_get(tokens, i, &token); + dmnsn_print_token(file, &token); } fprintf(file, ")"); @@ -629,7 +626,7 @@ dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens) fprintf(file, "\n"); } -static const char * +const char * dmnsn_token_name(dmnsn_token_type token_type) { switch (token_type) { @@ -641,8 +638,8 @@ dmnsn_token_name(dmnsn_token_type token_type) /* Punctuation */ dmnsn_token_map(DMNSN_T_LBRACE, "{"); dmnsn_token_map(DMNSN_T_RBRACE, "}") - dmnsn_token_map(DMNSN_T_LPAREN, "\\("); - dmnsn_token_map(DMNSN_T_RPAREN, "\\)"); + dmnsn_token_map(DMNSN_T_LPAREN, "("); + dmnsn_token_map(DMNSN_T_RPAREN, ")"); dmnsn_token_map(DMNSN_T_LBRACKET, "["); dmnsn_token_map(DMNSN_T_RBRACKET, "]"); dmnsn_token_map(DMNSN_T_LT, "<"); diff --git a/dimension/tokenize.h b/dimension/tokenize.h index 7a36232..4e16909 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -99,6 +99,14 @@ struct dmnsn_token { unsigned int line, col; }; +/* The workhorse */ dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file); + +/* Free an array of tokens - use this rather than dmnsn_delete_array() */ void dmnsn_delete_tokens(dmnsn_array *tokens); -void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens); + +/* Print an S-expression of a list of tokens to `file' */ +void dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens); + +/* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */ +const char *dmnsn_token_name(dmnsn_token_type token_type); diff --git a/dimension/utility.c b/dimension/utility.c new file mode 100644 index 0000000..1961d3e --- /dev/null +++ b/dimension/utility.c @@ -0,0 +1,36 @@ +/************************************************************************* + * Copyright (C) 2009 Tavian Barnes * + * * + * This file is part of Dimension. * + * * + * Dimension is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 3 of the License, or (at * + * your option) any later version. * + * * + * Dimension is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +#include "utility.h" +#include +#include + +void +dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col, + const char *format, ...) +{ + va_list ap; + va_start(ap, format); + + fprintf(stderr, "%s:%u:%u: ", filename, line, col); + vfprintf(stderr, format, ap); + fprintf(stderr, "\n"); + + va_end(ap); +} diff --git a/dimension/utility.h b/dimension/utility.h new file mode 100644 index 0000000..5403b19 --- /dev/null +++ b/dimension/utility.h @@ -0,0 +1,22 @@ +/************************************************************************* + * Copyright (C) 2009 Tavian Barnes * + * * + * This file is part of Dimension. * + * * + * Dimension is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 3 of the License, or (at * + * your option) any later version. * + * * + * Dimension is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +/* Print a parsing diagnostic to stderr */ +void dmnsn_diagnostic(const char *filename, unsigned int line, unsigned int col, + const char *format, ...); -- cgit v1.2.3