From 43ab94e9a2f18b0e40b441fedde5b4ce88046539 Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Mon, 26 Oct 2009 20:26:29 -0400 Subject: Begin tokenizer. --- dimension/tokenize.h | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 dimension/tokenize.h (limited to 'dimension/tokenize.h') diff --git a/dimension/tokenize.h b/dimension/tokenize.h new file mode 100644 index 0000000..3798d1e --- /dev/null +++ b/dimension/tokenize.h @@ -0,0 +1,36 @@ +/************************************************************************* + * Copyright (C) 2009 Tavian Barnes * + * * + * This file is part of Dimension. * + * * + * Dimension is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 3 of the License, or (at * + * your option) any later version. * + * * + * Dimension is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +#include "../libdimension/dimension.h" + +typedef enum { + DMNSN_LBRACE, + DMNSN_RBRACE +} dmnsn_token_type; + +typedef struct dmnsn_token dmnsn_token; + +struct dmnsn_token { + dmnsn_token_type type; + char *value; +}; + +dmnsn_array *dmnsn_tokenize(FILE *file); +void dmnsn_delete_tokens(dmnsn_array *tokens); +void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens); -- cgit v1.2.3