From 281a8741c0407e3cc504e98dee7a1e10782b46b2 Mon Sep 17 00:00:00 2001 From: tavianator Date: Thu, 29 Oct 2009 12:35:32 -0400 Subject: Refactor tokenizer a bit. --- dimension/tokenize.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'dimension/tokenize.h') diff --git a/dimension/tokenize.h b/dimension/tokenize.h index 7a36232..4e16909 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -99,6 +99,14 @@ struct dmnsn_token { unsigned int line, col; }; +/* The workhorse */ dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file); + +/* Free an array of tokens - use this rather than dmnsn_delete_array() */ void dmnsn_delete_tokens(dmnsn_array *tokens); -void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens); + +/* Print an S-expression of a list of tokens to `file' */ +void dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens); + +/* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */ +const char *dmnsn_token_name(dmnsn_token_type token_type); -- cgit v1.2.3