summaryrefslogtreecommitdiffstats
path: root/dimension/tokenize.h
diff options
context:
space:
mode:
authortavianator <tavianator@antimatter.localdomain>2009-10-29 12:35:32 -0400
committertavianator <tavianator@antimatter.localdomain>2009-10-29 12:35:32 -0400
commit281a8741c0407e3cc504e98dee7a1e10782b46b2 (patch)
tree03061db1b4fffc8ee5fb021ab8461b5d555e5fe5 /dimension/tokenize.h
parent6aced77005b10311176a53d8db9862d4fff5dbc3 (diff)
downloaddimension-281a8741c0407e3cc504e98dee7a1e10782b46b2.tar.xz
Refactor tokenizer a bit.
Diffstat (limited to 'dimension/tokenize.h')
-rw-r--r--dimension/tokenize.h10
1 files changed, 9 insertions, 1 deletions
diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 7a36232..4e16909 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -99,6 +99,14 @@ struct dmnsn_token {
unsigned int line, col;
};
+/* The workhorse */
dmnsn_array *dmnsn_tokenize(const char *filename, FILE *file);
+
+/* Free an array of tokens - use this rather than dmnsn_delete_array() */
void dmnsn_delete_tokens(dmnsn_array *tokens);
-void dmnsn_print_token_sexpr(FILE *file, dmnsn_array *tokens);
+
+/* Print an S-expression of a list of tokens to `file' */
+void dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens);
+
+/* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */
+const char *dmnsn_token_name(dmnsn_token_type token_type);