From 50d844e91589a8f51cade42b732bcd1de889987b Mon Sep 17 00:00:00 2001
From: Tavian Barnes <tavianator@gmail.com>
Date: Tue, 27 Oct 2009 17:33:44 -0400
Subject: Support more symbols in tokenizer.

---
 dimension/tokenize.h | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

(limited to 'dimension/tokenize.h')

diff --git a/dimension/tokenize.h b/dimension/tokenize.h
index 3798d1e..e64b7eb 100644
--- a/dimension/tokenize.h
+++ b/dimension/tokenize.h
@@ -20,8 +20,19 @@
 #include "../libdimension/dimension.h"
 
 typedef enum {
-  DMNSN_LBRACE,
-  DMNSN_RBRACE
+  DMNSN_LBRACE,   /* { */
+  DMNSN_RBRACE,   /* } */
+  DMNSN_LPAREN,   /* ( */
+  DMNSN_RPAREN,   /* ) */
+  DMNSN_LBRACKET, /* [ */
+  DMNSN_RBRACKET, /* ] */
+  DMNSN_LT,       /* < */
+  DMNSN_GT,       /* > */
+  DMNSN_PLUS,     /* + */
+  DMNSN_MINUS,    /* - */
+  DMNSN_STAR,     /* * */
+  DMNSN_SLASH,    /* / */
+  DMNSN_COMMA,    /* , */
 } dmnsn_token_type;
 
 typedef struct dmnsn_token dmnsn_token;
-- 
cgit v1.2.3