diff options
| author | Tavian Barnes <tavianator@gmail.com> | 2009-10-29 01:34:28 -0400 |
|---|---|---|
| committer | Tavian Barnes <tavianator@gmail.com> | 2009-10-29 01:34:28 -0400 |
| commit | 87cb2d172843e114a8640de3fde61db3a2bf0a6a (patch) | |
| tree | 759747df1cde1f8647ea34e7738b3d013197128b /dimension/tokenize.h | |
| parent | 7e77347d6135a9cffe82f5b1384a34fe3cd4c5cc (diff) | |
| download | dimension-87cb2d172843e114a8640de3fde61db3a2bf0a6a.tar.xz | |
Even more tokenizer work.
The tokenizer is now able to tokenize all of POV-Ray 3.6's include files. It
turns out that some #includes may be dynamic (ex. with #writes beforehand), so
the tokenizer can't directly perform includes - the executor must do this.
Diffstat (limited to 'dimension/tokenize.h')
| -rw-r--r-- | dimension/tokenize.h | 36 |
1 files changed, 22 insertions, 14 deletions
diff --git a/dimension/tokenize.h b/dimension/tokenize.h index a90073b..7a36232 100644 --- a/dimension/tokenize.h +++ b/dimension/tokenize.h @@ -21,19 +21,27 @@ typedef enum { /* Punctuation */ - DMNSN_T_LBRACE, /* { */ - DMNSN_T_RBRACE, /* } */ - DMNSN_T_LPAREN, /* ( */ - DMNSN_T_RPAREN, /* ) */ - DMNSN_T_LBRACKET, /* [ */ - DMNSN_T_RBRACKET, /* ] */ - DMNSN_T_LT, /* < */ - DMNSN_T_GT, /* > */ - DMNSN_T_PLUS, /* + */ - DMNSN_T_MINUS, /* - */ - DMNSN_T_STAR, /* * */ - DMNSN_T_SLASH, /* / */ - DMNSN_T_COMMA, /* , */ + DMNSN_T_LBRACE, /* { */ + DMNSN_T_RBRACE, /* } */ + DMNSN_T_LPAREN, /* ( */ + DMNSN_T_RPAREN, /* ) */ + DMNSN_T_LBRACKET, /* [ */ + DMNSN_T_RBRACKET, /* ] */ + DMNSN_T_LT, /* < */ + DMNSN_T_GT, /* > */ + DMNSN_T_PLUS, /* + */ + DMNSN_T_MINUS, /* - */ + DMNSN_T_STAR, /* * */ + DMNSN_T_SLASH, /* / */ + DMNSN_T_COMMA, /* , */ + DMNSN_T_EQUALS, /* = */ + DMNSN_T_SEMICOLON, /* ; */ + DMNSN_T_QUESTION, /* ? */ + DMNSN_T_COLON, /* : */ + DMNSN_T_AND, /* & */ + DMNSN_T_EXCLAMATION, /* ! */ + DMNSN_T_DOT, /* . */ + DMNSN_T_PIPE, /* | */ /* Numeric values */ DMNSN_T_INT, @@ -59,7 +67,7 @@ typedef enum { DMNSN_T_IF, DMNSN_T_IFDEF, DMNSN_T_IFNDEF, - DMNSN_T_INCLUDE, /* Only used internally */ + DMNSN_T_INCLUDE, DMNSN_T_LOCAL, DMNSN_T_MACRO, DMNSN_T_RANGE, |