diff options
Diffstat (limited to 'dimension/lexer.l')
| -rw-r--r-- | dimension/lexer.l | 217 |
1 files changed, 118 insertions, 99 deletions
diff --git a/dimension/lexer.l b/dimension/lexer.l index bfe5b8a..b8bfbb0 100644 --- a/dimension/lexer.l +++ b/dimension/lexer.l @@ -20,12 +20,15 @@ %option reentrant stack yylineno noyywrap prefix="dmnsn_yy" outfile="lex.yy.c" %{ -#define YY_DECL static int yylex(const char *filename, dmnsn_array *tokens, \ - yyscan_t yyscanner) +#include "parse.h" #include "tokenize.h" #include "utility.h" #include <stdlib.h> #include <stdio.h> + +#define YY_DECL int yylex(dmnsn_parse_item *lvalp, \ + dmnsn_parse_location *llocp, \ + const char *filename, yyscan_t yyscanner) %} %x DMNSN_BLOCK_COMMENT @@ -38,34 +41,34 @@ %{ /* Some helpful macros that set fields of a token correctly, and other stuff */ -#define NEW_TOKEN(token_type) \ - do { \ - token.type = token_type; \ - token.filename = filename; \ - token.line = yylineno; \ - token.col = yycolumn; \ - token.value = NULL; \ +#define NEW_TOKEN(token_type) \ + do { \ + token = token_type; \ + lvalp->value = NULL; \ + llocp->first_filename = llocp->last_filename = filename; \ + llocp->first_line = llocp->last_line = yylineno; \ + llocp->first_column = llocp->last_column = yycolumn; \ } while (0) -#define CALCULATE_COLUMN() yycolumn += yyleng +#define CALCULATE_COLUMN() do { yycolumn += yyleng; } while (0) -#define PUSH() \ - do { \ - dmnsn_array_push(tokens, &token); \ - CALCULATE_COLUMN(); \ +#define RETURN() \ + do { \ + CALCULATE_COLUMN(); \ + return token; \ } while (0) -#define PUSH_TOKEN(token_type) \ +#define RETURN_TOKEN(token_type) \ do { \ NEW_TOKEN(token_type); \ - PUSH(); \ + RETURN(); \ } while (0) -#define PUSH_VALUE_TOKEN(token_type) \ +#define RETURN_VALUE_TOKEN(token_type) \ do { \ NEW_TOKEN(token_type); \ - token.value = strdup(yytext); \ - PUSH(); \ + lvalp->value = strdup(yytext); \ + RETURN(); \ } while (0) #define STRING_TOKEN() \ @@ -73,8 +76,8 @@ NEW_TOKEN(DMNSN_T_STRING); \ string_length = 0; \ string_extent = 8; \ - token.value = malloc(string_extent); \ - token.value[0] = '\0'; \ + lvalp->value = malloc(string_extent); \ + lvalp->value[0] = '\0'; \ CALCULATE_COLUMN(); \ } while (0) @@ -82,15 +85,15 @@ do { \ if (string_length + len + 1 >= string_length) { \ string_extent = 2*(string_length + len + 1); \ - token.value = realloc(token.value, string_extent); \ + lvalp->value = realloc(lvalp->value, string_extent); \ } \ \ - strncpy(token.value + string_length, str, len + 1); \ + strncpy(lvalp->value + string_length, str, len + 1); \ string_length += len; \ CALCULATE_COLUMN(); \ } while(0) -dmnsn_token token; +int token; size_t string_length, string_extent; unsigned long wchar; %} @@ -115,89 +118,89 @@ unsigned long wchar; <DMNSN_LINE_COMMENT>[^\n]+ CALCULATE_COLUMN(); (?# Punctuation) -"{" PUSH_TOKEN(DMNSN_T_LBRACE); -"}" PUSH_TOKEN(DMNSN_T_RBRACE); -"(" PUSH_TOKEN(DMNSN_T_LPAREN); -")" PUSH_TOKEN(DMNSN_T_RPAREN); -"[" PUSH_TOKEN(DMNSN_T_LBRACKET); -"]" PUSH_TOKEN(DMNSN_T_RBRACKET); -"+" PUSH_TOKEN(DMNSN_T_PLUS); -"-" PUSH_TOKEN(DMNSN_T_MINUS); -"*" PUSH_TOKEN(DMNSN_T_STAR); -"/" PUSH_TOKEN(DMNSN_T_SLASH); -"," PUSH_TOKEN(DMNSN_T_COMMA); -";" PUSH_TOKEN(DMNSN_T_SEMICOLON); -"?" PUSH_TOKEN(DMNSN_T_QUESTION); -":" PUSH_TOKEN(DMNSN_T_COLON); -"&" PUSH_TOKEN(DMNSN_T_AND); -"." PUSH_TOKEN(DMNSN_T_DOT); -"|" PUSH_TOKEN(DMNSN_T_PIPE); -"<" PUSH_TOKEN(DMNSN_T_LESS); -">" PUSH_TOKEN(DMNSN_T_GREATER); -"!" PUSH_TOKEN(DMNSN_T_BANG); -"=" PUSH_TOKEN(DMNSN_T_EQUALS); -"<=" PUSH_TOKEN(DMNSN_T_LESS_EQUAL); -">=" PUSH_TOKEN(DMNSN_T_GREATER_EQUAL); -"!=" PUSH_TOKEN(DMNSN_T_NOT_EQUAL); +"{" RETURN_TOKEN(DMNSN_T_LBRACE); +"}" RETURN_TOKEN(DMNSN_T_RBRACE); +"(" RETURN_TOKEN(DMNSN_T_LPAREN); +")" RETURN_TOKEN(DMNSN_T_RPAREN); +"[" RETURN_TOKEN(DMNSN_T_LBRACKET); +"]" RETURN_TOKEN(DMNSN_T_RBRACKET); +"+" RETURN_TOKEN(DMNSN_T_PLUS); +"-" RETURN_TOKEN(DMNSN_T_MINUS); +"*" RETURN_TOKEN(DMNSN_T_STAR); +"/" RETURN_TOKEN(DMNSN_T_SLASH); +"," RETURN_TOKEN(DMNSN_T_COMMA); +";" RETURN_TOKEN(DMNSN_T_SEMICOLON); +"?" RETURN_TOKEN(DMNSN_T_QUESTION); +":" RETURN_TOKEN(DMNSN_T_COLON); +"&" RETURN_TOKEN(DMNSN_T_AND); +"." RETURN_TOKEN(DMNSN_T_DOT); +"|" RETURN_TOKEN(DMNSN_T_PIPE); +"<" RETURN_TOKEN(DMNSN_T_LESS); +">" RETURN_TOKEN(DMNSN_T_GREATER); +"!" RETURN_TOKEN(DMNSN_T_BANG); +"=" RETURN_TOKEN(DMNSN_T_EQUALS); +"<=" RETURN_TOKEN(DMNSN_T_LESS_EQUAL); +">=" RETURN_TOKEN(DMNSN_T_GREATER_EQUAL); +"!=" RETURN_TOKEN(DMNSN_T_NOT_EQUAL); (?# Integers) [[:digit:]]+ | -0(x|X)[[:digit:]aAbBcCdDeEfF]+ PUSH_VALUE_TOKEN(DMNSN_T_INTEGER); +0(x|X)[[:digit:]aAbBcCdDeEfF]+ RETURN_VALUE_TOKEN(DMNSN_T_INTEGER); (?# Floats) [[:digit:]]*\.?[[:digit:]]+((e|E)(\+|-)?[[:digit:]]+)? { - PUSH_VALUE_TOKEN(DMNSN_T_FLOAT); + RETURN_VALUE_TOKEN(DMNSN_T_FLOAT); } (?# Keywords) -"angle" PUSH_TOKEN(DMNSN_T_ANGLE); -"background" PUSH_TOKEN(DMNSN_T_BACKGROUND); -"box" PUSH_TOKEN(DMNSN_T_BOX); -"blue" PUSH_TOKEN(DMNSN_T_BLUE); -"camera" PUSH_TOKEN(DMNSN_T_CAMERA); -"color" PUSH_TOKEN(DMNSN_T_COLOR); -"colour" PUSH_TOKEN(DMNSN_T_COLOR); -"direction" PUSH_TOKEN(DMNSN_T_DIRECTION); -"filter" PUSH_TOKEN(DMNSN_T_FILTER); -"gray" PUSH_TOKEN(DMNSN_T_GRAY); -"grey" PUSH_TOKEN(DMNSN_T_GRAY); -"green" PUSH_TOKEN(DMNSN_T_GREEN); -"location" PUSH_TOKEN(DMNSN_T_LOCATION); -"look_at" PUSH_TOKEN(DMNSN_T_LOOK_AT); -"light_source" PUSH_TOKEN(DMNSN_T_LIGHT_SOURCE); -"perspective" PUSH_TOKEN(DMNSN_T_PERSPECTIVE); -"pigment" PUSH_TOKEN(DMNSN_T_PIGMENT); -"red" PUSH_TOKEN(DMNSN_T_RED); -"rgb" PUSH_TOKEN(DMNSN_T_RGB); -"rgbf" PUSH_TOKEN(DMNSN_T_RGBF); -"rgbft" PUSH_TOKEN(DMNSN_T_RGBFT); -"rgbt" PUSH_TOKEN(DMNSN_T_RGBT); -"right" PUSH_TOKEN(DMNSN_T_RIGHT); -"rotate" PUSH_TOKEN(DMNSN_T_ROTATE); -"sphere" PUSH_TOKEN(DMNSN_T_SPHERE); -"sky" PUSH_TOKEN(DMNSN_T_SKY); -"t" PUSH_TOKEN(DMNSN_T_T); -"texture" PUSH_TOKEN(DMNSN_T_TEXTURE); -"transmit" PUSH_TOKEN(DMNSN_T_TRANSMIT); -"u" PUSH_TOKEN(DMNSN_T_U); -"up" PUSH_TOKEN(DMNSN_T_UP); -"v" PUSH_TOKEN(DMNSN_T_V); -"x" PUSH_TOKEN(DMNSN_T_X); -"y" PUSH_TOKEN(DMNSN_T_Y); -"z" PUSH_TOKEN(DMNSN_T_Z); +"angle" RETURN_TOKEN(DMNSN_T_ANGLE); +"background" RETURN_TOKEN(DMNSN_T_BACKGROUND); +"box" RETURN_TOKEN(DMNSN_T_BOX); +"blue" RETURN_TOKEN(DMNSN_T_BLUE); +"camera" RETURN_TOKEN(DMNSN_T_CAMERA); +"color" RETURN_TOKEN(DMNSN_T_COLOR); +"colour" RETURN_TOKEN(DMNSN_T_COLOR); +"direction" RETURN_TOKEN(DMNSN_T_DIRECTION); +"filter" RETURN_TOKEN(DMNSN_T_FILTER); +"gray" RETURN_TOKEN(DMNSN_T_GRAY); +"grey" RETURN_TOKEN(DMNSN_T_GRAY); +"green" RETURN_TOKEN(DMNSN_T_GREEN); +"location" RETURN_TOKEN(DMNSN_T_LOCATION); +"look_at" RETURN_TOKEN(DMNSN_T_LOOK_AT); +"light_source" RETURN_TOKEN(DMNSN_T_LIGHT_SOURCE); +"perspective" RETURN_TOKEN(DMNSN_T_PERSPECTIVE); +"pigment" RETURN_TOKEN(DMNSN_T_PIGMENT); +"red" RETURN_TOKEN(DMNSN_T_RED); +"rgb" RETURN_TOKEN(DMNSN_T_RGB); +"rgbf" RETURN_TOKEN(DMNSN_T_RGBF); +"rgbft" RETURN_TOKEN(DMNSN_T_RGBFT); +"rgbt" RETURN_TOKEN(DMNSN_T_RGBT); +"right" RETURN_TOKEN(DMNSN_T_RIGHT); +"rotate" RETURN_TOKEN(DMNSN_T_ROTATE); +"sphere" RETURN_TOKEN(DMNSN_T_SPHERE); +"sky" RETURN_TOKEN(DMNSN_T_SKY); +"t" RETURN_TOKEN(DMNSN_T_T); +"texture" RETURN_TOKEN(DMNSN_T_TEXTURE); +"transmit" RETURN_TOKEN(DMNSN_T_TRANSMIT); +"u" RETURN_TOKEN(DMNSN_T_U); +"up" RETURN_TOKEN(DMNSN_T_UP); +"v" RETURN_TOKEN(DMNSN_T_V); +"x" RETURN_TOKEN(DMNSN_T_X); +"y" RETURN_TOKEN(DMNSN_T_Y); +"z" RETURN_TOKEN(DMNSN_T_Z); (?# Directives) -"#include" PUSH_TOKEN(DMNSN_T_INCLUDE); -"#declare" PUSH_TOKEN(DMNSN_T_DECLARE); +"#include" RETURN_TOKEN(DMNSN_T_INCLUDE); +"#declare" RETURN_TOKEN(DMNSN_T_DECLARE); (?# Identifiers) -[[:alpha:]][[:alnum:]_]* PUSH_VALUE_TOKEN(DMNSN_T_IDENTIFIER); +[[:alpha:]][[:alnum:]_]* RETURN_VALUE_TOKEN(DMNSN_T_IDENTIFIER); (?# Strings) "\"" STRING_TOKEN(); yy_push_state(DMNSN_STRING, yyscanner); <DMNSN_STRING>[^\\\"\n]* STRCAT(yytext, yyleng); -<DMNSN_STRING>"\"" PUSH(); yy_pop_state(yyscanner); +<DMNSN_STRING>"\"" yy_pop_state(yyscanner); RETURN(); (?# String escape sequences) @@ -218,8 +221,8 @@ unsigned long wchar; <DMNSN_STRING_ESCAPE>"u"[[:digit:]aAbBcCdDeEfF]{4} { wchar = strtoul(yytext + 1, NULL, 16); STRCAT("", 2); - token.value[string_length - 2] = wchar/256; - token.value[string_length - 1] = wchar%256; + lvalp->value[string_length - 2] = wchar/256; + lvalp->value[string_length - 1] = wchar%256; yy_pop_state(yyscanner); } <DMNSN_STRING_ESCAPE>. { @@ -245,8 +248,11 @@ unsigned long wchar; %% dmnsn_array * -dmnsn_tokenize(const char *filename, FILE *file) +dmnsn_tokenize(FILE *file, const char *filename) { + dmnsn_token token; + dmnsn_parse_item item; + dmnsn_parse_location location; dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token)); yyscan_t scanner; @@ -254,24 +260,37 @@ dmnsn_tokenize(const char *filename, FILE *file) yylex_init(&scanner); yyset_in(file, scanner); - if (yylex(filename, tokens, scanner) != 0) { - dmnsn_delete_tokens(tokens); - tokens = NULL; + while ((token.type = yylex(&item, &location, filename, scanner)) != 0) { + if (token.type == 1 || token.type == 2) { + dmnsn_delete_tokens(tokens); + tokens = NULL; + } else { + token.value = item.value; + token.filename = location.first_filename; + token.line = location.first_line; + token.col = location.first_column; + dmnsn_array_push(tokens, &token); + } } yylex_destroy(scanner); - return tokens; } void +dmnsn_delete_token(dmnsn_token token) +{ + free(token.value); +} + +void dmnsn_delete_tokens(dmnsn_array *tokens) { - dmnsn_token *token; + dmnsn_token token; unsigned int i; for (i = 0; i < dmnsn_array_size(tokens); ++i) { - token = dmnsn_array_at(tokens, i); - free(token->value); + dmnsn_array_get(tokens, i, &token); + dmnsn_delete_token(token); } dmnsn_delete_array(tokens); } |