summaryrefslogtreecommitdiffstats
path: root/dimension/lexer.l
diff options
context:
space:
mode:
Diffstat (limited to 'dimension/lexer.l')
-rw-r--r--dimension/lexer.l217
1 files changed, 118 insertions, 99 deletions
diff --git a/dimension/lexer.l b/dimension/lexer.l
index bfe5b8a..b8bfbb0 100644
--- a/dimension/lexer.l
+++ b/dimension/lexer.l
@@ -20,12 +20,15 @@
%option reentrant stack yylineno noyywrap prefix="dmnsn_yy" outfile="lex.yy.c"
%{
-#define YY_DECL static int yylex(const char *filename, dmnsn_array *tokens, \
- yyscan_t yyscanner)
+#include "parse.h"
#include "tokenize.h"
#include "utility.h"
#include <stdlib.h>
#include <stdio.h>
+
+#define YY_DECL int yylex(dmnsn_parse_item *lvalp, \
+ dmnsn_parse_location *llocp, \
+ const char *filename, yyscan_t yyscanner)
%}
%x DMNSN_BLOCK_COMMENT
@@ -38,34 +41,34 @@
%{
/* Some helpful macros that set fields of a token correctly, and other stuff */
-#define NEW_TOKEN(token_type) \
- do { \
- token.type = token_type; \
- token.filename = filename; \
- token.line = yylineno; \
- token.col = yycolumn; \
- token.value = NULL; \
+#define NEW_TOKEN(token_type) \
+ do { \
+ token = token_type; \
+ lvalp->value = NULL; \
+ llocp->first_filename = llocp->last_filename = filename; \
+ llocp->first_line = llocp->last_line = yylineno; \
+ llocp->first_column = llocp->last_column = yycolumn; \
} while (0)
-#define CALCULATE_COLUMN() yycolumn += yyleng
+#define CALCULATE_COLUMN() do { yycolumn += yyleng; } while (0)
-#define PUSH() \
- do { \
- dmnsn_array_push(tokens, &token); \
- CALCULATE_COLUMN(); \
+#define RETURN() \
+ do { \
+ CALCULATE_COLUMN(); \
+ return token; \
} while (0)
-#define PUSH_TOKEN(token_type) \
+#define RETURN_TOKEN(token_type) \
do { \
NEW_TOKEN(token_type); \
- PUSH(); \
+ RETURN(); \
} while (0)
-#define PUSH_VALUE_TOKEN(token_type) \
+#define RETURN_VALUE_TOKEN(token_type) \
do { \
NEW_TOKEN(token_type); \
- token.value = strdup(yytext); \
- PUSH(); \
+ lvalp->value = strdup(yytext); \
+ RETURN(); \
} while (0)
#define STRING_TOKEN() \
@@ -73,8 +76,8 @@
NEW_TOKEN(DMNSN_T_STRING); \
string_length = 0; \
string_extent = 8; \
- token.value = malloc(string_extent); \
- token.value[0] = '\0'; \
+ lvalp->value = malloc(string_extent); \
+ lvalp->value[0] = '\0'; \
CALCULATE_COLUMN(); \
} while (0)
@@ -82,15 +85,15 @@
do { \
if (string_length + len + 1 >= string_length) { \
string_extent = 2*(string_length + len + 1); \
- token.value = realloc(token.value, string_extent); \
+ lvalp->value = realloc(lvalp->value, string_extent); \
} \
\
- strncpy(token.value + string_length, str, len + 1); \
+ strncpy(lvalp->value + string_length, str, len + 1); \
string_length += len; \
CALCULATE_COLUMN(); \
} while(0)
-dmnsn_token token;
+int token;
size_t string_length, string_extent;
unsigned long wchar;
%}
@@ -115,89 +118,89 @@ unsigned long wchar;
<DMNSN_LINE_COMMENT>[^\n]+ CALCULATE_COLUMN();
(?# Punctuation)
-"{" PUSH_TOKEN(DMNSN_T_LBRACE);
-"}" PUSH_TOKEN(DMNSN_T_RBRACE);
-"(" PUSH_TOKEN(DMNSN_T_LPAREN);
-")" PUSH_TOKEN(DMNSN_T_RPAREN);
-"[" PUSH_TOKEN(DMNSN_T_LBRACKET);
-"]" PUSH_TOKEN(DMNSN_T_RBRACKET);
-"+" PUSH_TOKEN(DMNSN_T_PLUS);
-"-" PUSH_TOKEN(DMNSN_T_MINUS);
-"*" PUSH_TOKEN(DMNSN_T_STAR);
-"/" PUSH_TOKEN(DMNSN_T_SLASH);
-"," PUSH_TOKEN(DMNSN_T_COMMA);
-";" PUSH_TOKEN(DMNSN_T_SEMICOLON);
-"?" PUSH_TOKEN(DMNSN_T_QUESTION);
-":" PUSH_TOKEN(DMNSN_T_COLON);
-"&" PUSH_TOKEN(DMNSN_T_AND);
-"." PUSH_TOKEN(DMNSN_T_DOT);
-"|" PUSH_TOKEN(DMNSN_T_PIPE);
-"<" PUSH_TOKEN(DMNSN_T_LESS);
-">" PUSH_TOKEN(DMNSN_T_GREATER);
-"!" PUSH_TOKEN(DMNSN_T_BANG);
-"=" PUSH_TOKEN(DMNSN_T_EQUALS);
-"<=" PUSH_TOKEN(DMNSN_T_LESS_EQUAL);
-">=" PUSH_TOKEN(DMNSN_T_GREATER_EQUAL);
-"!=" PUSH_TOKEN(DMNSN_T_NOT_EQUAL);
+"{" RETURN_TOKEN(DMNSN_T_LBRACE);
+"}" RETURN_TOKEN(DMNSN_T_RBRACE);
+"(" RETURN_TOKEN(DMNSN_T_LPAREN);
+")" RETURN_TOKEN(DMNSN_T_RPAREN);
+"[" RETURN_TOKEN(DMNSN_T_LBRACKET);
+"]" RETURN_TOKEN(DMNSN_T_RBRACKET);
+"+" RETURN_TOKEN(DMNSN_T_PLUS);
+"-" RETURN_TOKEN(DMNSN_T_MINUS);
+"*" RETURN_TOKEN(DMNSN_T_STAR);
+"/" RETURN_TOKEN(DMNSN_T_SLASH);
+"," RETURN_TOKEN(DMNSN_T_COMMA);
+";" RETURN_TOKEN(DMNSN_T_SEMICOLON);
+"?" RETURN_TOKEN(DMNSN_T_QUESTION);
+":" RETURN_TOKEN(DMNSN_T_COLON);
+"&" RETURN_TOKEN(DMNSN_T_AND);
+"." RETURN_TOKEN(DMNSN_T_DOT);
+"|" RETURN_TOKEN(DMNSN_T_PIPE);
+"<" RETURN_TOKEN(DMNSN_T_LESS);
+">" RETURN_TOKEN(DMNSN_T_GREATER);
+"!" RETURN_TOKEN(DMNSN_T_BANG);
+"=" RETURN_TOKEN(DMNSN_T_EQUALS);
+"<=" RETURN_TOKEN(DMNSN_T_LESS_EQUAL);
+">=" RETURN_TOKEN(DMNSN_T_GREATER_EQUAL);
+"!=" RETURN_TOKEN(DMNSN_T_NOT_EQUAL);
(?# Integers)
[[:digit:]]+ |
-0(x|X)[[:digit:]aAbBcCdDeEfF]+ PUSH_VALUE_TOKEN(DMNSN_T_INTEGER);
+0(x|X)[[:digit:]aAbBcCdDeEfF]+ RETURN_VALUE_TOKEN(DMNSN_T_INTEGER);
(?# Floats)
[[:digit:]]*\.?[[:digit:]]+((e|E)(\+|-)?[[:digit:]]+)? {
- PUSH_VALUE_TOKEN(DMNSN_T_FLOAT);
+ RETURN_VALUE_TOKEN(DMNSN_T_FLOAT);
}
(?# Keywords)
-"angle" PUSH_TOKEN(DMNSN_T_ANGLE);
-"background" PUSH_TOKEN(DMNSN_T_BACKGROUND);
-"box" PUSH_TOKEN(DMNSN_T_BOX);
-"blue" PUSH_TOKEN(DMNSN_T_BLUE);
-"camera" PUSH_TOKEN(DMNSN_T_CAMERA);
-"color" PUSH_TOKEN(DMNSN_T_COLOR);
-"colour" PUSH_TOKEN(DMNSN_T_COLOR);
-"direction" PUSH_TOKEN(DMNSN_T_DIRECTION);
-"filter" PUSH_TOKEN(DMNSN_T_FILTER);
-"gray" PUSH_TOKEN(DMNSN_T_GRAY);
-"grey" PUSH_TOKEN(DMNSN_T_GRAY);
-"green" PUSH_TOKEN(DMNSN_T_GREEN);
-"location" PUSH_TOKEN(DMNSN_T_LOCATION);
-"look_at" PUSH_TOKEN(DMNSN_T_LOOK_AT);
-"light_source" PUSH_TOKEN(DMNSN_T_LIGHT_SOURCE);
-"perspective" PUSH_TOKEN(DMNSN_T_PERSPECTIVE);
-"pigment" PUSH_TOKEN(DMNSN_T_PIGMENT);
-"red" PUSH_TOKEN(DMNSN_T_RED);
-"rgb" PUSH_TOKEN(DMNSN_T_RGB);
-"rgbf" PUSH_TOKEN(DMNSN_T_RGBF);
-"rgbft" PUSH_TOKEN(DMNSN_T_RGBFT);
-"rgbt" PUSH_TOKEN(DMNSN_T_RGBT);
-"right" PUSH_TOKEN(DMNSN_T_RIGHT);
-"rotate" PUSH_TOKEN(DMNSN_T_ROTATE);
-"sphere" PUSH_TOKEN(DMNSN_T_SPHERE);
-"sky" PUSH_TOKEN(DMNSN_T_SKY);
-"t" PUSH_TOKEN(DMNSN_T_T);
-"texture" PUSH_TOKEN(DMNSN_T_TEXTURE);
-"transmit" PUSH_TOKEN(DMNSN_T_TRANSMIT);
-"u" PUSH_TOKEN(DMNSN_T_U);
-"up" PUSH_TOKEN(DMNSN_T_UP);
-"v" PUSH_TOKEN(DMNSN_T_V);
-"x" PUSH_TOKEN(DMNSN_T_X);
-"y" PUSH_TOKEN(DMNSN_T_Y);
-"z" PUSH_TOKEN(DMNSN_T_Z);
+"angle" RETURN_TOKEN(DMNSN_T_ANGLE);
+"background" RETURN_TOKEN(DMNSN_T_BACKGROUND);
+"box" RETURN_TOKEN(DMNSN_T_BOX);
+"blue" RETURN_TOKEN(DMNSN_T_BLUE);
+"camera" RETURN_TOKEN(DMNSN_T_CAMERA);
+"color" RETURN_TOKEN(DMNSN_T_COLOR);
+"colour" RETURN_TOKEN(DMNSN_T_COLOR);
+"direction" RETURN_TOKEN(DMNSN_T_DIRECTION);
+"filter" RETURN_TOKEN(DMNSN_T_FILTER);
+"gray" RETURN_TOKEN(DMNSN_T_GRAY);
+"grey" RETURN_TOKEN(DMNSN_T_GRAY);
+"green" RETURN_TOKEN(DMNSN_T_GREEN);
+"location" RETURN_TOKEN(DMNSN_T_LOCATION);
+"look_at" RETURN_TOKEN(DMNSN_T_LOOK_AT);
+"light_source" RETURN_TOKEN(DMNSN_T_LIGHT_SOURCE);
+"perspective" RETURN_TOKEN(DMNSN_T_PERSPECTIVE);
+"pigment" RETURN_TOKEN(DMNSN_T_PIGMENT);
+"red" RETURN_TOKEN(DMNSN_T_RED);
+"rgb" RETURN_TOKEN(DMNSN_T_RGB);
+"rgbf" RETURN_TOKEN(DMNSN_T_RGBF);
+"rgbft" RETURN_TOKEN(DMNSN_T_RGBFT);
+"rgbt" RETURN_TOKEN(DMNSN_T_RGBT);
+"right" RETURN_TOKEN(DMNSN_T_RIGHT);
+"rotate" RETURN_TOKEN(DMNSN_T_ROTATE);
+"sphere" RETURN_TOKEN(DMNSN_T_SPHERE);
+"sky" RETURN_TOKEN(DMNSN_T_SKY);
+"t" RETURN_TOKEN(DMNSN_T_T);
+"texture" RETURN_TOKEN(DMNSN_T_TEXTURE);
+"transmit" RETURN_TOKEN(DMNSN_T_TRANSMIT);
+"u" RETURN_TOKEN(DMNSN_T_U);
+"up" RETURN_TOKEN(DMNSN_T_UP);
+"v" RETURN_TOKEN(DMNSN_T_V);
+"x" RETURN_TOKEN(DMNSN_T_X);
+"y" RETURN_TOKEN(DMNSN_T_Y);
+"z" RETURN_TOKEN(DMNSN_T_Z);
(?# Directives)
-"#include" PUSH_TOKEN(DMNSN_T_INCLUDE);
-"#declare" PUSH_TOKEN(DMNSN_T_DECLARE);
+"#include" RETURN_TOKEN(DMNSN_T_INCLUDE);
+"#declare" RETURN_TOKEN(DMNSN_T_DECLARE);
(?# Identifiers)
-[[:alpha:]][[:alnum:]_]* PUSH_VALUE_TOKEN(DMNSN_T_IDENTIFIER);
+[[:alpha:]][[:alnum:]_]* RETURN_VALUE_TOKEN(DMNSN_T_IDENTIFIER);
(?# Strings)
"\"" STRING_TOKEN(); yy_push_state(DMNSN_STRING, yyscanner);
<DMNSN_STRING>[^\\\"\n]* STRCAT(yytext, yyleng);
-<DMNSN_STRING>"\"" PUSH(); yy_pop_state(yyscanner);
+<DMNSN_STRING>"\"" yy_pop_state(yyscanner); RETURN();
(?# String escape sequences)
@@ -218,8 +221,8 @@ unsigned long wchar;
<DMNSN_STRING_ESCAPE>"u"[[:digit:]aAbBcCdDeEfF]{4} {
wchar = strtoul(yytext + 1, NULL, 16);
STRCAT("", 2);
- token.value[string_length - 2] = wchar/256;
- token.value[string_length - 1] = wchar%256;
+ lvalp->value[string_length - 2] = wchar/256;
+ lvalp->value[string_length - 1] = wchar%256;
yy_pop_state(yyscanner);
}
<DMNSN_STRING_ESCAPE>. {
@@ -245,8 +248,11 @@ unsigned long wchar;
%%
dmnsn_array *
-dmnsn_tokenize(const char *filename, FILE *file)
+dmnsn_tokenize(FILE *file, const char *filename)
{
+ dmnsn_token token;
+ dmnsn_parse_item item;
+ dmnsn_parse_location location;
dmnsn_array *tokens = dmnsn_new_array(sizeof(dmnsn_token));
yyscan_t scanner;
@@ -254,24 +260,37 @@ dmnsn_tokenize(const char *filename, FILE *file)
yylex_init(&scanner);
yyset_in(file, scanner);
- if (yylex(filename, tokens, scanner) != 0) {
- dmnsn_delete_tokens(tokens);
- tokens = NULL;
+ while ((token.type = yylex(&item, &location, filename, scanner)) != 0) {
+ if (token.type == 1 || token.type == 2) {
+ dmnsn_delete_tokens(tokens);
+ tokens = NULL;
+ } else {
+ token.value = item.value;
+ token.filename = location.first_filename;
+ token.line = location.first_line;
+ token.col = location.first_column;
+ dmnsn_array_push(tokens, &token);
+ }
}
yylex_destroy(scanner);
-
return tokens;
}
void
+dmnsn_delete_token(dmnsn_token token)
+{
+ free(token.value);
+}
+
+void
dmnsn_delete_tokens(dmnsn_array *tokens)
{
- dmnsn_token *token;
+ dmnsn_token token;
unsigned int i;
for (i = 0; i < dmnsn_array_size(tokens); ++i) {
- token = dmnsn_array_at(tokens, i);
- free(token->value);
+ dmnsn_array_get(tokens, i, &token);
+ dmnsn_delete_token(token);
}
dmnsn_delete_array(tokens);
}