1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
/*************************************************************************
* Copyright (C) 2010 Tavian Barnes <tavianator@gmail.com> *
* *
* This file is part of Dimension. *
* *
* Dimension is free software; you can redistribute it and/or modify it *
* under the terms of the GNU General Public License as published by the *
* Free Software Foundation; either version 3 of the License, or (at *
* your option) any later version. *
* *
* Dimension is distributed in the hope that it will be useful, but *
* WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
* General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
#ifndef TOKENIZE_H
#define TOKENIZE_H
#include "dimension.h"
#include "parse.h"
#define yytokentype dmnsn_yytokentype
#define YYSTYPE
#define YYLTYPE
#include "grammar.h"
#undef YYLTYPE
#undef YYSTYPE
#undef yytokentype
typedef enum dmnsn_yytokentype dmnsn_token_type;
typedef struct dmnsn_token dmnsn_token;
struct dmnsn_token {
dmnsn_token_type type;
char *value;
/* File name, and line and column numbers from source code */
const char *filename;
int line, col;
};
/* Scanner manipulation */
int dmnsn_yylex_init(void **scannerp);
void dmnsn_yyset_in(FILE *file, void *scanner);
int dmnsn_yylex_destroy(void *scanner);
void *dmnsn_yyget_extra(void *scanner);
void dmnsn_yyset_extra(void *arbitrary_data, void *scanner);
void *dmnsn_yy_make_buffer(FILE *file, void *scanner);
void *dmnsn_yy_make_string_buffer(const char *str, void *scanner);
void dmnsn_yy_push_buffer(void *buffer, void *scanner);
void dmnsn_yy_pop_buffer(void *scanner);
/* Actual lexer */
int dmnsn_yylex(dmnsn_parse_item *lvalp, dmnsn_parse_location *llocp,
const char *filename, dmnsn_symbol_table *symtable,
void *yyscanner);
void dmnsn_yylex_cleanup(void *yyscanner);
/* For debugging - returns an array of raw tokens */
dmnsn_array *dmnsn_tokenize(FILE *file, const char *filename);
/* Token destruction */
void dmnsn_delete_tokens(dmnsn_array *tokens);
/* Print an S-expression of a list of tokens to `file' */
void dmnsn_print_token_sexpr(FILE *file, const dmnsn_array *tokens);
/* Returns a readable name for a token type (ex. DMNSN_T_FLOAT -> float) */
const char *dmnsn_token_string(dmnsn_token_type token_type);
#endif /* TOKENIZE_H */
|
