974 lines
22 KiB
C++
974 lines
22 KiB
C++
|
/*************************************************************************/
|
||
|
|
/* gd_tokenizer.cpp */
|
||
|
|
/*************************************************************************/
|
||
|
|
/* This file is part of: */
|
||
|
|
/* GODOT ENGINE */
|
||
|
|
/* http://www.godotengine.org */
|
||
|
|
/*************************************************************************/
|
||
|
|
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
|
||
|
|
/* */
|
||
|
|
/* Permission is hereby granted, free of charge, to any person obtaining */
|
||
|
|
/* a copy of this software and associated documentation files (the */
|
||
|
|
/* "Software"), to deal in the Software without restriction, including */
|
||
|
|
/* without limitation the rights to use, copy, modify, merge, publish, */
|
||
|
|
/* distribute, sublicense, and/or sell copies of the Software, and to */
|
||
|
|
/* permit persons to whom the Software is furnished to do so, subject to */
|
||
|
|
/* the following conditions: */
|
||
|
|
/* */
|
||
|
|
/* The above copyright notice and this permission notice shall be */
|
||
|
|
/* included in all copies or substantial portions of the Software. */
|
||
|
|
/* */
|
||
|
|
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
|
||
|
|
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
|
||
|
|
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
|
||
|
|
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
|
||
|
|
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
|
||
|
|
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
|
||
|
|
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
|
||
|
|
/*************************************************************************/
|
||
|
|
#include "gd_tokenizer.h"
|
||
|
|
#include "print_string.h"
|
||
|
|
#include "gd_functions.h"
|
||
|
|
const char* GDTokenizer::token_names[TK_MAX]={
|
||
|
|
"Empty",
|
||
|
|
"Identifier",
|
||
|
|
"Constant",
|
||
|
|
"Self",
|
||
|
|
"Built-In Type",
|
||
|
|
"Built-In Func",
|
||
|
|
"In",
|
||
|
|
"'=='",
|
||
|
|
"'!='",
|
||
|
|
"'<'",
|
||
|
|
"'<='",
|
||
|
|
"'>'",
|
||
|
|
"'>='",
|
||
|
|
"'and'",
|
||
|
|
"'or'",
|
||
|
|
"'not'",
|
||
|
|
"'+'",
|
||
|
|
"'-'",
|
||
|
|
"'*'",
|
||
|
|
"'/'",
|
||
|
|
"'%'",
|
||
|
|
"'<<'",
|
||
|
|
"'>>'",
|
||
|
|
"'='",
|
||
|
|
"'+='",
|
||
|
|
"'-='",
|
||
|
|
"'*='",
|
||
|
|
"'/='",
|
||
|
|
"'%='",
|
||
|
|
"'<<='",
|
||
|
|
"'>>='",
|
||
|
|
"'&='",
|
||
|
|
"'|='",
|
||
|
|
"'^='",
|
||
|
|
"'&'",
|
||
|
|
"'|'",
|
||
|
|
"'^'",
|
||
|
|
"'~'",
|
||
|
|
//"Plus Plus",
|
||
|
|
//"Minus Minus",
|
||
|
|
"if",
|
||
|
|
"elif",
|
||
|
|
"else",
|
||
|
|
"for",
|
||
|
|
"do",
|
||
|
|
"while",
|
||
|
|
"switch",
|
||
|
|
"case",
|
||
|
|
"break",
|
||
|
|
"continue",
|
||
|
|
"pass",
|
||
|
|
"return",
|
||
|
|
"func",
|
||
|
|
"class",
|
||
|
|
"extends",
|
||
|
|
"tool",
|
||
|
|
"static",
|
||
|
|
"export",
|
||
|
|
"const",
|
||
|
|
"var",
|
||
|
|
"preload",
|
||
|
|
"assert",
|
||
|
|
"'['",
|
||
|
|
"']'",
|
||
|
|
"'{'",
|
||
|
|
"'}'",
|
||
|
|
"'('",
|
||
|
|
"')'",
|
||
|
|
"','",
|
||
|
|
"';'",
|
||
|
|
"'.'",
|
||
|
|
"'?'",
|
||
|
|
"':'",
|
||
|
|
"'\\n'",
|
||
|
|
"Error",
|
||
|
|
"EOF"};
|
||
|
|
|
||
|
|
const char *GDTokenizer::get_token_name(Token p_token) {
|
||
|
|
|
||
|
|
ERR_FAIL_INDEX_V(p_token,TK_MAX,"<error>");
|
||
|
|
return token_names[p_token];
|
||
|
|
}
|
||
|
|
|
||
|
|
static bool _is_text_char(CharType c) {
|
||
|
|
|
||
|
|
return (c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='_';
|
||
|
|
}
|
||
|
|
|
||
|
|
static bool _is_number(CharType c) {
|
||
|
|
|
||
|
|
return (c>='0' && c<='9');
|
||
|
|
}
|
||
|
|
|
||
|
|
static bool _is_hex(CharType c) {
|
||
|
|
|
||
|
|
return (c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F');
|
||
|
|
}
|
||
|
|
|
||
|
|
void GDTokenizer::_make_token(Token p_type) {
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
|
||
|
|
tk.type=p_type;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
}
|
||
|
|
void GDTokenizer::_make_identifier(const StringName& p_identifier) {
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
|
||
|
|
tk.type=TK_IDENTIFIER;
|
||
|
|
tk.identifier=p_identifier;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
void GDTokenizer::_make_built_in_func(GDFunctions::Function p_func) {
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
|
||
|
|
tk.type=TK_BUILT_IN_FUNC;
|
||
|
|
tk.func=p_func;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
|
||
|
|
}
|
||
|
|
void GDTokenizer::_make_constant(const Variant& p_constant) {
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
|
||
|
|
tk.type=TK_CONSTANT;
|
||
|
|
tk.constant=p_constant;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
void GDTokenizer::_make_type(const Variant::Type& p_type) {
|
||
|
|
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
|
||
|
|
tk.type=TK_BUILT_IN_TYPE;
|
||
|
|
tk.vtype=p_type;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
void GDTokenizer::_make_error(const String& p_error) {
|
||
|
|
|
||
|
|
error_flag=true;
|
||
|
|
last_error=p_error;
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
tk.type=TK_ERROR;
|
||
|
|
tk.constant=p_error;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
void GDTokenizer::_make_newline(int p_spaces) {
|
||
|
|
|
||
|
|
TokenData &tk=tk_rb[tk_rb_pos];
|
||
|
|
tk.type=TK_NEWLINE;
|
||
|
|
tk.constant=p_spaces;
|
||
|
|
tk.line=line;
|
||
|
|
tk.col=column;
|
||
|
|
tk_rb_pos=(tk_rb_pos+1)%TK_RB_SIZE;
|
||
|
|
}
|
||
|
|
|
||
|
|
void GDTokenizer::_advance() {
|
||
|
|
|
||
|
|
if (error_flag) {
|
||
|
|
//parser broke
|
||
|
|
_make_error(last_error);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (code_pos>=len) {
|
||
|
|
_make_token(TK_EOF);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
#define GETCHAR(m_ofs) ((m_ofs+code_pos)>=len?0:_code[m_ofs+code_pos])
|
||
|
|
#define INCPOS(m_amount) { code_pos+=m_amount; column+=m_amount; }
|
||
|
|
while (true) {
|
||
|
|
|
||
|
|
|
||
|
|
bool is_node_path=false;
|
||
|
|
|
||
|
|
switch(GETCHAR(0)) {
|
||
|
|
case 0:
|
||
|
|
_make_token(TK_EOF);
|
||
|
|
break;
|
||
|
|
case '\t':
|
||
|
|
case '\r':
|
||
|
|
case ' ':
|
||
|
|
INCPOS(1);
|
||
|
|
continue;
|
||
|
|
case '\n': {
|
||
|
|
line++;
|
||
|
|
INCPOS(1);
|
||
|
|
column=0;
|
||
|
|
int i=0;
|
||
|
|
while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
|
||
|
|
i++;
|
||
|
|
}
|
||
|
|
|
||
|
|
_make_newline(i);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
#if 1 //py style tokenizer
|
||
|
|
case '#': { // line comment skip
|
||
|
|
|
||
|
|
while(GETCHAR(0)!='\n') {
|
||
|
|
code_pos++;
|
||
|
|
if (GETCHAR(0)==0) { //end of file
|
||
|
|
_make_error("Unterminated Comment");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
INCPOS(1);
|
||
|
|
column=0;
|
||
|
|
line++;
|
||
|
|
int i=0;
|
||
|
|
while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
|
||
|
|
i++;
|
||
|
|
}
|
||
|
|
_make_newline(i);
|
||
|
|
return;
|
||
|
|
|
||
|
|
} break;
|
||
|
|
#endif
|
||
|
|
case '/': {
|
||
|
|
|
||
|
|
switch(GETCHAR(1)) {
|
||
|
|
#if 0 // c style tokenizer
|
||
|
|
case '*': { // block comment
|
||
|
|
int pos = code_pos+2;
|
||
|
|
int new_line=line;
|
||
|
|
int new_col=column+2;
|
||
|
|
|
||
|
|
while(true) {
|
||
|
|
if (_code[pos]=='0') {
|
||
|
|
_make_error("Unterminated Comment");
|
||
|
|
code_pos=pos;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
if (_code[pos]=='*' && _code[pos+1]=='/') {
|
||
|
|
new_col+=2;
|
||
|
|
pos+=2; //compensate
|
||
|
|
break;
|
||
|
|
} else if (_code[pos]=='\n') {
|
||
|
|
new_line++;
|
||
|
|
new_col=0;
|
||
|
|
} else {
|
||
|
|
new_col++;
|
||
|
|
}
|
||
|
|
pos++;
|
||
|
|
}
|
||
|
|
|
||
|
|
column=new_col;
|
||
|
|
line=new_line;
|
||
|
|
code_pos=pos;
|
||
|
|
continue;
|
||
|
|
|
||
|
|
} break;
|
||
|
|
case '/': { // line comment skip
|
||
|
|
|
||
|
|
while(GETCHAR(0)!='\n') {
|
||
|
|
code_pos++;
|
||
|
|
if (GETCHAR(0)==0) { //end of file
|
||
|
|
_make_error("Unterminated Comment");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
INCPOS(1);
|
||
|
|
column=0;
|
||
|
|
line++;
|
||
|
|
continue;
|
||
|
|
|
||
|
|
} break;
|
||
|
|
#endif
|
||
|
|
case '=': { // diveq
|
||
|
|
|
||
|
|
_make_token(TK_OP_ASSIGN_DIV);
|
||
|
|
INCPOS(1);
|
||
|
|
|
||
|
|
} break;
|
||
|
|
default:
|
||
|
|
_make_token(TK_OP_DIV);
|
||
|
|
|
||
|
|
}
|
||
|
|
} break;
|
||
|
|
case '=': {
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_EQUAL);
|
||
|
|
INCPOS(1);
|
||
|
|
|
||
|
|
} else
|
||
|
|
_make_token(TK_OP_ASSIGN);
|
||
|
|
|
||
|
|
} break;
|
||
|
|
case '<': {
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
|
||
|
|
_make_token(TK_OP_LESS_EQUAL);
|
||
|
|
INCPOS(1);
|
||
|
|
} else if (GETCHAR(1)=='<') {
|
||
|
|
if (GETCHAR(2)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_SHIFT_LEFT);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_SHIFT_LEFT);
|
||
|
|
}
|
||
|
|
INCPOS(1);
|
||
|
|
} else
|
||
|
|
_make_token(TK_OP_LESS);
|
||
|
|
|
||
|
|
} break;
|
||
|
|
case '>': {
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_GREATER_EQUAL);
|
||
|
|
INCPOS(1);
|
||
|
|
} else if (GETCHAR(1)=='>') {
|
||
|
|
if (GETCHAR(2)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_SHIFT_RIGHT);
|
||
|
|
INCPOS(1);
|
||
|
|
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_SHIFT_RIGHT);
|
||
|
|
}
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_GREATER);
|
||
|
|
}
|
||
|
|
|
||
|
|
} break;
|
||
|
|
case '!': {
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_NOT_EQUAL);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_NOT);
|
||
|
|
}
|
||
|
|
|
||
|
|
} break;
|
||
|
|
//case '"' //string - no strings in shader
|
||
|
|
//case '\'' //string - no strings in shader
|
||
|
|
case '{':
|
||
|
|
_make_token(TK_CURLY_BRACKET_OPEN);
|
||
|
|
break;
|
||
|
|
case '}':
|
||
|
|
_make_token(TK_CURLY_BRACKET_CLOSE);
|
||
|
|
break;
|
||
|
|
case '[':
|
||
|
|
_make_token(TK_BRACKET_OPEN);
|
||
|
|
break;
|
||
|
|
case ']':
|
||
|
|
_make_token(TK_BRACKET_CLOSE);
|
||
|
|
break;
|
||
|
|
case '(':
|
||
|
|
_make_token(TK_PARENTHESIS_OPEN);
|
||
|
|
break;
|
||
|
|
case ')':
|
||
|
|
_make_token(TK_PARENTHESIS_CLOSE);
|
||
|
|
break;
|
||
|
|
case ',':
|
||
|
|
_make_token(TK_COMMA);
|
||
|
|
break;
|
||
|
|
case ';':
|
||
|
|
_make_token(TK_SEMICOLON);
|
||
|
|
break;
|
||
|
|
case '?':
|
||
|
|
_make_token(TK_QUESTION_MARK);
|
||
|
|
break;
|
||
|
|
case ':':
|
||
|
|
_make_token(TK_COLON); //for methods maybe but now useless.
|
||
|
|
break;
|
||
|
|
case '^': {
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_BIT_XOR);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_BIT_XOR);
|
||
|
|
}
|
||
|
|
|
||
|
|
} break;
|
||
|
|
case '~':
|
||
|
|
_make_token(TK_OP_BIT_INVERT);
|
||
|
|
break;
|
||
|
|
case '&': {
|
||
|
|
if (GETCHAR(1)=='&') {
|
||
|
|
|
||
|
|
_make_token(TK_OP_AND);
|
||
|
|
INCPOS(1);
|
||
|
|
} else if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_BIT_AND);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_BIT_AND);
|
||
|
|
}
|
||
|
|
} break;
|
||
|
|
case '|': {
|
||
|
|
if (GETCHAR(1)=='|') {
|
||
|
|
|
||
|
|
_make_token(TK_OP_OR);
|
||
|
|
INCPOS(1);
|
||
|
|
} else if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_BIT_OR);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_BIT_OR);
|
||
|
|
}
|
||
|
|
} break;
|
||
|
|
case '*': {
|
||
|
|
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_MUL);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_MUL);
|
||
|
|
}
|
||
|
|
} break;
|
||
|
|
case '+': {
|
||
|
|
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_ADD);
|
||
|
|
INCPOS(1);
|
||
|
|
//} else if (GETCHAR(1)=='+') {
|
||
|
|
// _make_token(TK_OP_PLUS_PLUS);
|
||
|
|
// INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_ADD);
|
||
|
|
}
|
||
|
|
|
||
|
|
} break;
|
||
|
|
case '-': {
|
||
|
|
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_SUB);
|
||
|
|
INCPOS(1);
|
||
|
|
//} else if (GETCHAR(1)=='-') {
|
||
|
|
// _make_token(TK_OP_MINUS_MINUS);
|
||
|
|
// INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_SUB);
|
||
|
|
}
|
||
|
|
} break;
|
||
|
|
case '%': {
|
||
|
|
|
||
|
|
if (GETCHAR(1)=='=') {
|
||
|
|
_make_token(TK_OP_ASSIGN_MOD);
|
||
|
|
INCPOS(1);
|
||
|
|
} else {
|
||
|
|
_make_token(TK_OP_MOD);
|
||
|
|
}
|
||
|
|
} break;
|
||
|
|
case '@':
|
||
|
|
if (CharType(GETCHAR(1))!='"') {
|
||
|
|
_make_error("Unexpected '@'");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
INCPOS(1);
|
||
|
|
is_node_path=true;
|
||
|
|
case '"': {
|
||
|
|
|
||
|
|
int i=1;
|
||
|
|
String str;
|
||
|
|
while(true) {
|
||
|
|
if (CharType(GETCHAR(i)==0)) {
|
||
|
|
|
||
|
|
_make_error("Unterminated String");
|
||
|
|
return;
|
||
|
|
} else if (CharType(GETCHAR(i)=='"')) {
|
||
|
|
break;
|
||
|
|
} else if (CharType(GETCHAR(i)=='\\')) {
|
||
|
|
//escaped characters...
|
||
|
|
i++;
|
||
|
|
CharType next = GETCHAR(i);
|
||
|
|
if (next==0) {
|
||
|
|
_make_error("Unterminated String");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
CharType res=0;
|
||
|
|
|
||
|
|
switch(next) {
|
||
|
|
|
||
|
|
case 'a': res=7; break;
|
||
|
|
case 'b': res=8; break;
|
||
|
|
case 't': res=9; break;
|
||
|
|
case 'n': res=10; break;
|
||
|
|
case 'v': res=11; break;
|
||
|
|
case 'f': res=12; break;
|
||
|
|
case 'r': res=13; break;
|
||
|
|
case '\'': res='\''; break;
|
||
|
|
case '\"': res='\"'; break;
|
||
|
|
case '\\': res='\\'; break;
|
||
|
|
case 'x': {
|
||
|
|
//hexnumbarh - oct is deprecated
|
||
|
|
|
||
|
|
int read=0;
|
||
|
|
for(int j=0;j<4;j++) {
|
||
|
|
CharType c = GETCHAR(i+j);
|
||
|
|
if (c==0) {
|
||
|
|
_make_error("Unterminated String");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
if (!_is_hex(c)) {
|
||
|
|
if (j==0 || !(j&1)) {
|
||
|
|
_make_error("Malformed hex constant in string");
|
||
|
|
return;
|
||
|
|
} else
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
CharType v;
|
||
|
|
if (c>='0' && c<='9') {
|
||
|
|
v=c-'0';
|
||
|
|
} else if (c>='a' && c<='f') {
|
||
|
|
v=c-'a';
|
||
|
|
v+=10;
|
||
|
|
} else if (c>='A' && c<='F') {
|
||
|
|
v=c-'A';
|
||
|
|
v+=10;
|
||
|
|
} else {
|
||
|
|
ERR_PRINT("BUG");
|
||
|
|
v=0;
|
||
|
|
}
|
||
|
|
|
||
|
|
res<<=4;
|
||
|
|
res|=v;
|
||
|
|
|
||
|
|
read++;
|
||
|
|
}
|
||
|
|
i+=read-1;
|
||
|
|
|
||
|
|
|
||
|
|
} break;
|
||
|
|
default: {
|
||
|
|
|
||
|
|
_make_error("Invalid escape sequence");
|
||
|
|
return;
|
||
|
|
} break;
|
||
|
|
}
|
||
|
|
|
||
|
|
str+=res;
|
||
|
|
|
||
|
|
} else {
|
||
|
|
str+=CharType(GETCHAR(i));
|
||
|
|
}
|
||
|
|
i++;
|
||
|
|
}
|
||
|
|
INCPOS(i);
|
||
|
|
|
||
|
|
if (is_node_path) {
|
||
|
|
_make_constant(NodePath(str));
|
||
|
|
} else {
|
||
|
|
_make_constant(str);
|
||
|
|
}
|
||
|
|
|
||
|
|
} break;
|
||
|
|
default: {
|
||
|
|
|
||
|
|
if (_is_number(GETCHAR(0)) || (GETCHAR(0)=='.' && _is_number(GETCHAR(1)))) {
|
||
|
|
// parse number
|
||
|
|
bool period_found=false;
|
||
|
|
bool exponent_found=false;
|
||
|
|
bool hexa_found=false;
|
||
|
|
bool sign_found=false;
|
||
|
|
|
||
|
|
String str;
|
||
|
|
int i=0;
|
||
|
|
|
||
|
|
while(true) {
|
||
|
|
if (GETCHAR(i)=='.') {
|
||
|
|
if (period_found || exponent_found) {
|
||
|
|
_make_error("Invalid numeric constant at '.'");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
period_found=true;
|
||
|
|
} else if (GETCHAR(i)=='x') {
|
||
|
|
if (hexa_found || str.length()!=1 || !( (i==1 && str[0]=='0') || (i==2 && str[1]=='0' && str[0]=='-') ) ) {
|
||
|
|
_make_error("Invalid numeric constant at 'x'");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
hexa_found=true;
|
||
|
|
} else if (!hexa_found && GETCHAR(i)=='e') {
|
||
|
|
if (hexa_found || exponent_found) {
|
||
|
|
_make_error("Invalid numeric constant at 'e'");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
exponent_found=true;
|
||
|
|
} else if (_is_number(GETCHAR(i))) {
|
||
|
|
//all ok
|
||
|
|
} else if (hexa_found && _is_hex(GETCHAR(i))) {
|
||
|
|
|
||
|
|
} else if ((GETCHAR(i)=='-' || GETCHAR(i)=='+') && exponent_found) {
|
||
|
|
if (sign_found) {
|
||
|
|
_make_error("Invalid numeric constant at '-'");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
sign_found=true;
|
||
|
|
} else
|
||
|
|
break;
|
||
|
|
|
||
|
|
str+=CharType(GETCHAR(i));
|
||
|
|
i++;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!( _is_number(str[str.length()-1]) || (hexa_found && _is_hex(str[str.length()-1])))) {
|
||
|
|
_make_error("Invalid numeric constant: "+str);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
INCPOS(str.length());
|
||
|
|
if (hexa_found) {
|
||
|
|
int val = str.hex_to_int();
|
||
|
|
_make_constant(val);
|
||
|
|
} else if (period_found) {
|
||
|
|
real_t val = str.to_double();
|
||
|
|
//print_line("*%*%*%*% to convert: "+str+" result: "+rtos(val));
|
||
|
|
_make_constant(val);
|
||
|
|
} else {
|
||
|
|
int val = str.to_int();
|
||
|
|
_make_constant(val);
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (GETCHAR(0)=='.') {
|
||
|
|
//parse period
|
||
|
|
_make_token(TK_PERIOD);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (_is_text_char(GETCHAR(0))) {
|
||
|
|
// parse identifier
|
||
|
|
String str;
|
||
|
|
str+=CharType(GETCHAR(0));
|
||
|
|
|
||
|
|
int i=1;
|
||
|
|
while(_is_text_char(GETCHAR(i))) {
|
||
|
|
str+=CharType(GETCHAR(i));
|
||
|
|
i++;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool identifier=false;
|
||
|
|
|
||
|
|
if (str=="null") {
|
||
|
|
_make_constant(Variant());
|
||
|
|
|
||
|
|
} else if (str=="true") {
|
||
|
|
_make_constant(true);
|
||
|
|
|
||
|
|
} else if (str=="false") {
|
||
|
|
_make_constant(false);
|
||
|
|
} else {
|
||
|
|
|
||
|
|
bool found=false;
|
||
|
|
|
||
|
|
struct _bit { Variant::Type type; const char *text;};
|
||
|
|
//built in types
|
||
|
|
|
||
|
|
static const _bit type_list[]={
|
||
|
|
//types
|
||
|
|
{Variant::BOOL,"bool"},
|
||
|
|
{Variant::INT,"int"},
|
||
|
|
{Variant::REAL,"float"},
|
||
|
|
{Variant::STRING,"String"},
|
||
|
|
{Variant::VECTOR2,"vec2"},
|
||
|
|
{Variant::VECTOR2,"Vector2"},
|
||
|
|
{Variant::RECT2,"Rect2"},
|
||
|
|
{Variant::MATRIX32,"Matrix32"},
|
||
|
|
{Variant::MATRIX32,"mat32"},
|
||
|
|
{Variant::VECTOR3,"vec3"},
|
||
|
|
{Variant::VECTOR3,"Vector3"},
|
||
|
|
{Variant::_AABB,"AABB"},
|
||
|
|
{Variant::_AABB,"Rect3"},
|
||
|
|
{Variant::PLANE,"Plane"},
|
||
|
|
{Variant::QUAT,"Quat"},
|
||
|
|
{Variant::MATRIX3,"mat3"},
|
||
|
|
{Variant::MATRIX3,"Matrix3"},
|
||
|
|
{Variant::TRANSFORM,"trn"},
|
||
|
|
{Variant::TRANSFORM,"Transform"},
|
||
|
|
{Variant::COLOR,"Color"},
|
||
|
|
{Variant::IMAGE,"Image"},
|
||
|
|
{Variant::_RID,"RID"},
|
||
|
|
{Variant::OBJECT,"Object"},
|
||
|
|
{Variant::INPUT_EVENT,"InputEvent"},
|
||
|
|
{Variant::DICTIONARY,"dict"},
|
||
|
|
{Variant::DICTIONARY,"Dictionary"},
|
||
|
|
{Variant::ARRAY,"Array"},
|
||
|
|
{Variant::RAW_ARRAY,"RawArray"},
|
||
|
|
{Variant::INT_ARRAY,"IntArray"},
|
||
|
|
{Variant::REAL_ARRAY,"FloatArray"},
|
||
|
|
{Variant::STRING_ARRAY,"StringArray"},
|
||
|
|
{Variant::VECTOR2_ARRAY,"Vector2Array"},
|
||
|
|
{Variant::VECTOR3_ARRAY,"Vector3Array"},
|
||
|
|
{Variant::COLOR_ARRAY,"ColorArray"},
|
||
|
|
{Variant::VARIANT_MAX,NULL},
|
||
|
|
};
|
||
|
|
|
||
|
|
{
|
||
|
|
|
||
|
|
|
||
|
|
int idx=0;
|
||
|
|
|
||
|
|
while(type_list[idx].text) {
|
||
|
|
|
||
|
|
if (str==type_list[idx].text) {
|
||
|
|
_make_type(type_list[idx].type);
|
||
|
|
found=true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
idx++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!found) {
|
||
|
|
|
||
|
|
//built in func?
|
||
|
|
|
||
|
|
for(int i=0;i<GDFunctions::FUNC_MAX;i++) {
|
||
|
|
|
||
|
|
if (str==GDFunctions::get_func_name(GDFunctions::Function(i))) {
|
||
|
|
|
||
|
|
_make_built_in_func(GDFunctions::Function(i));
|
||
|
|
found=true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
//keywor
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!found) {
|
||
|
|
|
||
|
|
|
||
|
|
struct _kws { Token token; const char *text;};
|
||
|
|
|
||
|
|
static const _kws keyword_list[]={
|
||
|
|
//ops
|
||
|
|
{TK_OP_IN,"in"},
|
||
|
|
{TK_OP_NOT,"not"},
|
||
|
|
{TK_OP_OR,"or"},
|
||
|
|
{TK_OP_AND,"and"},
|
||
|
|
//func
|
||
|
|
{TK_PR_FUNCTION,"func"},
|
||
|
|
{TK_PR_FUNCTION,"function"},
|
||
|
|
{TK_PR_CLASS,"class"},
|
||
|
|
{TK_PR_EXTENDS,"extends"},
|
||
|
|
{TK_PR_TOOL,"tool"},
|
||
|
|
{TK_PR_STATIC,"static"},
|
||
|
|
{TK_PR_EXPORT,"export"},
|
||
|
|
{TK_PR_VAR,"var"},
|
||
|
|
{TK_PR_PRELOAD,"preload"},
|
||
|
|
{TK_PR_ASSERT,"assert"},
|
||
|
|
{TK_PR_CONST,"const"},
|
||
|
|
//controlflow
|
||
|
|
{TK_CF_IF,"if"},
|
||
|
|
{TK_CF_ELIF,"elif"},
|
||
|
|
{TK_CF_ELSE,"else"},
|
||
|
|
{TK_CF_FOR,"for"},
|
||
|
|
{TK_CF_WHILE,"while"},
|
||
|
|
{TK_CF_DO,"do"},
|
||
|
|
{TK_CF_SWITCH,"switch"},
|
||
|
|
{TK_CF_BREAK,"break"},
|
||
|
|
{TK_CF_CONTINUE,"continue"},
|
||
|
|
{TK_CF_RETURN,"return"},
|
||
|
|
{TK_CF_PASS,"pass"},
|
||
|
|
{TK_SELF,"self"},
|
||
|
|
{TK_ERROR,NULL}
|
||
|
|
};
|
||
|
|
|
||
|
|
int idx=0;
|
||
|
|
found=false;
|
||
|
|
|
||
|
|
while(keyword_list[idx].text) {
|
||
|
|
|
||
|
|
if (str==keyword_list[idx].text) {
|
||
|
|
_make_token(keyword_list[idx].token);
|
||
|
|
found=true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
idx++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!found)
|
||
|
|
identifier=true;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
if (identifier) {
|
||
|
|
_make_identifier(str);
|
||
|
|
}
|
||
|
|
INCPOS(str.length());
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
_make_error("Unknown character");
|
||
|
|
return;
|
||
|
|
|
||
|
|
} break;
|
||
|
|
}
|
||
|
|
|
||
|
|
INCPOS(1);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
void GDTokenizer::set_code(const String& p_code) {
|
||
|
|
|
||
|
|
code=p_code;
|
||
|
|
len = p_code.length();
|
||
|
|
if (len) {
|
||
|
|
_code=&code[0];
|
||
|
|
} else {
|
||
|
|
_code=NULL;
|
||
|
|
}
|
||
|
|
code_pos=0;
|
||
|
|
line=1; //it is stand-ar-ized that lines begin in 1 in code..
|
||
|
|
column=0;
|
||
|
|
tk_rb_pos=0;
|
||
|
|
error_flag=false;
|
||
|
|
last_error="";
|
||
|
|
for(int i=0;i<MAX_LOOKAHEAD+1;i++)
|
||
|
|
_advance();
|
||
|
|
}
|
||
|
|
|
||
|
|
GDTokenizer::Token GDTokenizer::get_token(int p_offset) const {
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, TK_ERROR);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, TK_ERROR);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
return tk_rb[ofs].type;
|
||
|
|
}
|
||
|
|
|
||
|
|
int GDTokenizer::get_token_line(int p_offset) const {
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
return tk_rb[ofs].line;
|
||
|
|
}
|
||
|
|
|
||
|
|
int GDTokenizer::get_token_column(int p_offset) const {
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, -1);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, -1);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
return tk_rb[ofs].col;
|
||
|
|
}
|
||
|
|
|
||
|
|
const Variant& GDTokenizer::get_token_constant(int p_offset) const {
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, tk_rb[0].constant);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, tk_rb[0].constant);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_CONSTANT,tk_rb[0].constant);
|
||
|
|
return tk_rb[ofs].constant;
|
||
|
|
}
|
||
|
|
StringName GDTokenizer::get_token_identifier(int p_offset) const {
|
||
|
|
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, StringName());
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, StringName());
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_IDENTIFIER,StringName());
|
||
|
|
return tk_rb[ofs].identifier;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
GDFunctions::Function GDTokenizer::get_token_built_in_func(int p_offset) const {
|
||
|
|
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, GDFunctions::FUNC_MAX);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, GDFunctions::FUNC_MAX);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_FUNC,GDFunctions::FUNC_MAX);
|
||
|
|
return tk_rb[ofs].func;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
Variant::Type GDTokenizer::get_token_type(int p_offset) const {
|
||
|
|
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, Variant::NIL);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, Variant::NIL);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_BUILT_IN_TYPE,Variant::NIL);
|
||
|
|
return tk_rb[ofs].vtype;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
int GDTokenizer::get_token_line_indent(int p_offset) const {
|
||
|
|
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, 0);
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, 0);
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_NEWLINE,0);
|
||
|
|
return tk_rb[ofs].constant;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
String GDTokenizer::get_token_error(int p_offset) const {
|
||
|
|
|
||
|
|
ERR_FAIL_COND_V( p_offset <= -MAX_LOOKAHEAD, String());
|
||
|
|
ERR_FAIL_COND_V( p_offset >= MAX_LOOKAHEAD, String());
|
||
|
|
|
||
|
|
int ofs = (TK_RB_SIZE + tk_rb_pos + p_offset - MAX_LOOKAHEAD -1)%TK_RB_SIZE;
|
||
|
|
ERR_FAIL_COND_V(tk_rb[ofs].type!=TK_ERROR,String());
|
||
|
|
return tk_rb[ofs].constant;
|
||
|
|
}
|
||
|
|
|
||
|
|
void GDTokenizer::advance(int p_amount) {
|
||
|
|
|
||
|
|
ERR_FAIL_COND( p_amount <=0 );
|
||
|
|
for(int i=0;i<p_amount;i++)
|
||
|
|
_advance();
|
||
|
|
}
|