asgardius/virtualx-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Fix shader tokenizer/compiler reporting wrong error location

Browse source 
Raname shader tokenizer methods for clarity
This commit is contained in:
Pedro J. Estébanez 2016-10-10 11:51:28 +02:00
parent 85a7105345
commit a130520a7c
2 changed files with 74 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
servers/visual/shader_language.cpp
View file

@ -460,27 +460,35 @@ Error ShaderLanguage::tokenize(const String& p_text,Vector<Token> *p_tokens,Stri
int pos=0; int pos=0;
int line=0; int line=0;
int col=0;
while(pos<len) { while(pos<len) {
int advance=0; int advance=0;
int prev_line=line; int prev_line=line;
Token t = read_token(&p_text[pos],len-pos,line,advance); Token t = read_token(&p_text[pos],len-pos,line,advance);
t.line=prev_line; t.line=line;
t.col=col;
if (t.type==TK_ERROR) { if (t.type==TK_ERROR) {
if (r_error) { if (r_error) {
*r_error=t.text; *r_error=t.text;
*r_err_line=line; *r_err_line=line;
*r_err_column=col;
return ERR_COMPILATION_FAILED; return ERR_COMPILATION_FAILED;
} }
} }
if (line==prev_line) {
col+=advance;
} else {
col=0;
//p_tokens->push_back(Token(TK_LINE,itos(line)))
}
if (t.type!=TK_EMPTY) if (t.type!=TK_EMPTY)
p_tokens->push_back(t); p_tokens->push_back(t);
//if (prev_line!=line)
// p_tokens->push_back(Token(TK_LINE,itos(line)));
pos+=advance; pos+=advance;
@ -589,7 +597,7 @@ bool ShaderLanguage::is_token_nonvoid_datatype(TokenType p_type) {
bool ShaderLanguage::parser_is_at_function(Parser& parser) { bool ShaderLanguage::parser_is_at_function(Parser& parser) {
return (is_token_datatype(parser.get_token_type(0)) && parser.get_token_type(1)==TK_INDENTIFIER && parser.get_token_type(2)==TK_PARENTHESIS_OPEN); return (is_token_datatype(parser.get_next_token_type(0)) && parser.get_next_token_type(1)==TK_INDENTIFIER && parser.get_next_token_type(2)==TK_PARENTHESIS_OPEN);
} }
@ -662,7 +670,7 @@ Error ShaderLanguage::parse_function(Parser& parser,BlockNode *p_block) {
ProgramNode *program = (ProgramNode*)p_block->parent; ProgramNode *program = (ProgramNode*)p_block->parent;
StringName name = parser.get_token(1).text; StringName name = parser.get_next_token(1).text;
if (test_existing_identifier(p_block,name)) { if (test_existing_identifier(p_block,name)) {
@ -677,7 +685,7 @@ Error ShaderLanguage::parse_function(Parser& parser,BlockNode *p_block) {
function->name=name; function->name=name;
function->return_type=get_token_datatype(parser.get_token_type(0)); function->return_type=get_token_datatype(parser.get_next_token_type(0));
{ //add to programnode { //add to programnode
ProgramNode::Function f; ProgramNode::Function f;
@ -691,13 +699,13 @@ Error ShaderLanguage::parse_function(Parser& parser,BlockNode *p_block) {
while(true) { while(true) {
//end of arguments //end of arguments
if (parser.get_token_type(ofs)==TK_PARENTHESIS_CLOSE) { if (parser.get_next_token_type(ofs)==TK_PARENTHESIS_CLOSE) {
ofs++; ofs++;
break; break;
} }
//next argument awaits //next argument awaits
if (parser.get_token_type(ofs)==TK_COMMA) { if (parser.get_next_token_type(ofs)==TK_COMMA) {
if (!is_token_nonvoid_datatype(parser.get_token_type(ofs+1))) { if (!is_token_nonvoid_datatype(parser.get_next_token_type(ofs+1))) {
parser.set_error("Expected Identifier or ')' following ','"); parser.set_error("Expected Identifier or ')' following ','");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -707,19 +715,19 @@ Error ShaderLanguage::parse_function(Parser& parser,BlockNode *p_block) {
if (!is_token_nonvoid_datatype(parser.get_token_type(ofs+0))) { if (!is_token_nonvoid_datatype(parser.get_next_token_type(ofs+0))) {
parser.set_error("Invalid Argument Type"); parser.set_error("Invalid Argument Type");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
DataType identtype=get_token_datatype(parser.get_token_type(ofs+0)); DataType identtype=get_token_datatype(parser.get_next_token_type(ofs+0));
if (parser.get_token_type(ofs+1)!=TK_INDENTIFIER) { if (parser.get_next_token_type(ofs+1)!=TK_INDENTIFIER) {
parser.set_error("Expected Argument Identifier"); parser.set_error("Expected Argument Identifier");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
StringName identname=parser.get_token(ofs+1).text; StringName identname=parser.get_next_token(ofs+1).text;
if (test_existing_identifier(function,identname)) { if (test_existing_identifier(function,identname)) {
parser.set_error("Duplicate Argument Identifier: "+identname); parser.set_error("Duplicate Argument Identifier: "+identname);
@ -737,7 +745,7 @@ Error ShaderLanguage::parse_function(Parser& parser,BlockNode *p_block) {
parser.advance(ofs); parser.advance(ofs);
// match { // match {
if (parser.get_token_type()!=TK_CURLY_BRACKET_OPEN) { if (parser.get_next_token_type()!=TK_CURLY_BRACKET_OPEN) {
parser.set_error("Expected '{'"); parser.set_error("Expected '{'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -1527,14 +1535,14 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
Node *expr=NULL; Node *expr=NULL;
if (parser.get_token_type()==TK_PARENTHESIS_OPEN) { if (parser.get_next_token_type()==TK_PARENTHESIS_OPEN) {
//handle subexpression //handle subexpression
parser.advance(); parser.advance();
Error err = parse_expression(parser,p_parent,&expr); Error err = parse_expression(parser,p_parent,&expr);
if (err) if (err)
return err; return err;
if (parser.get_token_type()!=TK_PARENTHESIS_CLOSE) { if (parser.get_next_token_type()!=TK_PARENTHESIS_CLOSE) {
parser.set_error("Expected ')' in expression"); parser.set_error("Expected ')' in expression");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
@ -1542,15 +1550,15 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
parser.advance(); parser.advance();
} else if (parser.get_token_type()==TK_REAL_CONSTANT) { } else if (parser.get_next_token_type()==TK_REAL_CONSTANT) {
ConstantNode *constant = parser.create_node<ConstantNode>(p_parent); ConstantNode *constant = parser.create_node<ConstantNode>(p_parent);
constant->value=parser.get_token().text.operator String().to_double(); constant->value=parser.get_next_token().text.operator String().to_double();
constant->datatype=TYPE_FLOAT; constant->datatype=TYPE_FLOAT;
expr=constant; expr=constant;
parser.advance(); parser.advance();
} else if (parser.get_token_type()==TK_TRUE) { } else if (parser.get_next_token_type()==TK_TRUE) {
//print_line("found true"); //print_line("found true");
//handle true constant //handle true constant
@ -1559,7 +1567,7 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
constant->datatype=TYPE_BOOL; constant->datatype=TYPE_BOOL;
expr=constant; expr=constant;
parser.advance(); parser.advance();
} else if (parser.get_token_type()==TK_FALSE) { } else if (parser.get_next_token_type()==TK_FALSE) {
//handle false constant //handle false constant
ConstantNode *constant = parser.create_node<ConstantNode>(p_parent); ConstantNode *constant = parser.create_node<ConstantNode>(p_parent);
@ -1567,21 +1575,21 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
constant->datatype=TYPE_BOOL; constant->datatype=TYPE_BOOL;
expr=constant; expr=constant;
parser.advance(); parser.advance();
} else if (parser.get_token_type()==TK_TYPE_VOID) { } else if (parser.get_next_token_type()==TK_TYPE_VOID) {
//make sure void is not used in expression //make sure void is not used in expression
parser.set_error("Void value not allowed in Expression"); parser.set_error("Void value not allowed in Expression");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} else if (parser.get_token_type(1)==TK_PARENTHESIS_OPEN && (is_token_nonvoid_datatype(parser.get_token_type()) || parser.get_token_type()==TK_INDENTIFIER)) { } else if (parser.get_next_token_type(1)==TK_PARENTHESIS_OPEN && (is_token_nonvoid_datatype(parser.get_next_token_type()) || parser.get_next_token_type()==TK_INDENTIFIER)) {
//function or constructor //function or constructor
StringName name; StringName name;
DataType constructor=TYPE_VOID; DataType constructor=TYPE_VOID;
if (is_token_nonvoid_datatype(parser.get_token_type())) { if (is_token_nonvoid_datatype(parser.get_next_token_type())) {
constructor=get_token_datatype(parser.get_token_type()); constructor=get_token_datatype(parser.get_next_token_type());
switch(get_token_datatype(parser.get_token_type())) { switch(get_token_datatype(parser.get_next_token_type())) {
case TYPE_BOOL: name="bool"; break; case TYPE_BOOL: name="bool"; break;
case TYPE_FLOAT: name="float"; break; case TYPE_FLOAT: name="float"; break;
case TYPE_VEC2: name="vec2"; break; case TYPE_VEC2: name="vec2"; break;
@ -1594,7 +1602,7 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
} }
} else { } else {
name=parser.get_token().text; name=parser.get_next_token().text;
} }
if (!test_existing_identifier(p_parent,name)) { if (!test_existing_identifier(p_parent,name)) {
@ -1615,7 +1623,7 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
//parse parameters //parse parameters
if (parser.get_token_type()==TK_PARENTHESIS_CLOSE) { if (parser.get_next_token_type()==TK_PARENTHESIS_CLOSE) {
parser.advance(); parser.advance();
} else { } else {
@ -1628,13 +1636,13 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
return err; return err;
func->arguments.push_back(arg); func->arguments.push_back(arg);
if (parser.get_token_type()==TK_PARENTHESIS_CLOSE) { if (parser.get_next_token_type()==TK_PARENTHESIS_CLOSE) {
parser.advance(); parser.advance();
break; break;
} else if (parser.get_token_type()==TK_COMMA) { } else if (parser.get_next_token_type()==TK_COMMA) {
if (parser.get_token_type(1)==TK_PARENTHESIS_CLOSE) { if (parser.get_next_token_type(1)==TK_PARENTHESIS_CLOSE) {
parser.set_error("Expression expected"); parser.set_error("Expression expected");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
@ -1658,14 +1666,14 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
} }
} else if (parser.get_token_type()==TK_INDENTIFIER) { } else if (parser.get_next_token_type()==TK_INDENTIFIER) {
//probably variable //probably variable
Node *node =p_parent; Node *node =p_parent;
bool existing=false; bool existing=false;
DataType datatype; DataType datatype;
StringName identifier=parser.get_token().text; StringName identifier=parser.get_next_token().text;
while(node) { while(node) {
@ -1728,10 +1736,10 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
parser.advance(); parser.advance();
expr=varname; expr=varname;
} else if (parser.get_token_type()==TK_OP_SUB || parser.get_token_type()==TK_OP_NOT) { } else if (parser.get_next_token_type()==TK_OP_SUB || parser.get_next_token_type()==TK_OP_NOT) {
//single prefix operators //single prefix operators
TokenType token_type=parser.get_token_type(); TokenType token_type=parser.get_next_token_type();
parser.advance(); parser.advance();
//Node *subexpr=NULL; //Node *subexpr=NULL;
//Error err = parse_expression(parser,p_parent,&subexpr); //Error err = parse_expression(parser,p_parent,&subexpr);
@ -1758,9 +1766,9 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
} else { } else {
print_line("found bug?"); print_line("found bug?");
print_line("misplaced token: "+String(token_names[parser.get_token_type()])); print_line("misplaced token: "+String(token_names[parser.get_next_token_type()]));
parser.set_error("Error parsing expression, misplaced: "+String(token_names[parser.get_token_type()])); parser.set_error("Error parsing expression, misplaced: "+String(token_names[parser.get_next_token_type()]));
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
//nothing //nothing
} }
@ -1773,15 +1781,15 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
/* OK now see what's NEXT to the operator.. */ /* OK now see what's NEXT to the operator.. */
if (parser.get_token_type()==TK_PERIOD) { if (parser.get_next_token_type()==TK_PERIOD) {
if (parser.get_token_type(1)!=TK_INDENTIFIER) { if (parser.get_next_token_type(1)!=TK_INDENTIFIER) {
parser.set_error("Expected identifier as member"); parser.set_error("Expected identifier as member");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
DataType dt = compute_node_type(expr); DataType dt = compute_node_type(expr);
String ident = parser.get_token(1).text; String ident = parser.get_next_token(1).text;
bool ok=true; bool ok=true;
DataType member_type; DataType member_type;
@ -1907,13 +1915,13 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
//creates a subindexing expression in place //creates a subindexing expression in place
} else if (parser.get_token_type()==TK_BRACKET_OPEN) { } else if (parser.get_next_token_type()==TK_BRACKET_OPEN) {
//todo //todo
//subindexing has priority over any operator //subindexing has priority over any operator
//creates a subindexing expression in place //creates a subindexing expression in place
} /*else if (parser.get_token_type()==TK_OP_PLUS_PLUS || parser.get_token_type()==TK_OP_MINUS_MINUS) { } /*else if (parser.get_next_token_type()==TK_OP_PLUS_PLUS || parser.get_next_token_type()==TK_OP_MINUS_MINUS) {
//todo //todo
//inc/dec operators have priority over any operator //inc/dec operators have priority over any operator
//creates a subindexing expression in place //creates a subindexing expression in place
@ -1927,11 +1935,11 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
expression.push_back(e); expression.push_back(e);
if (is_token_operator(parser.get_token_type())) { if (is_token_operator(parser.get_next_token_type())) {
Expression o; Expression o;
o.is_op=true; o.is_op=true;
o.op=parser.get_token_type(); o.op=parser.get_next_token_type();
expression.push_back(o); expression.push_back(o);
parser.advance(); parser.advance();
} else { } else {
@ -2132,9 +2140,9 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
return OK; return OK;
/* /*
TokenType token_type=parser.get_token_type(); TokenType token_type=parser.get_next_token_type();
OperatorNode *op = parser.create_node<OperatorNode>(p_parent); OperatorNode *op = parser.create_node<OperatorNode>(p_parent);
op->op=get_token_operator(parser.get_token_type()); op->op=get_token_operator(parser.get_next_token_type());
op->arguments.push_back(*r_expr); //expression goes as left op->arguments.push_back(*r_expr); //expression goes as left
parser.advance(); parser.advance();
@ -2156,10 +2164,10 @@ Error ShaderLanguage::parse_expression(Parser& parser,Node *p_parent,Node **r_ex
Error ShaderLanguage::parse_variable_declaration(Parser& parser,BlockNode *p_block) { Error ShaderLanguage::parse_variable_declaration(Parser& parser,BlockNode *p_block) {
bool uniform = parser.get_token(-1).type==TK_UNIFORM; bool uniform = parser.get_next_token(-1).type==TK_UNIFORM;
DataType type=get_token_datatype(parser.get_token_type(0)); DataType type=get_token_datatype(parser.get_next_token_type(0));
bool iscolor = parser.get_token_type(0)==TK_TYPE_COLOR; bool iscolor = parser.get_next_token_type(0)==TK_TYPE_COLOR;
if (type==TYPE_VOID) { if (type==TYPE_VOID) {
@ -2185,18 +2193,18 @@ Error ShaderLanguage::parse_variable_declaration(Parser& parser,BlockNode *p_blo
while(true) { while(true) {
if (found && parser.get_token_type()!=TK_COMMA) { if (found && parser.get_next_token_type()!=TK_COMMA) {
break; break;
} }
if (parser.get_token_type()!=TK_INDENTIFIER) { if (parser.get_next_token_type()!=TK_INDENTIFIER) {
parser.set_error("Identifier Expected"); parser.set_error("Identifier Expected");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
StringName name = parser.get_token().text; StringName name = parser.get_next_token().text;
if (test_existing_identifier(p_block,name)) { if (test_existing_identifier(p_block,name)) {
parser.set_error("Duplicate Identifier (existing variable/function): "+name); parser.set_error("Duplicate Identifier (existing variable/function): "+name);
@ -2207,7 +2215,7 @@ Error ShaderLanguage::parse_variable_declaration(Parser& parser,BlockNode *p_blo
parser.advance(); parser.advance();
//see if declaration has an initializer //see if declaration has an initializer
if (parser.get_token_type()==TK_OP_ASSIGN) { if (parser.get_next_token_type()==TK_OP_ASSIGN) {
parser.advance(); parser.advance();
OperatorNode * op = parser.create_node<OperatorNode>(p_block); OperatorNode * op = parser.create_node<OperatorNode>(p_block);
VariableNode * var = parser.create_node<VariableNode>(op); VariableNode * var = parser.create_node<VariableNode>(op);
@ -2298,7 +2306,7 @@ Error ShaderLanguage::parse_variable_declaration(Parser& parser,BlockNode *p_blo
} }
if (parser.get_token_type()!=TK_SEMICOLON) { if (parser.get_next_token_type()!=TK_SEMICOLON) {
parser.set_error("Expected ';'"); parser.set_error("Expected ';'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -2316,7 +2324,7 @@ Error ShaderLanguage::parse_flow_if(Parser& parser,Node *p_parent,Node **r_state
parser.advance(); parser.advance();
if (parser.get_token_type()!=TK_PARENTHESIS_OPEN) { if (parser.get_next_token_type()!=TK_PARENTHESIS_OPEN) {
parser.set_error("Expected '(' after 'if'"); parser.set_error("Expected '(' after 'if'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -2335,14 +2343,14 @@ Error ShaderLanguage::parse_flow_if(Parser& parser,Node *p_parent,Node **r_state
cf->statements.push_back(expression); cf->statements.push_back(expression);
if (parser.get_token_type()!=TK_PARENTHESIS_CLOSE) { if (parser.get_next_token_type()!=TK_PARENTHESIS_CLOSE) {
parser.set_error("Expected ')' after expression"); parser.set_error("Expected ')' after expression");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
parser.advance(); parser.advance();
if (parser.get_token_type()!=TK_CURLY_BRACKET_OPEN) { if (parser.get_next_token_type()!=TK_CURLY_BRACKET_OPEN) {
parser.set_error("Expected statement block after 'if()'"); parser.set_error("Expected statement block after 'if()'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -2354,11 +2362,11 @@ Error ShaderLanguage::parse_flow_if(Parser& parser,Node *p_parent,Node **r_state
cf->statements.push_back(substatement); cf->statements.push_back(substatement);
if (parser.get_token_type()==TK_CF_ELSE) { if (parser.get_next_token_type()==TK_CF_ELSE) {
parser.advance(); parser.advance();
if (parser.get_token_type()!=TK_CURLY_BRACKET_OPEN) { if (parser.get_next_token_type()!=TK_CURLY_BRACKET_OPEN) {
parser.set_error("Expected statement block after 'else'"); parser.set_error("Expected statement block after 'else'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -2426,7 +2434,7 @@ Error ShaderLanguage::parse_flow_return(Parser& parser,Node *p_parent,Node **r_s
*r_statement=cf; *r_statement=cf;
if (parser.get_token_type()!=TK_SEMICOLON) { if (parser.get_next_token_type()!=TK_SEMICOLON) {
parser.set_error("Expected ';'"); parser.set_error("Expected ';'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -2438,7 +2446,7 @@ Error ShaderLanguage::parse_statement(Parser& parser,Node *p_parent,Node **r_sta
*r_statement=NULL; *r_statement=NULL;
TokenType token_type = parser.get_token_type(); TokenType token_type = parser.get_next_token_type();
if (token_type==TK_CURLY_BRACKET_OPEN) { if (token_type==TK_CURLY_BRACKET_OPEN) {
//sub-block //sub-block
@ -2462,7 +2470,7 @@ Error ShaderLanguage::parse_statement(Parser& parser,Node *p_parent,Node **r_sta
if (err) if (err)
return err; return err;
if (parser.get_token_type()!=TK_SEMICOLON) { if (parser.get_next_token_type()!=TK_SEMICOLON) {
parser.set_error("Expected ';'"); parser.set_error("Expected ';'");
return ERR_PARSE_ERROR; return ERR_PARSE_ERROR;
} }
@ -2484,7 +2492,7 @@ Error ShaderLanguage::parse_block(Parser& parser,BlockNode *p_block) {
return OK; //bye return OK; //bye
} }
TokenType token_type = parser.get_token_type(); TokenType token_type = parser.get_next_token_type();
if (token_type==TK_CURLY_BRACKET_CLOSE) { if (token_type==TK_CURLY_BRACKET_CLOSE) {
if (p_block->parent->type==Node::TYPE_PROGRAM) { if (p_block->parent->type==Node::TYPE_PROGRAM) {

8
servers/visual/shader_language.h
View file

@ -332,14 +332,14 @@ private:
void get_error(String *r_error, int *r_line, int *r_column) { void get_error(String *r_error, int *r_line, int *r_column) {
*r_error=error; *r_error=error;
*r_line=get_token(pos).line; *r_line=get_next_token(0).line;
*r_column=get_token(pos).col; *r_column=get_next_token(0).col;
} }
Token get_token(int ofs=0) const { int idx=pos+ofs; if (idx<0 || idx>=tokens.size()) return Token(TK_ERROR); return tokens[idx]; } Token get_next_token(int ofs=0) const { int idx=pos+ofs; if (idx<0 || idx>=tokens.size()) return Token(TK_ERROR); return tokens[idx]; }
TokenType get_token_type(int ofs=0) const { int idx=pos+ofs; if (idx<0 || idx>=tokens.size()) return TK_ERROR; return tokens[idx].type; } TokenType get_next_token_type(int ofs=0) const { int idx=pos+ofs; if (idx<0 || idx>=tokens.size()) return TK_ERROR; return tokens[idx].type; }
void advance(int p_amount=1) { pos+=p_amount; } void advance(int p_amount=1) { pos+=p_amount; }
bool is_at_end() const { return pos>=tokens.size(); } bool is_at_end() const { return pos>=tokens.size(); }