asgardius/virtualx-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Improvements to GDScript identifier tokenization

Browse source
This commit is contained in:
Ninni Pipping 2023-02-13 10:32:56 +01:00
parent e2e870c611
commit f68beeb7fa
1 changed files with 19 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
modules/gdscript/gdscript_tokenizer.cpp
View file

@ -559,6 +559,24 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() {
return make_identifier(name); return make_identifier(name);
} }
if (!only_ascii) {
// Kept here in case the order with push_error matters.
Token id = make_identifier(name);
#ifdef DEBUG_ENABLED
// Additional checks for identifiers but only in debug and if it's available in TextServer.
if (TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {
int64_t confusable = TS->is_confusable(name, keyword_list);
if (confusable >= 0) {
push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));
}
}
#endif // DEBUG_ENABLED
// Cannot be a keyword, as keywords are ASCII only.
return id;
}
// Define some helper macros for the switch case. // Define some helper macros for the switch case.
#define KEYWORD_GROUP_CASE(char) \ #define KEYWORD_GROUP_CASE(char) \
break; \ break; \
@ -594,19 +612,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() {
} }
// Not a keyword, so must be an identifier. // Not a keyword, so must be an identifier.
Token id = make_identifier(name); return make_identifier(name);
#ifdef DEBUG_ENABLED
// Additional checks for identifiers but only in debug and if it's available in TextServer.
if (!only_ascii && TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {
int64_t confusable = TS->is_confusable(name, keyword_list);
if (confusable >= 0) {
push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));
}
}
#endif // DEBUG_ENABLED
return id;
#undef KEYWORD_GROUP_CASE #undef KEYWORD_GROUP_CASE
#undef KEYWORD #undef KEYWORD