Make --doctool locale aware

* Adds `indent(prefix)` to `String`
* Moves the loading of tool/doc translation into
  `editor/editor_translation.{h,cpp}`
* Makes use of doc translation when generating XML class references, and
  setup the translation locale based on `-l LOCALE` CLI parameter.

The XML class reference won't be translated if `-l LOCALE` parameter is
not given, or when it's `-l en`.

core/ustring.cpp

@@ -3144,6 +3144,27 @@ CharType String::ord_at(int p_idx) const {
 	return operator[](p_idx);
 }
 
+String String::indent(const String &p_prefix) const {
+	String new_string;
+	int line_start = 0;
+
+	for (int i = 0; i < length(); i++) {
+		const char32_t c = operator[](i);
+		if (c == '\n') {
+			if (i == line_start) {
+				new_string += c; // Leave empty lines empty.
+			} else {
+				new_string += p_prefix + substr(line_start, i - line_start + 1);
+			}
+			line_start = i + 1;
+		}
+	}
+	if (line_start != length()) {
+		new_string += p_prefix + substr(line_start);
+	}
+	return new_string;
+}
+
 String String::dedent() const {
 	String new_string;
 	String indent;

core/ustring.h

@@ -286,6 +286,7 @@ public:
 
 	String left(int p_pos) const;
 	String right(int p_pos) const;
+	String indent(const String &p_prefix) const;
 	String dedent() const;
 	String strip_edges(bool left = true, bool right = true) const;
 	String strip_escapes() const;

core/variant_call.cpp

@@ -272,6 +272,7 @@ struct _VariantCall {
 	VCALL_LOCALMEM0R(String, to_lower);
 	VCALL_LOCALMEM1R(String, left);
 	VCALL_LOCALMEM1R(String, right);
+	VCALL_LOCALMEM1R(String, indent);
 	VCALL_LOCALMEM0R(String, dedent);
 	VCALL_LOCALMEM2R(String, strip_edges);
 	VCALL_LOCALMEM0R(String, strip_escapes);
@@ -1681,6 +1682,7 @@ void register_variant_methods() {
 	ADDFUNC0R(STRING, STRING, String, get_basename, varray());
 	ADDFUNC1R(STRING, STRING, String, plus_file, STRING, "file", varray());
 	ADDFUNC1R(STRING, INT, String, ord_at, INT, "at", varray());
+	ADDFUNC1R(STRING, STRING, String, indent, STRING, "prefix", varray());
 	ADDFUNC0R(STRING, STRING, String, dedent, varray());
 	ADDFUNC2(STRING, NIL, String, erase, INT, "position", INT, "chars", varray());
 	ADDFUNC0R(STRING, INT, String, hash, varray());

doc/classes/String.xml

@@ -241,7 +241,7 @@
 		<method name="dedent">
 			<return type="String" />
 			<description>
-				Returns a copy of the string with indentation (leading tabs and spaces) removed.
+				Returns a copy of the string with indentation (leading tabs and spaces) removed. See also [method indent] to add indentation.
 			</description>
 		</method>
 		<method name="empty">
@@ -393,6 +393,15 @@
 				[/codeblock]
 			</description>
 		</method>
+		<method name="indent">
+			<return type="String" />
+			<argument index="0" name="prefix" type="String" />
+			<description>
+				Returns a copy of the string with lines indented with [code]prefix[/code].
+				For example, the string can be indented with two tabs using [code]"\t\t"[/code], or four spaces using [code]"    "[/code]. The prefix can be any string so it can also be used to comment out strings with e.g. [code]"# "[/code]. See also [method dedent] to remove indentation.
+				[b]Note:[/b] Empty lines are kept empty.
+			</description>
+		</method>
 		<method name="insert">
 			<return type="String" />
 			<argument index="0" name="position" type="int" />

editor/doc/doc_data.cpp

@@ -37,9 +37,39 @@
 #include "core/os/dir_access.h"
 #include "core/project_settings.h"
 #include "core/script_language.h"
+#include "core/translation.h"
 #include "core/version.h"
 #include "scene/resources/theme.h"
 
+static String _get_indent(const String &p_text) {
+	String indent;
+	bool has_text = false;
+	int line_start = 0;
+
+	for (int i = 0; i < p_text.length(); i++) {
+		const char32_t c = p_text[i];
+		if (c == '\n') {
+			line_start = i + 1;
+		} else if (c > 32) {
+			has_text = true;
+			indent = p_text.substr(line_start, i - line_start);
+			break; // Indentation of the first line that has text.
+		}
+	}
+	if (!has_text) {
+		return p_text;
+	}
+	return indent;
+}
+
+static String _translate_doc_string(const String &p_text) {
+	const String indent = _get_indent(p_text);
+	const String message = p_text.dedent().strip_edges();
+	const String translated = TranslationServer::get_singleton()->doc_translate(message);
+	// No need to restore stripped edges because they'll be stripped again later.
+	return translated.indent(indent);
+}
+
 void DocData::merge_from(const DocData &p_data) {
 	for (Map<String, ClassDoc>::Element *E = class_list.front(); E; E = E->next()) {
 		ClassDoc &c = E->get();
@@ -1057,11 +1087,11 @@ Error DocData::save_classes(const String &p_default_path, const Map<String, Stri
 		_write_string(f, 0, header);
 
 		_write_string(f, 1, "<brief_description>");
-		_write_string(f, 2, c.brief_description.strip_edges().xml_escape());
+		_write_string(f, 2, _translate_doc_string(c.brief_description).strip_edges().xml_escape());
 		_write_string(f, 1, "</brief_description>");
 
 		_write_string(f, 1, "<description>");
-		_write_string(f, 2, c.description.strip_edges().xml_escape());
+		_write_string(f, 2, _translate_doc_string(c.description).strip_edges().xml_escape());
 		_write_string(f, 1, "</description>");
 
 		_write_string(f, 1, "<tutorials>");
@@ -1110,7 +1140,7 @@ Error DocData::save_classes(const String &p_default_path, const Map<String, Stri
 			}
 
 			_write_string(f, 3, "<description>");
-			_write_string(f, 4, m.description.strip_edges().xml_escape());
+			_write_string(f, 4, _translate_doc_string(m.description).strip_edges().xml_escape());
 			_write_string(f, 3, "</description>");
 
 			_write_string(f, 2, "</method>");
@@ -1138,7 +1168,7 @@ Error DocData::save_classes(const String &p_default_path, const Map<String, Stri
 					_write_string(f, 2, "<member name=\"" + p.name + "\" type=\"" + p.type + "\" setter=\"" + p.setter + "\" getter=\"" + p.getter + "\" overrides=\"" + p.overrides + "\"" + additional_attributes + " />");
 				} else {
 					_write_string(f, 2, "<member name=\"" + p.name + "\" type=\"" + p.type + "\" setter=\"" + p.setter + "\" getter=\"" + p.getter + "\"" + additional_attributes + ">");
-					_write_string(f, 3, p.description.strip_edges().xml_escape());
+					_write_string(f, 3, _translate_doc_string(p.description).strip_edges().xml_escape());
 					_write_string(f, 2, "</member>");
 				}
 			}
@@ -1158,7 +1188,7 @@ Error DocData::save_classes(const String &p_default_path, const Map<String, Stri
 				}
 
 				_write_string(f, 3, "<description>");
-				_write_string(f, 4, m.description.strip_edges().xml_escape());
+				_write_string(f, 4, _translate_doc_string(m.description).strip_edges().xml_escape());
 				_write_string(f, 3, "</description>");
 
 				_write_string(f, 2, "</signal>");
@@ -1184,7 +1214,7 @@ Error DocData::save_classes(const String &p_default_path, const Map<String, Stri
 					_write_string(f, 2, "<constant name=\"" + k.name + "\" value=\"platform-dependent\">");
 				}
 			}
-			_write_string(f, 3, k.description.strip_edges().xml_escape());
+			_write_string(f, 3, _translate_doc_string(k.description).strip_edges().xml_escape());
 			_write_string(f, 2, "</constant>");
 		}
 
@@ -1203,7 +1233,7 @@ Error DocData::save_classes(const String &p_default_path, const Map<String, Stri
 					_write_string(f, 2, "<theme_item name=\"" + ti.name + "\" data_type=\"" + ti.data_type + "\" type=\"" + ti.type + "\">");
 				}
 
-				_write_string(f, 3, ti.description.strip_edges().xml_escape());
+				_write_string(f, 3, _translate_doc_string(ti.description).strip_edges().xml_escape());
 
 				_write_string(f, 2, "</theme_item>");
 			}

editor/editor_settings.cpp

@@ -31,22 +31,18 @@
 #include "editor_settings.h"
 
 #include "core/io/certs_compressed.gen.h"
-#include "core/io/compression.h"
 #include "core/io/config_file.h"
-#include "core/io/file_access_memory.h"
 #include "core/io/ip.h"
 #include "core/io/resource_loader.h"
 #include "core/io/resource_saver.h"
-#include "core/io/translation_loader_po.h"
 #include "core/os/dir_access.h"
 #include "core/os/file_access.h"
 #include "core/os/keyboard.h"
 #include "core/os/os.h"
 #include "core/project_settings.h"
 #include "core/version.h"
-#include "editor/doc_translations.gen.h"
 #include "editor/editor_node.h"
-#include "editor/editor_translations.gen.h"
+#include "editor/editor_translation.h"
 #include "scene/main/node.h"
 #include "scene/main/scene_tree.h"
 #include "scene/main/viewport.h"
@@ -268,17 +264,14 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 		const Vector<String> locales_to_skip = String("ar,bn,fa,he,hi,ml,si,ta,te,ur").split(",");
 
 		String best;
-
+		const Vector<String> &locales = get_editor_locales();
-		EditorTranslationList *etl = _editor_translations;
+		for (int i = 0; i < locales.size(); i++) {
-
+			const String &locale = locales[i];
-		while (etl->data) {
-			const String &locale = etl->lang;
 
 			// Skip locales which we can't render properly (see above comment).
 			// Test against language code without regional variants (e.g. ur_PK).
 			String lang_code = locale.get_slice("_", 0);
 			if (locales_to_skip.find(lang_code) != -1) {
-				etl++;
 				continue;
 			}
 
@@ -292,8 +285,6 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 			if (best == String() && host_lang.begins_with(locale)) {
 				best = locale;
 			}
-
-			etl++;
 		}
 
 		if (best == String()) {
@@ -1009,50 +1000,10 @@ void EditorSettings::setup_language() {
 	}
 
 	// Load editor translation for configured/detected locale.
-	EditorTranslationList *etl = _editor_translations;
+	load_editor_translations(lang);
-	while (etl->data) {
-		if (etl->lang == lang) {
-			Vector<uint8_t> data;
-			data.resize(etl->uncomp_size);
-			Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
-
-			FileAccessMemory *fa = memnew(FileAccessMemory);
-			fa->open_custom(data.ptr(), data.size());
-
-			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
-
-			if (tr.is_valid()) {
-				tr->set_locale(etl->lang);
-				TranslationServer::get_singleton()->set_tool_translation(tr);
-				break;
-			}
-		}
-
-		etl++;
-	}
 
 	// Load class reference translation.
-	DocTranslationList *dtl = _doc_translations;
+	load_doc_translations(lang);
-	while (dtl->data) {
-		if (dtl->lang == lang) {
-			Vector<uint8_t> data;
-			data.resize(dtl->uncomp_size);
-			Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
-
-			FileAccessMemory *fa = memnew(FileAccessMemory);
-			fa->open_custom(data.ptr(), data.size());
-
-			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
-
-			if (tr.is_valid()) {
-				tr->set_locale(dtl->lang);
-				TranslationServer::get_singleton()->set_doc_translation(tr);
-				break;
-			}
-		}
-
-		dtl++;
-	}
 }
 
 void EditorSettings::setup_network() {

editor/editor_translation.cpp

@@ -0,0 +1,99 @@
+/*************************************************************************/
+/*  editor_translation.cpp                                               */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "editor/editor_translation.h"
+
+#include "core/io/compression.h"
+#include "core/io/file_access_memory.h"
+#include "core/io/translation_loader_po.h"
+#include "editor/doc_translations.gen.h"
+#include "editor/editor_translations.gen.h"
+
+Vector<String> get_editor_locales() {
+	Vector<String> locales;
+
+	EditorTranslationList *etl = _editor_translations;
+	while (etl->data) {
+		const String &locale = etl->lang;
+		locales.push_back(locale);
+
+		etl++;
+	}
+
+	return locales;
+}
+
+void load_editor_translations(const String &p_locale) {
+	EditorTranslationList *etl = _editor_translations;
+	while (etl->data) {
+		if (etl->lang == p_locale) {
+			Vector<uint8_t> data;
+			data.resize(etl->uncomp_size);
+			Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
+
+			FileAccessMemory *fa = memnew(FileAccessMemory);
+			fa->open_custom(data.ptr(), data.size());
+
+			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
+
+			if (tr.is_valid()) {
+				tr->set_locale(etl->lang);
+				TranslationServer::get_singleton()->set_tool_translation(tr);
+				break;
+			}
+		}
+
+		etl++;
+	}
+}
+
+void load_doc_translations(const String &p_locale) {
+	DocTranslationList *dtl = _doc_translations;
+	while (dtl->data) {
+		if (dtl->lang == p_locale) {
+			Vector<uint8_t> data;
+			data.resize(dtl->uncomp_size);
+			Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
+
+			FileAccessMemory *fa = memnew(FileAccessMemory);
+			fa->open_custom(data.ptr(), data.size());
+
+			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
+
+			if (tr.is_valid()) {
+				tr->set_locale(dtl->lang);
+				TranslationServer::get_singleton()->set_doc_translation(tr);
+				break;
+			}
+		}
+
+		dtl++;
+	}
+}

editor/editor_translation.h

@@ -0,0 +1,41 @@
+/*************************************************************************/
+/*  editor_translation.h                                                 */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef EDITOR_TRANSLATION_H
+#define EDITOR_TRANSLATION_H
+
+#include "core/ustring.h"
+#include "core/vector.h"
+
+Vector<String> get_editor_locales();
+void load_editor_translations(const String &p_locale);
+void load_doc_translations(const String &p_locale);
+
+#endif // EDITOR_TRANSLATION_H

main/main.cpp

@@ -76,6 +76,7 @@
 #include "editor/doc/doc_data_class_path.gen.h"
 #include "editor/editor_node.h"
 #include "editor/editor_settings.h"
+#include "editor/editor_translation.h"
 #include "editor/progress_dialog.h"
 #include "editor/project_manager.h"
 #ifndef NO_EDITOR_SPLASH
@@ -1523,7 +1524,6 @@ Error Main::setup2(Thread::ID p_main_tid_override) {
 	VisualServer::get_singleton()->callbacks_register(visual_server_callbacks);
 
 	_start_success = true;
-	locale = String();
 
 	ClassDB::set_current_api(ClassDB::API_NONE); //no more api is registered at this point
 
@@ -1631,6 +1631,11 @@ bool Main::start() {
 	if (doc_tool_path != "") {
 		Engine::get_singleton()->set_editor_hint(true); // Needed to instance editor-only classes for their default values
 
+		// Translate the class reference only when `-l LOCALE` parameter is given.
+		if (!locale.empty() && locale != "en") {
+			load_doc_translations(locale);
+		}
+
 		{
 			DirAccessRef da = DirAccess::open(doc_tool_path);
 			ERR_FAIL_COND_V_MSG(!da, false, "Argument supplied to --doctool must be a valid directory path.");