virtualx-engine/doc/translations/extract.py

#!/usr/bin/env python3

import argparse
import os
import re
import shutil
from collections import OrderedDict

EXTRACT_TAGS = ["description", "brief_description", "member", "constant", "theme_item", "link"]
HEADER = '''\
# LANGUAGE translation of the Godot Engine class reference.
# Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.
# Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).
# This file is distributed under the same license as the Godot source code.
#
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: Godot Engine class reference\\n"
"Report-Msgid-Bugs-To: https://github.com/godotengine/godot\\n"
"MIME-Version: 1.0\\n"
"Content-Type: text/plain; charset=UTF-8\\n"
"Content-Transfer-Encoding: 8-bit\\n"

'''
# Some strings used by makerst.py are normally part of the editor translations,
# so we need to include them manually here for the online docs.
BASE_STRINGS = [
    "Description",
    "Tutorials",
    "Properties",
    "Methods",
    "Theme Properties",
    "Signals",
    "Enumerations",
    "Constants",
    "Property Descriptions",
    "Method Descriptions",
]

## <xml-line-number-hack from="https://stackoverflow.com/a/36430270/10846399">
import sys
sys.modules['_elementtree'] = None
import xml.etree.ElementTree as ET

## override the parser to get the line number
class LineNumberingParser(ET.XMLParser):
    def _start(self, *args, **kwargs):
        ## Here we assume the default XML parser which is expat
        ## and copy its element position attributes into output Elements
        element = super(self.__class__, self)._start(*args, **kwargs)
        element._start_line_number = self.parser.CurrentLineNumber
        element._start_column_number = self.parser.CurrentColumnNumber
        element._start_byte_index = self.parser.CurrentByteIndex
        return element

    def _end(self, *args, **kwargs):
        element = super(self.__class__, self)._end(*args, **kwargs)
        element._end_line_number = self.parser.CurrentLineNumber
        element._end_column_number = self.parser.CurrentColumnNumber
        element._end_byte_index = self.parser.CurrentByteIndex
        return element
## </xml-line-number-hack>

class Desc:
    def __init__(self, line_no, msg, desc_list=None):
        ## line_no   : the line number where the desc is
        ## msg       : the description string
        ## desc_list : the DescList it belongs to
        self.line_no = line_no
        self.msg = msg
        self.desc_list = desc_list

class DescList:
    def __init__(self, doc, path):
        ## doc  : root xml element of the document
        ## path : file path of the xml document
        ## list : list of Desc objects for this document
        self.doc = doc
        self.path = path
        self.list = []

def print_error(error):
    print("ERROR: {}".format(error))

## build classes with xml elements recursively
def _collect_classes_dir(path, classes):
    if not os.path.isdir(path):
        print_error("Invalid directory path: {}".format(path))
        exit(1)
    for _dir in map(lambda dir : os.path.join(path, dir), os.listdir(path)):
        if os.path.isdir(_dir):
            _collect_classes_dir(_dir, classes)
        elif os.path.isfile(_dir):
            if not _dir.endswith(".xml"):
                #print("Got non-.xml file '{}', skipping.".format(path))
                continue
            _collect_classes_file(_dir, classes)

## opens a file and parse xml add to classes
def _collect_classes_file(path, classes):
    if not os.path.isfile(path) or not path.endswith(".xml"):
        print_error("Invalid xml file path: {}".format(path))
        exit(1)
    print('Collecting file: {}'.format(os.path.basename(path)))

    try:
        tree = ET.parse(path, parser=LineNumberingParser())
    except ET.ParseError as e:
        print_error("Parse error reading file '{}': {}".format(path, e))
        exit(1)

    doc = tree.getroot()

    if 'name' in doc.attrib:
        if 'version' not in doc.attrib:
            print_error("Version missing from 'doc', file: {}".format(path))

        name = doc.attrib["name"]
        if name in classes:
            print_error("Duplicate class {} at path {}".format(name, path))
            exit(1)
        classes[name] = DescList(doc, path)
    else:
        print_error('Unknown XML file {}, skipping'.format(path))


## regions are list of tuples with size 3 (start_index, end_index, indent)
## indication in string where the codeblock starts, ends, and it's indent
## if i inside the region returns the indent, else returns -1
def _get_xml_indent(i, regions):
    for region in regions:
        if region[0] < i < region[1] :
            return region[2]
    return -1

## find and build all regions of codeblock which we need later
def _make_codeblock_regions(desc, path=''):
    code_block_end = False
    code_block_index = 0
    code_block_regions = []
    while not code_block_end:
        code_block_index = desc.find("[codeblock]", code_block_index)
        if code_block_index < 0: break
        xml_indent=0
        while True :
            ## [codeblock] always have a trailing new line and some tabs
            ## those tabs are belongs to xml indentations not code indent
            if desc[code_block_index+len("[codeblock]\n")+xml_indent] == '\t':
                xml_indent+=1
            else: break
        end_index = desc.find("[/codeblock]", code_block_index)
        if end_index < 0 :
            print_error('Non terminating codeblock: {}'.format(path))
            exit(1)
        code_block_regions.append( (code_block_index, end_index, xml_indent) )
        code_block_index += 1
    return code_block_regions

def _strip_and_split_desc(desc, code_block_regions):
    desc_strip = ''   ## a stripped desc msg
    total_indent = 0  ## code indent = total indent - xml indent
    for i in range(len(desc)):
        c = desc[i]
        if c == '\n' : c = '\\n'
        if c == '"': c = '\\"'
        if c == '\\': c = '\\\\' ## <element \> is invalid for msgmerge
        if c == '\t':
            xml_indent = _get_xml_indent(i, code_block_regions)
            if xml_indent >= 0:
                total_indent += 1
                if xml_indent < total_indent:
                    c = '\\t'
                else:
                    continue
            else:
                continue
        desc_strip += c
        if c == '\\n':
            total_indent = 0
    return desc_strip

## make catalog strings from xml elements
def _make_translation_catalog(classes):
    unique_msgs = OrderedDict()
    for class_name in classes:
        desc_list = classes[class_name]
        for elem in desc_list.doc.iter():
            if elem.tag in EXTRACT_TAGS:
                if not elem.text or len(elem.text) == 0 : continue
                line_no = elem._start_line_number if elem.text[0]!='\n' else elem._start_line_number+1
                desc_str = elem.text.strip()
                code_block_regions = _make_codeblock_regions(desc_str, desc_list.path)
                desc_msg = _strip_and_split_desc(desc_str, code_block_regions)
                desc_obj = Desc(line_no, desc_msg, desc_list)
                desc_list.list.append(desc_obj)

                if desc_msg not in unique_msgs:
                    unique_msgs[desc_msg] = [desc_obj]
                else:
                    unique_msgs[desc_msg].append(desc_obj)
    return unique_msgs

## generate the catalog file
def _generate_translation_catalog_file(unique_msgs, output):
    with open(output, 'w', encoding='utf8') as f:
        f.write(HEADER)
        for msg in BASE_STRINGS:
            f.write('#: doc/tools/makerst.py\n')
            f.write('msgid "{}"\n'.format(msg))
            f.write('msgstr ""\n\n')
        for msg in unique_msgs:
            if len(msg) == 0 or msg in BASE_STRINGS:
                continue

            f.write('#:')
            desc_list = unique_msgs[msg]
            for desc in desc_list:
                path = desc.desc_list.path.replace('\\', '/')
                if path.startswith('./'):
                    path = path[2:]
                f.write(' {}:{}'.format(path, desc.line_no))
            f.write('\n')

            f.write('msgid "{}"\n'.format(msg))
            f.write('msgstr ""\n\n')

    ## TODO: what if 'nt'?
    if (os.name == "posix"):
        print("Wrapping template at 79 characters for compatibility with Weblate.")
        os.system("msgmerge -w79 {0} {0} > {0}.wrap".format(output))
        shutil.move("{}.wrap".format(output), output)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--path", "-p", default=".", help="The directory containing XML files to collect.")
    parser.add_argument("--output", "-o", default="translation_catalog.pot", help="The path to the output file.")
    args = parser.parse_args()

    output = os.path.abspath(args.output)
    if not os.path.isdir(os.path.dirname(output)) or not output.endswith('.pot'):
        print_error("Invalid output path: {}".format(output))
        exit(1)
    if not os.path.isdir(args.path):
        print_error("Invalid working directory path: {}".format(args.path))
        exit(1)

    os.chdir(args.path)
    print("Current working dir: {}\n".format(os.getcwd()))

    classes = OrderedDict() ## dictionary of key=class_name, value=DescList objects
    _collect_classes_dir('.', classes)
    classes = OrderedDict(sorted(classes.items(), key = lambda kv: kv[0].lower() ))
    unique_msgs = _make_translation_catalog(classes)
    _generate_translation_catalog_file(unique_msgs, output)

if __name__ == '__main__':
    main()