From 50dbe77900c955d1a0d440447dfdcf987067774a Mon Sep 17 00:00:00 2001 From: KoffeinFlummi Date: Mon, 12 Jan 2015 11:24:17 +0100 Subject: [PATCH] Add some helpful python scripts from agm --- scripts/stringtablediag.py | 119 ++++++++++++++++++++++++++ scripts/stringtableduplicates.py | 56 +++++++++++++ scripts/stringtablemerger.py | 140 +++++++++++++++++++++++++++++++ scripts/versionnumbers.py | 66 +++++++++++++++ 4 files changed, 381 insertions(+) create mode 100755 scripts/stringtablediag.py create mode 100644 scripts/stringtableduplicates.py create mode 100755 scripts/stringtablemerger.py create mode 100644 scripts/versionnumbers.py diff --git a/scripts/stringtablediag.py b/scripts/stringtablediag.py new file mode 100755 index 0000000000..3e4d906158 --- /dev/null +++ b/scripts/stringtablediag.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +import os +import sys + +from xml.dom import minidom + +# STRINGTABLE DIAG TOOL +# Author: KoffeinFlummi +# --------------------- +# Checks for missing translations and all that jazz. + +def get_all_languages(projectpath): + """ Checks what languages exist in the repo. """ + languages = [] + + for module in os.listdir(projectpath): + if module[0] == ".": + continue + + stringtablepath = os.path.join(projectpath, module, "stringtable.xml") + try: + xmldoc = minidom.parse(stringtablepath) + except: + continue + + keys = xmldoc.getElementsByTagName("Key") + for key in keys: + for child in key.childNodes: + try: + if not child.tagName in languages: + languages.append(child.tagName) + except: + continue + + return languages + +def check_module(projectpath, module, languages): + """ Checks the given module for all the different languages. """ + localized = [] + + stringtablepath = os.path.join(projectpath, module, "stringtable.xml") + try: + xmldoc = minidom.parse(stringtablepath) + except: + return 0, localized + + keynumber = len(xmldoc.getElementsByTagName("Key")) + + for language in languages: + localized.append(len(xmldoc.getElementsByTagName(language))) + + return keynumber, localized + +def main(): + scriptpath = os.path.realpath(__file__) + projectpath = os.path.dirname(os.path.dirname(scriptpath)) + projectpath = os.path.join(projectpath, "addons") + + print("#########################") + print("# Stringtable Diag Tool #") + print("#########################") + + languages = get_all_languages(projectpath) + + print("\nLanguages present in the repo:") + print(", ".join(languages)) + + keysum = 0 + localizedsum = list(map(lambda x: 0, languages)) + missing = list(map(lambda x: [], languages)) + + for module in os.listdir(projectpath): + keynumber, localized = check_module(projectpath, module, languages) + + if keynumber == 0: + continue + + print("\n# " + module) + + keysum += keynumber + for i in range(len(localized)): + print(" %s %s / %i" % ((languages[i]+":").ljust(10), str(localized[i]).ljust(3), keynumber)) + localizedsum[i] += localized[i] + if localized[i] < keynumber: + missing[i].append(module) + + print("\n###########") + print("# RESULTS #") + print("###########") + + print("\nTotal number of keys: %i\n" % (keysum)) + + for i in range(len(languages)): + if localizedsum[i] == keysum: + print("%s No missing stringtable entries." % ((languages[i] + ":").ljust(12))) + else: + print("%s %s missing stringtable entry/entries." % ((languages[i] + ":").ljust(12), str(keysum - localizedsum[i]).rjust(4)), end="") + print(" ("+", ".join(missing[i])+")") + + print("\n\n### MARKDOWN ###") + + print("\nTotal number of keys: %i\n" % (keysum)) + + print("| Language | Missing Entries | Relevant Modules | % done |") + print("|----------|----------------:|------------------|--------|") + + for i, language in enumerate(languages): + if localizedsum[i] == keysum: + print("| {} | 0 | - | 100% |".format(language)) + else: + print("| {} | {} | {} | {}% |".format( + language, + keysum - localizedsum[i], + ", ".join(missing[i]), + round(100 * localizedsum[i] / keysum))) + +if __name__ == "__main__": + main() diff --git a/scripts/stringtableduplicates.py b/scripts/stringtableduplicates.py new file mode 100644 index 0000000000..d72cc3e6f6 --- /dev/null +++ b/scripts/stringtableduplicates.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import os +import sys +from xml.dom import minidom + +# STRINGTABLE DUPLICATE FINDER +# Author: KoffeinFlummi +# ---------------------------- +# Counts duplicate stringtable entries + +def main(): + scriptpath = os.path.realpath(__file__) + projectpath = os.path.dirname(os.path.dirname(scriptpath)) + projectpath = os.path.join(projectpath, "addons") + + entries = {} + + for module in os.listdir(projectpath): + if module[0] == ".": + continue + stringtablepath = os.path.join(projectpath, module, "stringtable.xml") + try: + xmldoc = minidom.parse(stringtablepath) + except: + continue + + keys = xmldoc.getElementsByTagName("English") + for key in keys: + text = key.firstChild.wholeText + parentid = key.parentNode.getAttribute("ID") + if text in entries: + entries[text].append(parentid) + else: + entries[text] = [parentid] + + entries = {k: v for k, v in entries.items() if len(v) > 1} + output = list([[k, v] for k, v in entries.items()]) + output = sorted(output, key=lambda x: len(x[1])*-1) + + print("Potential duplicate stringtable entries:\n") + for l in output: + k, v = l + print(k.ljust(50), end=" ") + print("Listed %i times in: %s" % (len(v), ", ".join(v))) + + print("\n# MARKDOWN\n") + + print("| Text | # Occurences | Containing Entries |") + print("|------|-------------:|--------------------|") + + for l in output: + print("| %s | %i | %s |" % (l[0], len(l[1]), ", ".join(l[1]))) + +if __name__ == "__main__": + main() diff --git a/scripts/stringtablemerger.py b/scripts/stringtablemerger.py new file mode 100755 index 0000000000..9558ec081b --- /dev/null +++ b/scripts/stringtablemerger.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import os +import sys +import re + +from xml.dom import minidom + +# STRINGTABLE MERGER TOOL +# Author: KoffeinFlummi +# -------------------------- +# Automatically merges all stringtable entries +# in the given language from the given dir. + +def get_modules(projectpath): + """ Get all the modules of the project. """ + modules = [] + + for i in os.listdir(projectpath): + path = os.path.join(projectpath, i) + if not os.path.isdir(path): + continue + if i[0] == ".": + continue + modules.append(i) + + return modules + +def contains_language(key, language): + """ Checks whether a given key contains a certain language. """ + for child in key.childNodes: + try: + assert(child.tagName == language) + return True + except: + pass + + return False + +def get_entry_by_id(keys, keyid): + """ Returns the first child of keys with ID='keyid'. """ + for key in keys: + if key.getAttribute("ID") == keyid: + return key + + return False + +def replace_entries(oldpath, newpath, language, breakdown): + """ Replaces all new entries of the given language in the given module. """ + oldfile = minidom.parse(oldpath) + newfile = minidom.parse(newpath) + + oldkeys = oldfile.getElementsByTagName("Key") + newkeys = newfile.getElementsByTagName("Key") + newkeys = list(filter(lambda x: contains_language(x, language), newkeys)) + + for newkey in newkeys: + keyid = newkey.getAttribute("ID") + oldkey = get_entry_by_id(oldkeys, keyid) + + if not oldkey: + continue + + if breakdown: + print(" Merging %s translation for %s" % (language, keyid)) + + newentry = newkey.getElementsByTagName(language)[0].firstChild + + try: + # An entry for this language already exists, overwrite it + oldentry = oldkey.getElementsByTagName(language)[0].firstChild + oldentry.replaceWholeText(newentry.wholeText) + except: + # There is no entry for this language yet, make one + oldentry = oldfile.createElement(language) + oldentry.appendChild(oldfile.createTextNode(newentry.wholeText)) + # Some whitespace tetris to maintain file structure + oldkey.insertBefore(oldfile.createTextNode("\n "), oldkey.lastChild) + oldkey.insertBefore(oldentry, oldkey.lastChild) + + # Make a nice string + xmlstring = oldfile.toxml() + xmlstring = xmlstring.replace('" ?>', '" encoding="utf-8"?>') + + # Replace the newlines that minidom swallows + xmlstring = xmlstring.replace("><", ">\n<") + xmlstring += "\n" + + fhandle = open(oldpath, "w") + fhandle.write(xmlstring) + fhandle.close() + + return len(newkeys) + +def main(sourcepath, language, breakdown): + scriptpath = os.path.realpath(__file__) + projectpath = os.path.dirname(os.path.dirname(scriptpath)) + projectpath = os.path.join(projectpath, "addons") + + modules = get_modules(projectpath) + modulecounter = 0 + keycounter = 0 + + for module in modules: + oldpath = os.path.join(projectpath, module, "stringtable.xml") + newpath = os.path.join(sourcepath, module, "stringtable.xml") + + # Some translators extract the lowercase PBOs, so the module name might + # be lowercase (obviously only matters on Linux) + if not os.path.exists(newpath): + newpath = os.path.join(sourcepath, module.lower(), "stringtable.xml") + + # Translator didn't include this module, skip + if not os.path.exists(newpath): + continue + + keynum = replace_entries(oldpath, newpath, language, breakdown) + modulecounter += 1 + keycounter += keynum + + print("# Merged %i entry/entries in %s" % (keynum, module)) + if breakdown: + print("") + + print("") + print("# Merged %i entry/entries in %i modules" % (keycounter, modulecounter)) + +if __name__ == "__main__": + try: + sourcepath = os.path.normpath(os.path.join(os.getcwd(), sys.argv[1])) + language = sys.argv[2] + + assert(os.path.exists(sourcepath)) + except: + print("ERROR: Missing arguments of invalid path.") + print("\nUsage:") + print("[script] [path to new project] [language]") + sys.exit(1) + + main(sourcepath, language, "--breakdown" in sys.argv) diff --git a/scripts/versionnumbers.py b/scripts/versionnumbers.py new file mode 100644 index 0000000000..357f4a950b --- /dev/null +++ b/scripts/versionnumbers.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import os +import sys +import re + +# VERSION NUMBERING TOOL +# Author: KoffeinFlummi +# ---------------------- +# Allows for easy changes of, you know, version numbers + +def get_all_modules(projectpath): + modules = os.listdir(projectpath) + modules = list(filter(lambda x: os.path.exists(os.path.join(projectpath, x, "config.cpp")), modules)) + return modules + +def replace_version_number(filepath, version, versionAr): + fileInput = open(filepath, "r", encoding="utf-8") + content = fileInput.read() + + # version + p = re.compile("CfgPatches(.*?)version(.*?)\"(.*?)\"", re.DOTALL) + content = p.sub(r'CfgPatches\1version\2"' + version + '"', content) + + # versionStr + p = re.compile("CfgPatches(.*?)versionStr(.*?)\"(.*?)\"", re.DOTALL) + content = p.sub(r'CfgPatches\1versionStr\2"' + version + '"', content) + + # versionAr + p = re.compile("CfgPatches(.*?)versionAr\[\](.*?)\{(.*?)\}", re.DOTALL) + content = p.sub(r'CfgPatches\1versionAr[]\2{' + versionAr + '}', content) + + fileOutput = open(filepath, "w", encoding="utf-8") + fileOutput.write(content) + +def main(version): + versionAr = version.split(".") + try: + versionAr = list(map(lambda x: int(x), versionAr)) + except: + print("ERROR: Version number doesn't consist of numbers.") + sys.exit(1) + while len(versionAr) < 3: + versionAr.append(0) + versionAr = list(map(lambda x: str(x), versionAr)) + versionAr = ",".join(versionAr) + + scriptpath = os.path.realpath(__file__) + projectpath = os.path.dirname(os.path.dirname(scriptpath)) + projectpath = os.path.join(projectpath, "addons") + modules = get_all_modules(projectpath) + + for module in modules: + print("# Updating %s" % (module)) + fullpath = os.path.join(projectpath, module, "config.cpp") + try: + replace_version_number(fullpath, version, versionAr) + except: + print(" Failed to update %s" % (module)) + +if __name__ == "__main__": + try: + main(sys.argv[1]) + except: + print("No version number specified.") + sys.exit(1)