Add some helpful python scripts from agm

scripts/stringtablediag.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+from xml.dom import minidom
+
+# STRINGTABLE DIAG TOOL
+# Author: KoffeinFlummi
+# ---------------------
+# Checks for missing translations and all that jazz.
+
+def get_all_languages(projectpath):
+  """ Checks what languages exist in the repo. """
+  languages = []
+
+  for module in os.listdir(projectpath):
+    if module[0] == ".":
+      continue
+
+    stringtablepath = os.path.join(projectpath, module, "stringtable.xml")
+    try:
+      xmldoc = minidom.parse(stringtablepath)
+    except:
+      continue
+
+    keys = xmldoc.getElementsByTagName("Key")
+    for key in keys:
+      for child in key.childNodes:
+        try:
+          if not child.tagName in languages:
+            languages.append(child.tagName)
+        except:
+          continue
+
+  return languages
+
+def check_module(projectpath, module, languages):
+  """ Checks the given module for all the different languages. """
+  localized = []
+
+  stringtablepath = os.path.join(projectpath, module, "stringtable.xml")
+  try:
+    xmldoc = minidom.parse(stringtablepath)
+  except:
+    return 0, localized
+
+  keynumber = len(xmldoc.getElementsByTagName("Key"))
+
+  for language in languages:
+    localized.append(len(xmldoc.getElementsByTagName(language)))
+
+  return keynumber, localized
+
+def main():
+  scriptpath = os.path.realpath(__file__)
+  projectpath = os.path.dirname(os.path.dirname(scriptpath))
+  projectpath = os.path.join(projectpath, "addons")
+
+  print("#########################")
+  print("# Stringtable Diag Tool #")
+  print("#########################")
+
+  languages = get_all_languages(projectpath)
+
+  print("\nLanguages present in the repo:")
+  print(", ".join(languages))
+
+  keysum = 0
+  localizedsum = list(map(lambda x: 0, languages))
+  missing = list(map(lambda x: [], languages))
+
+  for module in os.listdir(projectpath):
+    keynumber, localized = check_module(projectpath, module, languages)
+
+    if keynumber == 0:
+      continue
+
+    print("\n# " + module)
+
+    keysum += keynumber
+    for i in range(len(localized)):
+      print("  %s %s / %i" % ((languages[i]+":").ljust(10), str(localized[i]).ljust(3), keynumber))
+      localizedsum[i] += localized[i]
+      if localized[i] < keynumber:
+        missing[i].append(module)
+
+  print("\n###########")
+  print("# RESULTS #")
+  print("###########")
+
+  print("\nTotal number of keys: %i\n" % (keysum))
+
+  for i in range(len(languages)):
+    if localizedsum[i] == keysum:
+      print("%s No missing stringtable entries." % ((languages[i] + ":").ljust(12)))
+    else:
+      print("%s %s missing stringtable entry/entries." % ((languages[i] + ":").ljust(12), str(keysum - localizedsum[i]).rjust(4)), end="")
+      print(" ("+", ".join(missing[i])+")")
+
+  print("\n\n### MARKDOWN ###")
+
+  print("\nTotal number of keys: %i\n" % (keysum))
+
+  print("| Language | Missing Entries | Relevant Modules | % done |")
+  print("|----------|----------------:|------------------|--------|")
+
+  for i, language in enumerate(languages):
+    if localizedsum[i] == keysum:
+      print("| {} | 0 | - | 100% |".format(language))
+    else:
+      print("| {} | {} | {} | {}% |".format(
+        language,
+        keysum - localizedsum[i],
+        ", ".join(missing[i]),
+        round(100 * localizedsum[i] / keysum)))
+
+if __name__ == "__main__":
+  main()

scripts/stringtableduplicates.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+from xml.dom import minidom
+
+# STRINGTABLE DUPLICATE FINDER
+# Author: KoffeinFlummi
+# ----------------------------
+# Counts duplicate stringtable entries
+
+def main():
+  scriptpath = os.path.realpath(__file__)
+  projectpath = os.path.dirname(os.path.dirname(scriptpath))
+  projectpath = os.path.join(projectpath, "addons")
+
+  entries = {}
+
+  for module in os.listdir(projectpath):
+    if module[0] == ".":
+      continue
+    stringtablepath = os.path.join(projectpath, module, "stringtable.xml")
+    try:
+      xmldoc = minidom.parse(stringtablepath)
+    except:
+      continue
+
+    keys = xmldoc.getElementsByTagName("English")
+    for key in keys:
+      text = key.firstChild.wholeText
+      parentid = key.parentNode.getAttribute("ID")
+      if text in entries:
+        entries[text].append(parentid)
+      else:
+        entries[text] = [parentid]
+
+  entries = {k: v for k, v in entries.items() if len(v) > 1}
+  output = list([[k, v] for k, v in entries.items()])
+  output = sorted(output, key=lambda x: len(x[1])*-1)
+
+  print("Potential duplicate stringtable entries:\n")
+  for l in output:
+    k, v = l
+    print(k.ljust(50), end=" ")
+    print("Listed %i times in: %s" % (len(v), ", ".join(v)))
+
+  print("\n# MARKDOWN\n")
+
+  print("| Text | # Occurences | Containing Entries |")
+  print("|------|-------------:|--------------------|")
+
+  for l in output:
+    print("| %s | %i | %s |" % (l[0], len(l[1]), ", ".join(l[1])))
+
+if __name__ == "__main__":
+  main()

scripts/stringtablemerger.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import re
+
+from xml.dom import minidom
+
+# STRINGTABLE MERGER TOOL
+# Author: KoffeinFlummi
+# --------------------------
+# Automatically merges all stringtable entries
+# in the given language from the given dir.
+
+def get_modules(projectpath):
+  """ Get all the modules of the project. """
+  modules = []
+
+  for i in os.listdir(projectpath):
+    path = os.path.join(projectpath, i)
+    if not os.path.isdir(path):
+      continue
+    if i[0] == ".":
+      continue
+    modules.append(i)
+
+  return modules
+
+def contains_language(key, language):
+  """ Checks whether a given key contains a certain language. """
+  for child in key.childNodes:
+    try:
+      assert(child.tagName == language)
+      return True
+    except:
+      pass
+
+  return False
+
+def get_entry_by_id(keys, keyid):
+  """ Returns the first child of keys with ID='keyid'. """
+  for key in keys:
+    if key.getAttribute("ID") == keyid:
+      return key
+
+  return False
+
+def replace_entries(oldpath, newpath, language, breakdown):
+  """ Replaces all new entries of the given language in the given module. """
+  oldfile = minidom.parse(oldpath)
+  newfile = minidom.parse(newpath)
+
+  oldkeys = oldfile.getElementsByTagName("Key")
+  newkeys = newfile.getElementsByTagName("Key")
+  newkeys = list(filter(lambda x: contains_language(x, language), newkeys))
+
+  for newkey in newkeys:
+    keyid = newkey.getAttribute("ID")
+    oldkey = get_entry_by_id(oldkeys, keyid)
+
+    if not oldkey:
+      continue
+
+    if breakdown:
+      print("  Merging %s translation for %s" % (language, keyid))
+
+    newentry = newkey.getElementsByTagName(language)[0].firstChild
+
+    try:
+      # An entry for this language already exists, overwrite it
+      oldentry = oldkey.getElementsByTagName(language)[0].firstChild
+      oldentry.replaceWholeText(newentry.wholeText)
+    except:
+      # There is no entry for this language yet, make one
+      oldentry = oldfile.createElement(language)
+      oldentry.appendChild(oldfile.createTextNode(newentry.wholeText))
+      # Some whitespace tetris to maintain file structure
+      oldkey.insertBefore(oldfile.createTextNode("\n      "), oldkey.lastChild)
+      oldkey.insertBefore(oldentry, oldkey.lastChild)
+
+  # Make a nice string
+  xmlstring = oldfile.toxml()
+  xmlstring = xmlstring.replace('" ?>', '" encoding="utf-8"?>')
+
+  # Replace the newlines that minidom swallows
+  xmlstring = xmlstring.replace("><", ">\n<")
+  xmlstring += "\n"
+
+  fhandle = open(oldpath, "w")
+  fhandle.write(xmlstring)
+  fhandle.close()
+
+  return len(newkeys)
+
+def main(sourcepath, language, breakdown):
+  scriptpath = os.path.realpath(__file__)
+  projectpath = os.path.dirname(os.path.dirname(scriptpath))
+  projectpath = os.path.join(projectpath, "addons")
+
+  modules = get_modules(projectpath)
+  modulecounter = 0
+  keycounter = 0
+
+  for module in modules:
+    oldpath = os.path.join(projectpath, module, "stringtable.xml")
+    newpath = os.path.join(sourcepath, module, "stringtable.xml")
+
+    # Some translators extract the lowercase PBOs, so the module name might
+    # be lowercase (obviously only matters on Linux)
+    if not os.path.exists(newpath):
+      newpath = os.path.join(sourcepath, module.lower(), "stringtable.xml")
+
+    # Translator didn't include this module, skip
+    if not os.path.exists(newpath):
+      continue
+
+    keynum = replace_entries(oldpath, newpath, language, breakdown)
+    modulecounter += 1
+    keycounter += keynum
+
+    print("# Merged %i entry/entries in %s" % (keynum, module))
+    if breakdown:
+      print("")
+
+  print("")
+  print("# Merged %i entry/entries in %i modules" % (keycounter, modulecounter))
+
+if __name__ == "__main__":
+  try:
+    sourcepath = os.path.normpath(os.path.join(os.getcwd(), sys.argv[1]))
+    language = sys.argv[2]
+  
+    assert(os.path.exists(sourcepath))
+  except:
+    print("ERROR: Missing arguments of invalid path.")
+    print("\nUsage:")
+    print("[script] [path to new project] [language]")
+    sys.exit(1)
+
+  main(sourcepath, language, "--breakdown" in sys.argv)

scripts/versionnumbers.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import re
+
+# VERSION NUMBERING TOOL
+# Author: KoffeinFlummi
+# ----------------------
+# Allows for easy changes of, you know, version numbers
+
+def get_all_modules(projectpath):
+  modules = os.listdir(projectpath)
+  modules = list(filter(lambda x: os.path.exists(os.path.join(projectpath, x, "config.cpp")), modules))
+  return modules
+
+def replace_version_number(filepath, version, versionAr):
+  fileInput = open(filepath, "r", encoding="utf-8")
+  content = fileInput.read()
+
+  # version
+  p = re.compile("CfgPatches(.*?)version(.*?)\"(.*?)\"", re.DOTALL)
+  content = p.sub(r'CfgPatches\1version\2"' + version + '"', content)
+
+  # versionStr
+  p = re.compile("CfgPatches(.*?)versionStr(.*?)\"(.*?)\"", re.DOTALL)
+  content = p.sub(r'CfgPatches\1versionStr\2"' + version + '"', content)
+
+  # versionAr
+  p = re.compile("CfgPatches(.*?)versionAr\[\](.*?)\{(.*?)\}", re.DOTALL)
+  content = p.sub(r'CfgPatches\1versionAr[]\2{' + versionAr + '}', content)
+
+  fileOutput = open(filepath, "w", encoding="utf-8")
+  fileOutput.write(content)
+
+def main(version):
+  versionAr = version.split(".")
+  try:
+    versionAr = list(map(lambda x: int(x), versionAr))
+  except:
+    print("ERROR: Version number doesn't consist of numbers.")
+    sys.exit(1)
+  while len(versionAr) < 3:
+    versionAr.append(0)
+  versionAr = list(map(lambda x: str(x), versionAr))
+  versionAr = ",".join(versionAr)
+
+  scriptpath = os.path.realpath(__file__)
+  projectpath = os.path.dirname(os.path.dirname(scriptpath))
+  projectpath = os.path.join(projectpath, "addons")
+  modules = get_all_modules(projectpath)
+
+  for module in modules:
+    print("# Updating %s" % (module))
+    fullpath = os.path.join(projectpath, module, "config.cpp")
+    try:
+      replace_version_number(fullpath, version, versionAr)
+    except:
+      print("  Failed to update %s" % (module))
+
+if __name__ == "__main__":
+  try:
+    main(sys.argv[1])
+  except:
+    print("No version number specified.")
+    sys.exit(1)