Arma3_Epoch/ACE3
1
0
Fork 0
mirror of https://github.com/acemod/ACE3.git synced 2024-08-30 18:23:18 +00:00
Code Issues Packages Projects Releases Wiki Activity
e6683e8077
ACE3/tools/make.py
Daniel Jupp 5a4c439c22 Fix Mikero tools dupe key issue (#5853) 
Mikero tools seams to not cleanup regkeys properly and leaves a key with no value behind making. Making find_depbo_tools break
2017-12-04 20:15:24 +01:00

1495 lines
58 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# vim: set fileencoding=utf-8 :
# make.py
# An Arma 3 addon build system
###############################################################################
# The MIT License (MIT)
# Copyright (c) 2013-2014 Ryan Schultz
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
###############################################################################
__version__ = "0.9"
import sys
if sys.version_info[0] == 2:
print("Python 3 is required.")
sys.exit(1)
import os
import os.path
import shutil
import platform
import glob
import subprocess
import hashlib
import configparser
import json
import traceback
import time
import timeit
import re
import fileinput
if sys.platform == "win32":
import winreg
######## GLOBALS #########
project = "@ace"
project_version = "3.0.0"
arma3tools_path = ""
work_drive = ""
module_root = ""
make_root = ""
release_dir = ""
module_root_parent = ""
optionals_root = ""
key_name = "ace"
key = ""
dssignfile = ""
prefix = "ace"
pbo_name_prefix = "ace_"
signature_blacklist = []
importantFiles = ["mod.cpp", "README.md", "docs\\README_DE.md", "docs\\README_PL.md", "AUTHORS.txt", "LICENSE", "logo_ace3_ca.paa", "meta.cpp"]
versionFiles = ["README.md", "docs\\README_DE.md", "docs\\README_PL.md", "mod.cpp"]
ciBuild = False # Used for CI builds
###############################################################################
# http://akiscode.com/articles/sha-1directoryhash.shtml
# Copyright (c) 2009 Stephen Akiki
# MIT License (Means you can do whatever you want with this)
# See http://www.opensource.org/licenses/mit-license.php
# Error Codes:
# -1 -> Directory does not exist
# -2 -> General error (see stack traceback)
def get_directory_hash(directory):
directory_hash = hashlib.sha1()
if not os.path.exists (directory):
return -1
try:
for root, dirs, files in os.walk(directory):
for names in files:
path = os.path.join(root, names)
try:
f = open(path, 'rb')
except:
# You can't open the file for some reason
f.close()
continue
while 1:
# Read file in as little chunks
buf = f.read(4096)
if not buf: break
new = hashlib.sha1(buf)
directory_hash.update(new.digest())
f.close()
except:
# Print the stack traceback
traceback.print_exc()
return -2
retVal = directory_hash.hexdigest()
#print_yellow("Hash Value for {} is {}".format(directory,retVal))
return directory_hash.hexdigest()
def Fract_Sec(s):
temp = float()
temp = float(s) / (60*60*24)
d = int(temp)
temp = (temp - d) * 24
h = int(temp)
temp = (temp - h) * 60
m = int(temp)
temp = (temp - m) * 60
sec = temp
return d,h,m,sec
#endef Fract_Sec
# Copyright (c) André Burgaud
# http://www.burgaud.com/bring-colors-to-the-windows-console-with-python/
if sys.platform == "win32":
from ctypes import windll, Structure, c_short, c_ushort, byref
SHORT = c_short
WORD = c_ushort
class COORD(Structure):
"""struct in wincon.h."""
_fields_ = [
("X", SHORT),
("Y", SHORT)]
class SMALL_RECT(Structure):
"""struct in wincon.h."""
_fields_ = [
("Left", SHORT),
("Top", SHORT),
("Right", SHORT),
("Bottom", SHORT)]
class CONSOLE_SCREEN_BUFFER_INFO(Structure):
"""struct in wincon.h."""
_fields_ = [
("dwSize", COORD),
("dwCursorPosition", COORD),
("wAttributes", WORD),
("srWindow", SMALL_RECT),
("dwMaximumWindowSize", COORD)]
# winbase.h
STD_INPUT_HANDLE = -10
STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12
# wincon.h
FOREGROUND_BLACK = 0x0000
FOREGROUND_BLUE = 0x0001
FOREGROUND_GREEN = 0x0002
FOREGROUND_CYAN = 0x0003
FOREGROUND_RED = 0x0004
FOREGROUND_MAGENTA = 0x0005
FOREGROUND_YELLOW = 0x0006
FOREGROUND_GREY = 0x0007
FOREGROUND_INTENSITY = 0x0008 # foreground color is intensified.
BACKGROUND_BLACK = 0x0000
BACKGROUND_BLUE = 0x0010
BACKGROUND_GREEN = 0x0020
BACKGROUND_CYAN = 0x0030
BACKGROUND_RED = 0x0040
BACKGROUND_MAGENTA = 0x0050
BACKGROUND_YELLOW = 0x0060
BACKGROUND_GREY = 0x0070
BACKGROUND_INTENSITY = 0x0080 # background color is intensified.
stdout_handle = windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute
GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo
def get_text_attr():
"""Returns the character attributes (colors) of the console screen
buffer."""
csbi = CONSOLE_SCREEN_BUFFER_INFO()
GetConsoleScreenBufferInfo(stdout_handle, byref(csbi))
return csbi.wAttributes
def set_text_attr(color):
"""Sets the character attributes (colors) of the console screen
buffer. Color is a combination of foreground and background color,
foreground and background intensity."""
SetConsoleTextAttribute(stdout_handle, color)
###############################################################################
def find_bi_tools(work_drive):
"""Find BI tools."""
reg = winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER)
try:
k = winreg.OpenKey(reg, r"Software\bohemia interactive\arma 3 tools")
arma3tools_path = winreg.QueryValueEx(k, "path")[0]
winreg.CloseKey(k)
except:
raise Exception("BadTools","Arma 3 Tools are not installed correctly or the P: drive needs to be created.")
addonbuilder_path = os.path.join(arma3tools_path, "AddonBuilder", "AddonBuilder.exe")
dssignfile_path = os.path.join(arma3tools_path, "DSSignFile", "DSSignFile.exe")
dscreatekey_path = os.path.join(arma3tools_path, "DSSignFile", "DSCreateKey.exe")
cfgconvert_path = os.path.join(arma3tools_path, "CfgConvert", "CfgConvert.exe")
if os.path.isfile(addonbuilder_path) and os.path.isfile(dssignfile_path) and os.path.isfile(dscreatekey_path) and os.path.isfile(cfgconvert_path):
return [addonbuilder_path, dssignfile_path, dscreatekey_path, cfgconvert_path]
else:
raise Exception("BadTools","Arma 3 Tools are not installed correctly or the P: drive needs to be created.")
def find_depbo_tools():
"""Use registry entries to find DePBO-based tools."""
requiredToolPaths = {"pboProject": None, "rapify": None, "MakePbo": None}
failed = False
for tool in requiredToolPaths:
try:
try:
k = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Mikero\{}".format(tool))
path = winreg.QueryValueEx(k, "exe")[0]
except FileNotFoundError:
k = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"Software\Mikero\{}".format(tool))
path = winreg.QueryValueEx(k, "exe")[0]
except FileNotFoundError:
print_error("Could not find {}".format(tool))
failed = True
else:
#Strip any quotations from the path due to a MikeRo tool bug which leaves a trailing space in some of its registry paths.
requiredToolPaths[tool] = path.strip('"')
print_green("Found {}.".format(tool))
finally:
winreg.CloseKey(k)
if failed:
raise Exception("BadDePBO", "DePBO tools not installed correctly")
return requiredToolPaths
def color(color):
"""Set the color. Works on Win32 and normal terminals."""
if sys.platform == "win32":
if color == "green":
set_text_attr(FOREGROUND_GREEN | get_text_attr() & 0x0070 | FOREGROUND_INTENSITY)
elif color == "yellow":
set_text_attr(FOREGROUND_YELLOW | get_text_attr() & 0x0070 | FOREGROUND_INTENSITY)
elif color == "red":
set_text_attr(FOREGROUND_RED | get_text_attr() & 0x0070 | FOREGROUND_INTENSITY)
elif color == "blue":
set_text_attr(FOREGROUND_BLUE | get_text_attr() & 0x0070 | FOREGROUND_INTENSITY)
elif color == "reset":
set_text_attr(FOREGROUND_GREY | get_text_attr() & 0x0070)
elif color == "grey":
set_text_attr(FOREGROUND_GREY | get_text_attr() & 0x0070)
else :
if color == "green":
sys.stdout.write('\033[92m')
elif color == "red":
sys.stdout.write('\033[91m')
elif color == "blue":
sys.stdout.write('\033[94m')
elif color == "reset":
sys.stdout.write('\033[0m')
def print_error(msg):
color("red")
print("ERROR: {}".format(msg))
color("reset")
global printedErrors
printedErrors += 1
def print_green(msg):
color("green")
print(msg)
color("reset")
def print_blue(msg):
color("blue")
print(msg)
color("reset")
def print_yellow(msg):
color("yellow")
print(msg)
color("reset")
def copy_important_files(source_dir,destination_dir):
originalDir = os.getcwd()
# Copy importantFiles
try:
print_blue("\nSearching for important files in {}".format(source_dir))
print("Source_dir: {}".format(source_dir))
print("Destination_dir: {}".format(destination_dir))
for file in importantFiles:
filePath = os.path.join(module_root_parent, file)
if os.path.exists(filePath):
print_green("Copying file => {}".format(filePath))
shutil.copy(os.path.join(source_dir,filePath), destination_dir)
else:
missingFiles.append("{}".format(filePath))
print_error("Failed copying file => {}".format(filePath))
except:
print_error("COPYING IMPORTANT FILES.")
raise
# Copy all extension DLL's
try:
os.chdir(os.path.join(source_dir))
print_blue("\nSearching for DLLs in {}".format(os.getcwd()))
filenames = glob.glob("*.dll")
if not filenames:
print ("Empty SET")
for dll in filenames:
print_green("Copying dll => {}".format(os.path.join(source_dir,dll)))
if os.path.isfile(dll):
shutil.copyfile(os.path.join(source_dir,dll),os.path.join(destination_dir,dll))
except:
print_error("COPYING DLL FILES.")
raise
finally:
os.chdir(originalDir)
def copy_optionals_for_building(mod,pbos):
src_directories = os.listdir(optionals_root)
current_dir = os.getcwd()
print_blue("\nChecking optionals folder...")
try:
#special server.pbo processing
files = glob.glob(os.path.join(release_dir, project, "optionals", "*.pbo"))
for file in files:
file_name = os.path.basename(file)
#print ("Adding the following file: {}".format(file_name))
pbos.append(file_name)
pbo_path = os.path.join(release_dir, project, "optionals", file_name)
sigFile_name = file_name +"."+ key_name + ".bisign"
sig_path = os.path.join(release_dir, project, "optionals", sigFile_name)
if (os.path.isfile(pbo_path)):
print("Moving {} for processing.".format(pbo_path))
shutil.move(pbo_path, os.path.join(release_dir, project, "addons", file_name))
if (os.path.isfile(sig_path)):
#print("Moving {} for processing.".format(sig_path))
shutil.move(sig_path, os.path.join(release_dir, project, "addons", sigFile_name))
except:
print_error("Error in moving")
raise
finally:
os.chdir(current_dir)
try:
for dir_name in src_directories:
mod.append(dir_name)
#userconfig requires special handling since it is not a PBO source folder.
#CfgConvert fails to build server.pbo if userconfig is not found in P:\
if (dir_name == "userconfig"):
if (os.path.exists(os.path.join(release_dir, project, "optionals", dir_name))):
shutil.rmtree(os.path.join(release_dir, project, "optionals", dir_name), True)
shutil.copytree(os.path.join(optionals_root,dir_name), os.path.join(release_dir, project, "optionals", dir_name))
destination = os.path.join(work_drive,dir_name)
else:
destination = os.path.join(module_root,dir_name)
print("Temporarily copying {} => {} for building.".format(os.path.join(optionals_root,dir_name),destination))
if (os.path.exists(destination)):
shutil.rmtree(destination, True)
shutil.copytree(os.path.join(optionals_root,dir_name), destination)
except:
print_error("Copy Optionals Failed")
raise
finally:
os.chdir(current_dir)
def cleanup_optionals(mod):
print("")
try:
for dir_name in mod:
#userconfig requires special handling since it is not a PBO source folder.
if (dir_name == "userconfig"):
destination = os.path.join(work_drive,dir_name)
else:
destination = os.path.join(module_root,dir_name)
print("Cleaning {}".format(destination))
try:
file_name = "{}{}.pbo".format(pbo_name_prefix,dir_name)
src_file_path = os.path.join(release_dir, project, "addons", file_name)
dst_file_path = os.path.join(release_dir, project, "optionals", file_name)
sigFile_name = "{}.{}.bisign".format(file_name,key_name)
src_sig_path = os.path.join(release_dir, project, "addons", sigFile_name)
dst_sig_path = os.path.join(release_dir, project, "optionals", sigFile_name)
if (os.path.isfile(src_file_path)):
#print("Preserving {}".format(file_name))
os.renames(src_file_path,dst_file_path)
if (os.path.isfile(src_sig_path)):
#print("Preserving {}".format(sigFile_name))
os.renames(src_sig_path,dst_sig_path)
except FileExistsError:
print_error("{} already exists".format(file_name))
continue
shutil.rmtree(destination)
except FileNotFoundError:
print_yellow("{} file not found".format(file_name))
except:
print_error("Cleaning Optionals Failed")
raise
def purge(dir, pattern, friendlyPattern="files"):
print_green("Deleting {} files from directory: {}".format(friendlyPattern,dir))
if os.path.exists(dir):
for f in os.listdir(dir):
if re.search(pattern, f):
os.remove(os.path.join(dir, f))
def build_signature_file(file_name):
global key
global dssignfile
global signature_blacklist
ret = 0
baseFile = os.path.basename(file_name)
#print_yellow("Sig_fileName: {}".format(baseFile))
if not (baseFile in signature_blacklist):
print("Signing with {}.".format(key))
ret = subprocess.call([dssignfile, key, file_name])
if ret == 0:
return True
else:
return False
def check_for_obsolete_pbos(addonspath, file):
module = file[len(pbo_name_prefix):-4]
if not os.path.exists(os.path.join(addonspath, module)):
return True
return False
def backup_config(module):
#backup original $PBOPREFIX$
global work_drive
global prefix
try:
configpath = os.path.join(work_drive, prefix, module, "$PBOPREFIX$")
if os.path.isfile(configpath):
shutil.copyfile(configpath, os.path.join(work_drive, prefix, module, "$PBOPREFIX$.backup"))
else:
print_error("$PBOPREFIX$ Does not exist for module: {}.".format(module))
except:
print_error("Error creating backup of $PBOPREFIX$ for module {}.".format(module))
return True
def addon_restore(modulePath):
#restore original $PBOPREFIX$
try:
if os.path.isfile(os.path.join(modulePath, "$PBOPREFIX$.backup")):
if os.path.isfile(os.path.join(modulePath, "$PBOPREFIX$")):
os.remove(os.path.join(modulePath, "$PBOPREFIX$"))
os.rename(os.path.join(modulePath, "$PBOPREFIX$.backup"), os.path.join(modulePath, "$PBOPREFIX$"))
except:
print_yellow("Some error occurred. Check your addon folder {} for integrity".format(modulePath))
return True
def get_project_version(version_increments=[]):
global project_version
versionStamp = project_version
#do the magic based on https://github.com/acemod/ACE3/issues/806#issuecomment-95639048
try:
scriptModPath = os.path.join(module_root, "main\script_version.hpp")
if os.path.isfile(scriptModPath):
f = open(scriptModPath, "r")
hpptext = f.read()
f.close()
if hpptext:
majorText = re.search(r"#define MAJOR (.*\b)", hpptext).group(1)
minorText = re.search(r"#define MINOR (.*\b)", hpptext).group(1)
patchText = re.search(r"#define PATCHLVL (.*\b)", hpptext).group(1)
buildText = re.search(r"#define BUILD (.*\b)", hpptext).group(1)
# Increment version (reset all below except build)
if version_increments != []:
if "major" in version_increments:
majorText = int(majorText) + 1
minorText = 0
patchText = 0
elif "minor" in version_increments:
minorText = int(minorText) + 1
patchText = 0
elif "patch" in version_increments:
patchText = int(patchText) + 1
# Always increment build
if "build" in version_increments:
buildText = int(buildText) + 1
print_green("Incrementing version to {}.{}.{}.{}".format(majorText,minorText,patchText,buildText))
with open(scriptModPath, "w", newline="\n") as file:
file.writelines([
"#define MAJOR {}\n".format(majorText),
"#define MINOR {}\n".format(minorText),
"#define PATCHLVL {}\n".format(patchText),
"#define BUILD {}\n".format(buildText)
])
if majorText:
versionStamp = "{}.{}.{}.{}".format(majorText,minorText,patchText,buildText)
else:
print_error("A Critical file seems to be missing or inaccessible: {}".format(scriptModPath))
raise FileNotFoundError("File Not Found: {}".format(scriptModPath))
except Exception as e:
print_error("Get_project_version error: {}".format(e))
print_error("Check the integrity of the file: {}".format(scriptModPath))
versionStamp = project_version
print_error("Resetting to the default version stamp: {}".format(versionStamp))
input("Press Enter to continue...")
print("Resuming build...")
print_yellow("{} VERSION set to {}".format(project.lstrip("@").upper(),versionStamp))
project_version = versionStamp
return project_version
def replace_file(filePath, oldSubstring, newSubstring):
for line in fileinput.input(filePath, inplace=True):
# Use stdout directly, print() adds newlines automatically
sys.stdout.write(line.replace(oldSubstring,newSubstring))
def set_version_in_files():
newVersion = project_version # MAJOR.MINOR.PATCH.BUILD
newVersionArr = newVersion.split(".")
newVersionShort = ".".join((newVersionArr[0],newVersionArr[1],newVersionArr[2])) # MAJOR.MINOR.PATCH
# Regex patterns
pattern = re.compile(r"([\d]+\.[\d]+\.[\d]+\.[\d]+)") # MAJOR.MINOR.PATCH.BUILD
patternShort = re.compile(r"([\d]+\.[\d]+\.[\d]+)") # MAJOR.MINOR.PATCH
# Change versions in files containing version
for i in versionFiles:
filePath = os.path.join(module_root_parent, i)
try:
# Save the file contents to a variable if the file exists
if os.path.isfile(filePath):
f = open(filePath, "r+")
fileText = f.read()
f.close()
if fileText:
# Version string files
# Search and save version stamp
versionsFound = re.findall(pattern, fileText) + re.findall(patternShort, fileText)
# Filter out sub-versions of other versions
versionsFound = [j for i, j in enumerate(versionsFound) if all(j not in k for k in versionsFound[i + 1:])]
# Replace version stamp if any of the new version parts is higher than the one found
for versionFound in versionsFound:
if versionFound:
# Use the same version length as the one found
newVersionUsed = "" # In case undefined
if versionFound.count(".") == newVersion.count("."):
newVersionUsed = newVersion
if versionFound.count(".") == newVersionShort.count("."):
newVersionUsed = newVersionShort
# Print change and modify the file if changed
if newVersionUsed and versionFound != newVersionUsed:
print_green("Changing version {} => {} in {}".format(versionFound, newVersionUsed, filePath))
replace_file(filePath, versionFound, newVersionUsed)
except WindowsError as e:
# Temporary file is still "in use" by Python, pass this exception
pass
except Exception as e:
print_error("set_version_in_files error: {}".format(e))
raise
return True
def stash_version_files_for_building():
try:
for file in versionFiles:
filePath = os.path.join(module_root_parent, file)
if os.path.exists(filePath):
# Take only file name for stash location if in subfolder (otherwise it gets removed when removing folders from release dir)
if "\\" in file:
count = file.count("\\")
file = file.split("\\", count)[-1]
stashPath = os.path.join(release_dir, file)
print("Temporarily stashing {} => {}.bak for version update".format(filePath, stashPath))
shutil.copy(filePath, "{}.bak".format(stashPath))
else:
print_error("Failed temporarily stashing {} for version update".format(filePath))
missingFiles.append("{}".format(filePath))
except:
print_error("Stashing version files failed")
raise
# Set version
set_version_in_files()
return True
def restore_version_files():
try:
print_blue("\nRestoring version files...")
for file in versionFiles:
filePath = os.path.join(module_root_parent, file)
# Take only file name for stash path if in subfolder (otherwise it gets removed when removing folders from release dir)
if "\\" in file:
count = file.count("\\")
file = file.split("\\", count)[-1]
stashPath = os.path.join(release_dir, file)
if os.path.exists(filePath):
print("Restoring {}".format(filePath))
shutil.move("{}.bak".format(stashPath), filePath)
except:
print_error("Restoring version files failed")
raise
return True
def get_private_keyname(commitID,module="main"):
global pbo_name_prefix
global project_version
keyName = str("{prefix}{version}-{commit_id}".format(prefix=pbo_name_prefix,version=project_version,commit_id=commitID))
return keyName
def get_commit_ID():
# Get latest commit ID
global make_root
curDir = os.getcwd()
commit_id = ""
try:
# Verify if Git repository
gitpath = os.path.join(os.path.dirname(make_root), ".git")
assert os.path.exists(gitpath)
# Try to get commit ID through Git client
os.chdir(make_root)
commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"])
commit_id = str(commit_id, "utf-8")[:8]
except FileNotFoundError:
# Try to get commit ID from git files (subprocess failed - eg. no Git client)
head_path = os.path.join(gitpath, "HEAD")
if os.path.exists(head_path):
with open(head_path, "r") as head_file:
branch_path = head_file.readline().split(": ")
# Commit ID is written in HEAD file directly when in detached state
if len(branch_path) == 1:
commit_id = branch_path[0]
else:
branch_path = branch_path[-1].strip()
ref_path = os.path.join(gitpath, branch_path)
if os.path.exists(ref_path):
with open(ref_path, "r") as ref_file:
commit_id = ref_file.readline()
if commit_id != "":
commit_id = commit_id.strip()[:8]
else:
raise
except:
# All other exceptions (eg. AssertionException)
if commit_id == "":
raise
finally:
pass
if commit_id == "":
print_error("Failed to determine commit ID - folder is not a Git repository.")
commit_id = "NOGIT"
os.chdir(curDir)
print_yellow("COMMIT ID set to {}".format(commit_id))
return commit_id
def version_stamp_pboprefix(module,commitID):
### Update pboPrefix with the correct version stamp. Use commit_id as the build number.
#This function will not handle any $PBOPREFIX$ backup or cleanup.
global work_drive
global prefix
configpath = os.path.join(work_drive, prefix, module, "$PBOPREFIX$")
try:
f = open(configpath, "r")
configtext = f.read()
f.close()
if configtext:
if re.search(r"version=(.*?)$", configtext, re.DOTALL):
if configtext:
configtext = re.sub(r"version=(.*?)$", "version={}\n".format(commitID), configtext, flags=re.DOTALL)
f = open(configpath, "w")
f.write(configtext)
f.close()
else:
os.remove(os.path.join(work_drive, prefix, module, "$PBOPREFIX$"))
os.rename(os.path.join(work_drive, prefix, module, "$PBOPREFIX$.backup"), os.path.join(work_drive, prefix, module, "$PBOPREFIX$"))
else:
if configtext:
#append version info
f = open(configpath, "a")
f.write("\nversion = {}".format(commitID))
f.close()
else:
os.remove(os.path.join(work_drive, prefix, module, "$PBOPREFIX$"))
os.rename(os.path.join(work_drive, prefix, module, "$PBOPREFIX$.backup"), os.path.join(work_drive, prefix, module, "$PBOPREFIX$"))
except Exception as e:
print_error("Failed to include build number: {}".format(e))
return False
return True
###############################################################################
def main(argv):
"""Build an Arma addon suite in a directory from rules in a make.cfg file."""
print_blue("\nmake.py for Arma, modified for Advanced Combat Environment v{}".format(__version__))
global project_version
global arma3tools_path
global work_drive
global module_root
global make_root
global release_dir
global module_root_parent
global optionals_root
global key_name
global key
global dssignfile
global prefix
global pbo_name_prefix
global ciBuild
global missingFiles
global failedBuilds
global printedErrors
printedErrors = 0
if sys.platform != "win32":
print_error("Non-Windows platform (Cygwin?). Please re-run from cmd.")
sys.exit(1)
reg = winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER)
try:
k = winreg.OpenKey(reg, r"Software\bohemia interactive\arma 3 tools")
arma3tools_path = winreg.QueryValueEx(k, "path")[0]
winreg.CloseKey(k)
except:
raise Exception("BadTools","Arma 3 Tools are not installed correctly or the P: drive needs to be created.")
# Default behaviors
test = False # Copy to Arma 3 directory?
arg_modules = False # Only build modules on command line?
use_pboproject = True # Default to pboProject build tool
make_target = "DEFAULT" # Which section in make.cfg to use for the build
new_key = True # Make a new key and use it to sign?
quiet = False # Suppress output from build tool?
# Parse arguments
if "help" in argv or "-h" in argv or "--help" in argv:
print ("""
make.py [help] [test] [force] [key <name>] [target <name>] [release <version>]
[module name] [module name] [...]
test -- Copy result to Arma 3.
release <version> -- Make archive with <version>.
force -- Ignore cache and build all.
checkexternal -- Check External Files
target <name> -- Use rules in make.cfg under heading [<name>] rather than
default [Make]
key <name> -- Use key in working directory with <name> to sign. If it does not
exist, create key.
quiet -- Suppress command line output from build tool.
If module names are specified, only those modules will be built.
Examples:
make.py force test
Build all modules (ignoring cache) and copy the mod folder to the Arma 3
directory.
make.py mymodule_gun
Only build the module named 'mymodule_gun'.
make.py force key MyNewKey release 1.0
Build all modules (ignoring cache), sign them with NewKey, and pack them
into a zip file for release with version 1.0.
If a file called $NOBIN$ is found in the module directory, that module will not be binarized.
See the make.cfg file for additional build options.
""")
sys.exit(0)
if "force" in argv:
argv.remove("force")
force_build = True
else:
force_build = False
if "test" in argv:
test = True
argv.remove("test")
if "release" in argv:
make_release_zip = True
argv.remove("release")
else:
make_release_zip = False
if "target" in argv:
make_target = argv[argv.index("target") + 1]
argv.remove("target")
argv.remove(make_target)
force_build = True
if "key" in argv:
new_key = True
key_name = argv[argv.index("key") + 1]
argv.remove("key")
argv.remove(key_name)
if "quiet" in argv:
quiet = True
argv.remove("quiet")
if "checkexternal" in argv:
argv.remove("checkexternal")
check_external = True
else:
check_external = False
if "version" in argv:
argv.remove("version")
version_update = True
else:
version_update = False
version_increments = []
if "increment_build" in argv:
argv.remove("increment_build")
version_increments.append("build")
if "increment_patch" in argv:
argv.remove("increment_patch")
version_increments.append("patch")
if "increment_minor" in argv:
argv.remove("increment_minor")
version_increments.append("minor")
if "increment_major" in argv:
argv.remove("increment_major")
version_increments.append("major")
if "ci" in argv:
argv.remove("ci")
ciBuild = True
print_yellow("\nCheck external references is set to {}".format(str(check_external)))
# Get the directory the make script is in.
make_root = os.path.dirname(os.path.realpath(__file__))
make_root_parent = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
os.chdir(make_root)
cfg = configparser.ConfigParser();
try:
cfg.read(os.path.join(make_root, "make.cfg"))
# Project name (with @ symbol)
project = cfg.get(make_target, "project", fallback="@"+os.path.basename(os.getcwd()))
# BI Tools work drive on Windows
work_drive = cfg.get(make_target, "work_drive", fallback="P:\\")
# Private key path
key = cfg.get(make_target, "key", fallback=None)
# Private key creation directory
private_key_path = cfg.get(make_target, "private_key_path", fallback=os.path.join(work_drive, "private_keys"))
# Project prefix (folder path)
prefix = cfg.get(make_target, "prefix", fallback="")
# Release archive prefix
zipPrefix = cfg.get(make_target, "zipPrefix", fallback=project.lstrip("@").lower())
# Should we autodetect modules on a complete build?
module_autodetect = cfg.getboolean(make_target, "module_autodetect", fallback=True)
# Manual list of modules to build for a complete build
modules = cfg.get(make_target, "modules", fallback=None)
# Parse it out
if modules:
modules = [x.strip() for x in modules.split(',')]
else:
modules = []
# List of directories to ignore when detecting
ignore = [x.strip() for x in cfg.get(make_target, "ignore", fallback="release").split(',')]
# Which build tool should we use?
build_tool = cfg.get(make_target, "build_tool", fallback="addonbuilder").lower()
# Release/build directory, relative to script dir
release_dir = cfg.get(make_target, "release_dir", fallback="release")
#Directory to copy the final built PBO's for a test run.
test_dir = cfg.get(make_target, "test_dir", fallback=os.path.join(os.environ["USERPROFILE"],r"documents\Arma 3"))
# Project PBO file prefix (files are renamed to prefix_name.pbo)
pbo_name_prefix = cfg.get(make_target, "pbo_name_prefix", fallback=None)
# Project module Root
module_root_parent = os.path.abspath(os.path.join(os.path.join(work_drive, prefix), os.pardir))
module_root = cfg.get(make_target, "module_root", fallback=os.path.join(make_root_parent, "addons"))
optionals_root = os.path.join(module_root_parent, "optionals")
extensions_root = os.path.join(module_root_parent, "extensions")
if (os.path.isdir(module_root)):
os.chdir(module_root)
else:
print_error ("Directory {} does not exist.".format(module_root))
sys.exit(1)
commit_id = get_commit_ID()
get_project_version(version_increments)
key_name = versionStamp = get_private_keyname(commit_id)
print_green ("module_root: {}".format(module_root))
if (os.path.isdir(optionals_root)):
print_green ("optionals_root: {}".format(optionals_root))
else:
print("optionals_root does not exist: {}".format(optionals_root))
print_green ("release_dir: {}".format(release_dir))
except:
raise
print_error("Could not parse make.cfg.")
sys.exit(1)
# See if we have been given specific modules to build from command line.
if len(argv) > 1 and not make_release_zip:
arg_modules = True
modules = [a for a in argv[1:] if a[0] != "-"]
# Find the tools we need.
try:
tools = find_bi_tools(work_drive)
addonbuilder = tools[0]
dssignfile = tools[1]
dscreatekey = tools[2]
cfgconvert = tools[3]
except:
print_error("Arma 3 Tools are not installed correctly or the P: drive has not been created.")
sys.exit(1)
if build_tool == "pboproject":
try:
depbo_tools = find_depbo_tools()
pboproject = depbo_tools["pboProject"]
rapifyTool = depbo_tools["rapify"]
makepboTool = depbo_tools["MakePbo"]
except:
raise
print_error("Could not find dePBO tools. Download the needed tools from: https://dev.withsix.com/projects/mikero-pbodll/files")
sys.exit(1)
# Try to open and deserialize build cache file.
try:
cache = {}
with open(os.path.join(make_root, "make.cache"), 'r') as f:
cache_raw = f.read()
cache = json.loads(cache_raw)
except:
print ("No cache found.")
cache = {}
# Check the build version (from main) with cached version - forces a full rebuild when version changes
cacheVersion = "None";
if 'cacheVersion' in cache:
cacheVersion = cache['cacheVersion']
if (project_version != cacheVersion):
cache = {}
print("Reseting Cache {0} to New Version {1}".format(cacheVersion, project_version))
cache['cacheVersion'] = project_version
if not os.path.isdir(os.path.join(release_dir, project, "addons")):
try:
os.makedirs(os.path.join(release_dir, project, "addons"))
except:
print_error("Cannot create release directory")
raise
if not os.path.isdir(os.path.join(release_dir, project, "keys")):
try:
os.makedirs(os.path.join(release_dir, project, "keys"))
except:
print_error("Cannot create release directory")
raise
failedBuilds = []
missingFiles = []
# Update version stamp in all files that contain it
# Update version only for release if full update not requested (backup and restore files)
print_blue("\nChecking for obsolete version numbers...")
if not version_update:
stash_version_files_for_building()
else:
# Set version
set_version_in_files();
print("Version in files has been changed, make sure you commit and push the updates!")
try:
# Temporarily copy optionals_root for building. They will be removed later.
if (os.path.isdir(optionals_root)):
optionals_modules = []
optional_files = []
copy_optionals_for_building(optionals_modules,optional_files)
# Get list of subdirs in make root.
dirs = next(os.walk(module_root))[1]
# Autodetect what directories to build.
if module_autodetect and not arg_modules:
modules = []
for path in dirs:
# Any dir that has a config.cpp in its root is an addon to build.
config_path = os.path.join(path, 'config.cpp')
if os.path.isfile(config_path) and not path in ignore:
modules.append(path)
# Make the key specified from command line if necessary.
if new_key:
if not os.path.isfile(os.path.join(private_key_path, key_name + ".biprivatekey")):
print_yellow("\nRequested key does not exist.")
try:
os.makedirs(private_key_path)
except:
pass
curDir = os.getcwd()
os.chdir(private_key_path)
ret = subprocess.call([dscreatekey, key_name]) # Created in make_root
os.chdir(curDir)
if ret == 0:
print_green("Created: {}".format(os.path.join(private_key_path, key_name + ".biprivatekey")))
print("Removing any old signature keys...")
purge(os.path.join(module_root, release_dir, project, "addons"), "^.*\.bisign$","*.bisign")
purge(os.path.join(module_root, release_dir, project, "optionals"), "^.*\.bisign$","*.bisign")
purge(os.path.join(module_root, release_dir, project, "keys"), "^.*\.bikey$","*.bikey")
else:
print_error("Failed to create key!")
else:
print_green("\nNOTE: Using key {}".format(os.path.join(private_key_path, key_name + ".biprivatekey")))
try:
print("Copying public key to release directory.")
try:
os.makedirs(os.path.join(module_root, release_dir, project, "keys"))
except:
pass
# Use biKeyNameAbrev to attempt to minimize problems from this BI Bug REFERENCE: http://feedback.arma3.com/view.php?id=22133
biKeyNameAbrev = key_name.split("-")[0]
shutil.copyfile(os.path.join(private_key_path, key_name + ".bikey"), os.path.join(module_root, release_dir, project, "keys", "{}.bikey".format(biKeyNameAbrev)))
except:
print_error("Could not copy key to release directory.")
raise
key = os.path.join(private_key_path, "{}.biprivatekey".format(key_name))
# Remove any obsolete files.
print_blue("\nChecking for obsolete files...")
obsolete_check_path = os.path.join(module_root, release_dir, project,"addons")
for file in os.listdir(obsolete_check_path):
if (file.endswith(".pbo") and os.path.isfile(os.path.join(obsolete_check_path,file))):
if check_for_obsolete_pbos(module_root, file):
fileName = os.path.splitext(file)[0]
print_yellow("Removing obsolete pbo => {}".format(file))
purge(obsolete_check_path, "{}\..".format(fileName), "{}.*".format(fileName))
obsolete_check_path = os.path.join(module_root, release_dir, project)
for file in os.listdir(obsolete_check_path):
if (file.endswith(".dll") and os.path.isfile(os.path.join(obsolete_check_path,file))):
if not os.path.exists(os.path.join(module_root_parent, file)):
print_yellow("Removing obsolete dll => {}".format(file))
try:
os.remove(os.path.join(obsolete_check_path,file))
except:
print_error("\nFailed to delete {}".format(os.path.join(obsolete_check_path,file)))
pass
# For each module, prep files and then build.
print_blue("\nBuilding...")
for module in modules:
print_green("\nMaking {}".format(module + "-"*max(1, (60-len(module)))))
missing = False
sigMissing = False
# Cache check
if module in cache:
old_sha = cache[module]
else:
old_sha = ""
# Hash the module
new_sha = get_directory_hash(os.path.join(module_root, module))
# Is the pbo or sig file missing?
missing = not os.path.isfile(os.path.join(release_dir, project, "addons", "{}{}.pbo".format(pbo_name_prefix,module)))
sigFile = "{}{}.pbo.{}.bisign".format(pbo_name_prefix,module,key_name)
sigMissing = not os.path.isfile(os.path.join(release_dir, project, "addons", sigFile))
if missing:
print_yellow("Missing PBO file {}{}.pbo. Building...".format(pbo_name_prefix,module))
# Check if it needs rebuilt
# print ("Hash:", new_sha)
if old_sha == new_sha and not missing:
if not force_build:
print("Module has not changed.")
if sigMissing:
if key:
print("Missing Signature key {}".format(sigFile))
build_signature_file(os.path.join(module_root, release_dir, project, "addons", "{}{}.pbo".format(pbo_name_prefix,module)))
# Skip everything else
continue
# Only do this if the project isn't stored directly on the work drive.
# Split the path at the drive name and see if they are on the same drive (usually P:)
if os.path.splitdrive(module_root)[0] != os.path.splitdrive(work_drive)[0]:
try:
# Remove old work drive version (ignore errors)
shutil.rmtree(os.path.join(work_drive, prefix, module), True)
# Copy module to the work drive
shutil.copytree(module, os.path.join(work_drive, prefix, module))
except:
raise
print_error("Could not copy module to work drive. Does the module exist?")
input("Press Enter to continue...")
print("Resuming build...")
continue
#else:
#print("WARNING: Module is stored on work drive ({}).".format(work_drive))
try:
# Remove the old pbo, key, and log
old = os.path.join(module_root, release_dir, project, "addons", "{}{}".format(pbo_name_prefix,module)) + "*"
files = glob.glob(old)
for f in files:
os.remove(f)
if pbo_name_prefix:
old = os.path.join(module_root, release_dir, project, "addons", "{}{}".format(pbo_name_prefix,module)) + "*"
files = glob.glob(old)
for f in files:
os.remove(f)
except:
raise
print_error("Could not copy module to work drive. Does the module exist?")
input("Press Enter to continue...")
print("Resuming build...")
continue
# Build the module into a pbo
print_blue("Building: {}".format(os.path.join(work_drive, prefix, module)))
print_blue("Destination: {}".format(os.path.join(module_root, release_dir, project, "addons")))
# Make destination folder (if needed)
try:
os.makedirs(os.path.join(module_root, release_dir, project, "addons"))
except:
pass
# Run build tool
build_successful = False
if build_tool == "pboproject":
try:
nobinFilePath = os.path.join(work_drive, prefix, module, "$NOBIN$")
backup_config(module)
version_stamp_pboprefix(module,commit_id)
if os.path.isfile(nobinFilePath):
print_green("$NOBIN$ Found. Proceeding with non-binarizing!")
cmd = [makepboTool, "-P","-A","-G","-N","-X=*.backup", os.path.join(work_drive, prefix, module),os.path.join(module_root, release_dir, project,"addons")]
else:
if check_external:
cmd = [pboproject, "-P", os.path.join(work_drive, prefix, module), "+Engine=Arma3", "-S","+Noisy", "+X", "+Clean", "+Mod="+os.path.join(module_root, release_dir, project), "-Key"]
else:
cmd = [pboproject, "-P", os.path.join(work_drive, prefix, module), "+Engine=Arma3", "-S","+Noisy", "-X", "+Clean", "+Mod="+os.path.join(module_root, release_dir, project), "-Key"]
color("grey")
if quiet:
devnull = open(os.devnull, 'w')
ret = subprocess.call(cmd, stdout=devnull)
devnull.close()
else:
ret = subprocess.call(cmd)
color("reset")
if ret == 0:
print_green("pboProject return code == {}".format(str(ret)))
# Prettyprefix rename the PBO if requested.
if pbo_name_prefix:
try:
os.rename(os.path.join(module_root, release_dir, project, "addons", "{}.pbo".format(module)), os.path.join(module_root, release_dir, project, "addons", "{}{}.pbo".format(pbo_name_prefix,module)))
except:
raise
print_error("Could not rename built PBO with prefix.")
# Sign result
if (key and not "{}{}.pbo".format(pbo_name_prefix,module) in signature_blacklist):
print("Signing with {}.".format(key))
if pbo_name_prefix:
ret = subprocess.call([dssignfile, key, os.path.join(module_root, release_dir, project, "addons", "{}{}.pbo".format(pbo_name_prefix,module))])
else:
ret = subprocess.call([dssignfile, key, os.path.join(module_root, release_dir, project, "addons", "{}.pbo".format(module))])
if ret == 0:
build_successful = True
else:
build_successful = True
if not build_successful:
print_error("pboProject return code == {}".format(str(ret)))
print_error("Module not successfully built/signed. Check your {}temp\{}_packing.log for more info.".format(work_drive,module))
print ("Resuming build...")
failedBuilds.append("{}".format(module))
continue
# Back to the root
os.chdir(module_root)
except:
raise
print_error("Could not run Addon Builder.")
input("Press Enter to continue...")
print ("Resuming build...")
continue
finally:
addon_restore(os.path.join(work_drive, prefix, module))
elif build_tool== "addonbuilder":
# Detect $NOBIN$ and do not binarize if found.
if os.path.isfile(os.path.join(work_drive, prefix, module, "$NOBIN$")):
do_binarize = False
print("$NOBIN$ file found in module, packing only.")
else:
do_binarize = True
try:
# Call AddonBuilder
os.chdir("P:\\")
cmd = [addonbuilder, os.path.join(work_drive, prefix, module), os.path.join(make_root, release_dir, project, "addons"), "-clear", "-project="+work_drive]
if not do_binarize:
cmd.append("-packonly")
if quiet:
previousDirectory = os.getcwd()
os.chdir(arma3tools_path)
devnull = open(os.devnull, 'w')
ret = subprocess.call(cmd, stdout=devnull)
devnull.close()
os.chdir(previousDirectory)
else:
previousDirectory = os.getcwd()
os.chdir(arma3tools_path)
print_error("Current directory - {}".format(os.getcwd()))
ret = subprocess.call(cmd)
os.chdir(previousDirectory)
print_error("Current directory - {}".format(os.getcwd()))
color("reset")
print_green("completed")
# Prettyprefix rename the PBO if requested.
if pbo_name_prefix:
try:
os.rename(os.path.join(make_root, release_dir, project, "addons", "{}.pbo".format(module)), os.path.join(make_root, release_dir, project, "addons", "{}{}.pbo".format(pbo_name_prefix,module)))
except:
raise
print_error("Could not rename built PBO with prefix.")
if ret == 0:
# Sign result
#print_yellow("Sig_fileName: ace_{}.pbo".format(module))
if (key and not "{}{}.pbo".format(pbo_name_prefix,module) in signature_blacklist) :
print("Signing with {}.".format(key))
if pbo_name_prefix:
ret = subprocess.call([dssignfile, key, os.path.join(make_root, release_dir, project, "addons","{}{}.pbo".format(pbo_name_prefix,module))])
else:
ret = subprocess.call([dssignfile, key, os.path.join(make_root, release_dir, project, "addons", "{}.pbo".format(module))])
if ret == 0:
build_successful = True
else:
build_successful = True
if not build_successful:
print_error("Module not successfully built. Check your {}temp\{}_packing.log for more info.".format(work_drive,module))
# Back to the root
os.chdir(make_root)
except:
raise
print_error("Could not run Addon Builder.")
input("Press Enter to continue...")
print ("Resuming build...")
continue
else:
print_error("Unknown build_tool {}!".format(build_tool))
# Update the hash for a successfully built module
if build_successful:
cache[module] = new_sha
except Exception as e:
print_yellow("Cancel or some error detected: {}".format(e))
finally:
copy_important_files(module_root_parent,os.path.join(release_dir, project))
if (os.path.isdir(optionals_root)):
cleanup_optionals(optionals_modules)
if not version_update:
restore_version_files()
# Done building all modules!
# Write out the cache state
cache_out = json.dumps(cache)
with open(os.path.join(make_root, "make.cache"), 'w') as f:
f.write(cache_out)
# Delete the pboproject temp files if building a release.
if make_release_zip and build_tool == "pboproject":
try:
shutil.rmtree(os.path.join(release_dir, project, "temp"), True)
except:
print_error("ERROR: Could not delete pboProject temp files.")
# Make release
if make_release_zip:
release_name = "{}_{}".format(zipPrefix, project_version.rsplit(".", 1)[0])
try:
# Delete all log files
for root, dirs, files in os.walk(os.path.join(release_dir, project, "addons")):
for currentFile in files:
if currentFile.lower().endswith("log"):
os.remove(os.path.join(root, currentFile))
# Remove all zip files from release folder to prevent zipping the zip
for file in os.listdir(release_dir):
if file.endswith(".zip"):
os.remove(os.path.join(release_dir, file))
# Create a zip with the contents of release folder in it
print_blue("\nMaking release: {}.zip ...".format(release_name))
print("Packing...")
release_zip = shutil.make_archive("{}".format(release_name), "zip", release_dir)
# Move release zip to release folder
shutil.copy(release_zip, release_dir)
os.remove(release_zip)
except:
raise
print_error("Could not make release.")
# Copy to Arma 3 folder for testing
if test:
print_blue("\nCopying to Arma 3.")
if sys.platform == "win32":
reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE)
try:
k = winreg.OpenKey(reg, r"SOFTWARE\Wow6432Node\Bohemia Interactive\Arma 3")
a3_path = winreg.EnumValue(k, 1)[1]
winreg.CloseKey(k)
except:
print_error("Could not find Arma 3's directory in the registry.")
else:
a3_path = cygwin_a3path
print_yellow("Path from the registry => {}".format(a3_path))
a3_path = test_dir
print_yellow("Copying build files to {}".format(a3_path))
if os.path.exists(a3_path):
try:
shutil.rmtree(os.path.join(a3_path, project), True)
shutil.copytree(os.path.join(module_root, release_dir, project), os.path.join(a3_path, project))
except:
print_error("Could not copy files. Is Arma 3 running?")
tracedErrors = len(failedBuilds) + len(missingFiles)
if printedErrors > 0: # printedErrors includes tracedErrors
printedOnlyErrors = printedErrors - tracedErrors
print()
print_error("Failed with {} errors.".format(printedErrors))
if len(failedBuilds) > 0:
for failedBuild in failedBuilds:
print("- {} build failed!".format(failedBuild))
if len(missingFiles) > 0:
for missingFile in missingFiles:
print("- {} not found!".format(missingFile))
if printedOnlyErrors > 0:
print_yellow("- {} untraced error(s)!".format(printedOnlyErrors))
else:
print_green("\nCompleted with 0 errors.")
if __name__ == "__main__":
start_time = timeit.default_timer()
main(sys.argv)
d,h,m,s = Fract_Sec(timeit.default_timer() - start_time)
print("\nTotal Program time elapsed: {0:2}h {1:2}m {2:4.5f}s".format(h,m,s))
if ciBuild:
sys.exit(0)
input("Press Enter to continue...")
Reference in New Issue View Git Blame Copy Permalink