#506 two-file bundled offline installer generation

build/_offline.lua

@@ -0,0 +1,114 @@
+---@diagnostic disable: undefined-global
+
+local b64_lookup = {
+    ['A'] = 0,  ['B'] = 1,  ['C'] = 2,  ['D'] = 3,  ['E'] = 4,  ['F'] = 5,  ['G'] = 6,  ['H'] = 7,  ['I'] = 8,  ['J'] = 9,  ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, ['Y'] = 24, ['Z'] = 25,
+    ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31, ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39, ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47, ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51,
+    ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55, ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63
+}
+
+local BYTE  = 0xFF
+local CHAR  = string.char
+local BOR   = bit.bor           ---@type function
+local BAND  = bit.band          ---@type function
+local LSHFT = bit.blshift       ---@type function
+local RSHFT = bit.blogic_rshift ---@type function
+
+-- decode a base64 string
+---@param input string
+local function b64_decode(input)
+---@diagnostic disable-next-line: undefined-field
+    local t_start = os.epoch("local")
+
+    local decoded = {}
+
+    local c_idx, idx = 1, 1
+
+    for _ = 1, input:len() / 4 do
+        local block = input:sub(idx, idx + 4)
+        local word = 0x0
+
+        -- build the 24-bit sequence from the 4 characters
+        for i = 1, 4 do
+            local num = b64_lookup[block:sub(i, i)]
+
+            if num then
+                word = BOR(word, LSHFT(b64_lookup[block:sub(i, i)], (4 - i) * 6))
+            end
+        end
+
+        -- decode the 24-bit sequence as 8 bytes
+        for i = 1, 3 do
+            local char = BAND(BYTE, RSHFT(word, (3 - i) * 8))
+
+            if char ~= 0 then
+                decoded[c_idx] = CHAR(char)
+                c_idx = c_idx + 1
+            end
+        end
+
+        idx = idx + 4
+    end
+
+---@diagnostic disable-next-line: undefined-field
+    local elapsed = (os.epoch("local") - t_start)
+    local decoded_str = table.concat(decoded)
+
+    return decoded_str, elapsed
+end
+
+-- write files recursively from base64 encodings in a table
+---@param files table
+---@param path string
+local function write_files(files, path)
+    fs.makeDir(path)
+
+    for k, v in pairs(files) do
+        if type(v) == "table" then
+            if k == "system" then
+                -- write system files to root
+                write_files(v, "/")
+            else
+                -- descend into directories
+                write_files(v, path .. "/" .. k .. "/")
+            end
+
+---@diagnostic disable-next-line: undefined-field
+            os.sleep(0.05)
+        else
+            local handle = fs.open(path .. k, "w")
+            local text, time = b64_decode(v)
+
+            print("decoded '" .. k .. "' in " .. time .. "ms")
+
+            handle.write(text)
+            handle.close()
+        end
+    end
+end
+
+local function write_install()
+    local handle = fs.open("install_manifest.json", "w")
+    handle.write(b64_decode(install_manifest))
+    handle.close()
+
+    handle = fs.open("ccmsim.lua", "w")
+    handle.write(b64_decode(ccmsi_offline))
+    handle.close()
+end
+
+lgray()
+
+-- write both app and dependency files
+write_files(app_files, "/")
+write_files(dep_files, "/")
+
+-- write a install manifest and offline installer
+write_install()
+
+green()
+print("Done!")
+white()
+print("Use can use the freshly installed 'ccmsim' program to manage your off-line installation.")
+lgray()
+print("You do not need to run it now though, your system is all ready to start! Launch with 'startup'.")
+white()

build/bundle.py

@@ -1,10 +1,24 @@
 import base64
 import json
 import os
+import subprocess
 import sys
 
 path_prefix = "./_minified/"
 
+# get git build info
+build = subprocess.check_output(["git", "describe", "--tags"]).strip().decode("utf-8")
+
+# list files in a directory
+def list_files(path):
+    list = []
+
+    for (root, dirs, files) in os.walk(path):
+        for f in files:
+            list.append((root[2:] + "/" + f).replace('\\','/'))
+
+    return list
+
 # recursively encode files with base64
 def encode_recursive(path):
     list = {}
@@ -34,12 +48,46 @@ def encode_files(files):
 
     return list
 
+# get the version of an application at the provided path
+def get_version(path, is_lib = False):
+    ver = ""
+    string = ".version = \""
+
+    if not is_lib:
+        string = "_VERSION = \""
+
+    f = open(path, "r")
+
+    for line in f:
+        pos = line.find(string)
+        if pos >= 0:
+            ver = line[(pos + len(string)):(len(line) - 2)]
+            break
+
+    f.close()
+
+    return ver
+
 # file manifest (reflects imgen.py)
 manifest = {
+    "common_versions" : {
+        "bootloader" : get_version("./startup.lua"),
+        "common" : get_version("./scada-common/util.lua", True),
+        "comms" : get_version("./scada-common/comms.lua", True),
+        "graphics" : get_version("./graphics/core.lua", True),
+        "lockbox" : get_version("./lockbox/init.lua", True),
+    },
+    "app_versions" : {
+        "reactor-plc" : get_version("./reactor-plc/startup.lua"),
+        "rtu" : get_version("./rtu/startup.lua"),
+        "supervisor" : get_version("./supervisor/startup.lua"),
+        "coordinator" : get_version("./coordinator/startup.lua"),
+        "pocket" : get_version("./pocket/startup.lua")
+    },
     "files" : {
         # common files
         "system" : encode_files([ "initenv.lua", "startup.lua", "configure.lua", "LICENSE" ]),
-        "common" : encode_recursive(path_prefix + "./scada-common"),
+        "scada-common" : encode_recursive(path_prefix + "./scada-common"),
         "graphics" : encode_recursive(path_prefix + "./graphics"),
         "lockbox" : encode_recursive(path_prefix + "./lockbox"),
         # platform files
@@ -49,13 +97,20 @@ manifest = {
         "coordinator" : encode_recursive(path_prefix + "./coordinator"),
         "pocket" : encode_recursive(path_prefix + "./pocket"),
     },
-    "depends" : {
+    "install_files" : {
-        "reactor-plc" : [ "reactor-plc", "system", "common", "graphics", "lockbox" ],
+        # common files
-        "rtu" : [ "rtu", "system", "common", "graphics", "lockbox" ],
+        "system" : [ "initenv.lua", "startup.lua", "configure.lua", "LICENSE" ],
-        "supervisor" : [ "supervisor", "system", "common", "graphics", "lockbox" ],
+        "scada-common" : list_files("./scada-common"),
-        "coordinator" : [ "coordinator", "system", "common", "graphics", "lockbox" ],
+        "graphics" : list_files("./graphics"),
-        "pocket" : [ "pocket", "system", "common", "graphics", "lockbox" ]
+        "lockbox" : list_files("./lockbox"),
-    }
+        # platform files
+        "reactor-plc" : list_files("./reactor-plc"),
+        "rtu" : list_files("./rtu"),
+        "supervisor" : list_files("./supervisor"),
+        "coordinator" : list_files("./coordinator"),
+        "pocket" : list_files("./pocket"),
+    },
+    "depends" : [ "system", "scada-common", "graphics", "lockbox" ],
 }
 
 # write the application installation items as Lua tables
@@ -71,13 +126,89 @@ def write_items(body, items, indent):
 
     return body
 
+# create output directory
+if not os.path.exists("./BUNDLE"):
+    os.makedirs("./BUNDLE")
 
+# get offline installer
+ccmsim_file = open("./build/ccmsim.lua", "r")
+ccmsim_script = ccmsim_file.read()
+ccmsim_file.close()
 
+# create dependency bundled file
+dep_file = "common_" + build + ".lua"
+f_d = open("./BUNDLE/" + dep_file, "w")
+
+body_b = "local dep_files = {\n"
+
+for depend in manifest["depends"]:
+    body_b = body_b + write_items("", { f"{depend}": manifest["files"][depend] }, 4)
+body_b = body_b + "}\n"
+
+body_b = body_b + f"""
+if select("#", ...) == 0 then
+    term.setTextColor(colors.red)
+    print("You must run the other file you should have uploaded (it has the app in its name).")
+    term.setTextColor(colors.white)
+end
+
+return dep_files
+"""
+
+f_d.write(body_b)
+f_d.close()
+
+# application bundled files
 for app in [ "reactor-plc", "rtu", "supervisor", "coordinator", "pocket" ]:
-    f = open("_" + app + ".lua", "w")
+    app_file = app + "_" + build + ".lua"
-    body = "local application = {\n"
+
-    for depend in manifest["depends"][app]:
+    f_script = open("./build/_offline.lua", "r")
-        body = body + write_items("", { f"{depend}": manifest["files"][depend] }, 4)
+    script = f_script.read()
-    body = body + "}\n\n"
+    f_script.close()
-    f.write(body)
+
-    f.close()
+    f_a = open("./BUNDLE/" + app_file, "w")
+
+    body_a = "local app_files = {\n"
+
+    body_a = body_a + write_items("", { f"{app}": manifest["files"][app] }, 4) + "}\n"
+
+    versions = manifest["common_versions"].copy()
+    versions[app] = manifest["app_versions"][app]
+
+    depends = manifest["depends"].copy()
+    depends.append(app)
+
+    install_manifest = json.dumps({ "versions" : versions, "files" : manifest["install_files"], "depends" : depends })
+
+    body_a = body_a + f"""
+-- install manifest JSON and offline installer
+local install_manifest = "{base64.b64encode(bytes(install_manifest, 'UTF-8')).decode('ASCII')}"
+local ccmsi_offline = "{base64.b64encode(bytes(ccmsim_script, 'UTF-8')).decode('ASCII')}"
+
+local function red() term.setTextColor(colors.red) end
+local function green() term.setTextColor(colors.green) end
+local function white() term.setTextColor(colors.white) end
+local function lgray() term.setTextColor(colors.lightGray) end
+
+if not fs.exists("{dep_file}") then
+    red()
+    print("Missing '{dep_file}'! Please upload it, then run this file again.")
+    white()
+    return
+end
+
+-- rename the dependency file
+fs.move("{dep_file}", "install_depends.lua")
+
+-- load the other file
+local dep_files = require("install_depends")
+
+-- delete the uploaded files to free up space to actually install
+fs.delete("{app_file}")
+fs.delete("install_depends.lua")
+
+-- get started installing
+{script}"""
+
+    f_a.write(body_a)
+    f_a.close()

build/ccmsim.lua

@@ -0,0 +1,240 @@
+local function println(message) print(tostring(message)) end
+local function print(message) term.write(tostring(message)) end
+
+local opts = { ... }
+local mode, app
+
+local function red() term.setTextColor(colors.red) end
+local function orange() term.setTextColor(colors.orange) end
+local function yellow() term.setTextColor(colors.yellow) end
+local function green() term.setTextColor(colors.green) end
+local function blue() term.setTextColor(colors.blue) end
+local function white() term.setTextColor(colors.white) end
+local function lgray() term.setTextColor(colors.lightGray) end
+
+-- get command line option in list
+local function get_opt(opt, options)
+    for _, v in pairs(options) do if opt == v then return v end end
+    return nil
+end
+
+-- wait for any key to be pressed
+---@diagnostic disable-next-line: undefined-field
+local function any_key() os.pullEvent("key_up") end
+
+-- ask the user yes or no
+local function ask_y_n(question, default)
+    print(question)
+    if default == true then print(" (Y/n)? ") else print(" (y/N)? ") end
+    local response = read();any_key()
+    if response == "" then return default
+    elseif response == "Y" or response == "y" then return true
+    elseif response == "N" or response == "n" then return false
+    else return nil end
+end
+
+-- read the local manifest file
+local function read_local_manifest()
+    local local_ok = false
+    local local_manifest = {}
+    local imfile = fs.open("install_manifest.json", "r")
+    if imfile ~= nil then
+        local_ok, local_manifest = pcall(function () return textutils.unserializeJSON(imfile.readAll()) end)
+        imfile.close()
+    end
+    return local_ok, local_manifest
+end
+
+-- recursively build a tree out of the file manifest
+local function gen_tree(manifest, log)
+    local function _tree_add(tree, split)
+        if #split > 1 then
+            local name = table.remove(split, 1)
+            if tree[name] == nil then tree[name] = {} end
+            table.insert(tree[name], _tree_add(tree[name], split))
+        else return split[1] end
+        return nil
+    end
+
+    local list, tree = { log }, {}
+
+    -- make a list of each and every file
+    for _, files in pairs(manifest.files) do for i = 1, #files do table.insert(list, files[i]) end end
+
+    for i = 1, #list do
+        local split = {}
+---@diagnostic disable-next-line: discard-returns
+        string.gsub(list[i], "([^/]+)", function(c) split[#split + 1] = c end)
+        if #split == 1 then table.insert(tree, list[i])
+        else table.insert(tree, _tree_add(tree, split)) end
+    end
+
+    return tree
+end
+
+local function _in_array(val, array)
+    for _, v in pairs(array) do if v == val then return true end end
+    return false
+end
+
+local function _clean_dir(dir, tree)
+    if tree == nil then tree = {} end
+    local ls = fs.list(dir)
+    for _, val in pairs(ls) do
+        local path = dir.."/"..val
+        if fs.isDir(path) then
+            _clean_dir(path, tree[val])
+            if #fs.list(path) == 0 then fs.delete(path);println("deleted "..path) end
+        elseif (not _in_array(val, tree)) and (val ~= "config.lua" ) then
+            fs.delete(path)
+            println("deleted "..path)
+        end
+    end
+end
+
+-- go through app/common directories to delete unused files
+local function clean(manifest)
+    local log = nil
+    if fs.exists(app..".settings") and settings.load(app..".settings") then
+        log = settings.get("LogPath")
+        if log:sub(1, 1) == "/" then log = log:sub(2) end
+    end
+
+    local tree = gen_tree(manifest, log)
+
+    table.insert(tree, "install_manifest.json")
+    table.insert(tree, "ccmsim.lua")
+
+    local ls = fs.list("/")
+    for _, val in pairs(ls) do
+        if fs.isDriveRoot(val) then
+            yellow();println("skipped mount '"..val.."'")
+        elseif fs.isDir(val) then
+            if tree[val] ~= nil then lgray();_clean_dir("/"..val, tree[val])
+            else white(); if ask_y_n("delete the unused directory '"..val.."'") then lgray();_clean_dir("/"..val) end end
+            if #fs.list(val) == 0 then fs.delete(val);lgray();println("deleted empty directory '"..val.."'") end
+        elseif not _in_array(val, tree) and (string.find(val, ".settings") == nil) then
+            white();if ask_y_n("delete the unused file '"..val.."'") then fs.delete(val);lgray();println("deleted "..val) end
+        end
+    end
+
+    white()
+end
+
+-- get and validate command line options
+
+println("-- CC Mekanism SCADA Install Manager (Off-Line) --")
+
+if #opts == 0 or opts[1] == "help" then
+    println("usage: ccmsim <mode>")
+    println("<mode>")
+    lgray()
+    println(" check     - check your installed versions")
+    println(" update-rm - delete everything except the config,")
+    println("             so that you can upload files for a")
+    println("             new two-file/off-line update")
+    println(" uninstall - delete all app files and config")
+    return
+else
+    mode = get_opt(opts[1], { "check", "update-rm", "uninstall" })
+    if mode == nil then
+        red();println("Unrecognized mode.");white()
+        return
+    end
+end
+
+-- run selected mode
+
+if mode == "check" then
+    local local_ok, manifest = read_local_manifest()
+    if not local_ok then
+        yellow();println("failed to load local installation information");white()
+    end
+
+    -- list all versions
+    for key, value in pairs(manifest.versions) do
+        term.setTextColor(colors.purple)
+        print(string.format("%-14s", "["..key.."]"))
+        blue();println(value);white()
+    end
+elseif mode == "update-rm" or mode == "uninstall" then
+    local ok, manifest = read_local_manifest()
+    if not ok then
+        red();println("Error parsing local installation manifest.");white()
+        return
+    end
+
+    app = manifest.depends[#manifest.depends]
+
+    if mode == "uninstall" then
+        orange();println("Uninstalling all app files...")
+    else
+        orange();println("Deleting all app files except for configuration...")
+    end
+
+    -- ask for confirmation
+    if not ask_y_n("Continue", false) then return end
+
+    -- delete unused files first
+    clean(manifest)
+
+    local file_list = manifest.files
+    local dependencies = manifest.depends
+
+    -- delete all installed files
+    for _, dependency in pairs(dependencies) do
+        local files = file_list[dependency]
+        for _, file in pairs(files) do
+            if fs.exists(file) then fs.delete(file);println("deleted "..file) end
+        end
+
+        local folder = files[1]
+        while true do
+            local dir = fs.getDir(folder)
+            if dir == "" or dir == ".." then break else folder = dir end
+        end
+
+        if fs.isDir(folder) then
+            fs.delete(folder)
+            println("deleted directory "..folder)
+        end
+    end
+
+    -- delete log file
+    local log_deleted = false
+    local settings_file = app..".settings"
+
+    lgray()
+    if fs.exists(settings_file) and settings.load(settings_file) then
+        local log = settings.get("LogPath")
+        if log ~= nil then
+            log_deleted = true
+            if fs.exists(log) then
+                fs.delete(log)
+                println("deleted log file "..log)
+            end
+        end
+    end
+
+    if not log_deleted then
+        red();println("Failed to delete log file.")
+        white();println("press any key to continue...")
+        any_key();lgray()
+    end
+
+    if mode == "uninstall" then
+        if fs.exists(settings_file) then
+            fs.delete(settings_file);println("deleted "..settings_file)
+        end
+
+        fs.delete("install_manifest.json")
+        println("deleted install_manifest.json")
+
+        fs.delete("ccmsim.lua")
+        println("deleted ccmsim.lua")
+    end
+
+    green();println("Done!")
+end
+
+white()