Merge pull request #509 from MikaylaFischler/465-safe-lua-minifier

465 safe lua minifier

.github/workflows/manifest.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Generate manifest and shields for main branch
         id: manifest-main
         if: ${{ (success() || failure()) && steps.checkout-main.outcome == 'success' }}
-        run: python imgen.py shields
+        run: python ./build/imgen.py shields
       - name: Save main's manifest
         if: ${{ (success() || failure()) && steps.manifest-main.outcome == 'success' }}
         run: mv install_manifest.json deploy/manifests/main
@@ -61,7 +61,7 @@ jobs:
       - name: Generate manifest for devel
         id: manifest-devel
         if: ${{ (success() || failure()) && steps.checkout-devel.outcome == 'success' }}
-        run: python imgen.py
+        run: python ./build/imgen.py
       - name: Save devel's manifest
         if: ${{ (success() || failure()) && steps.manifest-devel.outcome == 'success' }}
         run: mv install_manifest.json deploy/manifests/devel

.gitignore

@@ -1,2 +1,2 @@
-_notes/
+_*/
 /*program.sh

build/imgen.py

@@ -44,9 +44,11 @@ def get_version(path, is_lib = False):
 
 # generate installation manifest object
 def make_manifest(size):
+    os.chdir('./_minified')
+
     manifest = {
         "versions" : {
-            "installer" : get_version("./ccmsi.lua"),
+            "installer" : get_version("../ccmsi.lua"),
             "bootloader" : get_version("./startup.lua"),
             "common" : get_version("./scada-common/util.lua", True),
             "comms" : get_version("./scada-common/comms.lua", True),
@@ -95,6 +97,8 @@ def make_manifest(size):
         }
     }
 
+    os.chdir('../')
+
     return manifest
 
 # write initial manifest with placeholder size

build/package_zip.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Create zips to attach to GitHub releases.
+# These can be extracted onto a computer and will include all files CCMSI would otherwise install.
+
+tag=$(git describe --tags)
+apps=(coordinator pocket reactor-plc rtu supervisor)
+
+for app in "${apps[@]}" do
+    mkdir ${tag}_${app}
+    cp -R $app scada-common graphics lockbox configure.lua initenv.lua startup.lua LICENSE ${tag}_${app}
+    zip -r ${tag}_${app}.zip ${tag}_${app}
+    rm -R ${tag}_${app}
+done

build/safemin.py

@@ -0,0 +1,73 @@
+import os
+import re
+
+# minify files in a directory
+def min_files(path):
+    start_sum, end_sum = 0, 0
+
+    for (root, _, files) in os.walk(path):
+        os.makedirs('_minified/' + root, exist_ok=True)
+
+        for f in files:
+            start, end = minify(root + "/" + f)
+
+            start_sum = start_sum + start
+            end_sum = end_sum + end
+
+    delta = start_sum - end_sum
+
+    print(f"> done with '{path}': shrunk from {start_sum} bytes to {end_sum} bytes (saved {delta} bytes, or {(100*delta/start_sum):.2f}%)")
+
+    return list
+
+# minify a file
+def minify(path: str):
+    size_start = os.stat(path).st_size
+
+    f = open(path, "r")
+    contents = f.read()
+    f.close()
+
+    if re.search(r'--+\[+', contents) != None:
+        # absolutely not dealing with lua multiline comments
+        # - there are more important things to do
+        # - this minification is intended to be 100% safe, so working with multiline comments is asking for trouble
+        # - the project doesn't use them as of writing this (except in test/), and it might as well stay that way
+        raise Exception(f"no multiline comments allowed! (offending file: {path})")
+
+    if re.search(r'\\$', contents, flags=re.MULTILINE) != None:
+        # '\' allows for multiline strings, which would require reverting to processing syntax line by line to support them
+        raise Exception(f"no escaping newlines! (offending file: {path})")
+
+    # drop the comments, unless the line has quotes, because quotes are scary
+    # (quotes are scary since we could actually be inside a string: "-- ..." shouldn't get deleted)
+    # -> whitespace before '--' and anything after that, which includes '---' comments
+    minified = re.sub(r'\s*--+(?!.*[\'"]).*', '', contents)
+
+    # drop leading whitespace on each line
+    minified = re.sub(r'^ +', '', minified, flags=re.MULTILINE)
+
+    # drop blank lines
+    while minified != re.sub(r'\n\n', '\n', minified):
+        minified = re.sub(r'\n\n', '\n', minified)
+
+    # write the minified file
+    f_min = open(f"_minified/{path}", "w")
+    f_min.write(minified)
+    f_min.close()
+
+    size_end = os.stat(f"_minified/{path}").st_size
+
+    print(f">> shrunk '{path}' from {size_start} bytes to {size_end} bytes (saved {size_start-size_end} bytes)")
+
+    return size_start, size_end
+
+# minify applications and libraries
+dirs = [ 'scada-common', 'graphics', 'lockbox', 'reactor-plc', 'rtu', 'supervisor', 'coordinator', 'pocket' ]
+for _, d in enumerate(dirs):
+    min_files(d)
+
+# minify root files
+minify("startup.lua")
+minify("initenv.lua")
+minify("configure.lua")