#124 debug stack trace on error

.vscode/settings.json

@@ -13,6 +13,7 @@
         "window",
         "read",
         "periphemu",
-        "mekanismEnergyHelper"
+        "mekanismEnergyHelper",
+        "_HOST"
     ]
 }

coordinator/startup.lua

@@ -4,6 +4,7 @@
 
 require("/initenv").init_env()
 
+local crash        = require("scada-common.crash")
 local log          = require("scada-common.log")
 local ppm          = require("scada-common.ppm")
 local tcallbackdsp = require("scada-common.tcallbackdsp")
@@ -16,7 +17,7 @@ local config       = require("coordinator.config")
 local coordinator  = require("coordinator.coordinator")
 local renderer     = require("coordinator.renderer")
 
-local COORDINATOR_VERSION = "alpha-v0.6.11"
+local COORDINATOR_VERSION = "alpha-v0.6.12"
 
 local print = util.print
 local println = util.println
@@ -57,67 +58,74 @@ log.info("BOOTING coordinator.startup " .. COORDINATOR_VERSION)
 log.info("========================================")
 println(">> SCADA Coordinator " .. COORDINATOR_VERSION .. " <<")
 
+crash.set_env("coordinator", COORDINATOR_VERSION)
+
 ----------------------------------------
--- system startup
+-- main application
 ----------------------------------------
 
--- mount connected devices
+local function main()
-ppm.mount_all()
+    ----------------------------------------
+    -- system startup
+    ----------------------------------------
 
--- setup monitors
+    -- mount connected devices
-local configured, monitors = coordinator.configure_monitors(config.NUM_UNITS)
+    ppm.mount_all()
-if not configured or monitors == nil then
+
+    -- setup monitors
+    local configured, monitors = coordinator.configure_monitors(config.NUM_UNITS)
+    if not configured or monitors == nil then
         println("boot> monitor setup failed")
         log.fatal("monitor configuration failed")
         return
-end
+    end
 
-log.info("monitors ready, dmesg output incoming...")
+    log.info("monitors ready, dmesg output incoming...")
 
--- init renderer
+    -- init renderer
-renderer.set_displays(monitors)
+    renderer.set_displays(monitors)
-renderer.reset(config.RECOLOR)
+    renderer.reset(config.RECOLOR)
-renderer.init_dmesg()
+    renderer.init_dmesg()
 
-log_graphics("displays connected and reset")
+    log_graphics("displays connected and reset")
-log_sys("system start on " .. os.date("%c"))
+    log_sys("system start on " .. os.date("%c"))
-log_boot("starting " .. COORDINATOR_VERSION)
+    log_boot("starting " .. COORDINATOR_VERSION)
 
-----------------------------------------
+    ----------------------------------------
--- setup communications
+    -- setup communications
-----------------------------------------
+    ----------------------------------------
 
--- get the communications modem
+    -- get the communications modem
-local modem = ppm.get_wireless_modem()
+    local modem = ppm.get_wireless_modem()
-if modem == nil then
+    if modem == nil then
         log_comms("wireless modem not found")
         println("boot> wireless modem not found")
         log.fatal("no wireless modem on startup")
         return
-else
+    else
         log_comms("wireless modem connected")
-end
+    end
 
--- create connection watchdog
+    -- create connection watchdog
-local conn_watchdog = util.new_watchdog(5)
+    local conn_watchdog = util.new_watchdog(5)
-conn_watchdog.cancel()
+    conn_watchdog.cancel()
-log.debug("boot> conn watchdog created")
+    log.debug("boot> conn watchdog created")
 
--- start comms, open all channels
+    -- start comms, open all channels
-local coord_comms = coordinator.comms(COORDINATOR_VERSION, modem, config.SCADA_SV_PORT, config.SCADA_SV_LISTEN, config.SCADA_API_LISTEN, conn_watchdog)
+    local coord_comms = coordinator.comms(COORDINATOR_VERSION, modem, config.SCADA_SV_PORT, config.SCADA_SV_LISTEN, config.SCADA_API_LISTEN, conn_watchdog)
-log.debug("boot> comms init")
+    log.debug("boot> comms init")
-log_comms("comms initialized")
+    log_comms("comms initialized")
 
--- base loop clock (2Hz, 10 ticks)
+    -- base loop clock (2Hz, 10 ticks)
-local MAIN_CLOCK = 0.5
+    local MAIN_CLOCK = 0.5
-local loop_clock = util.new_clock(MAIN_CLOCK)
+    local loop_clock = util.new_clock(MAIN_CLOCK)
 
-----------------------------------------
+    ----------------------------------------
--- connect to the supervisor
+    -- connect to the supervisor
-----------------------------------------
+    ----------------------------------------
 
--- attempt to connect to the supervisor or exit
+    -- attempt to connect to the supervisor or exit
-local function init_connect_sv()
+    local function init_connect_sv()
         local tick_waiting, task_done = log_comms_connecting("attempting to connect to configured supervisor on channel " .. config.SCADA_SV_PORT)
 
         -- attempt to establish a connection with the supervisory computer
@@ -128,23 +136,23 @@ local function init_connect_sv()
         end
 
         return true
-end
+    end
 
-if not init_connect_sv() then
+    if not init_connect_sv() then
         println("boot> failed to connect to supervisor")
         log_sys("system shutdown")
         return
-else
+    else
         log_sys("supervisor connected, proceeding to UI start")
-end
+    end
 
-----------------------------------------
+    ----------------------------------------
--- start the UI
+    -- start the UI
-----------------------------------------
+    ----------------------------------------
 
--- start up the UI
+    -- start up the UI
----@return boolean ui_ok started ok
+    ---@return boolean ui_ok started ok
-local function init_start_ui()
+    local function init_start_ui()
         log_graphics("starting UI...")
 
         local draw_start = util.time_ms()
@@ -163,26 +171,26 @@ local function init_start_ui()
         end
 
         return ui_ok
-end
+    end
 
-local ui_ok = init_start_ui()
+    local ui_ok = init_start_ui()
 
-----------------------------------------
+    ----------------------------------------
--- main event loop
+    -- main event loop
-----------------------------------------
+    ----------------------------------------
 
-local no_modem = false
+    local no_modem = false
 
-if ui_ok then
+    if ui_ok then
         -- start connection watchdog
         conn_watchdog.feed()
         log.debug("boot> conn watchdog started")
 
         log_sys("system started successfully")
-end
+    end
 
--- main event loop
+    -- main event loop
-while ui_ok do
+    while ui_ok do
         local event, param1, param2, param3, param4, param5 = util.pull_event()
 
         -- handle event
@@ -309,10 +317,13 @@ while ui_ok do
             log_comms("api sessions closed")
             break
         end
+    end
+
+    renderer.close_ui()
+    log_sys("system shutdown")
+
+    println_ts("exited")
+    log.info("exited")
 end
 
-renderer.close_ui()
+if not xpcall(main, crash.handler) then crash.exit() end
-log_sys("system shutdown")
-
-println_ts("exited")
-log.info("exited")

reactor-plc/startup.lua

@@ -4,6 +4,7 @@
 
 require("/initenv").init_env()
 
+local crash   = require("scada-common.crash")
 local log     = require("scada-common.log")
 local mqueue  = require("scada-common.mqueue")
 local ppm     = require("scada-common.ppm")
@@ -13,7 +14,7 @@ local config  = require("reactor-plc.config")
 local plc     = require("reactor-plc.plc")
 local threads = require("reactor-plc.threads")
 
-local R_PLC_VERSION = "beta-v0.9.6"
+local R_PLC_VERSION = "beta-v0.9.7"
 
 local print = util.print
 local println = util.println
@@ -45,16 +46,23 @@ log.info("BOOTING reactor-plc.startup " .. R_PLC_VERSION)
 log.info("========================================")
 println(">> Reactor PLC " .. R_PLC_VERSION .. " <<")
 
+crash.set_env("plc", R_PLC_VERSION)
+
 ----------------------------------------
--- startup
+-- main application
 ----------------------------------------
 
--- mount connected devices
+local function main()
-ppm.mount_all()
+    ----------------------------------------
+    -- startup
+    ----------------------------------------
 
--- shared memory across threads
+    -- mount connected devices
----@class plc_shared_memory
+    ppm.mount_all()
-local __shared_memory = {
+
+    -- shared memory across threads
+    ---@class plc_shared_memory
+    local __shared_memory = {
         -- networked setting
         networked = config.NETWORKED,   ---@type boolean
 
@@ -95,31 +103,31 @@ local __shared_memory = {
             mq_comms_tx = mqueue.new(),
             mq_comms_rx = mqueue.new()
         }
-}
+    }
 
-local smem_dev = __shared_memory.plc_dev
+    local smem_dev = __shared_memory.plc_dev
-local smem_sys = __shared_memory.plc_sys
+    local smem_sys = __shared_memory.plc_sys
 
-local plc_state = __shared_memory.plc_state
+    local plc_state = __shared_memory.plc_state
 
--- we need a reactor, can at least do some things even if it isn't formed though
+    -- we need a reactor, can at least do some things even if it isn't formed though
-if smem_dev.reactor == nil then
+    if smem_dev.reactor == nil then
         println("boot> fission reactor not found");
         log.warning("no reactor on startup")
 
         plc_state.init_ok = false
         plc_state.degraded = true
         plc_state.no_reactor = true
-elseif not smem_dev.reactor.isFormed() then
+    elseif not smem_dev.reactor.isFormed() then
         println("boot> fission reactor not formed");
         log.warning("reactor logic adapter present, but reactor is not formed")
 
         plc_state.degraded = true
         plc_state.reactor_formed = false
-end
+    end
 
--- modem is required if networked
+    -- modem is required if networked
-if __shared_memory.networked and smem_dev.modem == nil then
+    if __shared_memory.networked and smem_dev.modem == nil then
         println("boot> wireless modem not found")
         log.warning("no wireless modem on startup")
 
@@ -131,12 +139,12 @@ if __shared_memory.networked and smem_dev.modem == nil then
         plc_state.init_ok = false
         plc_state.degraded = true
         plc_state.no_modem = true
-end
+    end
 
--- PLC init
+    -- PLC init
---- 
+    --- 
---- EVENT_CONSUMER: this function consumes events
+    --- EVENT_CONSUMER: this function consumes events
-local function init()
+    local function init()
         if plc_state.init_ok then
             -- just booting up, no fission allowed (neutrons stay put thanks)
             if plc_state.reactor_formed and smem_dev.reactor.getStatus() then
@@ -169,20 +177,20 @@ local function init()
             println("boot> system in degraded state, awaiting devices...")
             log.warning("init> booted in a degraded state, awaiting peripheral connections...")
         end
-end
+    end
 
-----------------------------------------
+    ----------------------------------------
--- start system
+    -- start system
-----------------------------------------
+    ----------------------------------------
 
--- initialize PLC
+    -- initialize PLC
-init()
+    init()
 
--- init threads
+    -- init threads
-local main_thread = threads.thread__main(__shared_memory, init)
+    local main_thread = threads.thread__main(__shared_memory, init)
-local rps_thread  = threads.thread__rps(__shared_memory)
+    local rps_thread  = threads.thread__rps(__shared_memory)
 
-if __shared_memory.networked then
+    if __shared_memory.networked then
         -- init comms threads
         local comms_thread_tx = threads.thread__comms_tx(__shared_memory)
         local comms_thread_rx = threads.thread__comms_rx(__shared_memory)
@@ -201,10 +209,13 @@ if __shared_memory.networked then
             -- close connection
             smem_sys.plc_comms.close()
         end
-else
+    else
         -- run threads, excluding comms
         parallel.waitForAll(main_thread.p_exec, rps_thread.p_exec)
+    end
+
+    println_ts("exited")
+    log.info("exited")
 end
 
-println_ts("exited")
+if not xpcall(main, crash.handler) then crash.exit() end
-log.info("exited")

rtu/startup.lua

@@ -4,6 +4,7 @@
 
 require("/initenv").init_env()
 
+local crash        = require("scada-common.crash")
 local log          = require("scada-common.log")
 local mqueue       = require("scada-common.mqueue")
 local ppm          = require("scada-common.ppm")
@@ -24,7 +25,7 @@ local sna_rtu      = require("rtu.dev.sna_rtu")
 local sps_rtu      = require("rtu.dev.sps_rtu")
 local turbinev_rtu = require("rtu.dev.turbinev_rtu")
 
-local RTU_VERSION = "beta-v0.9.3"
+local RTU_VERSION = "beta-v0.9.4"
 
 local rtu_t = types.rtu_t
 
@@ -58,15 +59,22 @@ log.info("BOOTING rtu.startup " .. RTU_VERSION)
 log.info("========================================")
 println(">> RTU GATEWAY " .. RTU_VERSION .. " <<")
 
+crash.set_env("rtu", RTU_VERSION)
+
 ----------------------------------------
--- startup
+-- main application
 ----------------------------------------
 
--- mount connected devices
+local function main()
-ppm.mount_all()
+    ----------------------------------------
+    -- startup
+    ----------------------------------------
 
----@class rtu_shared_memory
+    -- mount connected devices
-local __shared_memory = {
+    ppm.mount_all()
+
+    ---@class rtu_shared_memory
+    local __shared_memory = {
         -- RTU system state flags
         ---@class rtu_state
         rtu_state = {
@@ -90,29 +98,29 @@ local __shared_memory = {
         q = {
             mq_comms = mqueue.new()
         }
-}
+    }
 
-local smem_dev = __shared_memory.rtu_dev
+    local smem_dev = __shared_memory.rtu_dev
-local smem_sys = __shared_memory.rtu_sys
+    local smem_sys = __shared_memory.rtu_sys
 
--- get modem
+    -- get modem
-if smem_dev.modem == nil then
+    if smem_dev.modem == nil then
         println("boot> wireless modem not found")
         log.fatal("no wireless modem on startup")
         return
-end
+    end
 
-----------------------------------------
+    ----------------------------------------
--- interpret config and init units
+    -- interpret config and init units
-----------------------------------------
+    ----------------------------------------
 
-local units = __shared_memory.rtu_sys.units
+    local units = __shared_memory.rtu_sys.units
 
-local rtu_redstone = config.RTU_REDSTONE
+    local rtu_redstone = config.RTU_REDSTONE
-local rtu_devices = config.RTU_DEVICES
+    local rtu_devices = config.RTU_DEVICES
 
--- configure RTU gateway based on config file definitions
+    -- configure RTU gateway based on config file definitions
-local function configure()
+    local function configure()
         -- redstone interfaces
         for entry_idx = 1, #rtu_redstone do
             local rs_rtu = redstone_rtu.new()
@@ -359,15 +367,15 @@ local function configure()
 
         -- we made it through all that trusting-user-to-write-a-config-file chaos
         return true
-end
+    end
 
-----------------------------------------
+    ----------------------------------------
--- start system
+    -- start system
-----------------------------------------
+    ----------------------------------------
 
-log.debug("boot> running configure()")
+    log.debug("boot> running configure()")
 
-if configure() then
+    if configure() then
         -- start connection watchdog
         smem_sys.conn_watchdog = util.new_watchdog(5)
         log.debug("boot> conn watchdog started")
@@ -390,9 +398,12 @@ if configure() then
 
         -- run threads
         parallel.waitForAll(table.unpack(_threads))
-else
+    else
         println("configuration failed, exiting...")
+    end
+
+    println_ts("exited")
+    log.info("exited")
 end
 
-println_ts("exited")
+if not xpcall(main, crash.handler) then crash.exit() end
-log.info("exited")

scada-common/crash.lua

@@ -0,0 +1,46 @@
+--
+-- Crash Handler
+--
+
+local comms = require("scada-common.comms")
+local log   = require("scada-common.log")
+local util  = require("scada-common.util")
+
+local crash = {}
+
+local app = "unknown"
+local ver = "v0.0.0"
+local err = ""
+
+-- set crash environment
+---@param application string app name
+---@param version string version
+function crash.set_env(application, version)
+    app = application
+    ver = version
+end
+
+-- handle a crash error
+---@param error string error message
+function crash.handler(error)
+    err = error
+    log.info("=====> FATAL SOFTWARE FAULT <=====")
+    log.fatal(error)
+    log.info("----------------------------------")
+    log.info(util.c("RUNTIME:          ", _HOST))
+    log.info(util.c("LUA VERSION:      ", _VERSION))
+    log.info(util.c("APPLICATION:      ", app))
+    log.info(util.c("FIRMWARE VERSION: ", ver))
+    log.info(util.c("COMMS VERSION:    ", comms.version))
+    log.info("----------------------------------")
+    log.info(debug.traceback("--- begin debug trace ---", 1))
+    log.info("--- end debug trace ---")
+end
+
+-- final error print on failed xpcall, app exits here
+function crash.exit()
+    util.println("fatal error occured in main application:")
+    error(err, 0)
+end
+
+return crash

supervisor/startup.lua

@@ -4,6 +4,7 @@
 
 require("/initenv").init_env()
 
+local crash      = require("scada-common.crash")
 local log        = require("scada-common.log")
 local ppm        = require("scada-common.ppm")
 local util       = require("scada-common.util")
@@ -13,7 +14,7 @@ local svsessions = require("supervisor.session.svsessions")
 local config     = require("supervisor.config")
 local supervisor = require("supervisor.supervisor")
 
-local SUPERVISOR_VERSION = "beta-v0.7.5"
+local SUPERVISOR_VERSION = "beta-v0.7.6"
 
 local print = util.print
 local println = util.println
@@ -57,33 +58,40 @@ log.info("BOOTING supervisor.startup " .. SUPERVISOR_VERSION)
 log.info("========================================")
 println(">> SCADA Supervisor " .. SUPERVISOR_VERSION .. " <<")
 
+crash.set_env("supervisor", SUPERVISOR_VERSION)
+
 ----------------------------------------
--- startup
+-- main application
 ----------------------------------------
 
--- mount connected devices
+local function main()
-ppm.mount_all()
+    ----------------------------------------
+    -- startup
+    ----------------------------------------
 
-local modem = ppm.get_wireless_modem()
+    -- mount connected devices
-if modem == nil then
+    ppm.mount_all()
+
+    local modem = ppm.get_wireless_modem()
+    if modem == nil then
         println("boot> wireless modem not found")
         log.fatal("no wireless modem on startup")
         return
-end
+    end
 
--- start comms, open all channels
+    -- start comms, open all channels
-local superv_comms = supervisor.comms(SUPERVISOR_VERSION, config.NUM_REACTORS, config.REACTOR_COOLING, modem,
+    local superv_comms = supervisor.comms(SUPERVISOR_VERSION, config.NUM_REACTORS, config.REACTOR_COOLING, modem,
                                             config.SCADA_DEV_LISTEN, config.SCADA_SV_LISTEN)
 
--- base loop clock (6.67Hz, 3 ticks)
+    -- base loop clock (6.67Hz, 3 ticks)
-local MAIN_CLOCK = 0.15
+    local MAIN_CLOCK = 0.15
-local loop_clock = util.new_clock(MAIN_CLOCK)
+    local loop_clock = util.new_clock(MAIN_CLOCK)
 
--- start clock
+    -- start clock
-loop_clock.start()
+    loop_clock.start()
 
--- event loop
+    -- event loop
-while true do
+    while true do
         local event, param1, param2, param3, param4, param5 = util.pull_event()
 
         -- handle event
@@ -145,7 +153,10 @@ while true do
             log.info("sessions closed")
             break
         end
+    end
+
+    println_ts("exited")
+    log.info("exited")
 end
 
-println_ts("exited")
+if not xpcall(main, crash.handler) then crash.exit() end
-log.info("exited")