From 31a663e86b446b4e08f983b8d03ca7ec1870d751 Mon Sep 17 00:00:00 2001 From: Mikayla Fischler Date: Sun, 7 Apr 2024 19:37:06 -0400 Subject: [PATCH] #460 threaded coordinator and moved UI start to render thread --- coordinator/iocontrol.lua | 7 + coordinator/startup.lua | 285 +++++---------------- coordinator/threads.lua | 346 ++++++++++++++++++++++++++ coordinator/ui/layout/front_panel.lua | 8 + 4 files changed, 428 insertions(+), 218 deletions(-) create mode 100644 coordinator/threads.lua diff --git a/coordinator/iocontrol.lua b/coordinator/iocontrol.lua index 0cb50e4..9be448a 100644 --- a/coordinator/iocontrol.lua +++ b/coordinator/iocontrol.lua @@ -376,6 +376,13 @@ function iocontrol.fp_monitor_state(id, connected) end end +-- report thread (routine) statuses +---@param thread string thread name +---@param ok boolean thread state +function iocontrol.fp_rt_status(thread, ok) + io.fp.ps.publish(util.c("routine__", thread), ok) +end + -- report PKT firmware version and PKT session connection state ---@param session_id integer PKT session ---@param fw string firmware version diff --git a/coordinator/startup.lua b/coordinator/startup.lua index 152d37b..750dde0 100644 --- a/coordinator/startup.lua +++ b/coordinator/startup.lua @@ -7,22 +7,19 @@ require("/initenv").init_env() local comms = require("scada-common.comms") local crash = require("scada-common.crash") local log = require("scada-common.log") +local mqueue = require("scada-common.mqueue") local network = require("scada-common.network") local ppm = require("scada-common.ppm") -local tcd = require("scada-common.tcd") local util = require("scada-common.util") -local core = require("graphics.core") - local configure = require("coordinator.configure") local coordinator = require("coordinator.coordinator") local iocontrol = require("coordinator.iocontrol") local renderer = require("coordinator.renderer") local sounder = require("coordinator.sounder") +local threads = require("coordinator.threads") -local apisessions = require("coordinator.session.apisessions") - -local COORDINATOR_VERSION = "v1.3.5" +local COORDINATOR_VERSION = "v1.4.0" local CHUNK_LOAD_DELAY_S = 30.0 @@ -133,12 +130,54 @@ local function main() log_sys("system start on " .. os.date("%c")) log_boot("starting " .. COORDINATOR_VERSION) + ---------------------------------------- + -- memory allocation + ---------------------------------------- + + -- shared memory across threads + ---@class crd_shared_memory + local __shared_memory = { + -- time and date format for display + date_format = util.trinary(config.Time24Hour, "%X \x04 %A, %B %d %Y", "%r \x04 %A, %B %d %Y"), + + -- coordinator system state flags + ---@class crd_state + crd_state = { + fp_ok = false, + ui_ok = false, + link_fail = false, + shutdown = false + }, + + -- core coordinator devices + crd_dev = { + speaker = ppm.get_device("speaker"), + modem = ppm.get_wireless_modem() + }, + + -- system objects + crd_sys = { + nic = nil, ---@type nic + coord_comms = nil, ---@type coord_comms + conn_watchdog = nil ---@type watchdog + }, + + -- message queues + q = { + mq_render = mqueue.new() + } + } + + local smem_dev = __shared_memory.crd_dev + local smem_sys = __shared_memory.crd_sys + + local crd_state = __shared_memory.crd_state + ---------------------------------------- -- setup alarm sounder subsystem ---------------------------------------- - local speaker = ppm.get_device("speaker") - if speaker == nil then + if smem_dev.speaker == nil then log_boot("annunciator alarm speaker not found") println("startup> speaker not found") log.fatal("no annunciator alarm speaker found") @@ -146,7 +185,7 @@ local function main() else local sounder_start = util.time_ms() log_boot("annunciator alarm speaker connected") - sounder.init(speaker, config.SpeakerVolume) + sounder.init(smem_dev.speaker, config.SpeakerVolume) log_boot("tone generation took " .. (util.time_ms() - sounder_start) .. "ms") log_sys("annunciator alarm configured") iocontrol.fp_has_speaker(true) @@ -163,8 +202,7 @@ local function main() end -- get the communications modem - local modem = ppm.get_wireless_modem() - if modem == nil then + if smem_dev.modem == nil then log_comms("wireless modem not found") println("startup> wireless modem not found") log.fatal("no wireless modem on startup") @@ -175,243 +213,54 @@ local function main() end -- create connection watchdog - local conn_watchdog = util.new_watchdog(config.SVR_Timeout) - conn_watchdog.cancel() + smem_sys.conn_watchdog = util.new_watchdog(config.SVR_Timeout) + smem_sys.conn_watchdog.cancel() log.debug("startup> conn watchdog created") -- create network interface then setup comms - local nic = network.nic(modem) - local coord_comms = coordinator.comms(COORDINATOR_VERSION, nic, conn_watchdog) + smem_sys.nic = network.nic(smem_dev.modem) + smem_sys.coord_comms = coordinator.comms(COORDINATOR_VERSION, smem_sys.nic, smem_sys.conn_watchdog) log.debug("startup> comms init") log_comms("comms initialized") - -- base loop clock (2Hz, 10 ticks) - local MAIN_CLOCK = 0.5 - local loop_clock = util.new_clock(MAIN_CLOCK) - ---------------------------------------- - -- start front panel & UI start function + -- start front panel ---------------------------------------- log_graphics("starting front panel UI...") - local fp_ok, fp_message = renderer.try_start_fp() - if not fp_ok then + local fp_message + crd_state.fp_ok, fp_message = renderer.try_start_fp() + if not crd_state.fp_ok then log_graphics(util.c("front panel UI error: ", fp_message)) println_ts("front panel UI creation failed") log.fatal(util.c("front panel GUI render failed with error ", fp_message)) return else log_graphics("front panel ready") end - -- start up the main UI - ---@return boolean ui_ok started ok - local function start_main_ui() - log_graphics("starting main UI...") - - local draw_start = util.time_ms() - - local ui_ok, ui_message = renderer.try_start_ui() - if not ui_ok then - log_graphics(util.c("main UI error: ", ui_message)) - log.fatal(util.c("main GUI render failed with error ", ui_message)) - else - log_graphics("main UI draw took " .. (util.time_ms() - draw_start) .. "ms") - end - - return ui_ok - end - ---------------------------------------- - -- main event loop + -- start system ---------------------------------------- - local link_failed = false - local ui_ok = true - local date_format = util.trinary(config.Time24Hour, "%X \x04 %A, %B %d %Y", "%r \x04 %A, %B %d %Y") + -- init threads + local main_thread = threads.thread__main(__shared_memory) + local render_thread = threads.thread__render(__shared_memory) - -- start clock - loop_clock.start() + log.info("startup> completed") - log_sys("system started successfully") - - -- main event loop - while true do - local event, param1, param2, param3, param4, param5 = util.pull_event() - - -- handle event - if event == "peripheral_detach" then - local type, device = ppm.handle_unmount(param1) - - if type ~= nil and device ~= nil then - if type == "modem" then - -- we only really care if this is our wireless modem - -- if it is another modem, handle other peripheral losses separately - if nic.is_modem(device) then - nic.disconnect() - log_sys("comms modem disconnected") - - local other_modem = ppm.get_wireless_modem() - if other_modem then - log_sys("found another wireless modem, using it for comms") - nic.connect(other_modem) - else - -- close out main UI - renderer.close_ui() - - -- alert user to status - log_sys("awaiting comms modem reconnect...") - - iocontrol.fp_has_modem(false) - end - else - log_sys("non-comms modem disconnected") - end - elseif type == "monitor" then - if renderer.handle_disconnect(device) then - log_sys("lost a configured monitor") - else - log_sys("lost an unused monitor") - end - elseif type == "speaker" then - log_sys("lost alarm sounder speaker") - iocontrol.fp_has_speaker(false) - end - end - elseif event == "peripheral" then - local type, device = ppm.mount(param1) - - if type ~= nil and device ~= nil then - if type == "modem" then - if device.isWireless() and not nic.is_connected() then - -- reconnected modem - log_sys("comms modem reconnected") - nic.connect(device) - iocontrol.fp_has_modem(true) - elseif device.isWireless() then - log.info("unused wireless modem reconnected") - else - log_sys("wired modem reconnected") - end - elseif type == "monitor" then - if renderer.handle_reconnect(param1, device) then - log_sys(util.c("configured monitor ", param1, " reconnected")) - else - log_sys(util.c("unused monitor ", param1, " connected")) - end - elseif type == "speaker" then - log_sys("alarm sounder speaker reconnected") - sounder.reconnect(device) - iocontrol.fp_has_speaker(true) - end - end - elseif event == "monitor_resize" then - local is_used, is_ok = renderer.handle_resize(param1) - if is_used then - log_sys(util.c("configured monitor ", param1, " resized, ", util.trinary(is_ok, "display still fits", "display no longer fits"))) - end - elseif event == "timer" then - if loop_clock.is_clock(param1) then - -- main loop tick - - -- toggle heartbeat - iocontrol.heartbeat() - - -- maintain connection - if nic.is_connected() then - local ok, start_ui = coord_comms.try_connect() - if not ok then - link_failed = true - log_sys("supervisor connection failed, shutting down...") - log.fatal("failed to connect to supervisor") - break - elseif start_ui then - log_sys("supervisor connected, proceeding to main UI start") - ui_ok = start_main_ui() - if not ui_ok then break end - end - end - - -- iterate sessions - apisessions.iterate_all() - - -- free any closed sessions - apisessions.free_all_closed() - - -- update date and time string for main display - if coord_comms.is_linked() then - iocontrol.get_db().facility.ps.publish("date_time", os.date(date_format)) - end - - loop_clock.start() - elseif conn_watchdog.is_timer(param1) then - -- supervisor watchdog timeout - log_comms("supervisor server timeout") - - -- close connection, main UI, and stop sounder - coord_comms.close() - renderer.close_ui() - sounder.stop() - else - -- a non-clock/main watchdog timer event - - -- check API watchdogs - apisessions.check_all_watchdogs(param1) - - -- notify timer callback dispatcher - tcd.handle(param1) - end - elseif event == "modem_message" then - -- got a packet - local packet = coord_comms.parse_packet(param1, param2, param3, param4, param5) - - -- handle then check if it was a disconnect - if coord_comms.handle_packet(packet) then - log_comms("supervisor closed connection") - - -- close connection, main UI, and stop sounder - coord_comms.close() - renderer.close_ui() - sounder.stop() - end - elseif event == "monitor_touch" or event == "mouse_click" or event == "mouse_up" or - event == "mouse_drag" or event == "mouse_scroll" or event == "double_click" then - -- handle a mouse event - renderer.handle_mouse(core.events.new_mouse_event(event, param1, param2, param3)) - elseif event == "speaker_audio_empty" then - -- handle speaker buffer emptied - sounder.continue() - end - - -- check for termination request - if event == "terminate" or ppm.should_terminate() then - -- handle supervisor connection - coord_comms.try_connect(true) - - if coord_comms.is_linked() then - log_comms("terminate requested, closing supervisor connection...") - else link_failed = true end - - coord_comms.close() - log_comms("supervisor connection closed") - - -- handle API sessions - log_comms("closing api sessions...") - apisessions.close_all() - log_comms("api sessions closed") - break - end - end + -- run threads + parallel.waitForAll(main_thread.p_exec, render_thread.p_exec) renderer.close_ui() renderer.close_fp() sounder.stop() log_sys("system shutdown") - if link_failed then println_ts("failed to connect to supervisor") end - if not ui_ok then println_ts("main UI creation failed") end + if crd_state.link_fail then println_ts("failed to connect to supervisor") end + if not crd_state.ui_ok then println_ts("main UI creation failed") end -- close on error exit (such as UI error) - if coord_comms.is_linked() then coord_comms.close() end + if smem_sys.coord_comms.is_linked() then smem_sys.coord_comms.close() end println_ts("exited") log.info("exited") diff --git a/coordinator/threads.lua b/coordinator/threads.lua new file mode 100644 index 0000000..6ec7d18 --- /dev/null +++ b/coordinator/threads.lua @@ -0,0 +1,346 @@ +local log = require("scada-common.log") +local mqueue = require("scada-common.mqueue") +local ppm = require("scada-common.ppm") +local tcd = require("scada-common.tcd") +local util = require("scada-common.util") + +local coordinator = require("coordinator.coordinator") +local iocontrol = require("coordinator.iocontrol") +local renderer = require("coordinator.renderer") +local sounder = require("coordinator.sounder") + +local apisessions = require("coordinator.session.apisessions") + +local core = require("graphics.core") + +local log_graphics = coordinator.log_graphics +local log_sys = coordinator.log_sys +local log_comms = coordinator.log_comms + +local threads = {} + +local MAIN_CLOCK = 0.5 -- (2Hz, 10 ticks) +local RENDER_SLEEP = 100 -- (100ms, 2 ticks) + +local MQ__RENDER_CMD = { + START_MAIN_UI = 1, + MON_DISCONNECT = 2, + MON_CONNECT = 3, + MON_RESIZE = 4, + UPDATE = 5 +} + +-- main thread +---@nodiscard +---@param smem crd_shared_memory +function threads.thread__main(smem) + ---@class parallel_thread + local public = {} + + -- execute thread + function public.exec() + iocontrol.fp_rt_status("main", true) + log.debug("main thread start") + + local loop_clock = util.new_clock(MAIN_CLOCK) + + -- start clock + loop_clock.start() + + log_sys("system started successfully") + + -- load in from shared memory + local crd_state = smem.crd_state + local nic = smem.crd_sys.nic + local coord_comms = smem.crd_sys.coord_comms + local conn_watchdog = smem.crd_sys.conn_watchdog + + -- event loop + while true do + local event, param1, param2, param3, param4, param5 = util.pull_event() + + -- handle event + if event == "peripheral_detach" then + local type, device = ppm.handle_unmount(param1) + + if type ~= nil and device ~= nil then + if type == "modem" then + -- we only really care if this is our wireless modem + -- if it is another modem, handle other peripheral losses separately + if nic.is_modem(device) then + nic.disconnect() + log_sys("comms modem disconnected") + + local other_modem = ppm.get_wireless_modem() + if other_modem then + log_sys("found another wireless modem, using it for comms") + nic.connect(other_modem) + else + -- close out main UI + renderer.close_ui() + + -- alert user to status + log_sys("awaiting comms modem reconnect...") + + iocontrol.fp_has_modem(false) + end + else + log_sys("non-comms modem disconnected") + end + elseif type == "monitor" then + if renderer.handle_disconnect(device) then + log_sys("lost a configured monitor") + else + log_sys("lost an unused monitor") + end + elseif type == "speaker" then + log_sys("lost alarm sounder speaker") + iocontrol.fp_has_speaker(false) + end + end + elseif event == "peripheral" then + local type, device = ppm.mount(param1) + + if type ~= nil and device ~= nil then + if type == "modem" then + if device.isWireless() and not nic.is_connected() then + -- reconnected modem + log_sys("comms modem reconnected") + nic.connect(device) + iocontrol.fp_has_modem(true) + elseif device.isWireless() then + log.info("unused wireless modem reconnected") + else + log_sys("wired modem reconnected") + end + elseif type == "monitor" then + if renderer.handle_reconnect(param1, device) then + log_sys(util.c("configured monitor ", param1, " reconnected")) + else + log_sys(util.c("unused monitor ", param1, " connected")) + end + elseif type == "speaker" then + log_sys("alarm sounder speaker reconnected") + sounder.reconnect(device) + iocontrol.fp_has_speaker(true) + end + end + elseif event == "monitor_resize" then + local is_used, is_ok = renderer.handle_resize(param1) + if is_used then + log_sys(util.c("configured monitor ", param1, " resized, ", util.trinary(is_ok, "display still fits", "display no longer fits"))) + end + elseif event == "timer" then + if loop_clock.is_clock(param1) then + -- main loop tick + + -- toggle heartbeat + iocontrol.heartbeat() + + -- maintain connection + if nic.is_connected() then + local ok, start_ui = coord_comms.try_connect() + if not ok then + crd_state.link_fail = true + log_sys("supervisor connection failed, shutting down...") + log.fatal("failed to connect to supervisor") + break + elseif start_ui then + log_sys("supervisor connected, dispatching main UI start") + smem.q.mq_render.push_command(MQ__RENDER_CMD.START_MAIN_UI) + --@todo if not ui_ok then break end + end + end + + -- iterate sessions + apisessions.iterate_all() + + -- free any closed sessions + apisessions.free_all_closed() + + -- update date and time string for main display + if coord_comms.is_linked() then + iocontrol.get_db().facility.ps.publish("date_time", os.date(smem.date_format)) + end + + loop_clock.start() + elseif conn_watchdog.is_timer(param1) then + -- supervisor watchdog timeout + log_comms("supervisor server timeout") + + -- close connection, main UI, and stop sounder + coord_comms.close() + renderer.close_ui() + sounder.stop() + else + -- a non-clock/main watchdog timer event + + -- check API watchdogs + apisessions.check_all_watchdogs(param1) + + -- notify timer callback dispatcher + tcd.handle(param1) + end + elseif event == "modem_message" then + -- got a packet + local packet = coord_comms.parse_packet(param1, param2, param3, param4, param5) + + -- handle then check if it was a disconnect + if coord_comms.handle_packet(packet) then + log_comms("supervisor closed connection") + + -- close connection, main UI, and stop sounder + coord_comms.close() + renderer.close_ui() + sounder.stop() + end + elseif event == "monitor_touch" or event == "mouse_click" or event == "mouse_up" or + event == "mouse_drag" or event == "mouse_scroll" or event == "double_click" then + -- handle a mouse event + renderer.handle_mouse(core.events.new_mouse_event(event, param1, param2, param3)) + elseif event == "speaker_audio_empty" then + -- handle speaker buffer emptied + sounder.continue() + end + + -- check for termination request + if event == "terminate" or ppm.should_terminate() then + crd_state.shutdown = true + log.info("terminate requested, main thread exiting") + + -- handle closing supervisor connection + coord_comms.try_connect(true) + + if coord_comms.is_linked() then + log_comms("terminate requested, closing supervisor connection...") + else crd_state.link_fail = true end + + coord_comms.close() + log_comms("supervisor connection closed") + + -- handle API sessions + log_comms("closing api sessions...") + apisessions.close_all() + log_comms("api sessions closed") + break + end + end + end + + -- execute the thread in a protected mode, retrying it on return if not shutting down + function public.p_exec() + local crd_state = smem.crd_state + + while not crd_state.shutdown do + local status, result = pcall(public.exec) + if status == false then + log.fatal(util.strval(result)) + end + + iocontrol.fp_rt_status("main", false) + + -- if status is true, then we are probably exiting, so this won't matter + -- if not, we need to restart the clock + -- this thread cannot be slept because it will miss events (namely "terminate" otherwise) + if not crd_state.shutdown then + log.info("main thread restarting now...") + end + end + end + + return public +end + +-- coordinator renderer thread +---@nodiscard +---@param smem crd_shared_memory +function threads.thread__render(smem) + ---@class parallel_thread + local public = {} + + -- execute thread + function public.exec() + iocontrol.fp_rt_status("render", true) + log.debug("render thread start") + + -- load in from shared memory + local crd_state = smem.crd_state + local render_queue = smem.q.mq_render + + local last_update = util.time() + + -- thread loop + while true do + -- check for messages in the message queue + while render_queue.ready() and not crd_state.shutdown do + local msg = render_queue.pop() + + if msg ~= nil then + if msg.qtype == mqueue.TYPE.COMMAND then + -- received a command + if msg.message == MQ__RENDER_CMD.START_MAIN_UI then + -- start up the main UI + log_graphics("starting main UI...") + + local draw_start = util.time_ms() + + local ui_message + crd_state.ui_ok, ui_message = renderer.try_start_ui() + if not crd_state.ui_ok then + log_graphics(util.c("main UI error: ", ui_message)) + log.fatal(util.c("main GUI render failed with error ", ui_message)) + else + log_graphics("main UI draw took " .. (util.time_ms() - draw_start) .. "ms") + end + elseif msg.message == MQ__RENDER_CMD.MON_DISCONNECT then + -- monitor lost + elseif msg.message == MQ__RENDER_CMD.MON_CONNECT then + -- monitor connected + elseif msg.message == MQ__RENDER_CMD.UPDATE then + -- new data + end + elseif msg.qtype == mqueue.TYPE.DATA then + -- received data + elseif msg.qtype == mqueue.TYPE.PACKET then + -- received a packet + end + end + + -- quick yield + util.nop() + end + + -- check for termination request + if crd_state.shutdown then + log.info("render thread exiting") + break + end + + -- delay before next check + last_update = util.adaptive_delay(RENDER_SLEEP, last_update) + end + end + + -- execute the thread in a protected mode, retrying it on return if not shutting down + function public.p_exec() + local crd_state = smem.crd_state + + while not crd_state.shutdown do + local status, result = pcall(public.exec) + if status == false then + log.fatal(util.strval(result)) + end + + iocontrol.fp_rt_status("render", false) + + if not crd_state.shutdown then + log.info("render thread restarting in 5 seconds...") + util.psleep(5) + end + end + end + + return public +end + +return threads diff --git a/coordinator/ui/layout/front_panel.lua b/coordinator/ui/layout/front_panel.lua index 7124083..7f2738a 100644 --- a/coordinator/ui/layout/front_panel.lua +++ b/coordinator/ui/layout/front_panel.lua @@ -100,6 +100,14 @@ local function init(panel, num_units) local speaker = LED{parent=system,label="SPEAKER",colors=led_grn} speaker.register(ps, "has_speaker", speaker.update) + system.line_break() + + local rt_main = LED{parent=system,label="RT MAIN",colors=led_grn} + local rt_render = LED{parent=system,label="RT RENDER",colors=led_grn} + + rt_main.register(ps, "routine__main", rt_main.update) + rt_render.register(ps, "routine__render", rt_render.update) + ---@diagnostic disable-next-line: undefined-field local comp_id = util.sprintf("(%d)", os.getComputerID()) TextBox{parent=system,x=9,y=4,width=6,height=1,text=comp_id,fg_bg=style.fp.disabled_fg}