#460 threaded coordinator and moved UI start to render thread

This commit is contained in:
Mikayla Fischler 2024-04-07 19:37:06 -04:00
parent 3537c59365
commit 31a663e86b
4 changed files with 428 additions and 218 deletions

View File

@ -376,6 +376,13 @@ function iocontrol.fp_monitor_state(id, connected)
end
end
-- report thread (routine) statuses
---@param thread string thread name
---@param ok boolean thread state
function iocontrol.fp_rt_status(thread, ok)
io.fp.ps.publish(util.c("routine__", thread), ok)
end
-- report PKT firmware version and PKT session connection state
---@param session_id integer PKT session
---@param fw string firmware version

View File

@ -7,22 +7,19 @@ require("/initenv").init_env()
local comms = require("scada-common.comms")
local crash = require("scada-common.crash")
local log = require("scada-common.log")
local mqueue = require("scada-common.mqueue")
local network = require("scada-common.network")
local ppm = require("scada-common.ppm")
local tcd = require("scada-common.tcd")
local util = require("scada-common.util")
local core = require("graphics.core")
local configure = require("coordinator.configure")
local coordinator = require("coordinator.coordinator")
local iocontrol = require("coordinator.iocontrol")
local renderer = require("coordinator.renderer")
local sounder = require("coordinator.sounder")
local threads = require("coordinator.threads")
local apisessions = require("coordinator.session.apisessions")
local COORDINATOR_VERSION = "v1.3.5"
local COORDINATOR_VERSION = "v1.4.0"
local CHUNK_LOAD_DELAY_S = 30.0
@ -133,12 +130,54 @@ local function main()
log_sys("system start on " .. os.date("%c"))
log_boot("starting " .. COORDINATOR_VERSION)
----------------------------------------
-- memory allocation
----------------------------------------
-- shared memory across threads
---@class crd_shared_memory
local __shared_memory = {
-- time and date format for display
date_format = util.trinary(config.Time24Hour, "%X \x04 %A, %B %d %Y", "%r \x04 %A, %B %d %Y"),
-- coordinator system state flags
---@class crd_state
crd_state = {
fp_ok = false,
ui_ok = false,
link_fail = false,
shutdown = false
},
-- core coordinator devices
crd_dev = {
speaker = ppm.get_device("speaker"),
modem = ppm.get_wireless_modem()
},
-- system objects
crd_sys = {
nic = nil, ---@type nic
coord_comms = nil, ---@type coord_comms
conn_watchdog = nil ---@type watchdog
},
-- message queues
q = {
mq_render = mqueue.new()
}
}
local smem_dev = __shared_memory.crd_dev
local smem_sys = __shared_memory.crd_sys
local crd_state = __shared_memory.crd_state
----------------------------------------
-- setup alarm sounder subsystem
----------------------------------------
local speaker = ppm.get_device("speaker")
if speaker == nil then
if smem_dev.speaker == nil then
log_boot("annunciator alarm speaker not found")
println("startup> speaker not found")
log.fatal("no annunciator alarm speaker found")
@ -146,7 +185,7 @@ local function main()
else
local sounder_start = util.time_ms()
log_boot("annunciator alarm speaker connected")
sounder.init(speaker, config.SpeakerVolume)
sounder.init(smem_dev.speaker, config.SpeakerVolume)
log_boot("tone generation took " .. (util.time_ms() - sounder_start) .. "ms")
log_sys("annunciator alarm configured")
iocontrol.fp_has_speaker(true)
@ -163,8 +202,7 @@ local function main()
end
-- get the communications modem
local modem = ppm.get_wireless_modem()
if modem == nil then
if smem_dev.modem == nil then
log_comms("wireless modem not found")
println("startup> wireless modem not found")
log.fatal("no wireless modem on startup")
@ -175,243 +213,54 @@ local function main()
end
-- create connection watchdog
local conn_watchdog = util.new_watchdog(config.SVR_Timeout)
conn_watchdog.cancel()
smem_sys.conn_watchdog = util.new_watchdog(config.SVR_Timeout)
smem_sys.conn_watchdog.cancel()
log.debug("startup> conn watchdog created")
-- create network interface then setup comms
local nic = network.nic(modem)
local coord_comms = coordinator.comms(COORDINATOR_VERSION, nic, conn_watchdog)
smem_sys.nic = network.nic(smem_dev.modem)
smem_sys.coord_comms = coordinator.comms(COORDINATOR_VERSION, smem_sys.nic, smem_sys.conn_watchdog)
log.debug("startup> comms init")
log_comms("comms initialized")
-- base loop clock (2Hz, 10 ticks)
local MAIN_CLOCK = 0.5
local loop_clock = util.new_clock(MAIN_CLOCK)
----------------------------------------
-- start front panel & UI start function
-- start front panel
----------------------------------------
log_graphics("starting front panel UI...")
local fp_ok, fp_message = renderer.try_start_fp()
if not fp_ok then
local fp_message
crd_state.fp_ok, fp_message = renderer.try_start_fp()
if not crd_state.fp_ok then
log_graphics(util.c("front panel UI error: ", fp_message))
println_ts("front panel UI creation failed")
log.fatal(util.c("front panel GUI render failed with error ", fp_message))
return
else log_graphics("front panel ready") end
-- start up the main UI
---@return boolean ui_ok started ok
local function start_main_ui()
log_graphics("starting main UI...")
local draw_start = util.time_ms()
local ui_ok, ui_message = renderer.try_start_ui()
if not ui_ok then
log_graphics(util.c("main UI error: ", ui_message))
log.fatal(util.c("main GUI render failed with error ", ui_message))
else
log_graphics("main UI draw took " .. (util.time_ms() - draw_start) .. "ms")
end
return ui_ok
end
----------------------------------------
-- main event loop
-- start system
----------------------------------------
local link_failed = false
local ui_ok = true
local date_format = util.trinary(config.Time24Hour, "%X \x04 %A, %B %d %Y", "%r \x04 %A, %B %d %Y")
-- init threads
local main_thread = threads.thread__main(__shared_memory)
local render_thread = threads.thread__render(__shared_memory)
-- start clock
loop_clock.start()
log.info("startup> completed")
log_sys("system started successfully")
-- main event loop
while true do
local event, param1, param2, param3, param4, param5 = util.pull_event()
-- handle event
if event == "peripheral_detach" then
local type, device = ppm.handle_unmount(param1)
if type ~= nil and device ~= nil then
if type == "modem" then
-- we only really care if this is our wireless modem
-- if it is another modem, handle other peripheral losses separately
if nic.is_modem(device) then
nic.disconnect()
log_sys("comms modem disconnected")
local other_modem = ppm.get_wireless_modem()
if other_modem then
log_sys("found another wireless modem, using it for comms")
nic.connect(other_modem)
else
-- close out main UI
renderer.close_ui()
-- alert user to status
log_sys("awaiting comms modem reconnect...")
iocontrol.fp_has_modem(false)
end
else
log_sys("non-comms modem disconnected")
end
elseif type == "monitor" then
if renderer.handle_disconnect(device) then
log_sys("lost a configured monitor")
else
log_sys("lost an unused monitor")
end
elseif type == "speaker" then
log_sys("lost alarm sounder speaker")
iocontrol.fp_has_speaker(false)
end
end
elseif event == "peripheral" then
local type, device = ppm.mount(param1)
if type ~= nil and device ~= nil then
if type == "modem" then
if device.isWireless() and not nic.is_connected() then
-- reconnected modem
log_sys("comms modem reconnected")
nic.connect(device)
iocontrol.fp_has_modem(true)
elseif device.isWireless() then
log.info("unused wireless modem reconnected")
else
log_sys("wired modem reconnected")
end
elseif type == "monitor" then
if renderer.handle_reconnect(param1, device) then
log_sys(util.c("configured monitor ", param1, " reconnected"))
else
log_sys(util.c("unused monitor ", param1, " connected"))
end
elseif type == "speaker" then
log_sys("alarm sounder speaker reconnected")
sounder.reconnect(device)
iocontrol.fp_has_speaker(true)
end
end
elseif event == "monitor_resize" then
local is_used, is_ok = renderer.handle_resize(param1)
if is_used then
log_sys(util.c("configured monitor ", param1, " resized, ", util.trinary(is_ok, "display still fits", "display no longer fits")))
end
elseif event == "timer" then
if loop_clock.is_clock(param1) then
-- main loop tick
-- toggle heartbeat
iocontrol.heartbeat()
-- maintain connection
if nic.is_connected() then
local ok, start_ui = coord_comms.try_connect()
if not ok then
link_failed = true
log_sys("supervisor connection failed, shutting down...")
log.fatal("failed to connect to supervisor")
break
elseif start_ui then
log_sys("supervisor connected, proceeding to main UI start")
ui_ok = start_main_ui()
if not ui_ok then break end
end
end
-- iterate sessions
apisessions.iterate_all()
-- free any closed sessions
apisessions.free_all_closed()
-- update date and time string for main display
if coord_comms.is_linked() then
iocontrol.get_db().facility.ps.publish("date_time", os.date(date_format))
end
loop_clock.start()
elseif conn_watchdog.is_timer(param1) then
-- supervisor watchdog timeout
log_comms("supervisor server timeout")
-- close connection, main UI, and stop sounder
coord_comms.close()
renderer.close_ui()
sounder.stop()
else
-- a non-clock/main watchdog timer event
-- check API watchdogs
apisessions.check_all_watchdogs(param1)
-- notify timer callback dispatcher
tcd.handle(param1)
end
elseif event == "modem_message" then
-- got a packet
local packet = coord_comms.parse_packet(param1, param2, param3, param4, param5)
-- handle then check if it was a disconnect
if coord_comms.handle_packet(packet) then
log_comms("supervisor closed connection")
-- close connection, main UI, and stop sounder
coord_comms.close()
renderer.close_ui()
sounder.stop()
end
elseif event == "monitor_touch" or event == "mouse_click" or event == "mouse_up" or
event == "mouse_drag" or event == "mouse_scroll" or event == "double_click" then
-- handle a mouse event
renderer.handle_mouse(core.events.new_mouse_event(event, param1, param2, param3))
elseif event == "speaker_audio_empty" then
-- handle speaker buffer emptied
sounder.continue()
end
-- check for termination request
if event == "terminate" or ppm.should_terminate() then
-- handle supervisor connection
coord_comms.try_connect(true)
if coord_comms.is_linked() then
log_comms("terminate requested, closing supervisor connection...")
else link_failed = true end
coord_comms.close()
log_comms("supervisor connection closed")
-- handle API sessions
log_comms("closing api sessions...")
apisessions.close_all()
log_comms("api sessions closed")
break
end
end
-- run threads
parallel.waitForAll(main_thread.p_exec, render_thread.p_exec)
renderer.close_ui()
renderer.close_fp()
sounder.stop()
log_sys("system shutdown")
if link_failed then println_ts("failed to connect to supervisor") end
if not ui_ok then println_ts("main UI creation failed") end
if crd_state.link_fail then println_ts("failed to connect to supervisor") end
if not crd_state.ui_ok then println_ts("main UI creation failed") end
-- close on error exit (such as UI error)
if coord_comms.is_linked() then coord_comms.close() end
if smem_sys.coord_comms.is_linked() then smem_sys.coord_comms.close() end
println_ts("exited")
log.info("exited")

346
coordinator/threads.lua Normal file
View File

@ -0,0 +1,346 @@
local log = require("scada-common.log")
local mqueue = require("scada-common.mqueue")
local ppm = require("scada-common.ppm")
local tcd = require("scada-common.tcd")
local util = require("scada-common.util")
local coordinator = require("coordinator.coordinator")
local iocontrol = require("coordinator.iocontrol")
local renderer = require("coordinator.renderer")
local sounder = require("coordinator.sounder")
local apisessions = require("coordinator.session.apisessions")
local core = require("graphics.core")
local log_graphics = coordinator.log_graphics
local log_sys = coordinator.log_sys
local log_comms = coordinator.log_comms
local threads = {}
local MAIN_CLOCK = 0.5 -- (2Hz, 10 ticks)
local RENDER_SLEEP = 100 -- (100ms, 2 ticks)
local MQ__RENDER_CMD = {
START_MAIN_UI = 1,
MON_DISCONNECT = 2,
MON_CONNECT = 3,
MON_RESIZE = 4,
UPDATE = 5
}
-- main thread
---@nodiscard
---@param smem crd_shared_memory
function threads.thread__main(smem)
---@class parallel_thread
local public = {}
-- execute thread
function public.exec()
iocontrol.fp_rt_status("main", true)
log.debug("main thread start")
local loop_clock = util.new_clock(MAIN_CLOCK)
-- start clock
loop_clock.start()
log_sys("system started successfully")
-- load in from shared memory
local crd_state = smem.crd_state
local nic = smem.crd_sys.nic
local coord_comms = smem.crd_sys.coord_comms
local conn_watchdog = smem.crd_sys.conn_watchdog
-- event loop
while true do
local event, param1, param2, param3, param4, param5 = util.pull_event()
-- handle event
if event == "peripheral_detach" then
local type, device = ppm.handle_unmount(param1)
if type ~= nil and device ~= nil then
if type == "modem" then
-- we only really care if this is our wireless modem
-- if it is another modem, handle other peripheral losses separately
if nic.is_modem(device) then
nic.disconnect()
log_sys("comms modem disconnected")
local other_modem = ppm.get_wireless_modem()
if other_modem then
log_sys("found another wireless modem, using it for comms")
nic.connect(other_modem)
else
-- close out main UI
renderer.close_ui()
-- alert user to status
log_sys("awaiting comms modem reconnect...")
iocontrol.fp_has_modem(false)
end
else
log_sys("non-comms modem disconnected")
end
elseif type == "monitor" then
if renderer.handle_disconnect(device) then
log_sys("lost a configured monitor")
else
log_sys("lost an unused monitor")
end
elseif type == "speaker" then
log_sys("lost alarm sounder speaker")
iocontrol.fp_has_speaker(false)
end
end
elseif event == "peripheral" then
local type, device = ppm.mount(param1)
if type ~= nil and device ~= nil then
if type == "modem" then
if device.isWireless() and not nic.is_connected() then
-- reconnected modem
log_sys("comms modem reconnected")
nic.connect(device)
iocontrol.fp_has_modem(true)
elseif device.isWireless() then
log.info("unused wireless modem reconnected")
else
log_sys("wired modem reconnected")
end
elseif type == "monitor" then
if renderer.handle_reconnect(param1, device) then
log_sys(util.c("configured monitor ", param1, " reconnected"))
else
log_sys(util.c("unused monitor ", param1, " connected"))
end
elseif type == "speaker" then
log_sys("alarm sounder speaker reconnected")
sounder.reconnect(device)
iocontrol.fp_has_speaker(true)
end
end
elseif event == "monitor_resize" then
local is_used, is_ok = renderer.handle_resize(param1)
if is_used then
log_sys(util.c("configured monitor ", param1, " resized, ", util.trinary(is_ok, "display still fits", "display no longer fits")))
end
elseif event == "timer" then
if loop_clock.is_clock(param1) then
-- main loop tick
-- toggle heartbeat
iocontrol.heartbeat()
-- maintain connection
if nic.is_connected() then
local ok, start_ui = coord_comms.try_connect()
if not ok then
crd_state.link_fail = true
log_sys("supervisor connection failed, shutting down...")
log.fatal("failed to connect to supervisor")
break
elseif start_ui then
log_sys("supervisor connected, dispatching main UI start")
smem.q.mq_render.push_command(MQ__RENDER_CMD.START_MAIN_UI)
--@todo if not ui_ok then break end
end
end
-- iterate sessions
apisessions.iterate_all()
-- free any closed sessions
apisessions.free_all_closed()
-- update date and time string for main display
if coord_comms.is_linked() then
iocontrol.get_db().facility.ps.publish("date_time", os.date(smem.date_format))
end
loop_clock.start()
elseif conn_watchdog.is_timer(param1) then
-- supervisor watchdog timeout
log_comms("supervisor server timeout")
-- close connection, main UI, and stop sounder
coord_comms.close()
renderer.close_ui()
sounder.stop()
else
-- a non-clock/main watchdog timer event
-- check API watchdogs
apisessions.check_all_watchdogs(param1)
-- notify timer callback dispatcher
tcd.handle(param1)
end
elseif event == "modem_message" then
-- got a packet
local packet = coord_comms.parse_packet(param1, param2, param3, param4, param5)
-- handle then check if it was a disconnect
if coord_comms.handle_packet(packet) then
log_comms("supervisor closed connection")
-- close connection, main UI, and stop sounder
coord_comms.close()
renderer.close_ui()
sounder.stop()
end
elseif event == "monitor_touch" or event == "mouse_click" or event == "mouse_up" or
event == "mouse_drag" or event == "mouse_scroll" or event == "double_click" then
-- handle a mouse event
renderer.handle_mouse(core.events.new_mouse_event(event, param1, param2, param3))
elseif event == "speaker_audio_empty" then
-- handle speaker buffer emptied
sounder.continue()
end
-- check for termination request
if event == "terminate" or ppm.should_terminate() then
crd_state.shutdown = true
log.info("terminate requested, main thread exiting")
-- handle closing supervisor connection
coord_comms.try_connect(true)
if coord_comms.is_linked() then
log_comms("terminate requested, closing supervisor connection...")
else crd_state.link_fail = true end
coord_comms.close()
log_comms("supervisor connection closed")
-- handle API sessions
log_comms("closing api sessions...")
apisessions.close_all()
log_comms("api sessions closed")
break
end
end
end
-- execute the thread in a protected mode, retrying it on return if not shutting down
function public.p_exec()
local crd_state = smem.crd_state
while not crd_state.shutdown do
local status, result = pcall(public.exec)
if status == false then
log.fatal(util.strval(result))
end
iocontrol.fp_rt_status("main", false)
-- if status is true, then we are probably exiting, so this won't matter
-- if not, we need to restart the clock
-- this thread cannot be slept because it will miss events (namely "terminate" otherwise)
if not crd_state.shutdown then
log.info("main thread restarting now...")
end
end
end
return public
end
-- coordinator renderer thread
---@nodiscard
---@param smem crd_shared_memory
function threads.thread__render(smem)
---@class parallel_thread
local public = {}
-- execute thread
function public.exec()
iocontrol.fp_rt_status("render", true)
log.debug("render thread start")
-- load in from shared memory
local crd_state = smem.crd_state
local render_queue = smem.q.mq_render
local last_update = util.time()
-- thread loop
while true do
-- check for messages in the message queue
while render_queue.ready() and not crd_state.shutdown do
local msg = render_queue.pop()
if msg ~= nil then
if msg.qtype == mqueue.TYPE.COMMAND then
-- received a command
if msg.message == MQ__RENDER_CMD.START_MAIN_UI then
-- start up the main UI
log_graphics("starting main UI...")
local draw_start = util.time_ms()
local ui_message
crd_state.ui_ok, ui_message = renderer.try_start_ui()
if not crd_state.ui_ok then
log_graphics(util.c("main UI error: ", ui_message))
log.fatal(util.c("main GUI render failed with error ", ui_message))
else
log_graphics("main UI draw took " .. (util.time_ms() - draw_start) .. "ms")
end
elseif msg.message == MQ__RENDER_CMD.MON_DISCONNECT then
-- monitor lost
elseif msg.message == MQ__RENDER_CMD.MON_CONNECT then
-- monitor connected
elseif msg.message == MQ__RENDER_CMD.UPDATE then
-- new data
end
elseif msg.qtype == mqueue.TYPE.DATA then
-- received data
elseif msg.qtype == mqueue.TYPE.PACKET then
-- received a packet
end
end
-- quick yield
util.nop()
end
-- check for termination request
if crd_state.shutdown then
log.info("render thread exiting")
break
end
-- delay before next check
last_update = util.adaptive_delay(RENDER_SLEEP, last_update)
end
end
-- execute the thread in a protected mode, retrying it on return if not shutting down
function public.p_exec()
local crd_state = smem.crd_state
while not crd_state.shutdown do
local status, result = pcall(public.exec)
if status == false then
log.fatal(util.strval(result))
end
iocontrol.fp_rt_status("render", false)
if not crd_state.shutdown then
log.info("render thread restarting in 5 seconds...")
util.psleep(5)
end
end
end
return public
end
return threads

View File

@ -100,6 +100,14 @@ local function init(panel, num_units)
local speaker = LED{parent=system,label="SPEAKER",colors=led_grn}
speaker.register(ps, "has_speaker", speaker.update)
system.line_break()
local rt_main = LED{parent=system,label="RT MAIN",colors=led_grn}
local rt_render = LED{parent=system,label="RT RENDER",colors=led_grn}
rt_main.register(ps, "routine__main", rt_main.update)
rt_render.register(ps, "routine__render", rt_render.update)
---@diagnostic disable-next-line: undefined-field
local comp_id = util.sprintf("(%d)", os.getComputerID())
TextBox{parent=system,x=9,y=4,width=6,height=1,text=comp_id,fg_bg=style.fp.disabled_fg}