#589 WIP reboot recovery

This commit is contained in:
Mikayla Fischler
2025-02-08 22:20:00 -05:00
parent b3cf40a01a
commit f32855084e
4 changed files with 53 additions and 24 deletions

View File

@ -80,7 +80,7 @@ function facility.new(config)
-- process control
recovery = RCV_STATE.INACTIVE, ---@type RECOVERY_STATE
recovery_boot_state = nil, ---@type sv_control_state|nil
last_unit_states = nil, ---@type boolean[]
last_unit_states = {}, ---@type boolean[]
units_ready = false,
mode = PROCESS.INACTIVE, ---@type PROCESS
last_mode = PROCESS.INACTIVE, ---@type PROCESS
@ -166,6 +166,13 @@ function facility.new(config)
table.insert(self.test_tone_states, false)
end
-- init next boot state
settings.set("LastProcessState", PROCESS.INACTIVE)
settings.set("LastUnitStates", self.last_unit_states)
if not settings.save("/supervisor.settings") then
log.warning("FAC: failed to save initial control state into supervisor settings file")
end
--#endregion
-- PRIVATE FUNCTIONS --
@ -316,19 +323,24 @@ function facility.new(config)
if self.recovery == RCV_STATE.RUNNING then
-- try to start auto control
if self.recovery_boot_state.mode ~= nil and self.units_ready then
if self.recovery_boot_state.mode ~= PROCESS.INACTIVE and self.recovery_boot_state.mode ~= PROCESS.SYSTEM_ALARM_IDLE then
self.mode = self.mode_set
log.info("FAC: process startup resume initiated")
end
self.recovery_boot_state.mode = nil
self.mode = self.mode_set
log.info("FAC: process startup resume initiated")
end
local recovered = self.recovery_boot_state.mode == nil
local recovered = self.recovery_boot_state.mode == nil or self.recovery_boot_state.mode == PROCESS.INACTIVE
-- restore manual control reactors
for i = 1, #self.units do
local u = self.units[i]
if self.recovery_boot_state.unit_states[i] and self.group_map[i] == AUTO_GROUP.MANUAL then
recovered = false
if self.units[i].get_control_inf().ready then
if u.get_control_inf().ready then
local plc_s = svsessions.get_reactor_session(i)
if plc_s ~= nil then
plc_s.in_queue.push_command(plc.PLC_S_CMDS.ENABLE)
@ -344,7 +356,7 @@ function facility.new(config)
if recovered then
self.recovery = RCV_STATE.STOPPED
self.recovery_boot_state = nil
log.info("FAC: startup resume complete")
log.info("FAC: startup resume sequence completed")
end
end
@ -378,29 +390,45 @@ function facility.new(config)
--#region Startup Recovery
-- on exit, use this to clear the boot state so we don't resume when exiting cleanly
function public.clear_boot_state()
settings.unset("LastProcessState")
settings.unset("LastUnitStates")
local saved = settings.save("/supervisor.settings")
if not saved then
log.warning("facility.clear_boot_state(): failed to save supervisor settings file")
else
log.debug("FAC: cleared boot state on exit")
end
end
-- initialize startup recovery
---@param state sv_control_state
function public.startup_recovery_init(state)
if self.recovery == RCV_STATE.INACTIVE then
self.recovery_boot_state = state
self.recovery = RCV_STATE.PRIMED
log.info("FAC: startup resume ready")
end
end
-- attempt startup recovery
---@param auto_cfg start_auto_config configuration
function public.startup_recovery_start(auto_cfg)
if self.recovery == RCV_STATE.PRIMED and self.recovery_boot_state and
self.recovery_boot_state.mode ~= PROCESS.INACTIVE and self.recovery_boot_state.mode ~= PROCESS.SYSTEM_ALARM_IDLE then
if self.recovery == RCV_STATE.PRIMED then
self.recovery = util.trinary(_auto_check_and_save(auto_cfg), RCV_STATE.RUNNING, RCV_STATE.STOPPED)
log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "ready", "failed")))
log.info(util.c("FAC: startup resume ", util.trinary(self.recovery == RCV_STATE.RUNNING, "started", "failed")))
else self.recovery = RCV_STATE.STOPPED end
end
-- used on certain coordinator commands to end reboot recovery (remain in current operational state)
function public.cancel_recovery()
self.recovery = RCV_STATE.STOPPED
self.recovery_boot_state = nil
log.info("FAC: process startup resume cancelled by user operation")
if self.recovery == RCV_STATE.RUNNING then
self.recovery = RCV_STATE.STOPPED
self.recovery_boot_state = nil
log.info("FAC: process startup resume cancelled by user operation")
end
end
--#endregion

View File

@ -817,9 +817,9 @@ function update.unit_mgmt()
need_emcool = true
end
-- check for control state changes to save
if self.last_unit_states[i] ~= u.get_control_state() then
self.last_unit_states[i] = u.get_control_state()
-- check for enabled state changes to save
if self.last_unit_states[i] ~= u.is_reactor_enabled() then
self.last_unit_states[i] = u.is_reactor_enabled()
write_state = true
end
end
@ -828,8 +828,7 @@ function update.unit_mgmt()
if write_state then
settings.set("LastUnitStates", self.last_unit_states)
local saved = settings.save("/supervisor.settings")
if not saved then
if not settings.save("/supervisor.settings") then
log.warning("facility_update.unit_mgmt(): failed to save supervisor settings file")
end
end

View File

@ -22,7 +22,7 @@ local supervisor = require("supervisor.supervisor")
local svsessions = require("supervisor.session.svsessions")
local SUPERVISOR_VERSION = "v1.6.2"
local SUPERVISOR_VERSION = "v1.6.3"
local println = util.println
local println_ts = util.println_ts
@ -240,6 +240,8 @@ local function main()
end
end
sv_facility.clear_boot_state()
renderer.close_ui()
util.println_ts("exited")

View File

@ -840,6 +840,12 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle)
return false
end
-- check the active state of the reactor (if connected)
---@nodiscard
function public.is_reactor_enabled()
if self.plc_i ~= nil then return self.plc_i.get_status().status else return false end
end
-- check if the reactor is connected, is stopped, the RPS is not tripped, and no alarms are active
---@nodiscard
function public.is_safe_idle()
@ -917,12 +923,6 @@ function unit.new(reactor_id, num_boilers, num_turbines, ext_idle)
return status
end
-- check the commanded control state of the reactor (if connected)
---@nodiscard
function public.get_control_state()
if self.plc_i ~= nil then return self.plc_i.get_db().control_state else return false end
end
-- get the current burn rate (actual rate)
---@nodiscard
function public.get_burn_rate()