From ccd9f4b6ccad5163a104425f55d9c0d47cff7019 Mon Sep 17 00:00:00 2001 From: Mikayla Fischler Date: Mon, 13 Feb 2023 18:08:32 -0500 Subject: [PATCH] #158 fixed race conditions and cleaned up ascram logic --- supervisor/session/facility.lua | 126 +++++++++++++++----------------- supervisor/session/plc.lua | 36 ++++----- supervisor/startup.lua | 2 +- 3 files changed, 78 insertions(+), 86 deletions(-) diff --git a/supervisor/session/facility.lua b/supervisor/session/facility.lua index ab658e4..b9b5ccd 100644 --- a/supervisor/session/facility.lua +++ b/supervisor/session/facility.lua @@ -69,6 +69,12 @@ function facility.new(num_reactors, cooling_conf) at_max_burn = false, ascram = false, ascram_reason = AUTO_SCRAM.NONE, + ascram_status = { + matrix_dc = false, + matrix_fill = false, + crit_alarm = false, + gen_fault = false + }, -- closed loop control charge_conversion = 1.0, time_start = 0.0, @@ -512,111 +518,95 @@ function facility.new(num_reactors, cooling_conf) -- Evaluate Automatic SCRAM -- ------------------------------ + local astatus = self.ascram_status + if (self.mode ~= PROCESS.INACTIVE) and (self.mode ~= PROCESS.UNIT_ALARM_IDLE) then - local scram = false - - if self.units_ready and self.ascram_reason == AUTO_SCRAM.GEN_FAULT then - self.ascram_reason = AUTO_SCRAM.NONE - end - if self.induction[1] ~= nil then local matrix = self.induction[1] ---@type unit_session local db = matrix.get_db() ---@type imatrix_session_db - if self.ascram_reason == AUTO_SCRAM.MATRIX_DC then - self.ascram_reason = AUTO_SCRAM.NONE - log.info("FAC: cleared automatic SCRAM trip due to prior induction matrix disconnect") + -- clear matrix disconnected + if astatus.matrix_dc then + astatus.matrix_dc = false + log.info("FAC: induction matrix reconnected, clearing ASCRAM condition") end - if (db.tanks.energy_fill >= HIGH_CHARGE) or - (self.ascram_reason == AUTO_SCRAM.MATRIX_FILL and db.tanks.energy_fill > RE_ENABLE_CHARGE) then - scram = true + -- check matrix fill too high + local was_fill = astatus.matrix_fill + astatus.matrix_fill = (db.tanks.energy_fill >= HIGH_CHARGE) or (astatus.matrix_fill and db.tanks.energy_fill > RE_ENABLE_CHARGE) - if self.mode ~= PROCESS.MATRIX_FAULT_IDLE then - self.return_mode = self.mode - next_mode = PROCESS.MATRIX_FAULT_IDLE - end - - if self.ascram_reason == AUTO_SCRAM.NONE then - self.ascram_reason = AUTO_SCRAM.MATRIX_FILL - end - elseif self.ascram_reason == AUTO_SCRAM.MATRIX_FILL then + if was_fill and not astatus.matrix_fill then log.info("FAC: charge state of induction matrix entered acceptable range <= " .. (RE_ENABLE_CHARGE * 100) .. "%") - self.ascram_reason = AUTO_SCRAM.NONE end + -- system not ready, will need to restart GEN_RATE mode + -- clears when we enter the fault waiting state + astatus.gen_fault = self.mode == PROCESS.GEN_RATE and not self.units_ready + + -- check for critical unit alarms for i = 1, #self.units do local u = self.units[i] ---@type reactor_unit if u.has_critical_alarm() then - scram = true - - if self.ascram_reason == AUTO_SCRAM.NONE then - self.ascram_reason = AUTO_SCRAM.CRIT_ALARM - end - - next_mode = PROCESS.UNIT_ALARM_IDLE - - log.info("FAC: emergency exit of process control due to critical unit alarm") + log.info(util.c("FAC: emergency exit of process control due to critical unit alarm (unit ", u.get_id(), ")")) break end end - - if (self.mode == PROCESS.GEN_RATE) and (not self.units_ready) then - -- system not ready, will need to restart GEN_RATE mode - scram = true - - if self.ascram_reason == AUTO_SCRAM.NONE then - self.ascram_reason = AUTO_SCRAM.GEN_FAULT - end - - next_mode = PROCESS.GEN_RATE_FAULT_IDLE - end else - scram = true - - if self.mode ~= PROCESS.MATRIX_FAULT_IDLE then - self.return_mode = self.mode - next_mode = PROCESS.MATRIX_FAULT_IDLE - end - - if self.ascram_reason == AUTO_SCRAM.NONE then - self.ascram_reason = AUTO_SCRAM.MATRIX_DC - end + astatus.matrix_dc = true end - -- SCRAM all units - if (not self.ascram) and scram then + -- log.debug(util.c("dc: ", astatus.matrix_dc, " fill: ", astatus.matrix_fill, " crit: ", astatus.crit_alarm, " gen: ", astatus.gen_fault)) + + local scram = astatus.matrix_dc or astatus.matrix_fill or astatus.crit_alarm or astatus.gen_fault + + if scram and not self.ascram then + -- SCRAM all units for i = 1, #self.prio_defs do for _, u in pairs(self.prio_defs[i]) do u.a_scram() end end - if self.ascram_reason == AUTO_SCRAM.MATRIX_DC then - log.info("FAC: automatic SCRAM due to induction matrix disconnection") - self.status_text = { "AUTOMATIC SCRAM", "induction matrix disconnected" } - elseif self.ascram_reason == AUTO_SCRAM.MATRIX_FILL then - log.info("FAC: automatic SCRAM due to induction matrix high charge") - self.status_text = { "AUTOMATIC SCRAM", "induction matrix fill high" } - elseif self.ascram_reason == AUTO_SCRAM.CRIT_ALARM then - log.info("FAC: automatic SCRAM due to critical unit alarm") + if astatus.crit_alarm then + -- highest priority alarm + next_mode = PROCESS.UNIT_ALARM_IDLE + self.ascram_reason = AUTO_SCRAM.CRIT_ALARM self.status_text = { "AUTOMATIC SCRAM", "critical unit alarm tripped" } - elseif self.ascram_reason == AUTO_SCRAM.GEN_FAULT then - log.info("FAC: automatic SCRAM due to unit problem while in GEN_RATE mode, will resume once all units are ready") + + log.info("FAC: automatic SCRAM due to critical unit alarm") + elseif astatus.matrix_dc then + next_mode = PROCESS.MATRIX_FAULT_IDLE + self.ascram_reason = AUTO_SCRAM.MATRIX_DC + self.status_text = { "AUTOMATIC SCRAM", "induction matrix disconnected" } + + if self.mode ~= PROCESS.MATRIX_FAULT_IDLE then self.return_mode = self.mode end + + log.info("FAC: automatic SCRAM due to induction matrix disconnection") + elseif astatus.matrix_fill then + next_mode = PROCESS.MATRIX_FAULT_IDLE + self.ascram_reason = AUTO_SCRAM.MATRIX_FILL + self.status_text = { "AUTOMATIC SCRAM", "induction matrix fill high" } + + if self.mode ~= PROCESS.MATRIX_FAULT_IDLE then self.return_mode = self.mode end + + log.info("FAC: automatic SCRAM due to induction matrix high charge") + elseif astatus.gen_fault then + -- lowest priority alarm + next_mode = PROCESS.GEN_RATE_FAULT_IDLE + self.ascram_reason = AUTO_SCRAM.GEN_FAULT self.status_text = { "GENERATION MODE IDLE", "paused: system not ready" } - else - log.error(util.c("FAC: automatic SCRAM reason (", self.ascram_reason, ") not set to a known value")) + + log.info("FAC: automatic SCRAM due to unit problem while in GEN_RATE mode, will resume once all units are ready") end end self.ascram = scram - -- clear PLC SCRAM if we should if not self.ascram then self.ascram_reason = AUTO_SCRAM.NONE - -- do not reset while in gen rate, we have to exit this mode first + -- reset PLC RPS trips if we should for i = 1, #self.units do local u = self.units[i] ---@type reactor_unit u.a_cond_rps_reset() diff --git a/supervisor/session/plc.lua b/supervisor/session/plc.lua index 7c07825..4d29c3c 100644 --- a/supervisor/session/plc.lua +++ b/supervisor/session/plc.lua @@ -62,7 +62,6 @@ function plc.new_session(id, for_reactor, in_queue, out_queue, timeout) commanded_burn_rate = 0.0, auto_cmd_token = 0, ramping_rate = false, - auto_scram = false, auto_lock = false, -- connection properties seq_num = 0, @@ -82,12 +81,14 @@ function plc.new_session(id, for_reactor, in_queue, out_queue, timeout) struct_req = (util.time() + 500), status_req = (util.time() + 500), scram_req = 0, + ascram_req = 0, burn_rate_req = 0, rps_reset_req = 0 }, -- command acknowledgements acks = { scram = true, + ascram = true, burn_rate = true, rps_reset = true }, @@ -398,7 +399,7 @@ function plc.new_session(id, for_reactor, in_queue, out_queue, timeout) -- automatic SCRAM acknowledgement local ack = _get_ack(pkt) if ack then - self.acks.scram = true + self.acks.ascram = true self.sDB.control_state = false elseif ack == false then log.debug(log_header .. " automatic SCRAM failed!") @@ -449,9 +450,7 @@ function plc.new_session(id, for_reactor, in_queue, out_queue, timeout) elseif pkt.type == RPLC_TYPES.RPS_AUTO_RESET then -- RPS auto control reset acknowledgement local ack = _get_ack(pkt) - if ack then - self.auto_scram = false - else + if not ack then log.debug(log_header .. "RPS auto reset failed") end elseif pkt.type == RPLC_TYPES.AUTO_BURN_RATE then @@ -608,23 +607,22 @@ function plc.new_session(id, for_reactor, in_queue, out_queue, timeout) end elseif cmd == PLC_S_CMDS.SCRAM then -- SCRAM reactor - self.auto_scram = false self.acks.scram = false self.retry_times.scram_req = util.time() + INITIAL_WAIT _send(RPLC_TYPES.RPS_SCRAM, {}) elseif cmd == PLC_S_CMDS.ASCRAM then -- SCRAM reactor - self.auto_scram = true - self.acks.scram = false - self.retry_times.scram_req = util.time() + INITIAL_WAIT + self.acks.ascram = false + self.retry_times.ascram_req = util.time() + INITIAL_WAIT _send(RPLC_TYPES.RPS_ASCRAM, {}) elseif cmd == PLC_S_CMDS.RPS_RESET then -- reset RPS + self.acks.ascram = true self.acks.rps_reset = false self.retry_times.rps_reset_req = util.time() + INITIAL_WAIT _send(RPLC_TYPES.RPS_RESET, {}) elseif cmd == PLC_S_CMDS.RPS_AUTO_RESET then - if self.auto_scram or self.sDB.rps_status.timeout then + if self.sDB.rps_status.automatic or self.sDB.rps_status.timeout then _send(RPLC_TYPES.RPS_AUTO_RESET, {}) end else @@ -763,17 +761,21 @@ function plc.new_session(id, for_reactor, in_queue, out_queue, timeout) -- SCRAM request retry if not self.acks.scram then - if rtimes.scram_req - util.time() <= 0 then - if self.auto_scram then - _send(RPLC_TYPES.RPS_ASCRAM, {}) - else - _send(RPLC_TYPES.RPS_SCRAM, {}) - end - + if rtimes.scram_req - util.time() <= 0 then + _send(RPLC_TYPES.RPS_SCRAM, {}) rtimes.scram_req = util.time() + RETRY_PERIOD end end + -- automatic SCRAM request retry + + if not self.acks.ascram then + if rtimes.ascram_req - util.time() <= 0 then + _send(RPLC_TYPES.RPS_ASCRAM, {}) + rtimes.ascram_req = util.time() + RETRY_PERIOD + end + end + -- RPS reset request retry if not self.acks.rps_reset then diff --git a/supervisor/startup.lua b/supervisor/startup.lua index e04f6b6..62c93ff 100644 --- a/supervisor/startup.lua +++ b/supervisor/startup.lua @@ -14,7 +14,7 @@ local svsessions = require("supervisor.session.svsessions") local config = require("supervisor.config") local supervisor = require("supervisor.supervisor") -local SUPERVISOR_VERSION = "beta-v0.11.2" +local SUPERVISOR_VERSION = "beta-v0.11.3" local print = util.print local println = util.println