diff --git a/reactor-plc/plc.lua b/reactor-plc/plc.lua index d53ad95..68671b9 100644 --- a/reactor-plc/plc.lua +++ b/reactor-plc/plc.lua @@ -22,49 +22,48 @@ local println_ts = util.println_ts -- identifies dangerous states and SCRAMs reactor if warranted -- autonomous from main SCADA supervisor/coordinator control plc.rps_init = function (reactor) + local state_keys = { + dmg_crit = 1, + high_temp = 2, + no_coolant = 3, + ex_waste = 4, + ex_hcoolant = 5, + no_fuel = 6, + fault = 7, + timeout = 8, + manual = 9 + } + local self = { reactor = reactor, - cache = { false, false, false, false, false, false, false }, - timed_out = false, + state = { false, false, false, false, false, false, false, false, false }, + reactor_enabled = false, tripped = false, trip_cause = "" } -- PRIVATE FUNCTIONS -- + -- set reactor access fault flag + local _set_fault = function () + self.state[state_keys.fault] = true + end + + -- clear reactor access fault flag + local _clear_fault = function () + self.state[state_keys.fault] = false + end + -- check for critical damage local _damage_critical = function () local damage_percent = self.reactor.getDamagePercent() if damage_percent == ppm.ACCESS_FAULT then -- lost the peripheral or terminated, handled later log.error("RPS: failed to check reactor damage") - return false + _set_fault() + self.state[state_keys.dmg_crit] = false else - return damage_percent >= 100 - end - end - - -- check for heated coolant backup - local _excess_heated_coolant = function () - local hc_needed = self.reactor.getHeatedCoolantNeeded() - if hc_needed == ppm.ACCESS_FAULT then - -- lost the peripheral or terminated, handled later - log.error("RPS: failed to check reactor heated coolant level") - return false - else - return hc_needed == 0 - end - end - - -- check for excess waste - local _excess_waste = function () - local w_needed = self.reactor.getWasteNeeded() - if w_needed == ppm.ACCESS_FAULT then - -- lost the peripheral or terminated, handled later - log.error("RPS: failed to check reactor waste level") - return false - else - return w_needed == 0 + self.state[state_keys.dmg_crit] = damage_percent >= 100 end end @@ -75,9 +74,49 @@ plc.rps_init = function (reactor) if temp == ppm.ACCESS_FAULT then -- lost the peripheral or terminated, handled later log.error("RPS: failed to check reactor temperature") - return false + _set_fault() + self.state[state_keys.high_temp] = false else - return temp >= 1200 + self.state[state_keys.high_temp] = temp >= 1200 + end + end + + -- check if there is no coolant (<2% filled) + local _no_coolant = function () + local coolant_filled = self.reactor.getCoolantFilledPercentage() + if coolant_filled == ppm.ACCESS_FAULT then + -- lost the peripheral or terminated, handled later + log.error("RPS: failed to check reactor coolant level") + _set_fault() + self.state[state_keys.no_coolant] = false + else + self.state[state_keys.no_coolant] = coolant_filled < 0.02 + end + end + + -- check for excess waste (>80% filled) + local _excess_waste = function () + local w_filled = self.reactor.getWasteFilledPercentage() + if w_filled == ppm.ACCESS_FAULT then + -- lost the peripheral or terminated, handled later + log.error("RPS: failed to check reactor waste level") + _set_fault() + self.state[state_keys.ex_waste] = false + else + self.state[state_keys.ex_waste] = w_filled > 0.8 + end + end + + -- check for heated coolant backup (>95% filled) + local _excess_heated_coolant = function () + local hc_filled = self.reactor.getHeatedCoolantFilledPercentage() + if hc_filled == ppm.ACCESS_FAULT then + -- lost the peripheral or terminated, handled later + log.error("RPS: failed to check reactor heated coolant level") + _set_fault() + state[state_keys.ex_hcoolant] = false + else + state[state_keys.ex_hcoolant] = hc_filled > 0.95 end end @@ -86,22 +125,11 @@ plc.rps_init = function (reactor) local fuel = self.reactor.getFuel() if fuel == ppm.ACCESS_FAULT then -- lost the peripheral or terminated, handled later - log.error("RPS: failed to check reactor fuel level") - return false + log.error("RPS: failed to check reactor fuel") + _set_fault() + state[state_keys.no_fuel] = false else - return fuel == 0 - end - end - - -- check if there is no coolant - local _no_coolant = function () - local coolant_filled = self.reactor.getCoolantFilledPercentage() - if coolant_filled == ppm.ACCESS_FAULT then - -- lost the peripheral or terminated, handled later - log.error("RPS: failed to check reactor coolant level") - return false - else - return coolant_filled < 0.02 + state[state_keys.no_fuel] = fuel.amount == 0 end end @@ -114,78 +142,114 @@ plc.rps_init = function (reactor) -- report a PLC comms timeout local trip_timeout = function () - self.timed_out = true + state[state_keys.timed_out] = true + end + + -- manually SCRAM the reactor + local trip_manual = function () + state[state_keys.manual] = true + end + + -- SCRAM the reactor now + local scram = function () + log.info("RPS: reactor SCRAM") + + self.reactor.scram() + if self.reactor.__p_is_faulted() then + log.error("RPS: failed reactor SCRAM") + return false + else + self.reactor_enabled = false + return true + end + end + + -- start the reactor + local activate = function () + if not self.tripped then + log.info("RPS: reactor start") + + self.reactor.activate() + if self.reactor.__p_is_faulted() then + log.error("RPS: failed reactor start") + else + self.reactor_enabled = true + return true + end + end + + return false end -- check all safety conditions local check = function () local status = rps_status_t.ok local was_tripped = self.tripped + local first_trip = false + + -- update state + parallel.waitForAll( + _damage_critical, + _high_temp, + _no_coolant, + _excess_waste, + _excess_heated_coolant, + _insufficient_fuel + ) - -- update cache - self.cache = { - _damage_critical(), - _excess_heated_coolant(), - _excess_waste(), - _high_temp(), - _insufficient_fuel(), - _no_coolant(), - self.timed_out - } - -- check system states in order of severity if self.tripped then status = self.trip_cause - elseif self.cache[1] then - log.warning("RPS: damage critical!") + elseif self.state[state_keys.dmg_crit] then + log.warning("RPS: damage critical") status = rps_status_t.dmg_crit - elseif self.cache[4] then - log.warning("RPS: high temperature!") + elseif self.state[state_keys.high_temp] then + log.warning("RPS: high temperature") status = rps_status_t.high_temp - elseif self.cache[2] then - log.warning("RPS: heated coolant backup!") - status = rps_status_t.ex_hcoolant - elseif self.cache[6] then - log.warning("RPS: no coolant!") + elseif self.state[state_keys.no_coolant] then + log.warning("RPS: no coolant") status = rps_status_t.no_coolant - elseif self.cache[3] then - log.warning("RPS: full waste!") + elseif self.state[state_keys.ex_waste] then + log.warning("RPS: full waste") status = rps_status_t.ex_waste - elseif self.cache[5] then - log.warning("RPS: no fuel!") + elseif self.state[state_keys.ex_hcoolant] then + log.warning("RPS: heated coolant backup") + status = rps_status_t.ex_hcoolant + elseif self.state[state_keys.no_fuel] then + log.warning("RPS: no fuel") status = rps_status_t.no_fuel - elseif self.cache[7] then - log.warning("RPS: supervisor connection timeout!") + elseif self.state[state_keys.fault] then + log.warning("RPS: reactor access fault") + status = rps_status_t.fault + elseif self.state[state_keys.timeout] then + log.warning("RPS: supervisor connection timeout") status = rps_status_t.timeout + elseif self.state[state_keys.manual] then + log.warning("RPS: manual SCRAM requested") + status = rps_status_t.manual else self.tripped = false end - - -- if a new trip occured... - local first_trip = false - if not was_tripped and status ~= rps_status_t.ok then - log.warning("RPS: reactor SCRAM") + -- if a new trip occured... + if (not was_tripped) and (status ~= rps_status_t.ok) then first_trip = true self.tripped = true self.trip_cause = status - self.reactor.scram() - if self.reactor.__p_is_faulted() then - log.error("RPS: failed reactor SCRAM") - end + scram() end - + return self.tripped, status, first_trip end -- get the RPS status - local status = function () return self.cache end + local status = function () return self.state end local is_tripped = function () return self.tripped end + local is_active = function () return self.reactor_enabled end -- reset the RPS local reset = function () - self.timed_out = false self.tripped = false self.trip_cause = rps_status_t.ok end @@ -193,9 +257,13 @@ plc.rps_init = function (reactor) return { reconnect_reactor = reconnect_reactor, trip_timeout = trip_timeout, + trip_manual = trip_manual, + scram = scram, + activate = activate, check = check, status = status, is_tripped = is_tripped, + is_active = is_active, reset = reset } end @@ -544,18 +612,6 @@ plc.comms = function (id, modem, local_port, server_port, reactor, rps) -- request for physical structure _send_struct() log.debug("sent out structure again, did supervisor miss it?") - elseif packet.type == RPLC_TYPES.MEK_SCRAM then - -- disable the reactor - self.scrammed = true - plc_state.scram = true - self.reactor.scram() - _send_ack(packet.type, self.reactor.__p_is_ok()) - elseif packet.type == RPLC_TYPES.MEK_ENABLE then - -- enable the reactor - self.scrammed = false - plc_state.scram = false - self.reactor.activate() - _send_ack(packet.type, self.reactor.__p_is_ok()) elseif packet.type == RPLC_TYPES.MEK_BURN_RATE then -- set the burn rate if packet.length == 1 then @@ -581,6 +637,15 @@ plc.comms = function (id, modem, local_port, server_port, reactor, rps) else log.debug("RPLC set burn rate packet length mismatch") end + elseif packet.type == RPLC_TYPES.RPS_ENABLE then + -- enable the reactor + self.scrammed = false + _send_ack(packet.type, self.rps.activate()) + elseif packet.type == RPLC_TYPES.RPS_SCRAM then + -- disable the reactor + self.scrammed = true + self.rps.trip_manual() + _send_ack(packet.type, true) elseif packet.type == RPLC_TYPES.RPS_RESET then -- reset the RPS status rps.reset() diff --git a/reactor-plc/startup.lua b/reactor-plc/startup.lua index e8654dc..cf2bb0f 100644 --- a/reactor-plc/startup.lua +++ b/reactor-plc/startup.lua @@ -11,7 +11,7 @@ local config = require("config") local plc = require("plc") local threads = require("threads") -local R_PLC_VERSION = "alpha-v0.6.1" +local R_PLC_VERSION = "alpha-v0.6.2" local print = util.print local println = util.println @@ -37,7 +37,6 @@ local __shared_memory = { plc_state = { init_ok = true, shutdown = false, - scram = true, degraded = false, no_reactor = false, no_modem = false diff --git a/reactor-plc/threads.lua b/reactor-plc/threads.lua index 5deaf10..42d8e33 100644 --- a/reactor-plc/threads.lua +++ b/reactor-plc/threads.lua @@ -13,7 +13,7 @@ local println_ts = util.println_ts local psleep = util.psleep local MAIN_CLOCK = 1 -- (1Hz, 20 ticks) -local RPS_SLEEP = 500 -- (500ms, 10 ticks) +local RPS_SLEEP = 250 -- (250ms, 5 ticks) local COMMS_SLEEP = 150 -- (150ms, 3 ticks) local SP_CTRL_SLEEP = 250 -- (250ms, 5 ticks) @@ -207,7 +207,7 @@ threads.thread__rps = function (smem) if plc_state.init_ok then -- SCRAM if no open connection if networked and not plc_comms.is_linked() then - plc_state.scram = true + rps.scram() if was_linked then was_linked = false rps.trip_timeout() @@ -219,21 +219,17 @@ threads.thread__rps = function (smem) -- if we tried to SCRAM but failed, keep trying -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check) - if not plc_state.no_reactor and plc_state.scram and reactor.getStatus() then - reactor.scram() + if not plc_state.no_reactor and rps.is_tripped() and reactor.getStatus() then + rps.scram() end -- if we are in standalone mode, continuously reset RPS -- RPS will trip again if there are faults, but if it isn't cleared, the user can't re-enable - if not networked then - plc_state.scram = false - rps.reset() - end + if not networked then rps.reset() end -- check safety (SCRAM occurs if tripped) if not plc_state.no_reactor then local rps_tripped, rps_status_string, rps_first = rps.check() - plc_state.scram = plc_state.scram or rps_tripped if rps_first then println_ts("[RPS] SCRAM! safety trip: " .. rps_status_string) @@ -250,26 +246,19 @@ threads.thread__rps = function (smem) if msg.qtype == mqueue.TYPE.COMMAND then -- received a command - if msg.message == MQ__RPS_CMD.SCRAM then - -- basic SCRAM - plc_state.scram = true - reactor.scram() - elseif msg.message == MQ__RPS_CMD.DEGRADED_SCRAM then - -- SCRAM with print - plc_state.scram = true - if reactor.scram() then - println_ts("successful reactor SCRAM") - log.error("successful reactor SCRAM") - else - println_ts("failed reactor SCRAM") - log.error("failed reactor SCRAM") + if plc_state.init_ok then + if msg.message == MQ__RPS_CMD.SCRAM then + -- SCRAM + rps.scram() + elseif msg.message == MQ__RPS_CMD.DEGRADED_SCRAM then + -- lost peripheral(s) + rps.trip_degraded() + elseif msg.message == MQ__RPS_CMD.TRIP_TIMEOUT then + -- watchdog tripped + rps.trip_timeout() + println_ts("server timeout") + log.warning("server timeout") end - elseif msg.message == MQ__RPS_CMD.TRIP_TIMEOUT then - -- watchdog tripped - plc_state.scram = true - rps.trip_timeout() - println_ts("server timeout") - log.warning("server timeout") end elseif msg.qtype == mqueue.TYPE.DATA then -- received data @@ -286,9 +275,7 @@ threads.thread__rps = function (smem) -- safe exit log.info("rps thread shutdown initiated") if plc_state.init_ok then - plc_state.scram = true - reactor.scram() - if reactor.__p_is_ok() then + if rps.scram() then println_ts("reactor disabled") log.info("rps thread reactor SCRAM OK") else @@ -368,6 +355,8 @@ threads.thread__comms_rx = function (smem) -- load in from shared memory local plc_state = smem.plc_state local setpoints = smem.setpoints + local plc_dev = smem.plc_dev + local rps = smem.plc_sys.rps local plc_comms = smem.plc_sys.plc_comms local conn_watchdog = smem.plc_sys.conn_watchdog @@ -388,7 +377,7 @@ threads.thread__comms_rx = function (smem) elseif msg.qtype == mqueue.TYPE.PACKET then -- received a packet -- handle the packet (setpoints passed to update burn rate setpoint) - -- (plc_state passed to allow clearing SCRAM flag and check if degraded) + -- (plc_state passed to check if degraded) -- (conn_watchdog passed to allow feeding the watchdog) plc_comms.handle_packet(msg.message, setpoints, plc_state, conn_watchdog) end @@ -421,6 +410,7 @@ threads.thread__setpoint_control = function (smem) local plc_state = smem.plc_state local setpoints = smem.setpoints local plc_dev = smem.plc_dev + local rps = smem.plc_sys.rps local last_update = util.time() local running = false @@ -433,7 +423,7 @@ threads.thread__setpoint_control = function (smem) -- check if we should start ramping if setpoints.burn_rate ~= last_sp_burn then - if not plc_state.scram then + if rps.is_active() then if math.abs(setpoints.burn_rate - last_sp_burn) <= 5 then -- update without ramp if <= 5 mB/t change log.debug("setting burn rate directly to " .. setpoints.burn_rate .. "mB/t") @@ -459,7 +449,7 @@ threads.thread__setpoint_control = function (smem) running = false -- adjust burn rate (setpoints.burn_rate) - if not plc_state.scram then + if rps.is_active() then local current_burn_rate = reactor.getBurnRate() if (current_burn_rate ~= ppm.ACCESS_FAULT) and (current_burn_rate ~= setpoints.burn_rate) then -- calculate new burn rate diff --git a/scada-common/comms.lua b/scada-common/comms.lua index da6f5d3..f762c3b 100644 --- a/scada-common/comms.lua +++ b/scada-common/comms.lua @@ -17,9 +17,9 @@ local RPLC_TYPES = { LINK_REQ = 1, -- linking requests STATUS = 2, -- reactor/system status MEK_STRUCT = 3, -- mekanism build structure - MEK_SCRAM = 4, -- SCRAM reactor - MEK_ENABLE = 5, -- enable reactor - MEK_BURN_RATE = 6, -- set burn rate + MEK_BURN_RATE = 4, -- set burn rate + RPS_ENABLE = 5, -- enable reactor + RPS_SCRAM = 6, -- SCRAM reactor RPS_STATUS = 7, -- RPS status RPS_ALARM = 8, -- RPS alarm broadcast RPS_RESET = 9 -- clear RPS trip (if in bad state, will trip immediately) @@ -229,9 +229,9 @@ comms.rplc_packet = function () self.type == RPLC_TYPES.LINK_REQ or self.type == RPLC_TYPES.STATUS or self.type == RPLC_TYPES.MEK_STRUCT or - self.type == RPLC_TYPES.MEK_SCRAM or - self.type == RPLC_TYPES.MEK_ENABLE or self.type == RPLC_TYPES.MEK_BURN_RATE or + self.type == RPLC_TYPES.RPS_ENABLE or + self.type == RPLC_TYPES.RPS_SCRAM or self.type == RPLC_TYPES.RPS_ALARM or self.type == RPLC_TYPES.RPS_STATUS or self.type == RPLC_TYPES.RPS_RESET diff --git a/scada-common/types.lua b/scada-common/types.lua index 855334e..5bd747e 100644 --- a/scada-common/types.lua +++ b/scada-common/types.lua @@ -17,12 +17,14 @@ types.rtu_t = { types.rps_status_t = { ok = "ok", dmg_crit = "dmg_crit", - ex_hcoolant = "heated_coolant_backup", - ex_waste = "full_waste", high_temp = "high_temp", - no_fuel = "no_fuel", no_coolant = "no_coolant", - timeout = "timeout" + ex_waste = "full_waste", + ex_hcoolant = "heated_coolant_backup", + no_fuel = "no_fuel", + fault = "fault", + timeout = "timeout", + manual = "manual" } -- MODBUS diff --git a/supervisor/session/plc.lua b/supervisor/session/plc.lua index bb79ac1..2ee2770 100644 --- a/supervisor/session/plc.lua +++ b/supervisor/session/plc.lua @@ -291,16 +291,15 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue) else log.debug(log_header .. "RPLC struct packet length mismatch") end - elseif pkt.type == RPLC_TYPES.MEK_SCRAM then - -- SCRAM acknowledgement + elseif pkt.type == RPLC_TYPES.MEK_BURN_RATE then + -- burn rate acknowledgement local ack = _get_ack(pkt) if ack then - self.acks.scram = true - self.sDB.control_state = false + self.acks.burn_rate = true elseif ack == false then - log.debug(log_header .. "SCRAM failed!") + log.debug(log_header .. "burn rate update failed!") end - elseif pkt.type == RPLC_TYPES.MEK_ENABLE then + elseif pkt.type == RPLC_TYPES.RPS_ENABLE then -- enable acknowledgement local ack = _get_ack(pkt) if ack then @@ -309,13 +308,14 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue) elseif ack == false then log.debug(log_header .. "enable failed!") end - elseif pkt.type == RPLC_TYPES.MEK_BURN_RATE then - -- burn rate acknowledgement + elseif pkt.type == RPLC_TYPES.RPS_SCRAM then + -- SCRAM acknowledgement local ack = _get_ack(pkt) if ack then - self.acks.burn_rate = true + self.acks.scram = true + self.sDB.control_state = false elseif ack == false then - log.debug(log_header .. "burn rate update failed!") + log.debug(log_header .. "SCRAM failed!") end elseif pkt.type == RPLC_TYPES.RPS_STATUS then -- RPS status packet received, copy data @@ -428,16 +428,16 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue) elseif message.qtype == mqueue.TYPE.COMMAND then -- handle instruction local cmd = message.message - if cmd == PLC_S_CMDS.SCRAM then - -- SCRAM reactor - self.acks.scram = false - self.retry_times.scram_req = util.time() + INITIAL_WAIT - _send(RPLC_TYPES.MEK_SCRAM, {}) - elseif cmd == PLC_S_CMDS.ENABLE then + if cmd == PLC_S_CMDS.ENABLE then -- enable reactor self.acks.enable = false self.retry_times.enable_req = util.time() + INITIAL_WAIT - _send(RPLC_TYPES.MEK_ENABLE, {}) + _send(RPLC_TYPES.RPS_ENABLE, {}) + elseif cmd == PLC_S_CMDS.SCRAM then + -- SCRAM reactor + self.acks.scram = false + self.retry_times.scram_req = util.time() + INITIAL_WAIT + _send(RPLC_TYPES.RPS_SCRAM, {}) elseif cmd == PLC_S_CMDS.RPS_RESET then -- reset RPS self.acks.rps_reset = false @@ -517,7 +517,7 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue) if not self.acks.scram then if rtimes.scram_req - util.time() <= 0 then - _send(RPLC_TYPES.MEK_SCRAM, {}) + _send(RPLC_TYPES.RPS_SCRAM, {}) rtimes.scram_req = util.time() + RETRY_PERIOD end end @@ -526,7 +526,7 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue) if not self.acks.enable then if rtimes.enable_req - util.time() <= 0 then - _send(RPLC_TYPES.MEK_ENABLE, {}) + _send(RPLC_TYPES.RPS_ENABLE, {}) rtimes.enable_req = util.time() + RETRY_PERIOD end end diff --git a/supervisor/startup.lua b/supervisor/startup.lua index a93dab9..72ac729 100644 --- a/supervisor/startup.lua +++ b/supervisor/startup.lua @@ -14,7 +14,7 @@ local svsessions = require("session.svsessions") local config = require("config") local supervisor = require("supervisor") -local SUPERVISOR_VERSION = "alpha-v0.3.1" +local SUPERVISOR_VERSION = "alpha-v0.3.2" local print = util.print local println = util.println