#4 PLC peripheral disconnect handling and small bugfixes/cleanup

reactor-plc/plc.lua

@@ -11,6 +11,10 @@ function iss_init(reactor)
         trip_cause = ""
     }
 
+    local reconnect_reactor = function (reactor)
+        self.reactor = reactor
+    end
+
     local check = function ()
         local status = "ok"
         local was_tripped = self.tripped
@@ -110,6 +114,7 @@ function iss_init(reactor)
     end
 
     return {
+        reconnect_reactor = reconnect_reactor,
         check = check,
         trip_timeout = trip_timeout,
         reset = reset,
@@ -197,7 +202,7 @@ function rplc_packet()
 end
 
 -- reactor PLC communications
-function rplc_comms(id, modem, local_port, server_port, reactor, iss)
+function comms_init(id, modem, local_port, server_port, reactor, iss)
     local self = {
         id = id,
         seq_num = 0,
@@ -211,6 +216,11 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss)
         linked = false
     }
 
+    -- open modem
+    if not self.modem.isOpen(self.l_port) then
+        self.modem.open(self.l_port)
+    end
+
     -- PRIVATE FUNCTIONS --
 
     local _send = function (msg)
@@ -323,6 +333,21 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss)
 
     -- PUBLIC FUNCTIONS --
 
+    -- reconnect a newly connected modem
+    local reconnect_modem = function (modem)
+        self.modem = modem
+
+        -- open modem
+        if not self.modem.isOpen(self.l_port) then
+            self.modem.open(self.l_port)
+        end
+    end
+
+    -- reconnect a newly connected reactor
+    local reconnect_reactor = function (reactor)
+        self.reactor = reactor
+    end
+
     -- parse an RPLC packet
     local parse_packet = function(side, sender, reply_to, message, distance)
         local pkt = nil
@@ -410,7 +435,7 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss)
                         local max_burn_rate = self.reactor.getMaxBurnRate()
                         local success = false
 
-                        if max_burn_rate is not nil then
+                        if max_burn_rate ~= nil then
                             if burn_rate > 0 and burn_rate <= max_burn_rate then
                                 success = self.reactor.setBurnRate(burn_rate)
                             end
@@ -508,6 +533,8 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss)
     local unlink = function () self.linked = false end
 
     return {
+        reconnect_modem = reconnect_modem,
+        reconnect_reactor = reconnect_reactor,
         parse_packet = parse_packet,
         handle_packet = handle_packet,
         send_link_req = send_link_req,

reactor-plc/startup.lua

@@ -22,35 +22,50 @@ ppm.mount_all()
 local reactor = ppm.get_device("fissionReactor")
 local modem = ppm.get_device("modem")
 
+local init_ok = true
+local control_degraded = { degraded = false, no_reactor = false, no_modem = false }
+
 -- we need a reactor and a modem
 if reactor == nil then
-    print("Fission reactor not found, exiting...");
-    return
-elseif modem == nil then
-    print("No modem found, disabling reactor and exiting...")
+    print_ts("Fission reactor not found. Running in a degraded state...\n");
+    log._warning("no reactor on startup")
+    init_ok = false
+    control_degraded.degraded = true
+    control_degraded.no_reactor = true
+end
+if modem == nil then
+    print_ts("No modem found. Disabling reactor and running in a degraded state...\n")
+    log._warning("no modem on startup")
     reactor.scram()
-    return
+    init_ok = false
+    control_degraded.degraded = true
+    control_degraded.no_modem = true
 end
 
--- just booting up, no fission allowed (neutrons stay put thanks)
-reactor.scram()
+::init::
+if init_ok then
+    -- just booting up, no fission allowed (neutrons stay put thanks)
+    reactor.scram()
 
--- init internal safety system
-local iss = plc.iss_init(reactor)
+    -- init internal safety system
+    local iss = plc.iss_init(reactor)
+    log._debug("iss init")
 
--- start comms
-if not modem.isOpen(config.LISTEN_PORT) then
-    modem.open(config.LISTEN_PORT)
+    -- start comms
+    local plc_comms = plc.comms_init(config.REACTOR_ID, modem, config.LISTEN_PORT, config.SERVER_PORT, reactor, iss)
+    log._debug("comms init")
+
+    -- comms watchdog, 3 second timeout
+    local conn_watchdog = watchdog.new_watchdog(3)
+    log._debug("conn watchdog started")
+
+    -- loop clock (10Hz, 2 ticks)
+    local loop_tick = os.startTimer(0.05)
+    log._debug("loop clock started")
+else
+    log._warning("booted in a degraded state, awaiting peripheral connections...")
 end
 
-local plc_comms = plc.rplc_comms(config.REACTOR_ID, modem, config.LISTEN_PORT, config.SERVER_PORT, reactor, iss)
-
--- comms watchdog, 3 second timeout
-local conn_watchdog = watchdog.new_watchdog(3)
-
--- loop clock (10Hz, 2 ticks)
-local loop_tick = os.startTimer(0.05)
-
 -- send status updates at ~3.33Hz (every 6 server ticks) (every 3 loop ticks)
 -- send link requests at 0.5Hz (every 40 server ticks) (every 20 loop ticks)
 local UPDATE_TICKS = 3
@@ -67,51 +82,112 @@ local reactor_scram = true      -- treated as latching e-stop
 while true do
     local event, param1, param2, param3, param4, param5 = os.pullEventRaw()
 
-    -- if we tried to SCRAM but failed, keep trying
-    -- if it disconnected, isPowered will return nil (and error logs will get spammed at 10Hz, so disable reporting)
-    -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check)
-    ppm.disable_reporting()
-    if reactor_scram and reactor.isPowered() then
-        reactor.scram()
+    if init_ok then
+        -- if we tried to SCRAM but failed, keep trying
+        -- if it disconnected, isPowered will return nil (and error logs will get spammed at 10Hz, so disable reporting)
+        -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check)
+        ppm.disable_reporting()
+        if reactor_scram and reactor.isPowered() then
+            reactor.scram()
+        end
+        ppm.enable_reporting()
     end
-    ppm.enable_reporting()
 
+    -- check for peripheral changes before ISS checks
     if event == "peripheral_detach" then
-        ppm.handle_unmount(param1)
+        local device = ppm.handle_unmount(param1)
 
-        -- try to scram reactor if it is still connected
-        reactor_scram = true
-        if reactor.scram() then
-            print_ts("[fatal] PLC lost a peripheral: successful SCRAM\n")
-        else
-            print_ts("[fatal] PLC lost a peripheral: failed SCRAM\n")
+        if device.type == "fissionReactor" then
+            print_ts("reactor disconnected!\n")
+            log._error("reactor disconnected!")
+            control_degraded.no_reactor = true
+            -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_PERI_DC) ?
+        elseif device.type == "modem" then
+            print_ts("modem disconnected!\n")
+            log._error("modem disconnected!")
+            control_degraded.no_modem = true
+
+            if init_ok then
+                -- try to scram reactor if it is still connected
+                reactor_scram = true
+                if reactor.scram() then
+                    print_ts("successful reactor SCRAM\n")
+                else
+                    print_ts("failed reactor SCRAM\n")
+                end
+            end
+
+            control_degraded.degraded = true
+        end
+    elseif event == "peripheral" then
+        local device = ppm.mount(param1)
+
+        if device.type == "fissionReactor" then
+            -- reconnected reactor
+            reactor_scram = true
+            device.scram()
+
+            print_ts("reactor reconnected.\n")
+            log._info("reactor reconnected.")
+            control_degraded.no_reactor = false
+
+            if init_ok then
+                iss.reconnect_reactor(device)
+                plc_comms.reconnect_reactor(device)
+            end
+
+            -- determine if we are still in a degraded state
+            if get_device("modem") not nil then
+                control_degraded.degraded = false
+            end
+        elseif device.type == "modem" then
+            -- reconnected modem
+            if init_ok then
+                plc_comms.reconnect_modem(device)
+            end
+
+            print_ts("modem reconnected.\n")
+            log._info("modem reconnected.")
+            control_degraded.no_modem = false
+
+            -- determine if we are still in a degraded state
+            if ppm.get_device("fissionReactor") not nil then
+                control_degraded.degraded = false
+            end
         end
 
-        -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_PERI_DC) ?
+        if not init_ok and not control_degraded.degraded then
+            init_ok = false
+            goto init
+        end
     end
 
     -- check safety (SCRAM occurs if tripped)
-    local iss_tripped, iss_status, iss_first = iss.check()
-    reactor_scram = reactor_scram or iss_tripped
-    if iss_first then
-        plc_comms.send_iss_alarm(iss_status)
+    if not control_degraded.degraded then
+        local iss_tripped, iss_status, iss_first = iss.check()
+        reactor_scram = reactor_scram or iss_tripped
+        if iss_first then
+            plc_comms.send_iss_alarm(iss_status)
+        end
     end
 
     -- handle event
     if event == "timer" and param1 == loop_tick then
-        -- basic event tick, send updated data if it is time (~3.33Hz)
-        -- iss was already checked (main reason for this tick rate)
-        ticks_to_update = ticks_to_update - 1
+        if not control_degraded.no_modem then
+            -- basic event tick, send updated data if it is time (~3.33Hz)
+            -- iss was already checked (main reason for this tick rate)
+            ticks_to_update = ticks_to_update - 1
 
-        if plc_comms.is_linked() then
-            if ticks_to_update <= 0 then
-                plc_comms.send_status(control_state, iss_tripped)
-                ticks_to_update = UPDATE_TICKS
-            end
-        else
-            if ticks_to_update <= 0 then
-                plc_comms.send_link_req()
-                ticks_to_update = LINK_TICKS
+            if plc_comms.is_linked() then
+                if ticks_to_update <= 0 then
+                    plc_comms.send_status(control_state, iss_tripped)
+                    ticks_to_update = UPDATE_TICKS
+                end
+            else
+                if ticks_to_update <= 0 then
+                    plc_comms.send_link_req()
+                    ticks_to_update = LINK_TICKS
+                end
             end
         end
     elseif event == "modem_message" then
@@ -130,14 +206,17 @@ while true do
         print_ts("[alert] server timeout, reactor disabled\n")
     elseif event == "terminate" then
         -- safe exit
-        reactor_scram = true
-        if reactor.scram() then
-            print_ts("[alert] exiting, reactor disabled\n")
-        else
-            -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_LOST_CONTROL) ?
-            print_ts("[alert] exiting, reactor failed to disable\n")
+        if init_ok then
+            reactor_scram = true
+            if reactor.scram() then
+                print_ts("[alert] exiting, reactor disabled\n")
+            else
+                -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_LOST_CONTROL) ?
+                print_ts("[alert] exiting, reactor failed to disable\n")
+            end
         end
         -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_SHUTDOWN) ?
+        print_ts("[alert] exited")
         return
     end
 end

scada-common/log.lua

@@ -34,6 +34,10 @@ function _debug(msg, trace)
     end
 end
 
+function _info(msg)
+    _log("[INF] " .. msg .. "\n")
+end
+
 function _warning(msg)
     _log("[WRN] " .. msg .. "\n")
 end

scada-common/ppm.lua

@@ -90,7 +90,7 @@ function handle_unmount(iface)
     
     log._warning("PPM: lost device " .. type .. " mounted to " .. iface)
 
-    return self.mounts[iface]
+    return lost_dev
 end
 
 -- list all available peripherals