#32 parallel reactor PLC code

reactor-plc/plc.lua

@@ -89,7 +89,7 @@ function iss_init(reactor)
             log._error("ISS: failed to check reactor coolant level")
             return false
         else
-            return coolant_filled < 2
+            return coolant_filled < 0.02
         end
     end
 
@@ -122,7 +122,9 @@ function iss_init(reactor)
         }
         
         -- check system states in order of severity
-        if self.cache[1] then
+        if self.tripped then
+            status = self.trip_cause
+        elseif self.cache[1] then
             log._warning("ISS: damage critical!")
             status = "dmg_crit"
         elseif self.cache[4] then
@@ -143,26 +145,25 @@ function iss_init(reactor)
         elseif self.timed_out then
             log._warning("ISS: supervisor connection timeout!")
             status = "timeout"
-        elseif self.tripped then
-            status = self.trip_cause
         else
             self.tripped = false
         end
     
-        -- if a trip occured...
+        -- if a new trip occured...
-        if status ~= "ok" then
+        local first_trip = false
+        if not was_tripped and status ~= "ok" then
             log._warning("ISS: reactor SCRAM")
+
+            first_trip = true
             self.tripped = true
             self.trip_cause = status
+
             self.reactor.scram()
             if self.reactor.__p_is_faulted() then
                 log._error("ISS: failed reactor SCRAM")
             end
         end
     
-        -- evaluate if this is a new trip
-        local first_trip = not was_tripped and self.tripped
-    
         return self.tripped, status, first_trip
     end
 

reactor-plc/startup.lua

@@ -9,8 +9,9 @@ os.loadAPI("scada-common/comms.lua")
 
 os.loadAPI("config.lua")
 os.loadAPI("plc.lua")
+os.loadAPI("threads.lua")
 
-local R_PLC_VERSION = "alpha-v0.2.8"
+local R_PLC_VERSION = "alpha-v0.3.0"
 
 local print = util.print
 local println = util.println
@@ -25,21 +26,37 @@ println(">> Reactor PLC " .. R_PLC_VERSION .. " <<")
 -- mount connected devices
 ppm.mount_all()
 
-local reactor = ppm.get_fission_reactor()
+-- shared memory across threads
-local modem = ppm.get_wireless_modem()
+local __shared_memory = {
+    networked = config.NETWORKED,
 
-local networked = config.NETWORKED
+    plc_state = {
-
-local plc_state = {
         init_ok = true,
         scram = true,
         degraded = false,
         no_reactor = false,
         no_modem = false
+    },
+    
+    plc_devices = {
+        reactor = ppm.get_fission_reactor(),
+        modem = ppm.get_wireless_modem()
+    },
+
+    system = {
+        iss = nil,
+        plc_comms = nil,
+        conn_watchdog = nil
+    }
 }
 
+local smem_dev = __shared_memory.plc_devices
+local smem_sys = __shared_memory.system
+
+local plc_state = __shared_memory.plc_state
+
 -- we need a reactor and a modem
-if reactor == nil then
+if smem_dev.reactor == nil then
     println("boot> fission reactor not found");
     log._warning("no reactor on startup")
 
@@ -47,12 +64,12 @@ if reactor == nil then
     plc_state.degraded = true
     plc_state.no_reactor = true
 end
-if networked and modem == nil then
+if networked and smem_dev.modem == nil then
     println("boot> wireless modem not found")
     log._warning("no wireless modem on startup")
 
-    if reactor ~= nil then
+    if smem_dev.reactor ~= nil then
-        reactor.scram()
+        smem_dev.reactor.scram()
     end
 
     plc_state.init_ok = false
@@ -60,43 +77,29 @@ if networked and modem == nil then
     plc_state.no_modem = true
 end
 
-local iss = nil
-local plc_comms = nil
-local conn_watchdog = nil
-
--- send status updates at ~3.33Hz (every 6 server ticks) (every 3 loop ticks)
--- send link requests at 0.5Hz (every 40 server ticks) (every 20 loop ticks)
-local UPDATE_TICKS = 3
-local LINK_TICKS = 20
-
-local loop_clock = nil
-local ticks_to_update = LINK_TICKS  -- start by linking
-
 function init()
     if plc_state.init_ok then
         -- just booting up, no fission allowed (neutrons stay put thanks)
-        reactor.scram()
+        smem_dev.reactor.scram()
 
         -- init internal safety system
-        iss = plc.iss_init(reactor)
+        smem_sys.iss = plc.iss_init(smem_dev.reactor)
         log._debug("iss init")
 
-        if networked then
+        if __shared_memory.networked then
             -- start comms
-            plc_comms = plc.comms_init(config.REACTOR_ID, modem, config.LISTEN_PORT, config.SERVER_PORT, reactor, iss)
+            smem_sys.plc_comms = plc.comms_init(config.REACTOR_ID, smem_dev.modem, config.LISTEN_PORT, config.SERVER_PORT, smem_dev.reactor, smem_sys.iss)
             log._debug("comms init")
 
             -- comms watchdog, 3 second timeout
-            conn_watchdog = util.new_watchdog(3)
+            smem_sys.conn_watchdog = util.new_watchdog(3)
             log._debug("conn watchdog started")
         else
             println("boot> starting in offline mode");
             log._debug("running without networking")
         end
 
-        -- loop clock (10Hz, 2 ticks)
+        os.queueEvent("clock_start")
-        loop_clock = os.startTimer(0.1)
-        log._debug("loop clock started")
 
         println("boot> completed");
     else
@@ -108,191 +111,13 @@ end
 -- initialize PLC
 init()
 
--- event loop
+-- init threads
-while true do
+local main_thread = threads.thread__main(__shared_memory, init)
-    local event, param1, param2, param3, param4, param5 = os.pullEventRaw()
+local iss_thread = threads.thread__iss(__shared_memory)
+-- local comms_thread = plc.thread__comms(__shared_memory)
 
-    if plc_state.init_ok then
+-- run threads
-        -- if we tried to SCRAM but failed, keep trying
+parallel.waitForAll(main_thread.exec, iss_thread.exec)
-        -- if it disconnected, isPowered will return nil (and error logs will get spammed at 10Hz, so disable reporting)
-        -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check)
-        ppm.disable_reporting()
-        if plc_state.scram and reactor.getStatus() then
-            reactor.scram()
-        end
-        ppm.enable_reporting()
-    end
-
-    -- check for peripheral changes before ISS checks
-    if event == "peripheral_detach" then
-        local device = ppm.handle_unmount(param1)
-
-        if device.type == "fissionReactor" then
-            println_ts("reactor disconnected!")
-            log._error("reactor disconnected!")
-            plc_state.no_reactor = true
-            plc_state.degraded = true
-            -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_PERI_DC) ?
-        elseif networked and device.type == "modem" then
-            -- we only care if this is our wireless modem
-            if device.dev == modem then
-                println_ts("wireless modem disconnected!")
-                log._error("comms modem disconnected!")
-                plc_state.no_modem = true
-
-                if plc_state.init_ok then
-                    -- try to scram reactor if it is still connected
-                    plc_state.scram = true
-                    if reactor.scram() then
-                        println_ts("successful reactor SCRAM")
-                        log._error("successful reactor SCRAM")
-                    else
-                        println_ts("failed reactor SCRAM")
-                        log._error("failed reactor SCRAM")
-                    end
-                end
-
-                plc_state.degraded = true
-            else
-                log._warning("non-comms modem disconnected")
-            end
-        end
-    elseif event == "peripheral" then
-        local type, device = ppm.mount(param1)
-
-        if type == "fissionReactor" then
-            -- reconnected reactor
-            reactor = device
-
-            plc_state.scram = true
-            reactor.scram()
-
-            println_ts("reactor reconnected.")
-            log._info("reactor reconnected.")
-            plc_state.no_reactor = false
-
-            if plc_state.init_ok then
-                iss.reconnect_reactor(reactor)
-                if networked then
-                    plc_comms.reconnect_reactor(reactor)
-                end
-            end
-
-            -- determine if we are still in a degraded state
-            if not networked or ppm.get_device("modem") ~= nil then
-                plc_state.degraded = false
-            end
-        elseif networked and type == "modem" then
-            if device.isWireless() then
-                -- reconnected modem
-                modem = device
-
-                if plc_state.init_ok then
-                    plc_comms.reconnect_modem(modem)
-                end
-
-                println_ts("wireless modem reconnected.")
-                log._info("comms modem reconnected.")
-                plc_state.no_modem = false
-
-                -- determine if we are still in a degraded state
-                if ppm.get_device("fissionReactor") ~= nil then
-                    plc_state.degraded = false
-                end
-            else
-                log._info("wired modem reconnected.")
-            end
-        end
-
-        if not plc_state.init_ok and not plc_state.degraded then
-            plc_state.init_ok = true
-            init()
-        end
-    end
-
-    -- ISS
-    if plc_state.init_ok then
-        -- if we are in standalone mode, continuously reset ISS
-        -- ISS will trip again if there are faults, but if it isn't cleared, the user can't re-enable
-        if not networked then
-            plc_state.scram = false
-            iss.reset()
-        end
-
-        -- check safety (SCRAM occurs if tripped)
-        if not plc_state.degraded then
-            local iss_tripped, iss_status_string, iss_first = iss.check()
-            plc_state.scram = plc_state.scram or iss_tripped
-
-            if iss_first then
-                println_ts("[ISS] reactor shutdown, safety tripped: " .. iss_status_string)
-                if networked then
-                    plc_comms.send_iss_alarm(iss_status_string)
-                end
-            end
-        elseif not plc_state.no_reactor then
-            -- degraded but we have a reactor
-            reactor.scram()
-        end
-    end
-
-    -- handle event
-    if event == "timer" and param1 == loop_clock then
-        -- basic event tick, send updated data if it is time (~3.33Hz)
-        -- iss was already checked (that's the main reason for this tick rate)
-        if networked and not plc_state.no_modem then
-            ticks_to_update = ticks_to_update - 1
-
-            if plc_comms.is_linked() then
-                if ticks_to_update <= 0 then
-                    plc_comms.send_status(iss_tripped, plc_state.degraded)
-                    plc_comms.send_iss_status()
-                    ticks_to_update = UPDATE_TICKS
-                end
-            else
-                if ticks_to_update <= 0 then
-                    plc_comms.send_link_req()
-                    ticks_to_update = LINK_TICKS
-                end
-            end
-        end
-
-        -- start next clock timer
-        loop_clock = os.startTimer(0.1)
-    elseif event == "modem_message" and networked and not plc_state.no_modem then
-        -- got a packet
-        -- feed the watchdog first so it doesn't uhh...eat our packets
-        conn_watchdog.feed()
-
-        -- handle the packet (plc_state passed to allow clearing SCRAM flag)
-        local packet = plc_comms.parse_packet(p1, p2, p3, p4, p5)
-        plc_comms.handle_packet(packet, plc_state)
-    elseif event == "timer" and networked and param1 == conn_watchdog.get_timer() then
-        -- haven't heard from server recently? shutdown reactor
-        plc_state.scram = true
-        plc_comms.unlink()
-        iss.trip_timeout()
-        println_ts("server timeout, reactor disabled")
-        log._warning("server timeout, reactor disabled")
-    end
-
-    -- check for termination request
-    if event == "terminate" or ppm.should_terminate() then
-        -- safe exit
-        log._warning("terminate requested, exiting...")
-        if plc_state.init_ok then
-            plc_state.scram = true
-            reactor.scram()
-            if reactor.__p_is_ok() then
-                println_ts("reactor disabled")
-            else
-                -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_LOST_CONTROL) ?
-                println_ts("exiting, reactor failed to disable")
-            end
-        end
-        break
-    end
-end
 
 -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_SHUTDOWN) ?
 println_ts("exited")

reactor-plc/threads.lua

@@ -0,0 +1,304 @@
+-- #REQUIRES comms.lua
+-- #REQUIRES ppm.lua
+-- #REQUIRES plc.lua
+-- #REQUIRES util.lua
+
+local print = util.print
+local println = util.println
+local print_ts = util.print_ts
+local println_ts = util.println_ts
+
+local async_wait = util.async_wait
+
+local MAIN_CLOCK = 0.5 -- (2Hz, 10 ticks)
+local ISS_CLOCK  = 0.5 -- (2Hz, 10 ticks)
+
+local ISS_EVENT = {
+    SCRAM = 1,
+    DEGRADED_SCRAM = 2,
+    TRIP_TIMEOUT = 3
+}
+
+-- main thread
+function thread__main(shared_memory, init)
+    -- execute thread
+    local exec = function ()
+        -- send status updates at 2Hz (every 10 server ticks) (every loop tick)
+        -- send link requests at 0.5Hz (every 40 server ticks) (every 4 loop ticks)
+        local LINK_TICKS = 4
+        
+        local loop_clock = nil
+        local ticks_to_update = LINK_TICKS  -- start by linking
+
+        -- load in from shared memory
+        local networked   = shared_memory.networked
+        local plc_state   = shared_memory.plc_state
+        local plc_devices = shared_memory.plc_devices
+
+        local iss           = shared_memory.system.iss
+        local plc_comms     = shared_memory.system.plc_comms
+        local conn_watchdog = shared_memory.system.conn_watchdog
+
+        -- debug
+        -- local last_update = util.time()
+
+        -- event loop
+        while true do
+            local event, param1, param2, param3, param4, param5 = os.pullEventRaw()
+
+            -- handle event
+            if event == "timer" and param1 == loop_clock then
+                -- core clock tick
+                if networked then
+                    -- start next clock timer
+                    loop_clock = os.startTimer(MAIN_CLOCK)
+
+                    -- send updated data
+                    if not plc_state.no_modem then
+
+                        if plc_comms.is_linked() then
+                            async_wait(function () 
+                                plc_comms.send_status(iss_tripped, plc_state.degraded)
+                                plc_comms.send_iss_status()
+                            end)
+                        else
+                            ticks_to_update = ticks_to_update - 1
+
+                            if ticks_to_update <= 0 then
+                                plc_comms.send_link_req()
+                                ticks_to_update = LINK_TICKS
+                            end
+                        end
+                    end
+
+                    -- debug
+                    -- print(util.time() - last_update)
+                    -- println("ms")
+                    -- last_update = util.time()
+                end
+            elseif event == "modem_message" and networked and not plc_state.no_modem then
+                -- got a packet
+                -- feed the watchdog first so it doesn't uhh...eat our packets
+                conn_watchdog.feed()
+
+                -- handle the packet (plc_state passed to allow clearing SCRAM flag)
+                local packet = plc_comms.parse_packet(p1, p2, p3, p4, p5)
+                async_wait(function () plc_comms.handle_packet(packet, plc_state) end)
+            elseif event == "timer" and networked and param1 == conn_watchdog.get_timer() then
+                -- haven't heard from server recently? shutdown reactor
+                println("timed out, passing event")
+                plc_comms.unlink()
+                os.queueEvent("iss_command", ISS_EVENT.TRIP_TIMEOUT)
+            elseif event == "peripheral_detach" then
+                -- peripheral disconnect
+                local device = ppm.handle_unmount(param1)
+
+                if device.type == "fissionReactor" then
+                    println_ts("reactor disconnected!")
+                    log._error("reactor disconnected!")
+                    plc_state.no_reactor = true
+                    plc_state.degraded = true
+                    -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_PERI_DC) ?
+                elseif networked and device.type == "modem" then
+                    -- we only care if this is our wireless modem
+                    if device.dev == modem then
+                        println_ts("wireless modem disconnected!")
+                        log._error("comms modem disconnected!")
+                        plc_state.no_modem = true
+
+                        if plc_state.init_ok then
+                            -- try to scram reactor if it is still connected
+                            os.queueEvent("iss_command", ISS_EVENT.DEGRADED_SCRAM)
+                        end
+
+                        plc_state.degraded = true
+                    else
+                        log._warning("non-comms modem disconnected")
+                    end
+                end
+            elseif event == "peripheral" then
+                -- peripheral connect
+                local type, device = ppm.mount(param1)
+
+                if type == "fissionReactor" then
+                    -- reconnected reactor
+                    plc_devices.reactor = device
+
+                    os.queueEvent("iss_command", ISS_EVENT.SCRAM)
+
+                    println_ts("reactor reconnected.")
+                    log._info("reactor reconnected.")
+                    plc_state.no_reactor = false
+
+                    if plc_state.init_ok then
+                        iss.reconnect_reactor(plc_devices.reactor)
+                        if networked then
+                            plc_comms.reconnect_reactor(plc_devices.reactor)
+                        end
+                    end
+
+                    -- determine if we are still in a degraded state
+                    if not networked or ppm.get_device("modem") ~= nil then
+                        plc_state.degraded = false
+                    end
+                elseif networked and type == "modem" then
+                    if device.isWireless() then
+                        -- reconnected modem
+                        plc_devices.modem = device
+
+                        if plc_state.init_ok then
+                            plc_comms.reconnect_modem(plc_devices.modem)
+                        end
+
+                        println_ts("wireless modem reconnected.")
+                        log._info("comms modem reconnected.")
+                        plc_state.no_modem = false
+
+                        -- determine if we are still in a degraded state
+                        if ppm.get_device("fissionReactor") ~= nil then
+                            plc_state.degraded = false
+                        end
+                    else
+                        log._info("wired modem reconnected.")
+                    end
+                end
+
+                if not plc_state.init_ok and not plc_state.degraded then
+                    plc_state.init_ok = true
+                    init()
+                end
+            elseif event == "clock_start" then
+                -- start loop clock
+                loop_clock = os.startTimer(MAIN_CLOCK)
+                log._debug("loop clock started")
+            end
+
+            -- check for termination request
+            if event == "terminate" or ppm.should_terminate() then
+                -- iss handles reactor shutdown
+                log._warning("terminate requested, main thread exiting")
+                break
+            end
+        end
+    end
+
+    return { exec = exec }
+end
+
+-- ISS monitor thread
+function thread__iss(shared_memory)
+    -- execute thread
+    local exec = function ()
+        local loop_clock = nil
+
+        -- load in from shared memory
+        local networked   = shared_memory.networked
+        local plc_state   = shared_memory.plc_state
+        local plc_devices = shared_memory.plc_devices
+
+        local iss         = shared_memory.system.iss
+        local plc_comms   = shared_memory.system.plc_comms
+
+        -- debug
+        -- local last_update = util.time()
+
+        -- event loop
+        while true do
+            local event, param1, param2, param3, param4, param5 = os.pullEventRaw()
+            
+            local reactor = shared_memory.plc_devices.reactor
+    
+            if event == "timer" and param1 == loop_clock then
+                -- start next clock timer
+                loop_clock = os.startTimer(ISS_CLOCK)
+
+                -- ISS checks
+                if plc_state.init_ok then
+                    -- if we tried to SCRAM but failed, keep trying
+                    -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check)
+                    async_wait(function ()
+                        if not plc_state.no_reactor and plc_state.scram and reactor.getStatus() then
+                            reactor.scram()
+                        end
+                    end)
+
+                    -- if we are in standalone mode, continuously reset ISS
+                    -- ISS will trip again if there are faults, but if it isn't cleared, the user can't re-enable
+                    if not networked then
+                        plc_state.scram = false
+                        iss.reset()
+                    end
+
+                    -- check safety (SCRAM occurs if tripped)
+                    async_wait(function () 
+                        if not plc_state.degraded then
+                            local iss_tripped, iss_status_string, iss_first = iss.check()
+                            plc_state.scram = plc_state.scram or iss_tripped
+    
+                            if iss_first then
+                                println_ts("[ISS] SCRAM! safety trip: " .. iss_status_string)
+                                if networked then
+                                    plc_comms.send_iss_alarm(iss_status_string)
+                                end
+                            end
+                        end
+                    end)
+                end
+
+                -- debug
+                -- print(util.time() - last_update)
+                -- println("ms")
+                -- last_update = util.time()
+            elseif event == "iss_command" then
+                -- handle ISS commands
+                println("got iss command?")
+                if param1 == ISS_EVENT.SCRAM then
+                    -- basic SCRAM
+                    plc_state.scram = true
+                    async_wait(reactor.scram)
+                elseif param1 == ISS_EVENT.DEGRADED_SCRAM then
+                    -- SCRAM with print
+                    plc_state.scram = true
+                    async_wait(function () 
+                        if reactor.scram() then
+                            println_ts("successful reactor SCRAM")
+                            log._error("successful reactor SCRAM")
+                        else
+                            println_ts("failed reactor SCRAM")
+                            log._error("failed reactor SCRAM")
+                        end
+                    end)
+                elseif param1 == ISS_EVENT.TRIP_TIMEOUT then
+                    -- watchdog tripped
+                    plc_state.scram = true
+                    iss.trip_timeout()
+                    println_ts("server timeout, reactor disabled")
+                    log._warning("server timeout, reactor disabled")
+                end
+            elseif event == "clock_start" then
+                -- start loop clock
+                loop_clock = os.startTimer(ISS_CLOCK)
+                log._debug("loop clock started")
+            end
+
+            -- check for termination request
+            if event == "terminate" or ppm.should_terminate() then
+                -- safe exit
+                log._warning("terminate requested, iss thread shutdown")
+                if plc_state.init_ok then
+                    plc_state.scram = true
+                    async_wait(reactor.scram)
+                    if reactor.__p_is_ok() then
+                        println_ts("reactor disabled")
+                    else
+                        -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_LOST_CONTROL) ?
+                        println_ts("exiting, reactor failed to disable")
+                    end
+                end
+                break
+            end
+        end
+    end
+
+    return { exec = exec }
+end

scada-common/util.lua

@@ -40,7 +40,7 @@ end
 -- PARALLELIZATION --
 
 -- block waiting for parallel call
-function task_wait(f)
+function async_wait(f)
     parallel.waitForAll(f)
 end