From 712c7a8f3b8aa903a788625b921ddd3ecd0ce099 Mon Sep 17 00:00:00 2001 From: Mikayla Date: Thu, 22 Jun 2023 19:46:17 +0000 Subject: [PATCH] #266 added health check to ppm and strengthened reliability of RTU hw state reporting --- coordinator/startup.lua | 2 +- install_manifest.json | 2 +- pocket/startup.lua | 2 +- reactor-plc/plc.lua | 4 +- reactor-plc/startup.lua | 2 +- rtu/dev/boilerv_rtu.lua | 1 + rtu/dev/envd_rtu.lua | 1 + rtu/dev/imatrix_rtu.lua | 1 + rtu/dev/sna_rtu.lua | 1 + rtu/dev/sps_rtu.lua | 1 + rtu/dev/turbinev_rtu.lua | 1 + rtu/startup.lua | 40 ++++++++-------- rtu/threads.lua | 98 +++++++++++++++++++++------------------- scada-common/ppm.lua | 15 ++++-- supervisor/startup.lua | 2 +- 15 files changed, 99 insertions(+), 74 deletions(-) diff --git a/coordinator/startup.lua b/coordinator/startup.lua index 5a68afc..838d9ec 100644 --- a/coordinator/startup.lua +++ b/coordinator/startup.lua @@ -20,7 +20,7 @@ local sounder = require("coordinator.sounder") local apisessions = require("coordinator.session.apisessions") -local COORDINATOR_VERSION = "v0.16.1" +local COORDINATOR_VERSION = "v0.16.2" local println = util.println local println_ts = util.println_ts diff --git a/install_manifest.json b/install_manifest.json index dc82370..b6c4c60 100644 --- a/install_manifest.json +++ b/install_manifest.json @@ -1 +1 @@ -{"versions": {"installer": "v1.4f", "bootloader": "0.2", "comms": "2.0.0", "graphics": "1.0.0", "reactor-plc": "v1.4.6", "rtu": "v1.3.6", "supervisor": "v0.17.8", "coordinator": "v0.16.1", "pocket": "alpha-v0.4.5"}, "files": {"system": ["initenv.lua", "startup.lua"], "common": ["scada-common/crypto.lua", "scada-common/ppm.lua", "scada-common/comms.lua", "scada-common/psil.lua", "scada-common/rsio.lua", "scada-common/constants.lua", "scada-common/mqueue.lua", "scada-common/tcd.lua", "scada-common/crash.lua", "scada-common/log.lua", "scada-common/types.lua", "scada-common/util.lua"], "graphics": ["graphics/element.lua", "graphics/events.lua", "graphics/flasher.lua", "graphics/core.lua", "graphics/elements/listbox.lua", "graphics/elements/textbox.lua", "graphics/elements/displaybox.lua", "graphics/elements/pipenet.lua", "graphics/elements/rectangle.lua", "graphics/elements/div.lua", "graphics/elements/multipane.lua", "graphics/elements/tiling.lua", "graphics/elements/colormap.lua", "graphics/elements/indicators/alight.lua", "graphics/elements/indicators/icon.lua", "graphics/elements/indicators/power.lua", "graphics/elements/indicators/rad.lua", "graphics/elements/indicators/state.lua", "graphics/elements/indicators/light.lua", "graphics/elements/indicators/vbar.lua", "graphics/elements/indicators/led.lua", "graphics/elements/indicators/coremap.lua", "graphics/elements/indicators/data.lua", "graphics/elements/indicators/ledpair.lua", "graphics/elements/indicators/hbar.lua", "graphics/elements/indicators/trilight.lua", "graphics/elements/indicators/ledrgb.lua", "graphics/elements/controls/switch_button.lua", "graphics/elements/controls/spinbox_numeric.lua", "graphics/elements/controls/hazard_button.lua", "graphics/elements/controls/push_button.lua", "graphics/elements/controls/radio_button.lua", "graphics/elements/controls/multi_button.lua", "graphics/elements/controls/tabbar.lua", "graphics/elements/controls/sidebar.lua", "graphics/elements/animations/waiting.lua"], "lockbox": ["lockbox/init.lua", "lockbox/LICENSE", "lockbox/kdf/pbkdf2.lua", "lockbox/util/bit.lua", "lockbox/util/array.lua", "lockbox/util/stream.lua", "lockbox/util/queue.lua", "lockbox/digest/sha2_224.lua", "lockbox/digest/sha1.lua", "lockbox/digest/sha2_256.lua", "lockbox/cipher/aes128.lua", "lockbox/cipher/aes256.lua", "lockbox/cipher/aes192.lua", "lockbox/cipher/mode/ofb.lua", "lockbox/cipher/mode/cbc.lua", "lockbox/cipher/mode/ctr.lua", "lockbox/cipher/mode/cfb.lua", "lockbox/mac/hmac.lua", "lockbox/padding/ansix923.lua", "lockbox/padding/pkcs7.lua", "lockbox/padding/zero.lua", "lockbox/padding/isoiec7816.lua"], "reactor-plc": ["reactor-plc/renderer.lua", "reactor-plc/threads.lua", "reactor-plc/databus.lua", "reactor-plc/plc.lua", "reactor-plc/config.lua", "reactor-plc/startup.lua", "reactor-plc/panel/front_panel.lua", "reactor-plc/panel/style.lua"], "rtu": ["rtu/renderer.lua", "rtu/threads.lua", "rtu/rtu.lua", "rtu/databus.lua", "rtu/modbus.lua", "rtu/config.lua", "rtu/startup.lua", "rtu/panel/front_panel.lua", "rtu/panel/style.lua", "rtu/dev/sps_rtu.lua", "rtu/dev/envd_rtu.lua", "rtu/dev/boilerv_rtu.lua", "rtu/dev/redstone_rtu.lua", "rtu/dev/sna_rtu.lua", "rtu/dev/imatrix_rtu.lua", "rtu/dev/turbinev_rtu.lua"], "supervisor": ["supervisor/renderer.lua", "supervisor/databus.lua", "supervisor/supervisor.lua", "supervisor/unit.lua", "supervisor/config.lua", "supervisor/startup.lua", "supervisor/unitlogic.lua", "supervisor/facility.lua", "supervisor/panel/pgi.lua", "supervisor/panel/front_panel.lua", "supervisor/panel/style.lua", "supervisor/panel/components/rtu_entry.lua", "supervisor/panel/components/pdg_entry.lua", "supervisor/session/coordinator.lua", "supervisor/session/svqtypes.lua", "supervisor/session/pocket.lua", "supervisor/session/svsessions.lua", "supervisor/session/rtu.lua", "supervisor/session/plc.lua", "supervisor/session/rsctl.lua", "supervisor/session/rtu/boilerv.lua", "supervisor/session/rtu/txnctrl.lua", "supervisor/session/rtu/unit_session.lua", "supervisor/session/rtu/turbinev.lua", "supervisor/session/rtu/envd.lua", "supervisor/session/rtu/imatrix.lua", "supervisor/session/rtu/sps.lua", "supervisor/session/rtu/qtypes.lua", "supervisor/session/rtu/sna.lua", "supervisor/session/rtu/redstone.lua"], "coordinator": ["coordinator/coordinator.lua", "coordinator/renderer.lua", "coordinator/iocontrol.lua", "coordinator/sounder.lua", "coordinator/config.lua", "coordinator/startup.lua", "coordinator/process.lua", "coordinator/ui/dialog.lua", "coordinator/ui/style.lua", "coordinator/ui/layout/main_view.lua", "coordinator/ui/layout/unit_view.lua", "coordinator/ui/components/reactor.lua", "coordinator/ui/components/processctl.lua", "coordinator/ui/components/unit_overview.lua", "coordinator/ui/components/boiler.lua", "coordinator/ui/components/unit_detail.lua", "coordinator/ui/components/imatrix.lua", "coordinator/ui/components/turbine.lua", "coordinator/session/pocket.lua", "coordinator/session/apisessions.lua"], "pocket": ["pocket/pocket.lua", "pocket/renderer.lua", "pocket/config.lua", "pocket/coreio.lua", "pocket/startup.lua", "pocket/ui/main.lua", "pocket/ui/style.lua", "pocket/ui/components/conn_waiting.lua", "pocket/ui/pages/turbine_page.lua", "pocket/ui/pages/reactor_page.lua", "pocket/ui/pages/home_page.lua", "pocket/ui/pages/unit_page.lua", "pocket/ui/pages/boiler_page.lua"]}, "depends": {"reactor-plc": ["system", "common", "graphics"], "rtu": ["system", "common", "graphics"], "supervisor": ["system", "common", "graphics"], "coordinator": ["system", "common", "graphics"], "pocket": ["system", "common", "graphics"]}, "sizes": {"manifest": 5803, "system": 1991, "common": 92736, "graphics": 144556, "lockbox": 100797, "reactor-plc": 97335, "rtu": 102240, "supervisor": 315388, "coordinator": 197991, "pocket": 37581}} \ No newline at end of file +{"versions": {"installer": "v1.4f", "bootloader": "0.2", "comms": "2.0.0", "graphics": "1.0.0", "reactor-plc": "v1.4.7", "rtu": "v1.3.7", "supervisor": "v0.17.10", "coordinator": "v0.16.2", "pocket": "alpha-v0.4.6"}, "files": {"system": ["initenv.lua", "startup.lua"], "common": ["scada-common/constants.lua", "scada-common/crypto.lua", "scada-common/types.lua", "scada-common/comms.lua", "scada-common/mqueue.lua", "scada-common/util.lua", "scada-common/psil.lua", "scada-common/crash.lua", "scada-common/tcd.lua", "scada-common/rsio.lua", "scada-common/ppm.lua", "scada-common/log.lua"], "graphics": ["graphics/core.lua", "graphics/flasher.lua", "graphics/element.lua", "graphics/events.lua", "graphics/elements/div.lua", "graphics/elements/colormap.lua", "graphics/elements/textbox.lua", "graphics/elements/pipenet.lua", "graphics/elements/listbox.lua", "graphics/elements/multipane.lua", "graphics/elements/displaybox.lua", "graphics/elements/rectangle.lua", "graphics/elements/tiling.lua", "graphics/elements/controls/tabbar.lua", "graphics/elements/controls/radio_button.lua", "graphics/elements/controls/spinbox_numeric.lua", "graphics/elements/controls/switch_button.lua", "graphics/elements/controls/sidebar.lua", "graphics/elements/controls/multi_button.lua", "graphics/elements/controls/push_button.lua", "graphics/elements/controls/hazard_button.lua", "graphics/elements/animations/waiting.lua", "graphics/elements/indicators/ledpair.lua", "graphics/elements/indicators/state.lua", "graphics/elements/indicators/icon.lua", "graphics/elements/indicators/hbar.lua", "graphics/elements/indicators/light.lua", "graphics/elements/indicators/led.lua", "graphics/elements/indicators/data.lua", "graphics/elements/indicators/rad.lua", "graphics/elements/indicators/trilight.lua", "graphics/elements/indicators/vbar.lua", "graphics/elements/indicators/coremap.lua", "graphics/elements/indicators/power.lua", "graphics/elements/indicators/ledrgb.lua", "graphics/elements/indicators/alight.lua"], "lockbox": ["lockbox/LICENSE", "lockbox/init.lua", "lockbox/digest/sha1.lua", "lockbox/digest/sha2_256.lua", "lockbox/digest/sha2_224.lua", "lockbox/cipher/aes128.lua", "lockbox/cipher/aes192.lua", "lockbox/cipher/aes256.lua", "lockbox/cipher/mode/cfb.lua", "lockbox/cipher/mode/cbc.lua", "lockbox/cipher/mode/ofb.lua", "lockbox/cipher/mode/ctr.lua", "lockbox/mac/hmac.lua", "lockbox/kdf/pbkdf2.lua", "lockbox/padding/ansix923.lua", "lockbox/padding/isoiec7816.lua", "lockbox/padding/zero.lua", "lockbox/padding/pkcs7.lua", "lockbox/util/stream.lua", "lockbox/util/bit.lua", "lockbox/util/queue.lua", "lockbox/util/array.lua"], "reactor-plc": ["reactor-plc/config.lua", "reactor-plc/plc.lua", "reactor-plc/renderer.lua", "reactor-plc/databus.lua", "reactor-plc/startup.lua", "reactor-plc/threads.lua", "reactor-plc/panel/style.lua", "reactor-plc/panel/front_panel.lua"], "rtu": ["rtu/modbus.lua", "rtu/rtu.lua", "rtu/config.lua", "rtu/renderer.lua", "rtu/databus.lua", "rtu/startup.lua", "rtu/threads.lua", "rtu/dev/redstone_rtu.lua", "rtu/dev/sps_rtu.lua", "rtu/dev/turbinev_rtu.lua", "rtu/dev/envd_rtu.lua", "rtu/dev/boilerv_rtu.lua", "rtu/dev/imatrix_rtu.lua", "rtu/dev/sna_rtu.lua", "rtu/panel/style.lua", "rtu/panel/front_panel.lua"], "supervisor": ["supervisor/unit.lua", "supervisor/supervisor.lua", "supervisor/config.lua", "supervisor/renderer.lua", "supervisor/databus.lua", "supervisor/startup.lua", "supervisor/facility.lua", "supervisor/unitlogic.lua", "supervisor/panel/style.lua", "supervisor/panel/front_panel.lua", "supervisor/panel/pgi.lua", "supervisor/panel/components/pdg_entry.lua", "supervisor/panel/components/rtu_entry.lua", "supervisor/session/coordinator.lua", "supervisor/session/rtu.lua", "supervisor/session/plc.lua", "supervisor/session/svsessions.lua", "supervisor/session/pocket.lua", "supervisor/session/svqtypes.lua", "supervisor/session/rsctl.lua", "supervisor/session/rtu/redstone.lua", "supervisor/session/rtu/imatrix.lua", "supervisor/session/rtu/envd.lua", "supervisor/session/rtu/qtypes.lua", "supervisor/session/rtu/txnctrl.lua", "supervisor/session/rtu/turbinev.lua", "supervisor/session/rtu/sna.lua", "supervisor/session/rtu/unit_session.lua", "supervisor/session/rtu/boilerv.lua", "supervisor/session/rtu/sps.lua"], "coordinator": ["coordinator/coordinator.lua", "coordinator/iocontrol.lua", "coordinator/config.lua", "coordinator/renderer.lua", "coordinator/startup.lua", "coordinator/sounder.lua", "coordinator/process.lua", "coordinator/session/pocket.lua", "coordinator/session/apisessions.lua", "coordinator/ui/style.lua", "coordinator/ui/dialog.lua", "coordinator/ui/components/processctl.lua", "coordinator/ui/components/turbine.lua", "coordinator/ui/components/reactor.lua", "coordinator/ui/components/imatrix.lua", "coordinator/ui/components/boiler.lua", "coordinator/ui/components/unit_detail.lua", "coordinator/ui/components/unit_overview.lua", "coordinator/ui/layout/main_view.lua", "coordinator/ui/layout/unit_view.lua"], "pocket": ["pocket/config.lua", "pocket/renderer.lua", "pocket/startup.lua", "pocket/coreio.lua", "pocket/pocket.lua", "pocket/ui/style.lua", "pocket/ui/main.lua", "pocket/ui/pages/reactor_page.lua", "pocket/ui/pages/boiler_page.lua", "pocket/ui/pages/home_page.lua", "pocket/ui/pages/unit_page.lua", "pocket/ui/pages/turbine_page.lua", "pocket/ui/components/conn_waiting.lua"]}, "depends": {"reactor-plc": ["system", "common", "graphics"], "rtu": ["system", "common", "graphics"], "supervisor": ["system", "common", "graphics"], "coordinator": ["system", "common", "graphics"], "pocket": ["system", "common", "graphics"]}, "sizes": {"manifest": 5804, "system": 1991, "common": 93200, "graphics": 144556, "lockbox": 100797, "reactor-plc": 97317, "rtu": 102353, "supervisor": 315477, "coordinator": 197991, "pocket": 37581}} \ No newline at end of file diff --git a/pocket/startup.lua b/pocket/startup.lua index 7afd208..3352605 100644 --- a/pocket/startup.lua +++ b/pocket/startup.lua @@ -17,7 +17,7 @@ local coreio = require("pocket.coreio") local pocket = require("pocket.pocket") local renderer = require("pocket.renderer") -local POCKET_VERSION = "alpha-v0.4.5" +local POCKET_VERSION = "alpha-v0.4.6" local println = util.println local println_ts = util.println_ts diff --git a/reactor-plc/plc.lua b/reactor-plc/plc.lua index 02e592a..fc41f80 100644 --- a/reactor-plc/plc.lua +++ b/reactor-plc/plc.lua @@ -639,7 +639,7 @@ function plc.comms(id, version, modem, plc_channel, svr_channel, range, reactor, parallel.waitForAll(table.unpack(tasks)) - if not reactor.__p_is_faulted() then + if reactor.__p_is_ok() then _send(RPLC_TYPE.MEK_STRUCT, mek_data) self.resend_build = false end @@ -836,7 +836,7 @@ function plc.comms(id, version, modem, plc_channel, svr_channel, range, reactor, success = true else reactor.setBurnRate(burn_rate) - success = not reactor.__p_is_faulted() + success = reactor.__p_is_ok() end else log.debug(burn_rate .. " rate outside of 0 < x <= " .. self.max_burn_rate) diff --git a/reactor-plc/startup.lua b/reactor-plc/startup.lua index 76185e9..a6a47a6 100644 --- a/reactor-plc/startup.lua +++ b/reactor-plc/startup.lua @@ -18,7 +18,7 @@ local plc = require("reactor-plc.plc") local renderer = require("reactor-plc.renderer") local threads = require("reactor-plc.threads") -local R_PLC_VERSION = "v1.4.6" +local R_PLC_VERSION = "v1.4.7" local println = util.println local println_ts = util.println_ts diff --git a/rtu/dev/boilerv_rtu.lua b/rtu/dev/boilerv_rtu.lua index 46ac7c2..6815a3c 100644 --- a/rtu/dev/boilerv_rtu.lua +++ b/rtu/dev/boilerv_rtu.lua @@ -10,6 +10,7 @@ function boilerv_rtu.new(boiler) local unit = rtu.init_unit() -- disable auto fault clearing + boiler.__p_clear_fault() boiler.__p_disable_afc() -- discrete inputs -- diff --git a/rtu/dev/envd_rtu.lua b/rtu/dev/envd_rtu.lua index 2894e2c..2d576bc 100644 --- a/rtu/dev/envd_rtu.lua +++ b/rtu/dev/envd_rtu.lua @@ -10,6 +10,7 @@ function envd_rtu.new(envd) local unit = rtu.init_unit() -- disable auto fault clearing + envd.__p_clear_fault() envd.__p_disable_afc() -- discrete inputs -- diff --git a/rtu/dev/imatrix_rtu.lua b/rtu/dev/imatrix_rtu.lua index 3b72a12..a20d1a5 100644 --- a/rtu/dev/imatrix_rtu.lua +++ b/rtu/dev/imatrix_rtu.lua @@ -10,6 +10,7 @@ function imatrix_rtu.new(imatrix) local unit = rtu.init_unit() -- disable auto fault clearing + imatrix.__p_clear_fault() imatrix.__p_disable_afc() -- discrete inputs -- diff --git a/rtu/dev/sna_rtu.lua b/rtu/dev/sna_rtu.lua index 16c0cfd..a3e678e 100644 --- a/rtu/dev/sna_rtu.lua +++ b/rtu/dev/sna_rtu.lua @@ -10,6 +10,7 @@ function sna_rtu.new(sna) local unit = rtu.init_unit() -- disable auto fault clearing + sna.__p_clear_fault() sna.__p_disable_afc() -- discrete inputs -- diff --git a/rtu/dev/sps_rtu.lua b/rtu/dev/sps_rtu.lua index 349550c..428f7bb 100644 --- a/rtu/dev/sps_rtu.lua +++ b/rtu/dev/sps_rtu.lua @@ -10,6 +10,7 @@ function sps_rtu.new(sps) local unit = rtu.init_unit() -- disable auto fault clearing + sps.__p_clear_fault() sps.__p_disable_afc() -- discrete inputs -- diff --git a/rtu/dev/turbinev_rtu.lua b/rtu/dev/turbinev_rtu.lua index 4f2ee48..f8d6d5e 100644 --- a/rtu/dev/turbinev_rtu.lua +++ b/rtu/dev/turbinev_rtu.lua @@ -10,6 +10,7 @@ function turbinev_rtu.new(turbine) local unit = rtu.init_unit() -- disable auto fault clearing + turbine.__p_clear_fault() turbine.__p_disable_afc() -- discrete inputs -- diff --git a/rtu/startup.lua b/rtu/startup.lua index 769e4d7..85f3220 100644 --- a/rtu/startup.lua +++ b/rtu/startup.lua @@ -28,7 +28,7 @@ local sna_rtu = require("rtu.dev.sna_rtu") local sps_rtu = require("rtu.dev.sps_rtu") local turbinev_rtu = require("rtu.dev.turbinev_rtu") -local RTU_VERSION = "v1.3.6" +local RTU_VERSION = "v1.3.7" local RTU_UNIT_TYPE = types.RTU_UNIT_TYPE local RTU_UNIT_HW_STATE = databus.RTU_UNIT_HW_STATE @@ -236,18 +236,19 @@ local function main() ---@class rtu_unit_registry_entry local unit = { - uid = 0, ---@type integer - name = "redstone_io", ---@type string - type = RTU_UNIT_TYPE.REDSTONE, ---@type RTU_UNIT_TYPE - index = entry_idx, ---@type integer - reactor = io_reactor, ---@type integer - device = capabilities, ---@type table use device field for redstone ports - is_multiblock = false, ---@type boolean - formed = nil, ---@type boolean|nil - rtu = rs_rtu, ---@type rtu_device|rtu_rs_device + uid = 0, ---@type integer + name = "redstone_io", ---@type string + type = RTU_UNIT_TYPE.REDSTONE, ---@type RTU_UNIT_TYPE + index = entry_idx, ---@type integer + reactor = io_reactor, ---@type integer + device = capabilities, ---@type table use device field for redstone ports + is_multiblock = false, ---@type boolean + formed = nil, ---@type boolean|nil + hw_state = RTU_UNIT_HW_STATE.OK, ---@type RTU_UNIT_HW_STATE + rtu = rs_rtu, ---@type rtu_device|rtu_rs_device modbus_io = modbus.new(rs_rtu, false), - pkt_queue = nil, ---@type mqueue|nil - thread = nil ---@type parallel_thread|nil + pkt_queue = nil, ---@type mqueue|nil + thread = nil ---@type parallel_thread|nil } table.insert(units, unit) @@ -261,7 +262,7 @@ local function main() unit.uid = #units - databus.tx_unit_hw_status(unit.uid, RTU_UNIT_HW_STATE.OK) + databus.tx_unit_hw_status(unit.uid, unit.hw_state) end end @@ -403,6 +404,7 @@ local function main() device = device, ---@type table is_multiblock = is_multiblock, ---@type boolean formed = formed, ---@type boolean|nil + hw_state = RTU_UNIT_HW_STATE.OFFLINE, ---@type RTU_UNIT_HW_STATE rtu = rtu_iface, ---@type rtu_device|rtu_rs_device modbus_io = modbus.new(rtu_iface, true), pkt_queue = mqueue.new(), ---@type mqueue|nil @@ -422,19 +424,21 @@ local function main() rtu_unit.uid = #units - -- report hardware status + -- determine hardware status if rtu_unit.type == RTU_UNIT_TYPE.VIRTUAL then - databus.tx_unit_hw_status(rtu_unit.uid, RTU_UNIT_HW_STATE.OFFLINE) + rtu_unit.hw_state = RTU_UNIT_HW_STATE.OFFLINE else if rtu_unit.is_multiblock then - databus.tx_unit_hw_status(rtu_unit.uid, util.trinary(rtu_unit.formed == true, RTU_UNIT_HW_STATE.OK, RTU_UNIT_HW_STATE.UNFORMED)) + rtu_unit.hw_state = util.trinary(rtu_unit.formed == true, RTU_UNIT_HW_STATE.OK, RTU_UNIT_HW_STATE.UNFORMED) elseif faulted then - databus.tx_unit_hw_status(rtu_unit.uid, RTU_UNIT_HW_STATE.FAULTED) + rtu_unit.hw_state = RTU_UNIT_HW_STATE.FAULTED else - databus.tx_unit_hw_status(rtu_unit.uid, RTU_UNIT_HW_STATE.OK) + rtu_unit.hw_state = RTU_UNIT_HW_STATE.OK end end + -- report hardware status + databus.tx_unit_hw_status(rtu_unit.uid, rtu_unit.hw_state) end -- we made it through all that trusting-user-to-write-a-config-file chaos diff --git a/rtu/threads.lua b/rtu/threads.lua index fdb82b3..6d51dec 100644 --- a/rtu/threads.lua +++ b/rtu/threads.lua @@ -105,13 +105,15 @@ function threads.thread__main(smem) for i = 1, #units do -- find disconnected device if units[i].device == device then - -- we are going to let the PPM prevent crashes - -- return fault flags/codes to MODBUS queries + -- will let the PPM prevent crashes, which will indicate failures in MODBUS queries local unit = units[i] ---@type rtu_unit_registry_entry local type_name = types.rtu_type_to_string(unit.type) + println_ts(util.c("lost the ", type_name, " on interface ", unit.name)) log.warning(util.c("lost the ", type_name, " unit peripheral on interface ", unit.name)) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.OFFLINE) + + unit.hw_state = UNIT_HW_STATE.OFFLINE + databus.tx_unit_hw_status(unit.uid, unit.hw_state) break end end @@ -144,6 +146,8 @@ function threads.thread__main(smem) -- note: cannot check isFormed as that would yield this coroutine and consume events if unit.name == param1 then local resend_advert = false + local faulted = false + local unknown = false -- found, re-link unit.device = device @@ -177,57 +181,58 @@ function threads.thread__main(smem) end if unit.type == RTU_UNIT_TYPE.BOILER_VALVE then - unit.rtu = boilerv_rtu.new(device) + unit.rtu, faulted = boilerv_rtu.new(device) -- if not formed, indexing the multiblock functions would have resulted in a PPM fault - unit.formed = util.trinary(device.__p_is_faulted(), false, nil) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) + unit.formed = util.trinary(faulted, false, nil) elseif unit.type == RTU_UNIT_TYPE.TURBINE_VALVE then - unit.rtu = turbinev_rtu.new(device) + unit.rtu, faulted = turbinev_rtu.new(device) -- if not formed, indexing the multiblock functions would have resulted in a PPM fault - unit.formed = util.trinary(device.__p_is_faulted(), false, nil) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) + unit.formed = util.trinary(faulted, false, nil) elseif unit.type == RTU_UNIT_TYPE.IMATRIX then - unit.rtu = imatrix_rtu.new(device) + unit.rtu, faulted = imatrix_rtu.new(device) -- if not formed, indexing the multiblock functions would have resulted in a PPM fault - unit.formed = util.trinary(device.__p_is_faulted(), false, nil) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) + unit.formed = util.trinary(faulted, false, nil) elseif unit.type == RTU_UNIT_TYPE.SPS then - unit.rtu = sps_rtu.new(device) + unit.rtu, faulted = sps_rtu.new(device) -- if not formed, indexing the multiblock functions would have resulted in a PPM fault - unit.formed = util.trinary(device.__p_is_faulted(), false, nil) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) + unit.formed = util.trinary(faulted, false, nil) elseif unit.type == RTU_UNIT_TYPE.SNA then - unit.rtu = sna_rtu.new(device) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.OK) + unit.rtu, faulted = sna_rtu.new(device) elseif unit.type == RTU_UNIT_TYPE.ENV_DETECTOR then - unit.rtu = envd_rtu.new(device) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.OK) + unit.rtu, faulted = envd_rtu.new(device) else + unknown = true log.error(util.c("failed to identify reconnected RTU unit type (", unit.name, ")"), true) end if unit.is_multiblock then - if (unit.formed == false) then + unit.hw_state = UNIT_HW_STATE.UNFORMED + if unit.formed == false then log.info(util.c("assuming ", unit.name, " is not formed due to PPM faults while initializing")) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) end - elseif device.__p_is_faulted() then - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.FAULTED) + elseif faulted then + unit.hw_state = UNIT_HW_STATE.FAULTED + elseif not unknown then + unit.hw_state = UNIT_HW_STATE.OK else - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.OK) + unit.hw_state = UNIT_HW_STATE.OFFLINE end - unit.modbus_io = modbus.new(unit.rtu, true) + databus.tx_unit_hw_status(unit.uid, unit.hw_state) - local type_name = types.rtu_type_to_string(unit.type) - local message = util.c("reconnected the ", type_name, " on interface ", unit.name) - println_ts(message) - log.info(message) + if not unknown then + unit.modbus_io = modbus.new(unit.rtu, true) - if resend_advert then - rtu_comms.send_advertisement(units) - else - rtu_comms.send_remounted(unit.uid) + local type_name = types.rtu_type_to_string(unit.type) + local message = util.c("reconnected the ", type_name, " on interface ", unit.name) + println_ts(message) + log.info(message) + + if resend_advert then + rtu_comms.send_advertisement(units) + else + rtu_comms.send_remounted(unit.uid) + end end end end @@ -391,13 +396,6 @@ function threads.thread__unit_comms(smem, unit) -- received a packet local _, reply = unit.modbus_io.handle_packet(msg.message) rtu_comms.send_modbus(reply) - - -- check if there was a problem and update the hardware state if so - local frame = reply.get() - if unit.formed and (bit.band(frame.func_code, types.MODBUS_FCODE.ERROR_FLAG) ~= 0) and - (frame.data[1] == types.MODBUS_EXCODE.SERVER_DEVICE_FAIL) then - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.FAULTED) - end end end @@ -413,12 +411,10 @@ function threads.thread__unit_comms(smem, unit) if unit.formed == nil then unit.formed = is_formed - if is_formed then databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.OK) end + if is_formed then unit.hw_state = UNIT_HW_STATE.OK end end - if not unit.formed then - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) - end + if not unit.formed then unit.hw_state = UNIT_HW_STATE.UNFORMED end if (not unit.formed) and is_formed then -- newly re-formed @@ -463,11 +459,11 @@ function threads.thread__unit_comms(smem, unit) if unit.formed and faulted then -- something is still wrong = can't mark as formed yet unit.formed = false + unit.hw_state = UNIT_HW_STATE.UNFORMED log.info(util.c("assuming ", unit.name, " is not formed due to PPM faults while initializing")) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.UNFORMED) else + unit.hw_state = UNIT_HW_STATE.OK rtu_comms.send_remounted(unit.uid) - databus.tx_unit_hw_status(unit.uid, UNIT_HW_STATE.OK) end local type_name = types.rtu_type_to_string(unit.type) @@ -484,6 +480,16 @@ function threads.thread__unit_comms(smem, unit) unit.formed = is_formed end + -- check hardware status + if unit.device.__p_is_healthy() then + if unit.hw_state == UNIT_HW_STATE.FAULTED then unit.hw_state = UNIT_HW_STATE.OK end + else + if unit.hw_state == UNIT_HW_STATE.OK then unit.hw_state = UNIT_HW_STATE.FAULTED end + end + + -- update hw status + databus.tx_unit_hw_status(unit.uid, unit.hw_state) + -- check for termination request if rtu_state.shutdown then log.info("rtu unit thread exiting -> " .. short_name) diff --git a/scada-common/ppm.lua b/scada-common/ppm.lua index fe9e026..90f0b17 100644 --- a/scada-common/ppm.lua +++ b/scada-common/ppm.lua @@ -101,22 +101,31 @@ local function peri_init(iface) end end - -- fault management functions + -- fault management & monitoring functions local function clear_fault() self.faulted = false end local function get_last_fault() return self.last_fault end local function is_faulted() return self.faulted end local function is_ok() return not self.faulted end + -- check if a peripheral has any faulted functions
+ -- contrasted with is_faulted() and is_ok() as those only check if the last operation failed, + -- unless auto fault clearing is disabled, at which point faults become sticky faults + local function is_healthy() + for _, v in pairs(self.fault_counts) do if v > 0 then return false end end + return true + end + local function enable_afc() self.auto_cf = true end local function disable_afc() self.auto_cf = false end - -- append to device functions + -- append PPM functions to device functions self.device.__p_clear_fault = clear_fault self.device.__p_last_fault = get_last_fault self.device.__p_is_faulted = is_faulted self.device.__p_is_ok = is_ok + self.device.__p_is_healthy = is_healthy self.device.__p_enable_afc = enable_afc self.device.__p_disable_afc = disable_afc @@ -156,7 +165,7 @@ local function peri_init(iface) return { type = self.type, - dev = self.device + dev = self.device } end diff --git a/supervisor/startup.lua b/supervisor/startup.lua index 0949573..d3f2d81 100644 --- a/supervisor/startup.lua +++ b/supervisor/startup.lua @@ -20,7 +20,7 @@ local supervisor = require("supervisor.supervisor") local svsessions = require("supervisor.session.svsessions") -local SUPERVISOR_VERSION = "v0.17.9" +local SUPERVISOR_VERSION = "v0.17.10" local println = util.println local println_ts = util.println_ts