From fd2624040c844743f54e9afe2a2a33306087fb98 Mon Sep 17 00:00:00 2001 From: saqriphnix Date: Fri, 16 Aug 2024 23:08:28 +0300 Subject: [PATCH 1/7] parse UPS data --- alerting/alerts.py | 1 + misc/enums.py | 20 ++++++++++++++++++++ misc/sensors.py | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 misc/enums.py diff --git a/alerting/alerts.py b/alerting/alerts.py index f387dc4..e403c5a 100644 --- a/alerting/alerts.py +++ b/alerting/alerts.py @@ -19,6 +19,7 @@ class AlertType(StrEnum): LOGIN = "LOGIN" # TODO SMART = "SMART" # TODO RAID = "RAID" # TODO + UPS = "UPS" class Severity(Enum): diff --git a/misc/enums.py b/misc/enums.py new file mode 100644 index 0000000..3d2cacc --- /dev/null +++ b/misc/enums.py @@ -0,0 +1,20 @@ +from enum import StrEnum + + +class UPSStatus(StrEnum): + """https://networkupstools.org/docs/developer-guide.chunked/new-drivers.html#_status_data + """ + + ON_LINE = "OL" + ON_BATTERY = "OB" + BATTERY_LOW = "LB" + BATTERY_HIGH = "HB" + BATTERY_REPLACE = "RB" + BATTERY_CHARGING = "CHRG" + BATTERY_DISCHARGING = "DISCHRG" + UPS_BYPASS = "BYPASS" + """Battery and connected devices are not protected from power outage!""" + UPS_OFFLINE = "OFF" + UPS_OVERLOAD = "OVER" + UPS_CALIBRATION = "CAL" + diff --git a/misc/sensors.py b/misc/sensors.py index b1b3f1b..f136847 100644 --- a/misc/sensors.py +++ b/misc/sensors.py @@ -1,7 +1,10 @@ +import subprocess from dataclasses import dataclass from psutil import cpu_percent, sensors_temperatures, virtual_memory +from .enums import UPSStatus + @dataclass class TemperatureSensor: @@ -27,6 +30,14 @@ class RamSensor: critical_avail: int = 2 * 1024**3 +@dataclass +class UPSSensor: + ups_status: UPSStatus = None + battery_charge_percentage: int = None + battery_warning_percentage: int = 20 + battery_critical_percentage: int = 10 + + class Sensors: @staticmethod def get_temperatures() -> dict[str, list[TemperatureSensor]]: @@ -96,10 +107,26 @@ class Sensors: @staticmethod def get_ram() -> RamSensor: ram = virtual_memory() - return RamSensor(current_avail=ram.available, - current_avail_percentage=ram.percent) + return RamSensor(current_avail=ram.available, current_avail_percentage=ram.percent) + @staticmethod + def get_ups() -> UPSSensor: + raw_data = subprocess.run(["upsc", "cp1300"], stdout=subprocess.PIPE, encoding="utf-8") -if __name__ == "__main__": - for i in Sensors.get_temperatures(): - print(i) + sensor_data = UPSSensor() + + for line in raw_data.stdout.splitlines(): + sensor, value = line.split(": ")[:2] + match sensor: + case "battery.charge": + sensor_data.battery_charge_percentage = int(value) + case "battery.charge.low": + sensor_data.battery_critical_percentage = int(value) + case "battery.charge.warning": + sensor_data.battery_warning_percentage = int(value) + case "ups.status": + sensor_data.ups_status = UPSStatus(value) + case _: + ... + + return sensor_data From 3a02adde569c1a222b11f47583ae45725a4045d4 Mon Sep 17 00:00:00 2001 From: saqriphnix Date: Sat, 17 Aug 2024 00:00:16 +0300 Subject: [PATCH 2/7] UPS check and runtime monitoring --- misc/checks.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ misc/sensors.py | 11 ++++++++-- service.py | 1 + 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/misc/checks.py b/misc/checks.py index 6efefd2..baee3bd 100644 --- a/misc/checks.py +++ b/misc/checks.py @@ -1,5 +1,8 @@ +from datetime import timedelta + from alerting import alerts from misc import sensors, vuln +from misc.enums import UPSStatus IS_TESTING = False @@ -92,3 +95,54 @@ async def vuln_check() -> list[alerts.Alert]: ) alert_list.append(alert) return alert_list + + +async def ups_check() -> list[alerts.Alert]: + sensor = sensors.Sensors.get_ups() + alert_list = [] + + if IS_TESTING or sensor.battery_charge_percentage < sensor.battery_critical_percentage: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Battery is under {sensor.battery_critical_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", + severity=alerts.Severity.CRITICAL + )) + elif IS_TESTING or sensor.battery_charge_percentage < sensor.battery_warning_percentage: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Battery is under {sensor.battery_warning_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", + severity=alerts.Severity.WARNING + )) + elif IS_TESTING or sensor.ups_status == UPSStatus.ON_BATTERY: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is on battery.\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", + severity=alerts.Severity.INFO + )) + + elif IS_TESTING or sensor.ups_status == UPSStatus.BATTERY_REPLACE: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS battery needs to be replaced ASAP!", + severity=alerts.Severity.CRITICAL + )) + elif IS_TESTING or sensor.ups_status == UPSStatus.UPS_OVERLOAD: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is overloaded!", + severity=alerts.Severity.CRITICAL + )) + elif IS_TESTING or sensor.ups_status == UPSStatus.UPS_BYPASS: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"BYPASS MODE: Battery and connected devices are not protected from power outage!", + severity=alerts.Severity.WARNING + )) + elif IS_TESTING or sensor.ups_status == UPSStatus.UPS_CALIBRATION: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is currently performing runtime calibration.", + severity=alerts.Severity.INFO + )) + + return alert_list diff --git a/misc/sensors.py b/misc/sensors.py index f136847..6169d44 100644 --- a/misc/sensors.py +++ b/misc/sensors.py @@ -36,6 +36,7 @@ class UPSSensor: battery_charge_percentage: int = None battery_warning_percentage: int = 20 battery_critical_percentage: int = 10 + battery_runtime: int = 1000 class Sensors: @@ -121,9 +122,15 @@ class Sensors: case "battery.charge": sensor_data.battery_charge_percentage = int(value) case "battery.charge.low": - sensor_data.battery_critical_percentage = int(value) + # ? in case we need to evaluate critical% from sensor + # sensor_data.battery_critical_percentage = int(value) + sensor_data.battery_critical_percentage = 25 case "battery.charge.warning": - sensor_data.battery_warning_percentage = int(value) + # ? in case we need to evaluate warning% from sensor + # sensor_data.battery_warning_percentage = int(value) + sensor_data.battery_warning_percentage = 50 + case "battery.runtime": + sensor_data.battery_runtime = int(value) case "ups.status": sensor_data.ups_status = UPSStatus(value) case _: diff --git a/service.py b/service.py index 59d8e60..fb63030 100755 --- a/service.py +++ b/service.py @@ -47,6 +47,7 @@ async def main(): checkers = ( checker(checks.temp_check, 5 * MINUTE, client), checker(checks.cpu_check, 5 * MINUTE, client), + checker(checks.ups_check, 5 * MINUTE, client), checker(checks.ram_check, 1 * MINUTE, client), checker(checks.vuln_check, 1 * DAY, client), ) From 66bc90da5c51b32882f15f53991664632dd64c24 Mon Sep 17 00:00:00 2001 From: saqriphnix Date: Sat, 17 Aug 2024 11:58:13 +0300 Subject: [PATCH 3/7] fix status is not actually a single word --- misc/checks.py | 61 +++++++++++++++++++++++++------------------------ misc/sensors.py | 4 ++-- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/misc/checks.py b/misc/checks.py index baee3bd..d4bad14 100644 --- a/misc/checks.py +++ b/misc/checks.py @@ -113,36 +113,37 @@ async def ups_check() -> list[alerts.Alert]: message=f"Battery is under {sensor.battery_warning_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", severity=alerts.Severity.WARNING )) - elif IS_TESTING or sensor.ups_status == UPSStatus.ON_BATTERY: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS is on battery.\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", - severity=alerts.Severity.INFO - )) - elif IS_TESTING or sensor.ups_status == UPSStatus.BATTERY_REPLACE: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS battery needs to be replaced ASAP!", - severity=alerts.Severity.CRITICAL - )) - elif IS_TESTING or sensor.ups_status == UPSStatus.UPS_OVERLOAD: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS is overloaded!", - severity=alerts.Severity.CRITICAL - )) - elif IS_TESTING or sensor.ups_status == UPSStatus.UPS_BYPASS: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"BYPASS MODE: Battery and connected devices are not protected from power outage!", - severity=alerts.Severity.WARNING - )) - elif IS_TESTING or sensor.ups_status == UPSStatus.UPS_CALIBRATION: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS is currently performing runtime calibration.", - severity=alerts.Severity.INFO - )) + for status in sensor.ups_status: + if IS_TESTING or status == UPSStatus.BATTERY_REPLACE: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS battery needs to be replaced ASAP!", + severity=alerts.Severity.CRITICAL + )) + elif IS_TESTING or status == UPSStatus.UPS_OVERLOAD: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is overloaded!", + severity=alerts.Severity.CRITICAL + )) + elif IS_TESTING or status == UPSStatus.UPS_BYPASS: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"BYPASS MODE: Battery and connected devices are not protected from power outage!", + severity=alerts.Severity.WARNING + )) + elif IS_TESTING or status == UPSStatus.UPS_CALIBRATION: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is currently performing runtime calibration.", + severity=alerts.Severity.INFO + )) + elif IS_TESTING or status == UPSStatus.ON_BATTERY: + alert_list.append(alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is on battery.\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", + severity=alerts.Severity.INFO + )) return alert_list diff --git a/misc/sensors.py b/misc/sensors.py index 6169d44..16e25e4 100644 --- a/misc/sensors.py +++ b/misc/sensors.py @@ -32,7 +32,7 @@ class RamSensor: @dataclass class UPSSensor: - ups_status: UPSStatus = None + ups_status: list[UPSStatus] = None battery_charge_percentage: int = None battery_warning_percentage: int = 20 battery_critical_percentage: int = 10 @@ -132,7 +132,7 @@ class Sensors: case "battery.runtime": sensor_data.battery_runtime = int(value) case "ups.status": - sensor_data.ups_status = UPSStatus(value) + sensor_data.ups_status = [UPSStatus(status) for status in value.split()] case _: ... From 17e5d1be0cefdbfeb4a50f71c97eb17ded8868c6 Mon Sep 17 00:00:00 2001 From: saqriphnix Date: Sat, 17 Aug 2024 12:32:25 +0300 Subject: [PATCH 4/7] alerting if upsc is not installed --- misc/checks.py | 5 ++++- misc/sensors.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/misc/checks.py b/misc/checks.py index d4bad14..70a9d37 100644 --- a/misc/checks.py +++ b/misc/checks.py @@ -98,7 +98,10 @@ async def vuln_check() -> list[alerts.Alert]: async def ups_check() -> list[alerts.Alert]: - sensor = sensors.Sensors.get_ups() + sensor = await sensors.Sensors.get_ups() + + if not sensor: return + alert_list = [] if IS_TESTING or sensor.battery_charge_percentage < sensor.battery_critical_percentage: diff --git a/misc/sensors.py b/misc/sensors.py index 16e25e4..ed554cb 100644 --- a/misc/sensors.py +++ b/misc/sensors.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from psutil import cpu_percent, sensors_temperatures, virtual_memory from .enums import UPSStatus +from alerting import alerts @dataclass @@ -111,8 +112,16 @@ class Sensors: return RamSensor(current_avail=ram.available, current_avail_percentage=ram.percent) @staticmethod - def get_ups() -> UPSSensor: - raw_data = subprocess.run(["upsc", "cp1300"], stdout=subprocess.PIPE, encoding="utf-8") + async def get_ups() -> None | UPSSensor: + try: + raw_data = subprocess.run(["upsc", "cp1300"], stdout=subprocess.PIPE, encoding="utf-8") + except FileNotFoundError: + await alerts.send_alert(alerts.Alert( + alert_type=alerts.AlertType.ERROR, + message="upsc is not installed!", + severity=alerts.Severity.CRITICAL + )) + return None sensor_data = UPSSensor() From 983caffc10172af0fdd67a20bd2d2e04cd03fe65 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 17 Aug 2024 13:33:22 +0300 Subject: [PATCH 5/7] add missing ups statuses --- misc/enums.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misc/enums.py b/misc/enums.py index 3d2cacc..2ea1fc1 100644 --- a/misc/enums.py +++ b/misc/enums.py @@ -2,8 +2,7 @@ from enum import StrEnum class UPSStatus(StrEnum): - """https://networkupstools.org/docs/developer-guide.chunked/new-drivers.html#_status_data - """ + """https://networkupstools.org/docs/developer-guide.chunked/new-drivers.html#_status_data""" ON_LINE = "OL" ON_BATTERY = "OB" @@ -17,4 +16,6 @@ class UPSStatus(StrEnum): UPS_OFFLINE = "OFF" UPS_OVERLOAD = "OVER" UPS_CALIBRATION = "CAL" - + UPS_TRIM = "TRIM" + UPS_BOOST = "BOOST" + UPS_FSD = "FSD" From df21b8dfdb4c1aa980e20c69c717fc088bdeb603 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 17 Aug 2024 13:41:39 +0300 Subject: [PATCH 6/7] add more ups alert events --- misc/checks.py | 115 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 36 deletions(-) diff --git a/misc/checks.py b/misc/checks.py index 70a9d37..d642cd4 100644 --- a/misc/checks.py +++ b/misc/checks.py @@ -100,53 +100,96 @@ async def vuln_check() -> list[alerts.Alert]: async def ups_check() -> list[alerts.Alert]: sensor = await sensors.Sensors.get_ups() - if not sensor: return + if not sensor: + return alert_list = [] if IS_TESTING or sensor.battery_charge_percentage < sensor.battery_critical_percentage: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"Battery is under {sensor.battery_critical_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", - severity=alerts.Severity.CRITICAL - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Battery is under {sensor.battery_critical_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", + severity=alerts.Severity.CRITICAL, + ) + ) elif IS_TESTING or sensor.battery_charge_percentage < sensor.battery_warning_percentage: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"Battery is under {sensor.battery_warning_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", - severity=alerts.Severity.WARNING - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Battery is under {sensor.battery_warning_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", + severity=alerts.Severity.WARNING, + ) + ) for status in sensor.ups_status: if IS_TESTING or status == UPSStatus.BATTERY_REPLACE: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS battery needs to be replaced ASAP!", - severity=alerts.Severity.CRITICAL - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS battery needs to be replaced ASAP!", + severity=alerts.Severity.CRITICAL, + ) + ) elif IS_TESTING or status == UPSStatus.UPS_OVERLOAD: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS is overloaded!", - severity=alerts.Severity.CRITICAL - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, message=f"UPS is overloaded!", severity=alerts.Severity.CRITICAL + ) + ) elif IS_TESTING or status == UPSStatus.UPS_BYPASS: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"BYPASS MODE: Battery and connected devices are not protected from power outage!", - severity=alerts.Severity.WARNING - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"BYPASS MODE: Battery and connected devices are not protected from power outage!", + severity=alerts.Severity.WARNING, + ) + ) elif IS_TESTING or status == UPSStatus.UPS_CALIBRATION: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS is currently performing runtime calibration.", - severity=alerts.Severity.INFO - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is currently performing runtime calibration.", + severity=alerts.Severity.INFO, + ) + ) elif IS_TESTING or status == UPSStatus.ON_BATTERY: - alert_list.append(alerts.Alert( - alert_type=alerts.AlertType.UPS, - message=f"UPS is on battery.\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", - severity=alerts.Severity.INFO - )) + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"UPS is on battery.\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", + severity=alerts.Severity.INFO, + ) + ) + elif IS_TESTING or status == UPSStatus.UPS_OFFLINE: + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, message=f"UPS seems to be offline.", severity=alerts.Severity.INFO + ) + ) + elif IS_TESTING or status == UPSStatus.UPS_TRIM: + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Overvoltage detected: trimming voltage to nominal.", + severity=alerts.Severity.INFO, + ) + ) + elif IS_TESTING or status == UPSStatus.UPS_BOOST: + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Undervoltage detected: boosting voltage to nominal.", + severity=alerts.Severity.INFO, + ) + ) + elif IS_TESTING or status == UPSStatus.UPS_FSD: + alert_list.append( + alerts.Alert( + alert_type=alerts.AlertType.UPS, + message=f"Shutdown imminent!", + severity=alerts.Severity.CRITICAL, + ) + ) return alert_list From 7fe016198963681ed65df13ed71cdff11758de18 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 17 Aug 2024 14:25:02 +0300 Subject: [PATCH 7/7] fix bracket --- misc/checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/checks.py b/misc/checks.py index d642cd4..e41c2ba 100644 --- a/misc/checks.py +++ b/misc/checks.py @@ -109,7 +109,7 @@ async def ups_check() -> list[alerts.Alert]: alert_list.append( alerts.Alert( alert_type=alerts.AlertType.UPS, - message=f"Battery is under {sensor.battery_critical_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", + message=f"Battery is under {sensor.battery_critical_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", severity=alerts.Severity.CRITICAL, ) ) @@ -117,7 +117,7 @@ async def ups_check() -> list[alerts.Alert]: alert_list.append( alerts.Alert( alert_type=alerts.AlertType.UPS, - message=f"Battery is under {sensor.battery_warning_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)} remaining.", + message=f"Battery is under {sensor.battery_warning_percentage}%\n{sensor.battery_charge_percentage}% ({timedelta(seconds=sensor.battery_runtime)}) remaining.", severity=alerts.Severity.WARNING, ) )