diff --git a/pyproject.toml b/pyproject.toml index aefda49..9d8410d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ dependencies = [ "alt-utils>=0.0.8", "humanize>=4.12.3", "psutil>=7.0.0", + "returns>=0.26.0", "telethon>=1.40.0", + "tenacity>=9.1.2", "uplink[aiohttp]>=0.10.0", ] diff --git a/src/lego_monitoring/__init__.py b/src/lego_monitoring/__init__.py index 29e5e56..385ad42 100644 --- a/src/lego_monitoring/__init__.py +++ b/src/lego_monitoring/__init__.py @@ -142,8 +142,9 @@ async def async_main(): if stopping: if "self" in config.enabled_check_sets: alert = checks.generate_stop_alert() - await sender.send_alert(alert) - await sender.send_healthchecks_status(alert) + async with asyncio.TaskGroup() as tg: + tg.create_task(sender.send_alert(alert)) + tg.create_task(sender.send_healthchecks_status(alert)) for c in checkers: try: await c.graceful_stop() diff --git a/src/lego_monitoring/alerting/sender.py b/src/lego_monitoring/alerting/sender.py index e06907f..0f17b8d 100644 --- a/src/lego_monitoring/alerting/sender.py +++ b/src/lego_monitoring/alerting/sender.py @@ -1,12 +1,13 @@ import logging -from socket import gethostname +import tenacity +from returns.result import Failure, Success from telethon import TelegramClient from telethon.sessions import MemorySession from uplink import AiohttpClient -from ..checks.utils import format_for_healthchecks_slug from ..core import cvars +from ..core.error_handling import log_errors_async from .alert import Alert from .clients.healthchecks import HealthchecksClient from .enum import SEVERITY_TO_EMOJI, Severity @@ -38,27 +39,26 @@ def format_message(alert: Alert, note: str) -> str: return message -async def send_alert(alert: Alert, note: str = "") -> None: +async def send_alert(alert: Alert, note: str = "") -> Success[None] | Failure[tenacity.RetryError]: + await log_errors_async(_send_alert(alert, note)) + + +async def send_healthchecks_status(alert: Alert) -> Success[None] | Failure[tenacity.RetryError]: + await log_errors_async(_send_healthchecks_status(alert)) + + +@tenacity.retry(wait=tenacity.wait_random_exponential(multiplier=1, max=60)) +async def _send_alert(alert: Alert, note: str = "") -> None: logging.debug(f"Sending {alert.alert_type} alert to Telegram") - try: - tg_client = cvars.tg_client.get() - except LookupError: # being called standalone - # cvars.config.set(get_config()) - # temp_client = True - # client = await get_tg_client() - # cvars.matrix_client.set(client) - raise NotImplementedError # TODO - else: - ... # temp_client = False + tg_client = cvars.tg_client.get() if tg_client is not None: room_id = cvars.config.get().alert_channels.telegram.room_id message = format_message(alert, note) await tg_client.send_message(entity=room_id, message=message) - # if temp_client: - # await client.close() -async def send_healthchecks_status(alert: Alert) -> None: +@tenacity.retry(wait=tenacity.wait_random_exponential(multiplier=1, max=60)) +async def _send_healthchecks_status(alert: Alert) -> None: def get_pinging_key(keys: dict[str, str]): if alert.healthchecks_slug in keys: return keys[alert.healthchecks_slug] diff --git a/src/lego_monitoring/core/checkers.py b/src/lego_monitoring/core/checkers.py index 29e041c..3bc1d97 100644 --- a/src/lego_monitoring/core/checkers.py +++ b/src/lego_monitoring/core/checkers.py @@ -70,21 +70,22 @@ class BaseChecker: return result async def _handle_alerts(self, alerts: list[Alert]) -> None: - if not self.is_reminder: - for alert in alerts: - await send_healthchecks_status(alert) + async with asyncio.TaskGroup() as tg: + if not self.is_reminder: + for alert in alerts: + tg.create_task(send_healthchecks_status(alert)) - if not self.persistent: - for alert in alerts: - if alert.severity != Severity.OK: - await send_alert(alert, "ongoing" if self.is_reminder else "") - return - old_severity, new_severity = self.current_alerts.update(alerts) - if (old_severity != new_severity or self.send_any_state) and not ( - old_severity == None and new_severity == Severity.OK - ): - for alert in alerts: - await send_alert(alert, note="ongoing") + if not self.persistent: + for alert in alerts: + if alert.severity != Severity.OK: + tg.create_task(send_alert(alert, "ongoing" if self.is_reminder else "")) + return + old_severity, new_severity = self.current_alerts.update(alerts) + if (old_severity != new_severity or self.send_any_state) and not ( + old_severity == None and new_severity == Severity.OK + ): + for alert in alerts: + tg.create_task(send_alert(alert, note="ongoing")) async def run_checker(self) -> None: raise NotImplementedError diff --git a/src/lego_monitoring/core/error_handling.py b/src/lego_monitoring/core/error_handling.py new file mode 100644 index 0000000..1b340aa --- /dev/null +++ b/src/lego_monitoring/core/error_handling.py @@ -0,0 +1,23 @@ +import logging +import traceback +from typing import Awaitable, Callable, TypeVar + +from returns.result import Failure, Success + +T = TypeVar("T") + + +def log_errors(function: Callable[..., T], *args, **kwargs) -> Success[T] | Failure[Exception]: + try: + return Success(function(args, kwargs)) + except Exception as e: + logging.error(traceback.format_exc()) + return Failure(e) + + +async def log_errors_async(awaitable: Awaitable[T]) -> Success[T] | Failure[Exception]: + try: + return Success(await awaitable) + except Exception as e: + logging.error(traceback.format_exc()) + return Failure(e) diff --git a/uv.lock b/uv.lock index f9daed7..1148674 100644 --- a/uv.lock +++ b/uv.lock @@ -287,7 +287,9 @@ dependencies = [ { name = "alt-utils" }, { name = "humanize" }, { name = "psutil" }, + { name = "returns" }, { name = "telethon" }, + { name = "tenacity" }, { name = "uplink", extra = ["aiohttp"] }, ] @@ -299,7 +301,9 @@ requires-dist = [ { name = "alt-utils", specifier = ">=0.0.8" }, { name = "humanize", specifier = ">=4.12.3" }, { name = "psutil", specifier = ">=7.0.0" }, + { name = "returns", specifier = ">=0.26.0" }, { name = "telethon", specifier = ">=1.40.0" }, + { name = "tenacity", specifier = ">=9.1.2" }, { name = "uplink", extras = ["aiohttp"], specifier = ">=0.10.0" }, ] @@ -514,6 +518,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, ] +[[package]] +name = "returns" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c2/6dda7ef39464568152e35c766a8b49ab1cdb1b03a5891441a7c2fa40dc61/returns-0.26.0.tar.gz", hash = "sha256:180320e0f6e9ea9845330ccfc020f542330f05b7250941d9b9b7c00203fcc3da", size = 105300, upload-time = "2025-07-24T13:11:21.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/4d/a7545bf6c62b0dbe5795f22ea9e88cc070fdced5c34663ebc5bed2f610c0/returns-0.26.0-py3-none-any.whl", hash = "sha256:7cae94c730d6c56ffd9d0f583f7a2c0b32cfe17d141837150c8e6cff3eb30d71", size = 160515, upload-time = "2025-07-24T13:11:20.041Z" }, +] + [[package]] name = "rsa" version = "4.9.1" @@ -549,6 +565,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/b0/78f74085b6c88c2bf2bec39c67267cd9ba6af24ceaea9654fb0c272a53da/telethon-1.40.0-py3-none-any.whl", hash = "sha256:1aebaca04fd8410968816645bdbcc0baeff55429b6d6bec37e647417bb8e8a2c", size = 744897, upload-time = "2025-09-01T15:32:34.212Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, +] + [[package]] name = "typing-extensions" version = "4.14.1"