account for alert sending failing

pyproject.toml

@@ -11,7 +11,9 @@ dependencies = [
     "alt-utils>=0.0.8",
     "humanize>=4.12.3",
     "psutil>=7.0.0",
+    "returns>=0.26.0",
     "telethon>=1.40.0",
+    "tenacity>=9.1.2",
     "uplink[aiohttp]>=0.10.0",
 ]
 

src/lego_monitoring/__init__.py

@@ -142,8 +142,9 @@ async def async_main():
             if stopping:
                 if "self" in config.enabled_check_sets:
                     alert = checks.generate_stop_alert()
-                    await sender.send_alert(alert)
-                    await sender.send_healthchecks_status(alert)
+                    async with asyncio.TaskGroup() as tg:
+                        tg.create_task(sender.send_alert(alert))
+                        tg.create_task(sender.send_healthchecks_status(alert))
                 for c in checkers:
                     try:
                         await c.graceful_stop()

src/lego_monitoring/alerting/sender.py

@@ -1,12 +1,13 @@
 import logging
-from socket import gethostname
 
+import tenacity
+from returns.result import Failure, Success
 from telethon import TelegramClient
 from telethon.sessions import MemorySession
 from uplink import AiohttpClient
 
-from ..checks.utils import format_for_healthchecks_slug
 from ..core import cvars
+from ..core.error_handling import log_errors_async
 from .alert import Alert
 from .clients.healthchecks import HealthchecksClient
 from .enum import SEVERITY_TO_EMOJI, Severity
@@ -38,27 +39,26 @@ def format_message(alert: Alert, note: str) -> str:
     return message
 
 
-async def send_alert(alert: Alert, note: str = "") -> None:
+async def send_alert(alert: Alert, note: str = "") -> Success[None] | Failure[tenacity.RetryError]:
+    await log_errors_async(_send_alert(alert, note))
+
+
+async def send_healthchecks_status(alert: Alert) -> Success[None] | Failure[tenacity.RetryError]:
+    await log_errors_async(_send_healthchecks_status(alert))
+
+
+@tenacity.retry(wait=tenacity.wait_random_exponential(multiplier=1, max=60))
+async def _send_alert(alert: Alert, note: str = "") -> None:
     logging.debug(f"Sending {alert.alert_type} alert to Telegram")
-    try:
-        tg_client = cvars.tg_client.get()
-    except LookupError:  # being called standalone
-        # cvars.config.set(get_config())
-        # temp_client = True
-        # client = await get_tg_client()
-        # cvars.matrix_client.set(client)
-        raise NotImplementedError  # TODO
-    else:
-        ...  # temp_client = False
+    tg_client = cvars.tg_client.get()
     if tg_client is not None:
         room_id = cvars.config.get().alert_channels.telegram.room_id
         message = format_message(alert, note)
         await tg_client.send_message(entity=room_id, message=message)
-        # if temp_client:
-        #     await client.close()
 
 
-async def send_healthchecks_status(alert: Alert) -> None:
+@tenacity.retry(wait=tenacity.wait_random_exponential(multiplier=1, max=60))
+async def _send_healthchecks_status(alert: Alert) -> None:
     def get_pinging_key(keys: dict[str, str]):
         if alert.healthchecks_slug in keys:
             return keys[alert.healthchecks_slug]

src/lego_monitoring/core/checkers.py

@@ -70,21 +70,22 @@ class BaseChecker:
         return result
 
     async def _handle_alerts(self, alerts: list[Alert]) -> None:
-        if not self.is_reminder:
-            for alert in alerts:
-                await send_healthchecks_status(alert)
+        async with asyncio.TaskGroup() as tg:
+            if not self.is_reminder:
+                for alert in alerts:
+                    tg.create_task(send_healthchecks_status(alert))
 
-        if not self.persistent:
-            for alert in alerts:
-                if alert.severity != Severity.OK:
-                    await send_alert(alert, "ongoing" if self.is_reminder else "")
-            return
-        old_severity, new_severity = self.current_alerts.update(alerts)
-        if (old_severity != new_severity or self.send_any_state) and not (
-            old_severity == None and new_severity == Severity.OK
-        ):
-            for alert in alerts:
-                await send_alert(alert, note="ongoing")
+            if not self.persistent:
+                for alert in alerts:
+                    if alert.severity != Severity.OK:
+                        tg.create_task(send_alert(alert, "ongoing" if self.is_reminder else ""))
+                return
+            old_severity, new_severity = self.current_alerts.update(alerts)
+            if (old_severity != new_severity or self.send_any_state) and not (
+                old_severity == None and new_severity == Severity.OK
+            ):
+                for alert in alerts:
+                    tg.create_task(send_alert(alert, note="ongoing"))
 
     async def run_checker(self) -> None:
         raise NotImplementedError

src/lego_monitoring/core/error_handling.py

@@ -0,0 +1,23 @@
+import logging
+import traceback
+from typing import Awaitable, Callable, TypeVar
+
+from returns.result import Failure, Success
+
+T = TypeVar("T")
+
+
+def log_errors(function: Callable[..., T], *args, **kwargs) -> Success[T] | Failure[Exception]:
+    try:
+        return Success(function(args, kwargs))
+    except Exception as e:
+        logging.error(traceback.format_exc())
+        return Failure(e)
+
+
+async def log_errors_async(awaitable: Awaitable[T]) -> Success[T] | Failure[Exception]:
+    try:
+        return Success(await awaitable)
+    except Exception as e:
+        logging.error(traceback.format_exc())
+        return Failure(e)

uv.lock

@@ -287,7 +287,9 @@ dependencies = [
     { name = "alt-utils" },
     { name = "humanize" },
     { name = "psutil" },
+    { name = "returns" },
     { name = "telethon" },
+    { name = "tenacity" },
     { name = "uplink", extra = ["aiohttp"] },
 ]
 
@@ -299,7 +301,9 @@ requires-dist = [
     { name = "alt-utils", specifier = ">=0.0.8" },
     { name = "humanize", specifier = ">=4.12.3" },
     { name = "psutil", specifier = ">=7.0.0" },
+    { name = "returns", specifier = ">=0.26.0" },
     { name = "telethon", specifier = ">=1.40.0" },
+    { name = "tenacity", specifier = ">=9.1.2" },
     { name = "uplink", extras = ["aiohttp"], specifier = ">=0.10.0" },
 ]
 
@@ -514,6 +518,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" },
 ]
 
+[[package]]
+name = "returns"
+version = "0.26.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/c2/6dda7ef39464568152e35c766a8b49ab1cdb1b03a5891441a7c2fa40dc61/returns-0.26.0.tar.gz", hash = "sha256:180320e0f6e9ea9845330ccfc020f542330f05b7250941d9b9b7c00203fcc3da", size = 105300, upload-time = "2025-07-24T13:11:21.772Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/57/4d/a7545bf6c62b0dbe5795f22ea9e88cc070fdced5c34663ebc5bed2f610c0/returns-0.26.0-py3-none-any.whl", hash = "sha256:7cae94c730d6c56ffd9d0f583f7a2c0b32cfe17d141837150c8e6cff3eb30d71", size = 160515, upload-time = "2025-07-24T13:11:20.041Z" },
+]
+
 [[package]]
 name = "rsa"
 version = "4.9.1"
@@ -549,6 +565,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ce/b0/78f74085b6c88c2bf2bec39c67267cd9ba6af24ceaea9654fb0c272a53da/telethon-1.40.0-py3-none-any.whl", hash = "sha256:1aebaca04fd8410968816645bdbcc0baeff55429b6d6bec37e647417bb8e8a2c", size = 744897, upload-time = "2025-09-01T15:32:34.212Z" },
 ]
 
+[[package]]
+name = "tenacity"
+version = "9.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" },
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.14.1"