add docker registry monitoring

This commit is contained in:
Alex 2024-10-27 22:29:32 +03:00
parent 618ca3c9aa
commit 13723a6bb4
9 changed files with 221 additions and 32 deletions

View file

@ -1,15 +1,16 @@
#!/usr/bin/env python3
import asyncio
import datetime
import json
import logging
import signal
from typing import Callable, Coroutine
import aiofiles
from alerting import alerts
from alerting.common import CONFIG_FILE
from misc import checks, cvars
from misc.checkers import interval_checker, scheduled_checker
logging.basicConfig(level=logging.INFO)
@ -21,29 +22,7 @@ def stop_gracefully(signum, frame):
stopping = True
async def checker(check: Callable | Coroutine, interval_secs: int, *args, **kwargs):
while True:
logging.info(f"Calling {check.__name__}")
if isinstance(check, Callable):
result = check(*args, **kwargs)
if isinstance(result, Coroutine):
result = await result
elif isinstance(check, Coroutine):
result = await check
else:
raise TypeError(f"check is {type(check)}, neither function nor coroutine")
logging.info(f"Got {len(result)} alerts")
for alert in result:
await alerts.send_alert(alert)
await asyncio.sleep(interval_secs)
async def main():
MINUTE = 60
HOUR = 60 * MINUTE
DAY = 24 * HOUR
WEEK = 7 * DAY
signal.signal(signal.SIGTERM, stop_gracefully)
async with aiofiles.open(CONFIG_FILE) as f:
@ -53,11 +32,14 @@ async def main():
client = await alerts.get_client()
cvars.matrix_client.set(client)
checkers = (
checker(checks.temp_check, 5 * MINUTE),
checker(checks.cpu_check, 5 * MINUTE),
checker(checks.ups_check, 5 * MINUTE),
checker(checks.ram_check, 1 * MINUTE),
checker(checks.vuln_check, 1 * DAY),
interval_checker(checks.temp_check, datetime.timedelta(minutes=5)),
interval_checker(checks.cpu_check, datetime.timedelta(minutes=5)),
interval_checker(checks.ups_check, datetime.timedelta(minutes=5)),
interval_checker(checks.ram_check, datetime.timedelta(minutes=1)),
interval_checker(checks.vuln_check, datetime.timedelta(days=1)),
scheduled_checker(
checks.docker_registry_check, period=datetime.timedelta(days=1), when=datetime.time(hour=0, minute=0)
),
)
async with asyncio.TaskGroup() as tg:
checker_tasks: set[asyncio.Task] = set()