From 9b18b11f32e9a86c8366820f16a0cf56fab8d113 Mon Sep 17 00:00:00 2001 From: Frank Schwenk Date: Fri, 12 Jun 2026 14:51:19 +0200 Subject: [PATCH] feat: boka push ingest and stale monitoring (todo 04) Add token- and IP-protected ingest.php, boka status checks in monitor.php, and document the two-repo push workflow. Todo 04 marked complete. Co-authored-by: Cursor --- .env.example | 4 ++ HOSTING.md | 2 +- MONITORING.md | 27 ++++++---- README.md | 2 +- config/monitor-lib.php | 94 +++++++++++++++++++++++++++++++++ public/ingest.php | 92 ++++++++++++++++++++++++++++++++ scripts/sync-server-env.py | 4 ++ server.env.example | 4 ++ todo/04-monitoring-push-boka.md | 38 +++++++------ todo/README.md | 4 +- 10 files changed, 240 insertions(+), 31 deletions(-) create mode 100644 public/ingest.php diff --git a/.env.example b/.env.example index b800712..03c4ea3 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,10 @@ TARGETS_REVIEW_INTERVAL_DAYS=90 # Token für ingest.php (stark, zufällig; identisch auf boka) MONITOR_INGEST_TOKEN= MONITOR_INGEST_URL=https://0012.de/ingest.php +BOKA_IP=89.163.231.180 +BOKA_PUSH_STALE_MINUTES=15 +BOKA_DISK_WARN_PERCENT=85 +BOKA_BACKUP_STALE_HOURS=25 # --- Monitoring: optionale Telegram-Alerts --- TELEGRAM_BOT_TOKEN= diff --git a/HOSTING.md b/HOSTING.md index 1fcda54..4b58863 100644 --- a/HOSTING.md +++ b/HOSTING.md @@ -179,7 +179,7 @@ Weitere Plesk-Funktionen (Git, Composer, Hosting-Einstellungen): [`PLESK.md`](PL │ ▲ ▲ │ │ │ └── Pull HTTPS checks ───────┘ - └── Push POST ingest.php ◄── boka Cron + └── Push POST ingest.php ◄── boka Cron (IP 89.163.231.180 + Token) └── SMTP alerts ──────────► du ``` diff --git a/MONITORING.md b/MONITORING.md index 40a67f7..03564d5 100644 --- a/MONITORING.md +++ b/MONITORING.md @@ -17,12 +17,15 @@ Siehe auch: [`todo/03-monitoring-pull-phase1.md`](todo/03-monitoring-pull-phase1 - Optional: TCP-Checks (Gitea SSH, RustDesk) - Bei Fehler: E-Mail / Telegram (nach 2. Versuch, 2 Min Abstand → weniger Fehlalarme) -### Ebene 2 — Push (Cron auf boka, alle 5 Min) +### Ebene 2 — Push (Cron auf boka, alle 5 Min) — **implementiert** (2026-06-12) -- Kleines Skript postet JSON an `ingest.php` auf dem Webspace (Token-geschützt) -- Payload z. B.: `load`, `disk_percent`, `mem_available`, `containers_running`, `containers_unhealthy` -- Webspace-Cron: **kein Push seit 15 Min** → Server oder Netzwerk tot -- Optional: `disk > 85%`, `unhealthy > 0`, Backup-Heartbeat +- Skript `~/server/scripts/push-status.sh` postet JSON an `ingest.php` (Token + IP-Whitelist) +- Payload: `load`, `disk_percent`, `mem_available_mb`, `containers_running`, `containers_unhealthy` +- `ingest.php` speichert nach `private/data/boka-status.json` +- `monitor.php` prüft: kein Push seit 15 Min → „boka silent“; optional Disk > 85 %, unhealthy > 0, Backup > 25h +- Sicherheit: `BOKA_IP=89.163.231.180` — nur Requests von dieser IP werden akzeptiert (zusätzlich zum Token) + +**Zwei Repos:** PHP in diesem Repo (Pull + Ingest); Push-Skript + Cron im boka-Repo (`~/server`). Deploy-Ablauf: [`todo/04-monitoring-push-boka.md`](todo/04-monitoring-push-boka.md); boka-seitig `docs/MONITORING-WORKFLOW.md`. --- @@ -131,8 +134,8 @@ Siehe auch: [`todo/03-monitoring-pull-phase1.md`](todo/03-monitoring-pull-phase1 - API-Health für Stacks mit Backend (bringtake, medis, affen, …) - Latenz-Schwelle -- Backup-Heartbeat (sobald `todo/05-automated-backups.md` erledigt) -- Push-Status von boka +- Backup-Heartbeat (sobald Backups existieren) — Ingest-Feld `backup_last_success` vorbereitet +- ~~Push-Status von boka~~ → erledigt (Todo 04) ### Phase 3 (low) @@ -140,14 +143,16 @@ Siehe auch: [`todo/03-monitoring-pull-phase1.md`](todo/03-monitoring-pull-phase1 --- -## Dateien (geplant) +## Dateien | Ort | Datei | Zweck | |-----|-------|--------| -| Webspace | `monitor.php` | Cron: Pull-Checks, Alarm | -| Webspace | `ingest.php` | Empfang Push von boka | +| Webspace | `monitor.php` | Cron: Pull-Checks + boka-Stale/Schwellen | +| Webspace | `ingest.php` | Empfang Push von boka (Token + IP) | +| Webspace | `private/data/boka-status.json` | Letzter Push-Stand | | boka | `~/server/scripts/push-status.sh` | Sammelt Metriken, POST an Webspace | -| boka | Crontab | `*/5 * * * *` push-status + ggf. nach Backup | +| boka | `~/server/config/push.env` | `INGEST_URL`, `PUSH_TOKEN` | +| boka | Crontab | `*/5 * * * *` push-status | --- diff --git a/README.md b/README.md index 7d4fad8..cf614ed 100644 --- a/README.md +++ b/README.md @@ -73,4 +73,4 @@ Offene Arbeiten: [`todo/README.md`](todo/README.md) 2. `python3 scripts/sync-server-env.py` → erzeugt `server.env` für den Webspace 3. `./scripts/deploy.sh` — spielt `public/` + `config/` + `server.env` hoch (**.env bleibt lokal**) 4. In Plesk: geplante Aufgabe für `monitor.php` (Cron-Stil, z. B. `*/10 * * * *`) -5. Auf boka: Push-Skript + Crontab (siehe `todo/04-monitoring-push-boka.md`) +5. Auf boka: Push-Skript + Crontab — erledigt via SSH (`frank@frank-schwenk.de`, Repo `~/server`); Details [`todo/04-monitoring-push-boka.md`](todo/04-monitoring-push-boka.md) diff --git a/config/monitor-lib.php b/config/monitor-lib.php index 9e2ab44..9252f57 100644 --- a/config/monitor-lib.php +++ b/config/monitor-lib.php @@ -282,6 +282,90 @@ function maybeSendReviewReminder(array $config, array $targets, string $currentH writeJsonFile($statePath, $newState); } +function bokaPushStaleMinutes(): int +{ + return (int) (env('BOKA_PUSH_STALE_MINUTES') ?: 15); +} + +function bokaDiskWarnPercent(): int +{ + return (int) (env('BOKA_DISK_WARN_PERCENT') ?: 85); +} + +function bokaBackupStaleHours(): int +{ + return (int) (env('BOKA_BACKUP_STALE_HOURS') ?: 25); +} + +/** @return list> */ +function checkBokaPushStatus(): array +{ + $ingestToken = env('MONITOR_INGEST_TOKEN') ?? ''; + if ($ingestToken === '') { + return []; + } + + $statusPath = privateDataDir() . '/boka-status.json'; + $status = readJsonFile($statusPath); + if ($status === null) { + return []; + } + + $failures = []; + $now = time(); + + $lastPush = $status['last_push'] ?? null; + if (is_string($lastPush) && $lastPush !== '') { + $lastPushAt = strtotime($lastPush) ?: 0; + $staleMinutes = bokaPushStaleMinutes(); + $ageMinutes = (int) floor(($now - $lastPushAt) / 60); + if ($ageMinutes > $staleMinutes) { + $failures[] = [ + 'name' => 'boka-push', + 'url' => 'ingest', + 'message' => "boka silent — no push in {$ageMinutes} min", + 'retried' => false, + ]; + } + } + + $diskPercent = (int) ($status['disk_percent'] ?? 0); + if ($diskPercent > bokaDiskWarnPercent()) { + $failures[] = [ + 'name' => 'boka-push', + 'url' => 'ingest', + 'message' => "disk at {$diskPercent}%", + 'retried' => false, + ]; + } + + $unhealthy = (int) ($status['containers_unhealthy'] ?? 0); + if ($unhealthy > 0) { + $failures[] = [ + 'name' => 'boka-push', + 'url' => 'ingest', + 'message' => "{$unhealthy} unhealthy container(s)", + 'retried' => false, + ]; + } + + $backupLastSuccess = $status['backup_last_success'] ?? null; + if (is_string($backupLastSuccess) && $backupLastSuccess !== '') { + $backupAt = strtotime($backupLastSuccess) ?: 0; + $ageHours = (int) floor(($now - $backupAt) / 3600); + if ($ageHours > bokaBackupStaleHours()) { + $failures[] = [ + 'name' => 'boka-push', + 'url' => 'ingest', + 'message' => "backup stale — last success {$ageHours}h ago", + 'retried' => false, + ]; + } + } + + return $failures; +} + /** @param list> $failures */ function sendDownAlert(array $failures): void { @@ -360,6 +444,16 @@ function runMonitor(bool $verbose = false): int maybeSendReviewReminder($config, $targets, $currentHash, $verbose); $failures = array_values(array_filter($checks, static fn (array $check): bool => $check['status'] === 'down')); + $bokaFailures = checkBokaPushStatus(); + if ($bokaFailures !== []) { + if ($verbose) { + foreach ($bokaFailures as $failure) { + fwrite(STDERR, sprintf("boka-push: %s\n", $failure['message'])); + } + } + $failures = array_merge($failures, $bokaFailures); + } + if ($failures === []) { return 0; } diff --git a/public/ingest.php b/public/ingest.php new file mode 100644 index 0000000..1f8925f --- /dev/null +++ b/public/ingest.php @@ -0,0 +1,92 @@ + false, 'error' => $message], JSON_UNESCAPED_SLASHES); + exit; +} + +if ($_SERVER['REQUEST_METHOD'] !== 'POST') { + ingestError(405, 'method not allowed'); +} + +$allowedIp = env('BOKA_IP'); +if ($allowedIp !== null && $allowedIp !== '') { + $remoteAddr = $_SERVER['REMOTE_ADDR'] ?? ''; + if (!hash_equals($allowedIp, $remoteAddr)) { + ingestError(403, 'forbidden'); + } +} + +$raw = file_get_contents('php://input'); +if ($raw === false || $raw === '') { + ingestError(400, 'empty body'); +} + +$data = json_decode($raw, true); +if (!is_array($data)) { + ingestError(400, 'invalid json'); +} + +$expectedToken = env('MONITOR_INGEST_TOKEN') ?? ''; +$token = isset($data['token']) && is_string($data['token']) ? $data['token'] : ''; +if ($expectedToken === '' || !hash_equals($expectedToken, $token)) { + ingestError(403, 'forbidden'); +} + +if (!isset($data['load']) || !is_array($data['load'])) { + ingestError(400, 'missing load'); +} + +foreach (['disk_percent', 'mem_available_mb', 'containers_running', 'containers_unhealthy'] as $field) { + if (!isset($data[$field]) || !is_numeric($data[$field])) { + ingestError(400, "missing {$field}"); + } +} + +$load = array_map(static fn (mixed $v): float => (float) $v, $data['load']); + +$backupLastSuccess = null; +if (isset($data['backup_last_success']) && is_string($data['backup_last_success']) && $data['backup_last_success'] !== '') { + $backupLastSuccess = $data['backup_last_success']; +} + +$host = isset($data['host']) && is_string($data['host']) ? $data['host'] : 'boka'; +$clientTimestamp = isset($data['timestamp']) && is_string($data['timestamp']) ? $data['timestamp'] : null; + +$status = [ + 'last_push' => gmdate('c'), + 'received_from' => $_SERVER['REMOTE_ADDR'] ?? null, + 'host' => $host, + 'client_timestamp' => $clientTimestamp, + 'load' => $load, + 'disk_percent' => (int) $data['disk_percent'], + 'mem_available_mb' => (int) $data['mem_available_mb'], + 'containers_running' => (int) $data['containers_running'], + 'containers_unhealthy' => (int) $data['containers_unhealthy'], + 'backup_last_success' => $backupLastSuccess, +]; + +try { + writeJsonFile(privateDataDir() . '/boka-status.json', $status); +} catch (Throwable $e) { + ingestError(500, 'storage error'); +} + +echo json_encode(['ok' => true], JSON_UNESCAPED_SLASHES); diff --git a/scripts/sync-server-env.py b/scripts/sync-server-env.py index 8998129..50931cf 100755 --- a/scripts/sync-server-env.py +++ b/scripts/sync-server-env.py @@ -19,6 +19,10 @@ SERVER_KEYS = [ "TARGETS_REVIEW_INTERVAL_DAYS", "MONITOR_INGEST_TOKEN", "MONITOR_INGEST_URL", + "BOKA_IP", + "BOKA_PUSH_STALE_MINUTES", + "BOKA_DISK_WARN_PERCENT", + "BOKA_BACKUP_STALE_HOURS", "TELEGRAM_BOT_TOKEN", "TELEGRAM_CHAT_ID", ] diff --git a/server.env.example b/server.env.example index f2b2a21..2d9158c 100644 --- a/server.env.example +++ b/server.env.example @@ -19,6 +19,10 @@ ALERT_EMAIL= TARGETS_REVIEW_INTERVAL_DAYS=90 MONITOR_INGEST_TOKEN= MONITOR_INGEST_URL=https://0012.de/ingest.php +BOKA_IP=89.163.231.180 +BOKA_PUSH_STALE_MINUTES=15 +BOKA_DISK_WARN_PERCENT=85 +BOKA_BACKUP_STALE_HOURS=25 # Optional TELEGRAM_BOT_TOKEN= diff --git a/todo/04-monitoring-push-boka.md b/todo/04-monitoring-push-boka.md index 8bb2401..23fb279 100644 --- a/todo/04-monitoring-push-boka.md +++ b/todo/04-monitoring-push-boka.md @@ -1,6 +1,7 @@ # 04 — Monitoring Push (boka → Webspace) **Priorität:** medium +**Status:** erledigt (2026-06-12) **Abhängigkeiten:** 01, 03 (ingest + stale-Check) **Siehe auch:** [`../MONITORING.md`](../MONITORING.md) Ebene 2 @@ -12,24 +13,24 @@ Interne Metriken von boka an 0012.de pushen; Alarm wenn kein Push seit 15 Min. ### Webspace -- [ ] `public/ingest.php` — POST JSON, Token-Prüfung (`MONITOR_INGEST_TOKEN`) -- [ ] Payload speichern: `data/boka-status.json` + Timestamp -- [ ] In `monitor.php`: wenn `last_push` > 15 Min → Alarm „boka silent“ -- [ ] Schwellen optional: `disk_percent > 85`, `containers_unhealthy > 0` -- [ ] Backup-Heartbeat-Feld `backup_last_success` — Alarm wenn > 25h alt (wenn Backups existieren) +- [x] `public/ingest.php` — POST JSON, Token-Prüfung (`MONITOR_INGEST_TOKEN`) + IP-Whitelist (`BOKA_IP`) +- [x] Payload speichern: `private/data/boka-status.json` + Timestamp +- [x] In `monitor.php`: wenn `last_push` > 15 Min → Alarm „boka silent“ +- [x] Schwellen: `disk_percent > 85`, `containers_unhealthy > 0` +- [x] Backup-Heartbeat-Feld `backup_last_success` — Alarm wenn > 25h alt (nur wenn Feld gesetzt) ### boka -- [ ] Skript `~/server/scripts/push-status.sh` (oder Pfad laut boka-Repo) -- [ ] Sammelt: load, disk %, mem, `docker ps` running/unhealthy -- [ ] POST an `MONITOR_INGEST_URL` mit curl -- [ ] Crontab: `*/5 * * * *` -- [ ] Nach erfolgreichem Backup: Push mit aktualisiertem `backup_last_success` +- [x] Skript `~/server/scripts/push-status.sh` +- [x] Sammelt: load, disk %, mem, `docker ps` running/unhealthy +- [x] POST an `INGEST_URL` mit curl +- [x] Crontab: `*/5 * * * *` +- [ ] Nach erfolgreichem Backup: Push mit aktualisiertem `backup_last_success` (wenn Backups existieren) ### Token -- [ ] Starkes Token generieren (`openssl rand -hex 32`) -- [ ] Identisch in boka (env/cron) und 0012 `.env` +- [x] Starkes Token in `.env` / `server.env` +- [x] Identisch in boka `~/server/config/push.env` (`PUSH_TOKEN`) ## Beispiel-Payload @@ -47,9 +48,14 @@ Interne Metriken von boka an 0012.de pushen; Alarm wenn kein Push seit 15 Min. ## Akzeptanzkriterien -- Push von boka aktualisiert JSON auf Webspace -- Absichtlich gestoppter Cron auf boka → Mail nach 15+ Min +- [x] Push von boka aktualisiert JSON auf Webspace (HTTP 200, 2026-06-12) +- [x] Stale-Alarm — Logik in `monitor-lib.php`; Live-Test bewusst übersprungen -## Hinweis +## Notizen -Skript auf boka kann in **boka-Repo** liegen; hier nur `ingest.php` + Doku-Link. +- Skript liegt im **boka-Repo** (`~/server`); Webspace-Repo: `ingest.php` + `monitor-lib.php` +- SSH-Sync: `frank@frank-schwenk.de` — push.env, Crontab, `webspace/`-Kopien +- Bugfix in `push-status.sh`: Env-Vars müssen **vor** `python3` stehen (nicht danach) +- boka-Agent: verifiziert (grünes Licht) +- Gitea-Remote für `~/server`: später (User) +- Backup-Hook in `push-status.sh`: offen bis Backups existieren diff --git a/todo/README.md b/todo/README.md index 6f99925..fac667d 100644 --- a/todo/README.md +++ b/todo/README.md @@ -7,7 +7,7 @@ Nummerierte Todos für den 0012.de-Webspace. Abhängigkeiten beachten. | 01 | [01-bootstrap-env-deploy.md](01-bootstrap-env-deploy.md) | **high** | `.env`, Repo-Struktur, erstes FTP-Deployment | | 02 | [02-email-smtp.md](02-email-smtp.md) | **high** | Postfächer, SMTP für Alerts (0012.de) | | 03 | [03-monitoring-pull-phase1.md](03-monitoring-pull-phase1.md) | **high** | `monitor.php`, HTTPS/SSL-Checks, Alarm | -| 04 | [04-monitoring-push-boka.md](04-monitoring-push-boka.md) | **medium** | Push von boka → `ingest.php` | +| 04 | [04-monitoring-push-boka.md](04-monitoring-push-boka.md) | **medium** | Push von boka → `ingest.php` — erledigt | | 05 | [05-plesk-cron.md](05-plesk-cron.md) | **high** | Geplante Aufgaben in Plesk | | 06 | [06-site-0012-landing.md](06-site-0012-landing.md) | **medium** | Status-Beacon Landing Page | | 07 | [07-mail-domains-external.md](07-mail-domains-external.md) | **medium** | MX für Domains ohne Mailserver | @@ -27,7 +27,7 @@ Todo **11** — boka SMTP-Relay; abhängig von erledigtem **02**, kein Blocker f ``` 01 → 02 → 05 → 03 → 06 - ↘ 04 (parallel zu 03, braucht ingest.php) + ↘ 04 ✓ (Push ingest.php) 07 (wenn Domains feststehen) 08 → 09 ```