Frank Schwenk
9b18b11f32
Add token- and IP-protected ingest.php, boka status checks in monitor.php, and document the two-repo push workflow. Todo 04 marked complete. Co-authored-by: Cursor <cursoragent@cursor.com>
473 lines
13 KiB
PHP
473 lines
13 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
require_once __DIR__ . '/bootstrap.php';
|
|
require_once __DIR__ . '/mail.php';
|
|
|
|
function loadTargetsConfig(): array
|
|
{
|
|
return require __DIR__ . '/targets.php';
|
|
}
|
|
|
|
/** @return list<array<string, mixed>> */
|
|
function getTargetList(array $config): array
|
|
{
|
|
return $config['targets'] ?? $config;
|
|
}
|
|
|
|
/** @param list<array<string, mixed>> $targets */
|
|
function targetsHash(array $targets): string
|
|
{
|
|
$normalized = array_map(static function (array $target): array {
|
|
return [
|
|
'name' => $target['name'],
|
|
'url' => $target['url'],
|
|
'expect' => $target['expect'] ?? 200,
|
|
];
|
|
}, $targets);
|
|
|
|
return hash('sha256', json_encode($normalized, JSON_UNESCAPED_SLASHES) ?: '');
|
|
}
|
|
|
|
/** @return list<int> */
|
|
function normalizeExpect(mixed $expect): array
|
|
{
|
|
if (is_array($expect)) {
|
|
return array_map(intval(...), $expect);
|
|
}
|
|
|
|
return [(int) $expect];
|
|
}
|
|
|
|
/** @param array<string, mixed> $target */
|
|
function checkTarget(array $target): array
|
|
{
|
|
$timeout = (int) ($target['timeout'] ?? 10);
|
|
$expect = normalizeExpect($target['expect'] ?? 200);
|
|
|
|
$result = checkHttp((string) $target['url'], $timeout, $expect);
|
|
$sslDays = checkSslDays((string) $target['url'], $timeout);
|
|
|
|
$result['name'] = (string) $target['name'];
|
|
$result['url'] = (string) $target['url'];
|
|
$result['ssl_days_remaining'] = $sslDays;
|
|
$result['checked_at'] = gmdate('c');
|
|
$result['retried'] = false;
|
|
|
|
if ($sslDays !== null && $sslDays < 14 && $result['status'] === 'ok') {
|
|
$result['status'] = 'warn';
|
|
$result['message'] = "SSL expires in {$sslDays} days";
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
/** @param list<int> $expect */
|
|
function checkHttp(string $url, int $timeout, array $expect): array
|
|
{
|
|
if (!function_exists('curl_init')) {
|
|
return httpResult('down', 0, 0, 'cURL not available');
|
|
}
|
|
|
|
$start = hrtime(true);
|
|
$handle = curl_init($url);
|
|
|
|
curl_setopt_array($handle, [
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
CURLOPT_FOLLOWLOCATION => true,
|
|
CURLOPT_MAXREDIRS => 5,
|
|
CURLOPT_TIMEOUT => $timeout,
|
|
CURLOPT_CONNECTTIMEOUT => $timeout,
|
|
CURLOPT_SSL_VERIFYPEER => true,
|
|
CURLOPT_SSL_VERIFYHOST => 2,
|
|
CURLOPT_USERAGENT => '0012-monitor/1.0',
|
|
]);
|
|
|
|
curl_exec($handle);
|
|
$httpCode = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE);
|
|
$curlError = curl_error($handle);
|
|
curl_close($handle);
|
|
|
|
$responseMs = (int) round((hrtime(true) - $start) / 1_000_000);
|
|
|
|
if ($curlError !== '') {
|
|
return httpResult('down', $httpCode, $responseMs, sanitizeError($curlError));
|
|
}
|
|
|
|
if (in_array($httpCode, $expect, true)) {
|
|
return httpResult('ok', $httpCode, $responseMs, null);
|
|
}
|
|
|
|
return httpResult('down', $httpCode, $responseMs, "HTTP {$httpCode}");
|
|
}
|
|
|
|
function httpResult(string $status, int $httpCode, int $responseMs, ?string $message): array
|
|
{
|
|
return [
|
|
'status' => $status,
|
|
'http_code' => $httpCode,
|
|
'response_ms' => $responseMs,
|
|
'message' => $message,
|
|
];
|
|
}
|
|
|
|
function checkSslDays(string $url, int $timeout): ?int
|
|
{
|
|
$host = parse_url($url, PHP_URL_HOST);
|
|
if (!is_string($host) || $host === '') {
|
|
return null;
|
|
}
|
|
|
|
$context = stream_context_create([
|
|
'ssl' => [
|
|
'capture_peer_cert' => true,
|
|
'verify_peer' => true,
|
|
'verify_peer_name' => true,
|
|
],
|
|
]);
|
|
|
|
$socket = @stream_socket_client(
|
|
"ssl://{$host}:443",
|
|
$errno,
|
|
$errstr,
|
|
$timeout,
|
|
STREAM_CLIENT_CONNECT,
|
|
$context
|
|
);
|
|
|
|
if ($socket === false) {
|
|
return null;
|
|
}
|
|
|
|
$params = stream_context_get_params($socket);
|
|
fclose($socket);
|
|
|
|
$certificate = $params['options']['ssl']['peer_certificate'] ?? null;
|
|
if ($certificate === false || $certificate === null) {
|
|
return null;
|
|
}
|
|
|
|
$info = openssl_x509_parse($certificate);
|
|
if (!is_array($info) || !isset($info['validTo_time_t'])) {
|
|
return null;
|
|
}
|
|
|
|
return (int) floor(((int) $info['validTo_time_t'] - time()) / 86400);
|
|
}
|
|
|
|
function sanitizeError(string $error): string
|
|
{
|
|
if (strlen($error) > 120) {
|
|
return substr($error, 0, 117) . '...';
|
|
}
|
|
|
|
return $error;
|
|
}
|
|
|
|
function writeJsonFile(string $path, array $data): void
|
|
{
|
|
$dir = dirname($path);
|
|
if (!is_dir($dir) && !mkdir($dir, 0750, true) && !is_dir($dir)) {
|
|
throw new RuntimeException("Failed to create directory {$dir}");
|
|
}
|
|
|
|
$json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
|
|
if ($json === false) {
|
|
throw new RuntimeException('Failed to encode JSON');
|
|
}
|
|
|
|
$tmp = $path . '.tmp';
|
|
if (file_put_contents($tmp, $json . "\n") === false) {
|
|
throw new RuntimeException("Failed to write {$tmp}");
|
|
}
|
|
|
|
if (!rename($tmp, $path)) {
|
|
throw new RuntimeException("Failed to rename {$tmp} to {$path}");
|
|
}
|
|
}
|
|
|
|
function readJsonFile(string $path): ?array
|
|
{
|
|
if (!is_readable($path)) {
|
|
return null;
|
|
}
|
|
|
|
$data = json_decode((string) file_get_contents($path), true);
|
|
|
|
return is_array($data) ? $data : null;
|
|
}
|
|
|
|
function effectiveLastReviewedAt(array $config, string $currentHash, ?array $reviewState): string
|
|
{
|
|
if (
|
|
$reviewState !== null
|
|
&& isset($reviewState['targets_hash'])
|
|
&& is_string($reviewState['targets_hash'])
|
|
&& $reviewState['targets_hash'] !== ''
|
|
&& $reviewState['targets_hash'] !== $currentHash
|
|
) {
|
|
return gmdate('Y-m-d');
|
|
}
|
|
|
|
$metaDate = $config['_meta']['last_reviewed_at'] ?? null;
|
|
if (is_string($metaDate) && $metaDate !== '') {
|
|
return $metaDate;
|
|
}
|
|
|
|
return gmdate('Y-m-d');
|
|
}
|
|
|
|
/** @param list<array<string, mixed>> $targets */
|
|
function maybeSendReviewReminder(array $config, array $targets, string $currentHash, bool $verbose): void
|
|
{
|
|
$intervalDays = (int) (env('TARGETS_REVIEW_INTERVAL_DAYS') ?: 90);
|
|
$statePath = privateDataDir() . '/review-state.json';
|
|
$state = readJsonFile($statePath) ?? [];
|
|
|
|
$lastReviewed = effectiveLastReviewedAt($config, $currentHash, $state);
|
|
$reviewedAt = strtotime($lastReviewed . ' UTC') ?: time();
|
|
$daysSinceReview = (int) floor((time() - $reviewedAt) / 86400);
|
|
|
|
$newState = [
|
|
'targets_hash' => $currentHash,
|
|
'last_reminder_at' => $state['last_reminder_at'] ?? null,
|
|
];
|
|
|
|
if ($daysSinceReview < $intervalDays) {
|
|
writeJsonFile($statePath, $newState);
|
|
|
|
return;
|
|
}
|
|
|
|
$lastReminder = $state['last_reminder_at'] ?? null;
|
|
if (is_string($lastReminder) && $lastReminder !== '') {
|
|
$reminderAt = strtotime($lastReminder) ?: 0;
|
|
$daysSinceReminder = (int) floor((time() - $reminderAt) / 86400);
|
|
if ($daysSinceReminder < 7) {
|
|
writeJsonFile($statePath, $newState);
|
|
|
|
return;
|
|
}
|
|
}
|
|
|
|
$domain = env('DOMAIN', '0012.de');
|
|
$names = array_map(static fn (array $target): string => (string) $target['name'], $targets);
|
|
|
|
$subject = "[{$domain}] Monitoring — Zeit für eine Target-Review?";
|
|
$body = implode("\n", [
|
|
"Deine Monitoring-Liste wurde zuletzt am {$lastReviewed} geprüft ({$daysSinceReview} Tage her).",
|
|
'Aktuell überwacht: ' . count($targets) . ' Targets (' . implode(', ', $names) . ').',
|
|
'',
|
|
'Vorschläge:',
|
|
'- Neue Stacks/Domains aus MONITORING.md (medium) nachziehen?',
|
|
'- Toten Eintrag entfernen?',
|
|
'- expect-Codes prüfen (401 bei Basic Auth)?',
|
|
'',
|
|
'Nach der Prüfung: last_reviewed_at in config/targets.php auf heute setzen und deployen.',
|
|
]);
|
|
|
|
try {
|
|
sendAlert($subject, $body);
|
|
$newState['last_reminder_at'] = gmdate('c');
|
|
if ($verbose) {
|
|
fwrite(STDERR, "Review reminder sent ({$daysSinceReview} days since last review)\n");
|
|
}
|
|
} catch (Throwable $e) {
|
|
if ($verbose) {
|
|
fwrite(STDERR, 'Review reminder failed: ' . $e->getMessage() . "\n");
|
|
}
|
|
}
|
|
|
|
writeJsonFile($statePath, $newState);
|
|
}
|
|
|
|
function bokaPushStaleMinutes(): int
|
|
{
|
|
return (int) (env('BOKA_PUSH_STALE_MINUTES') ?: 15);
|
|
}
|
|
|
|
function bokaDiskWarnPercent(): int
|
|
{
|
|
return (int) (env('BOKA_DISK_WARN_PERCENT') ?: 85);
|
|
}
|
|
|
|
function bokaBackupStaleHours(): int
|
|
{
|
|
return (int) (env('BOKA_BACKUP_STALE_HOURS') ?: 25);
|
|
}
|
|
|
|
/** @return list<array<string, mixed>> */
|
|
function checkBokaPushStatus(): array
|
|
{
|
|
$ingestToken = env('MONITOR_INGEST_TOKEN') ?? '';
|
|
if ($ingestToken === '') {
|
|
return [];
|
|
}
|
|
|
|
$statusPath = privateDataDir() . '/boka-status.json';
|
|
$status = readJsonFile($statusPath);
|
|
if ($status === null) {
|
|
return [];
|
|
}
|
|
|
|
$failures = [];
|
|
$now = time();
|
|
|
|
$lastPush = $status['last_push'] ?? null;
|
|
if (is_string($lastPush) && $lastPush !== '') {
|
|
$lastPushAt = strtotime($lastPush) ?: 0;
|
|
$staleMinutes = bokaPushStaleMinutes();
|
|
$ageMinutes = (int) floor(($now - $lastPushAt) / 60);
|
|
if ($ageMinutes > $staleMinutes) {
|
|
$failures[] = [
|
|
'name' => 'boka-push',
|
|
'url' => 'ingest',
|
|
'message' => "boka silent — no push in {$ageMinutes} min",
|
|
'retried' => false,
|
|
];
|
|
}
|
|
}
|
|
|
|
$diskPercent = (int) ($status['disk_percent'] ?? 0);
|
|
if ($diskPercent > bokaDiskWarnPercent()) {
|
|
$failures[] = [
|
|
'name' => 'boka-push',
|
|
'url' => 'ingest',
|
|
'message' => "disk at {$diskPercent}%",
|
|
'retried' => false,
|
|
];
|
|
}
|
|
|
|
$unhealthy = (int) ($status['containers_unhealthy'] ?? 0);
|
|
if ($unhealthy > 0) {
|
|
$failures[] = [
|
|
'name' => 'boka-push',
|
|
'url' => 'ingest',
|
|
'message' => "{$unhealthy} unhealthy container(s)",
|
|
'retried' => false,
|
|
];
|
|
}
|
|
|
|
$backupLastSuccess = $status['backup_last_success'] ?? null;
|
|
if (is_string($backupLastSuccess) && $backupLastSuccess !== '') {
|
|
$backupAt = strtotime($backupLastSuccess) ?: 0;
|
|
$ageHours = (int) floor(($now - $backupAt) / 3600);
|
|
if ($ageHours > bokaBackupStaleHours()) {
|
|
$failures[] = [
|
|
'name' => 'boka-push',
|
|
'url' => 'ingest',
|
|
'message' => "backup stale — last success {$ageHours}h ago",
|
|
'retried' => false,
|
|
];
|
|
}
|
|
}
|
|
|
|
return $failures;
|
|
}
|
|
|
|
/** @param list<array<string, mixed>> $failures */
|
|
function sendDownAlert(array $failures): void
|
|
{
|
|
$domain = env('DOMAIN', '0012.de');
|
|
$count = count($failures);
|
|
$subject = "[{$domain}] Monitoring alert — {$count} target(s) down";
|
|
|
|
$lines = [];
|
|
foreach ($failures as $failure) {
|
|
$message = (string) ($failure['message'] ?? 'unknown error');
|
|
$retried = !empty($failure['retried']) ? ' (retried)' : '';
|
|
$lines[] = "- {$failure['name']} ({$failure['url']}): {$message}{$retried}";
|
|
}
|
|
|
|
sendAlert($subject, implode("\n", $lines));
|
|
}
|
|
|
|
function runMonitor(bool $verbose = false): int
|
|
{
|
|
$started = hrtime(true);
|
|
$config = loadTargetsConfig();
|
|
$targets = getTargetList($config);
|
|
$currentHash = targetsHash($targets);
|
|
|
|
$results = [];
|
|
$failedIndexes = [];
|
|
|
|
foreach ($targets as $index => $target) {
|
|
$result = checkTarget($target);
|
|
$results[$index] = $result;
|
|
|
|
if ($result['status'] === 'down') {
|
|
$failedIndexes[] = $index;
|
|
}
|
|
|
|
if ($verbose) {
|
|
fwrite(STDERR, sprintf("%s: %s\n", $result['name'], $result['status']));
|
|
}
|
|
}
|
|
|
|
if ($failedIndexes !== []) {
|
|
if ($verbose) {
|
|
fwrite(STDERR, "Waiting 120s before retry...\n");
|
|
}
|
|
|
|
sleep(120);
|
|
|
|
foreach ($failedIndexes as $index) {
|
|
$result = checkTarget($targets[$index]);
|
|
$result['retried'] = true;
|
|
$results[$index] = $result;
|
|
|
|
if ($verbose) {
|
|
fwrite(STDERR, sprintf("%s (retry): %s\n", $result['name'], $result['status']));
|
|
}
|
|
}
|
|
}
|
|
|
|
$checks = array_values($results);
|
|
$payload = [
|
|
'updated_at' => gmdate('c'),
|
|
'duration_ms' => (int) round((hrtime(true) - $started) / 1_000_000),
|
|
'checks' => $checks,
|
|
];
|
|
|
|
try {
|
|
writeJsonFile(privateDataDir() . '/last-checks.json', $payload);
|
|
} catch (Throwable $e) {
|
|
if ($verbose) {
|
|
fwrite(STDERR, 'JSON write failed: ' . $e->getMessage() . "\n");
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
maybeSendReviewReminder($config, $targets, $currentHash, $verbose);
|
|
|
|
$failures = array_values(array_filter($checks, static fn (array $check): bool => $check['status'] === 'down'));
|
|
$bokaFailures = checkBokaPushStatus();
|
|
if ($bokaFailures !== []) {
|
|
if ($verbose) {
|
|
foreach ($bokaFailures as $failure) {
|
|
fwrite(STDERR, sprintf("boka-push: %s\n", $failure['message']));
|
|
}
|
|
}
|
|
$failures = array_merge($failures, $bokaFailures);
|
|
}
|
|
|
|
if ($failures === []) {
|
|
return 0;
|
|
}
|
|
|
|
try {
|
|
sendDownAlert($failures);
|
|
} catch (Throwable $e) {
|
|
if ($verbose) {
|
|
fwrite(STDERR, 'Down alert failed: ' . $e->getMessage() . "\n");
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
return 1;
|
|
}
|