feat(ops): alert on filesystem fill levels from the host worker
CI / check (push) Successful in 2m30s
CI / deploy (push) Successful in 1m25s
CI / publish-container (push) Successful in 1m5s

Add a pure df parser (disk-usage.ts) with 85/90/95% threshold classification, a diskPressure() alert (Discord/email/alert_logs, severity escalates with fill), and a 5-minute host-side probe in jobs-worker.ts that raises one alert per crossing mount, cooldown-gated per mount+level. Real mounts only: overlay/tmpfs pseudo filesystems are ignored.
This commit is contained in:
openhands committed 2026-09-13 13:12:37 +02:00
1 parent a0d7f42227
commit fe26ca3ff3
5 files changed
+333 -2

No files matched your search

+36
View File
@@ -2,6 +2,7 @@ import { env } from "@/env";
import { AlertLogs, db } from "@/lib/db";
import { logger } from "@/lib/logger";
import { sendMail } from "@/lib/services/email";
import { diskLevel, formatBytes } from "./disk-usage";
// === Alert service (AtomCMS → Next.js) ===========================================
//
@@ -274,3 +275,38 @@ export function ddosDetected(
context: { ip, count },
});
}
/**
* Raise a WARNING/ERROR/CRITICAL alert when a filesystem crosses a fill
* threshold (see DISK_THRESHOLDS in disk-usage.ts): 85% warning, 90% error,
* 95% critical. Includes the offending mount and a pointer to the Docker prune.
*/
export function diskPressure(usage: {
filesystem: string;
mount: string;
totalBytes: number;
usedBytes: number;
availableBytes: number;
percent: number;
}): Promise<SendAlertResult> {
const level = diskLevel(usage.percent);
const severity: AlertSeverity =
level >= 3 ? "critical" : level === 2 ? "error" : "warning";
return sendAlert({
type: "disk",
severity,
message:
`Disk ${level >= 3 ? "CRITICAL" : level === 2 ? "degraded" : "high"}: ` +
`${usage.mount} is ${usage.percent}% full ` +
`(${formatBytes(usage.usedBytes)} of ${formatBytes(usage.totalBytes)} used).`,
context: {
mount: usage.mount,
filesystem: usage.filesystem,
percentUsed: usage.percent,
used: formatBytes(usage.usedBytes),
total: formatBytes(usage.totalBytes),
available: formatBytes(usage.availableBytes),
hint: "Docker prune runs nightly; run scripts/docker-prune.sh to reclaim cache sooner.",
},
});
}