feat(ops): alert on filesystem fill levels from the host worker
Add a pure df parser (disk-usage.ts) with 85/90/95% threshold classification, a diskPressure() alert (Discord/email/alert_logs, severity escalates with fill), and a 5-minute host-side probe in jobs-worker.ts that raises one alert per crossing mount, cooldown-gated per mount+level. Real mounts only: overlay/tmpfs pseudo filesystems are ignored.
This commit is contained in:
1 parent
a0d7f42227
commit
fe26ca3ff3
5 files changed
+333
-2
No files matched your search
@@ -2,6 +2,7 @@ import { env } from "@/env";
|
||||
import { AlertLogs, db } from "@/lib/db";
|
||||
import { logger } from "@/lib/logger";
|
||||
import { sendMail } from "@/lib/services/email";
|
||||
import { diskLevel, formatBytes } from "./disk-usage";
|
||||
|
||||
// === Alert service (AtomCMS → Next.js) ===========================================
|
||||
//
|
||||
@@ -274,3 +275,38 @@ export function ddosDetected(
|
||||
context: { ip, count },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Raise a WARNING/ERROR/CRITICAL alert when a filesystem crosses a fill
|
||||
* threshold (see DISK_THRESHOLDS in disk-usage.ts): 85% warning, 90% error,
|
||||
* 95% critical. Includes the offending mount and a pointer to the Docker prune.
|
||||
*/
|
||||
export function diskPressure(usage: {
|
||||
filesystem: string;
|
||||
mount: string;
|
||||
totalBytes: number;
|
||||
usedBytes: number;
|
||||
availableBytes: number;
|
||||
percent: number;
|
||||
}): Promise<SendAlertResult> {
|
||||
const level = diskLevel(usage.percent);
|
||||
const severity: AlertSeverity =
|
||||
level >= 3 ? "critical" : level === 2 ? "error" : "warning";
|
||||
return sendAlert({
|
||||
type: "disk",
|
||||
severity,
|
||||
message:
|
||||
`Disk ${level >= 3 ? "CRITICAL" : level === 2 ? "degraded" : "high"}: ` +
|
||||
`${usage.mount} is ${usage.percent}% full ` +
|
||||
`(${formatBytes(usage.usedBytes)} of ${formatBytes(usage.totalBytes)} used).`,
|
||||
context: {
|
||||
mount: usage.mount,
|
||||
filesystem: usage.filesystem,
|
||||
percentUsed: usage.percent,
|
||||
used: formatBytes(usage.usedBytes),
|
||||
total: formatBytes(usage.totalBytes),
|
||||
available: formatBytes(usage.availableBytes),
|
||||
hint: "Docker prune runs nightly; run scripts/docker-prune.sh to reclaim cache sooner.",
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user