feat(ops): alert on filesystem fill levels from the host worker

Add a pure df parser (disk-usage.ts) with 85/90/95% threshold classification, a diskPressure() alert (Discord/email/alert_logs, severity escalates with fill), and a 5-minute host-side probe in jobs-worker.ts that raises one alert per crossing mount, cooldown-gated per mount+level. Real mounts only: overlay/tmpfs pseudo filesystems are ignored.

scripts/jobs-worker.ts

@@ -11,8 +11,13 @@ import {
 } from "../src/lib/db";
 import { logger } from "../src/lib/logger";
 import { redis } from "../src/lib/redis";
-import { emulatorOffline, healthDegraded } from "../src/lib/services/alert";
+import {
+	diskPressure,
+	emulatorOffline,
+	healthDegraded,
+} from "../src/lib/services/alert";
 import { runCatalogExport } from "../src/lib/services/catalog-git-export";
+import { diskLevel, parseDfOutput } from "../src/lib/services/disk-usage";
 import { invalidateNewsCache } from "../src/lib/services/news-cache";
 import { rcon } from "../src/lib/services/rcon";
 
@@ -88,6 +93,49 @@ async function checkOpsHealth(): Promise<void> {
 	}
 }
 
+/**
+ * Host-side: probe filesystem fill levels via `df -P -B1` and raise a
+ * diskPressure() alert per mount once it crosses 85/90/95% (cooldown-gated per
+ * mount+level so a stuck fill level does not spam Discord/email).
+ */
+async function checkDiskUsage(): Promise<void> {
+	const { execFile } = await import("node:child_process");
+
+	let dfOut: string;
+	try {
+		dfOut = await new Promise<string>((resolve, reject) => {
+			execFile(
+				"df",
+				["-P", "-B1"],
+				{
+					maxBuffer: 4 * 1024 * 1024,
+					timeout: 15_000,
+				},
+				(err, stdout) => (err ? reject(err) : resolve(stdout)),
+			);
+		});
+	} catch (err) {
+		captureWorkerError(err, "Disk probe failed (is df available?)");
+		return;
+	}
+
+	const mounts = parseDfOutput(dfOut);
+	if (mounts.length === 0) return;
+
+	for (const usage of mounts) {
+		const level = diskLevel(usage.percent);
+		if (level === 0) continue;
+		if (canAlert(`disk-${usage.mount}-${level}`)) {
+			logger.warn("Raising disk usage alert", {
+				module: "jobs",
+				mount: usage.mount,
+				percent: usage.percent,
+			});
+			await diskPressure(usage);
+		}
+	}
+}
+
 async function backupEmulatorJar(): Promise<void> {
 	if (!env.EMULATOR_JAR_PATH || !env.EMULATOR_BACKUP_DIR) return;
 
@@ -351,6 +399,11 @@ async function main() {
 	});
 	logger.info("Scheduled: ops health probe (every 5 min)", { module: "jobs" });
 
+	new Cron("*/5 * * * *", () => {
+		checkDiskUsage().catch((e) => captureWorkerError(e, "Disk check error"));
+	});
+	logger.info("Scheduled: disk usage probe (every 5 min)", { module: "jobs" });
+
 	new Cron("* * * * *", () => {
 		publishScheduledArticles().catch((e) =>
 			captureWorkerError(e, "ScheduledArticlePublish"),
@@ -369,6 +422,7 @@ async function main() {
 		cleanupOldLogs(),
 		cleanupOldSessions(),
 		checkOpsHealth(),
+		checkDiskUsage(),
 	]);
 }
 

src/lib/services/alert.test.ts

@@ -24,7 +24,7 @@ vi.mock("@/env", () => ({
 }));
 
 import { sendMail } from "@/lib/services/email";
-import { sendAlert } from "./alert";
+import { diskPressure, sendAlert } from "./alert";
 
 beforeEach(() => {
 	insertValues.mockReset();
@@ -64,3 +64,45 @@ describe("sendAlert", () => {
 		expect(sendMail).not.toHaveBeenCalled();
 	});
 });
+
+describe("diskPressure", () => {
+	const usage = (percent: number) => ({
+		filesystem: "/dev/vda1",
+		mount: "/",
+		totalBytes: 100000000000,
+		usedBytes: (percent / 100) * 100000000000,
+		availableBytes: 100000000000 - (percent / 100) * 100000000000,
+		percent,
+	});
+
+	it("raises a warning at 87% fill", async () => {
+		await diskPressure(usage(87));
+		expect(insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "disk",
+				severity: "warning",
+			}),
+		);
+	});
+
+	it("raises an error at 92% fill", async () => {
+		await diskPressure(usage(92));
+		expect(insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "disk",
+				severity: "error",
+			}),
+		);
+	});
+
+	it("raises a critical alert at 96% fill", async () => {
+		await diskPressure(usage(96));
+		expect(insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "disk",
+				severity: "critical",
+				context: expect.stringContaining('"mount":"/"'),
+			}),
+		);
+	});
+});

src/lib/services/alert.ts

@@ -2,6 +2,7 @@ import { env } from "@/env";
 import { AlertLogs, db } from "@/lib/db";
 import { logger } from "@/lib/logger";
 import { sendMail } from "@/lib/services/email";
+import { diskLevel, formatBytes } from "./disk-usage";
 
 // === Alert service (AtomCMS → Next.js) ===========================================
 //
@@ -274,3 +275,38 @@ export function ddosDetected(
 		context: { ip, count },
 	});
 }
+
+/**
+ * Raise a WARNING/ERROR/CRITICAL alert when a filesystem crosses a fill
+ * threshold (see DISK_THRESHOLDS in disk-usage.ts): 85% warning, 90% error,
+ * 95% critical. Includes the offending mount and a pointer to the Docker prune.
+ */
+export function diskPressure(usage: {
+	filesystem: string;
+	mount: string;
+	totalBytes: number;
+	usedBytes: number;
+	availableBytes: number;
+	percent: number;
+}): Promise<SendAlertResult> {
+	const level = diskLevel(usage.percent);
+	const severity: AlertSeverity =
+		level >= 3 ? "critical" : level === 2 ? "error" : "warning";
+	return sendAlert({
+		type: "disk",
+		severity,
+		message:
+			`Disk ${level >= 3 ? "CRITICAL" : level === 2 ? "degraded" : "high"}: ` +
+			`${usage.mount} is ${usage.percent}% full ` +
+			`(${formatBytes(usage.usedBytes)} of ${formatBytes(usage.totalBytes)} used).`,
+		context: {
+			mount: usage.mount,
+			filesystem: usage.filesystem,
+			percentUsed: usage.percent,
+			used: formatBytes(usage.usedBytes),
+			total: formatBytes(usage.totalBytes),
+			available: formatBytes(usage.availableBytes),
+			hint: "Docker prune runs nightly; run scripts/docker-prune.sh to reclaim cache sooner.",
+		},
+	});
+}

src/lib/services/disk-usage.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, it } from "vitest";
+import {
+	DISK_THRESHOLDS,
+	diskLevel,
+	formatBytes,
+	parseDfOutput,
+} from "./disk-usage";
+
+const SAMPLE = `Filesystem      1-blocks        Used   Available Capacity Mounted on
+/dev/vda1    10240000000   5120000000    5120000000      50% /
+/dev/vda2    20480000000  19456000000    1024000000      95% /var/www
+overlay      10240000000   5120000000    5120000000      50% /home/me
+tmpfs        10240000000   5120000000    5120000000      50% /dev/shm
+/dev/vdb1    51200000000  43929600000    7270400000      86% /var/lib/docker
+/dev/vdc1    10240000000   5120000000    5120000000      50% /mnt/with\\040space`;
+
+describe("parseDfOutput", () => {
+	it("parses real mounts and percent from the Capacity column", () => {
+		const mounts = parseDfOutput(SAMPLE);
+		expect(mounts).toHaveLength(4);
+		expect(mounts[0]).toEqual({
+			filesystem: "/dev/vda1",
+			mount: "/",
+			totalBytes: 10240000000,
+			usedBytes: 5120000000,
+			availableBytes: 5120000000,
+			percent: 50,
+		});
+		expect(mounts[1].mount).toBe("/var/www");
+		expect(mounts[1].percent).toBe(95);
+		expect(mounts[2].mount).toBe("/var/lib/docker");
+		expect(mounts[2].percent).toBe(86);
+	});
+
+	it("skips overlay and tmpfs pseudo filesystems", () => {
+		const mounts = parseDfOutput(SAMPLE);
+		expect(mounts.some((m) => m.filesystem === "overlay")).toBe(false);
+		expect(mounts.some((m) => m.filesystem === "tmpfs")).toBe(false);
+	});
+
+	it("unescapes \\040 mount points", () => {
+		const mounts = parseDfOutput(SAMPLE);
+		expect(mounts.some((m) => m.mount === "/mnt/with space")).toBe(true);
+	});
+
+	it("returns empty on header-only or empty input", () => {
+		expect(
+			parseDfOutput("Filesystem 1-blocks Used Available Capacity Mounted on\n"),
+		).toHaveLength(0);
+		expect(parseDfOutput("")).toHaveLength(0);
+	});
+
+	it("derives percent from bytes when the Capacity column is missing", () => {
+		const out = parseDfOutput("/dev/xd1 100000 75000 25000 - /data");
+		expect(out).toHaveLength(1);
+		expect(out[0].percent).toBe(75);
+	});
+});
+
+describe("diskLevel", () => {
+	it("mapps fill levels to the shared thresholds", () => {
+		expect(diskLevel(0)).toBe(0);
+		expect(diskLevel(50)).toBe(0);
+		expect(diskLevel(DISK_THRESHOLDS.warning - 1)).toBe(0);
+		expect(diskLevel(DISK_THRESHOLDS.warning)).toBe(1);
+		expect(diskLevel(DISK_THRESHOLDS.error - 1)).toBe(1);
+		expect(diskLevel(DISK_THRESHOLDS.error)).toBe(2);
+		expect(diskLevel(DISK_THRESHOLDS.critical - 1)).toBe(2);
+		expect(diskLevel(DISK_THRESHOLDS.critical)).toBe(3);
+		expect(diskLevel(99)).toBe(3);
+		expect(diskLevel(100)).toBe(3);
+	});
+
+	it("clamps out-of-range input", () => {
+		expect(diskLevel(-1)).toBe(0);
+		expect(diskLevel(101)).toBe(3);
+	});
+});
+
+describe("formatBytes", () => {
+	it("formats byte values with binary units", () => {
+		expect(formatBytes(0)).toBe("0 B");
+		expect(formatBytes(512)).toBe("512 B");
+		expect(formatBytes(1536)).toBe("1.5 KiB");
+		expect(formatBytes(2048)).toBe("2 KiB");
+		expect(formatBytes(1073741824)).toBe("1 GiB");
+		expect(formatBytes(-5)).toBe("0 B");
+	});
+});