feat(ops): alert on filesystem fill levels from the host worker

Add a pure df parser (disk-usage.ts) with 85/90/95% threshold classification, a diskPressure() alert (Discord/email/alert_logs, severity escalates with fill), and a 5-minute host-side probe in jobs-worker.ts that raises one alert per crossing mount, cooldown-gated per mount+level. Real mounts only: overlay/tmpfs pseudo filesystems are ignored.

scripts/jobs-worker.ts

@@ -11,8 +11,13 @@ import {
 } from "../src/lib/db";
 import { logger } from "../src/lib/logger";
 import { redis } from "../src/lib/redis";
-import { emulatorOffline, healthDegraded } from "../src/lib/services/alert";
+import {
+	diskPressure,
+	emulatorOffline,
+	healthDegraded,
+} from "../src/lib/services/alert";
 import { runCatalogExport } from "../src/lib/services/catalog-git-export";
+import { diskLevel, parseDfOutput } from "../src/lib/services/disk-usage";
 import { invalidateNewsCache } from "../src/lib/services/news-cache";
 import { rcon } from "../src/lib/services/rcon";
 
@@ -88,6 +93,49 @@ async function checkOpsHealth(): Promise<void> {
 	}
 }
 
+/**
+ * Host-side: probe filesystem fill levels via `df -P -B1` and raise a
+ * diskPressure() alert per mount once it crosses 85/90/95% (cooldown-gated per
+ * mount+level so a stuck fill level does not spam Discord/email).
+ */
+async function checkDiskUsage(): Promise<void> {
+	const { execFile } = await import("node:child_process");
+
+	let dfOut: string;
+	try {
+		dfOut = await new Promise<string>((resolve, reject) => {
+			execFile(
+				"df",
+				["-P", "-B1"],
+				{
+					maxBuffer: 4 * 1024 * 1024,
+					timeout: 15_000,
+				},
+				(err, stdout) => (err ? reject(err) : resolve(stdout)),
+			);
+		});
+	} catch (err) {
+		captureWorkerError(err, "Disk probe failed (is df available?)");
+		return;
+	}
+
+	const mounts = parseDfOutput(dfOut);
+	if (mounts.length === 0) return;
+
+	for (const usage of mounts) {
+		const level = diskLevel(usage.percent);
+		if (level === 0) continue;
+		if (canAlert(`disk-${usage.mount}-${level}`)) {
+			logger.warn("Raising disk usage alert", {
+				module: "jobs",
+				mount: usage.mount,
+				percent: usage.percent,
+			});
+			await diskPressure(usage);
+		}
+	}
+}
+
 async function backupEmulatorJar(): Promise<void> {
 	if (!env.EMULATOR_JAR_PATH || !env.EMULATOR_BACKUP_DIR) return;
 
@@ -351,6 +399,11 @@ async function main() {
 	});
 	logger.info("Scheduled: ops health probe (every 5 min)", { module: "jobs" });
 
+	new Cron("*/5 * * * *", () => {
+		checkDiskUsage().catch((e) => captureWorkerError(e, "Disk check error"));
+	});
+	logger.info("Scheduled: disk usage probe (every 5 min)", { module: "jobs" });
+
 	new Cron("* * * * *", () => {
 		publishScheduledArticles().catch((e) =>
 			captureWorkerError(e, "ScheduledArticlePublish"),
@@ -369,6 +422,7 @@ async function main() {
 		cleanupOldLogs(),
 		cleanupOldSessions(),
 		checkOpsHealth(),
+		checkDiskUsage(),
 	]);
 }
 

src/lib/services/alert.test.ts

@@ -24,7 +24,7 @@ vi.mock("@/env", () => ({
 }));
 
 import { sendMail } from "@/lib/services/email";
-import { sendAlert } from "./alert";
+import { diskPressure, sendAlert } from "./alert";
 
 beforeEach(() => {
 	insertValues.mockReset();
@@ -64,3 +64,45 @@ describe("sendAlert", () => {
 		expect(sendMail).not.toHaveBeenCalled();
 	});
 });
+
+describe("diskPressure", () => {
+	const usage = (percent: number) => ({
+		filesystem: "/dev/vda1",
+		mount: "/",
+		totalBytes: 100000000000,
+		usedBytes: (percent / 100) * 100000000000,
+		availableBytes: 100000000000 - (percent / 100) * 100000000000,
+		percent,
+	});
+
+	it("raises a warning at 87% fill", async () => {
+		await diskPressure(usage(87));
+		expect(insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "disk",
+				severity: "warning",
+			}),
+		);
+	});
+
+	it("raises an error at 92% fill", async () => {
+		await diskPressure(usage(92));
+		expect(insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "disk",
+				severity: "error",
+			}),
+		);
+	});
+
+	it("raises a critical alert at 96% fill", async () => {
+		await diskPressure(usage(96));
+		expect(insertValues).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "disk",
+				severity: "critical",
+				context: expect.stringContaining('"mount":"/"'),
+			}),
+		);
+	});
+});

src/lib/services/alert.ts

@@ -2,6 +2,7 @@ import { env } from "@/env";
 import { AlertLogs, db } from "@/lib/db";
 import { logger } from "@/lib/logger";
 import { sendMail } from "@/lib/services/email";
+import { diskLevel, formatBytes } from "./disk-usage";
 
 // === Alert service (AtomCMS → Next.js) ===========================================
 //
@@ -274,3 +275,38 @@ export function ddosDetected(
 		context: { ip, count },
 	});
 }
+
+/**
+ * Raise a WARNING/ERROR/CRITICAL alert when a filesystem crosses a fill
+ * threshold (see DISK_THRESHOLDS in disk-usage.ts): 85% warning, 90% error,
+ * 95% critical. Includes the offending mount and a pointer to the Docker prune.
+ */
+export function diskPressure(usage: {
+	filesystem: string;
+	mount: string;
+	totalBytes: number;
+	usedBytes: number;
+	availableBytes: number;
+	percent: number;
+}): Promise<SendAlertResult> {
+	const level = diskLevel(usage.percent);
+	const severity: AlertSeverity =
+		level >= 3 ? "critical" : level === 2 ? "error" : "warning";
+	return sendAlert({
+		type: "disk",
+		severity,
+		message:
+			`Disk ${level >= 3 ? "CRITICAL" : level === 2 ? "degraded" : "high"}: ` +
+			`${usage.mount} is ${usage.percent}% full ` +
+			`(${formatBytes(usage.usedBytes)} of ${formatBytes(usage.totalBytes)} used).`,
+		context: {
+			mount: usage.mount,
+			filesystem: usage.filesystem,
+			percentUsed: usage.percent,
+			used: formatBytes(usage.usedBytes),
+			total: formatBytes(usage.totalBytes),
+			available: formatBytes(usage.availableBytes),
+			hint: "Docker prune runs nightly; run scripts/docker-prune.sh to reclaim cache sooner.",
+		},
+	});
+}

src/lib/services/disk-usage.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, it } from "vitest";
+import {
+	DISK_THRESHOLDS,
+	diskLevel,
+	formatBytes,
+	parseDfOutput,
+} from "./disk-usage";
+
+const SAMPLE = `Filesystem      1-blocks        Used   Available Capacity Mounted on
+/dev/vda1    10240000000   5120000000    5120000000      50% /
+/dev/vda2    20480000000  19456000000    1024000000      95% /var/www
+overlay      10240000000   5120000000    5120000000      50% /home/me
+tmpfs        10240000000   5120000000    5120000000      50% /dev/shm
+/dev/vdb1    51200000000  43929600000    7270400000      86% /var/lib/docker
+/dev/vdc1    10240000000   5120000000    5120000000      50% /mnt/with\\040space`;
+
+describe("parseDfOutput", () => {
+	it("parses real mounts and percent from the Capacity column", () => {
+		const mounts = parseDfOutput(SAMPLE);
+		expect(mounts).toHaveLength(4);
+		expect(mounts[0]).toEqual({
+			filesystem: "/dev/vda1",
+			mount: "/",
+			totalBytes: 10240000000,
+			usedBytes: 5120000000,
+			availableBytes: 5120000000,
+			percent: 50,
+		});
+		expect(mounts[1].mount).toBe("/var/www");
+		expect(mounts[1].percent).toBe(95);
+		expect(mounts[2].mount).toBe("/var/lib/docker");
+		expect(mounts[2].percent).toBe(86);
+	});
+
+	it("skips overlay and tmpfs pseudo filesystems", () => {
+		const mounts = parseDfOutput(SAMPLE);
+		expect(mounts.some((m) => m.filesystem === "overlay")).toBe(false);
+		expect(mounts.some((m) => m.filesystem === "tmpfs")).toBe(false);
+	});
+
+	it("unescapes \\040 mount points", () => {
+		const mounts = parseDfOutput(SAMPLE);
+		expect(mounts.some((m) => m.mount === "/mnt/with space")).toBe(true);
+	});
+
+	it("returns empty on header-only or empty input", () => {
+		expect(
+			parseDfOutput("Filesystem 1-blocks Used Available Capacity Mounted on\n"),
+		).toHaveLength(0);
+		expect(parseDfOutput("")).toHaveLength(0);
+	});
+
+	it("derives percent from bytes when the Capacity column is missing", () => {
+		const out = parseDfOutput("/dev/xd1 100000 75000 25000 - /data");
+		expect(out).toHaveLength(1);
+		expect(out[0].percent).toBe(75);
+	});
+});
+
+describe("diskLevel", () => {
+	it("mapps fill levels to the shared thresholds", () => {
+		expect(diskLevel(0)).toBe(0);
+		expect(diskLevel(50)).toBe(0);
+		expect(diskLevel(DISK_THRESHOLDS.warning - 1)).toBe(0);
+		expect(diskLevel(DISK_THRESHOLDS.warning)).toBe(1);
+		expect(diskLevel(DISK_THRESHOLDS.error - 1)).toBe(1);
+		expect(diskLevel(DISK_THRESHOLDS.error)).toBe(2);
+		expect(diskLevel(DISK_THRESHOLDS.critical - 1)).toBe(2);
+		expect(diskLevel(DISK_THRESHOLDS.critical)).toBe(3);
+		expect(diskLevel(99)).toBe(3);
+		expect(diskLevel(100)).toBe(3);
+	});
+
+	it("clamps out-of-range input", () => {
+		expect(diskLevel(-1)).toBe(0);
+		expect(diskLevel(101)).toBe(3);
+	});
+});
+
+describe("formatBytes", () => {
+	it("formats byte values with binary units", () => {
+		expect(formatBytes(0)).toBe("0 B");
+		expect(formatBytes(512)).toBe("512 B");
+		expect(formatBytes(1536)).toBe("1.5 KiB");
+		expect(formatBytes(2048)).toBe("2 KiB");
+		expect(formatBytes(1073741824)).toBe("1 GiB");
+		expect(formatBytes(-5)).toBe("0 B");
+	});
+});

src/lib/services/disk-usage.ts

@@ -0,0 +1,110 @@
+// === Disk usage probing (pure helpers) ===========================================
+//
+// Parses `df -P -B1` output into per-mount DiskUsage records and classifies the
+// fill level against shared thresholds. Pure (no imports, no side effects) so the
+// parsing and threshold boundaries are trivially unit-testable; the host-side
+// jobs-worker calls them via execFile and raises diskPressure() alerts.
+
+export interface DiskUsage {
+	filesystem: string;
+	mount: string;
+	totalBytes: number;
+	usedBytes: number;
+	availableBytes: number;
+	/** Fill percentage (0-100), taken from the `Capacity` column of `df -P`. */
+	percent: number;
+}
+
+export type DiskLevel = 0 | 1 | 2 | 3;
+
+/** Fill thresholds that decide when an alert fires (see DiskLevel). */
+export const DISK_THRESHOLDS = {
+	warning: 85,
+	error: 90,
+	critical: 95,
+} as const;
+
+/**
+ * Pseudo/synthetic filesystems that mirror a backing store rather than owning
+ * disk space of their own; alerting on them is noise (overlay mirrors /,
+ * tmpfs/shm are RAM-backed, etc.).
+ */
+const IGNORED_FILESYSTEMS =
+	/^(?:overlay|tmpfs|shm|devtmpfs|devfs|proc|sysfs|cgroup|cgroup2|debugfs|devpts|mqueue|squashfs|none|udev|efivarfs|pstore|securityfs|configfs|bpf|tracefs|hugetlbfs|rpc_pipefs|nsfs|binfmt_misc)$/;
+
+/** Header column of `df -P` (first field) used to skip the title line. */
+const DF_HEADER = "Filesystem";
+
+/**
+ * Parse `df -P -B1` stdout into real filesystem records. Lines whose mount is a
+ * synthetic filesystem (overlay, tmpfs, ...) are skipped; `\040` escapes for
+ * spaces in mount points are unescaped. Malformed or zero-size rows are ignored.
+ */
+export function parseDfOutput(output: string): DiskUsage[] {
+	const result: DiskUsage[] = [];
+	for (const rawLine of output.split("\n")) {
+		const line = rawLine.trimEnd();
+		if (!line) continue;
+
+		const fields = line.split(/\s+/);
+		if (fields.length < 5) continue;
+
+		const [filesystem, totalStr, usedStr, availStr, capacityStr, ...rest] =
+			fields;
+		if (
+			!filesystem ||
+			filesystem === DF_HEADER ||
+			!totalStr ||
+			!usedStr ||
+			!availStr ||
+			!capacityStr
+		) {
+			continue;
+		}
+		if (IGNORED_FILESYSTEMS.test(filesystem)) continue;
+
+		const totalBytes = Number(totalStr);
+		const usedBytes = Number(usedStr);
+		const availableBytes = Number(availStr);
+		if (!Number.isFinite(totalBytes) || totalBytes <= 0) continue;
+
+		const capacity = capacityStr.endsWith("%")
+			? Number(capacityStr.slice(0, -1))
+			: Number(capacityStr);
+		const percent = Number.isFinite(capacity)
+			? capacity
+			: Math.round((usedBytes / totalBytes) * 100);
+
+		const mount = rest.join(" ").replace(/\\040/g, " ");
+		result.push({
+			filesystem,
+			mount,
+			totalBytes,
+			usedBytes,
+			availableBytes,
+			percent,
+		});
+	}
+	return result;
+}
+
+/** Classify a fill percentage against DISK_THRESHOLDS (0 = fine, 3 = critical). */
+export function diskLevel(percent: number): DiskLevel {
+	if (percent >= DISK_THRESHOLDS.critical) return 3;
+	if (percent >= DISK_THRESHOLDS.error) return 2;
+	if (percent >= DISK_THRESHOLDS.warning) return 1;
+	return 0;
+}
+
+/** Human-friendly byte formatting (KiB/MiB/GiB/…) for alert messages. */
+export function formatBytes(bytes: number): string {
+	if (!Number.isFinite(bytes) || bytes < 0) return "0 B";
+	if (bytes === 0) return "0 B";
+	const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+	const unit = Math.min(
+		Math.floor(Math.log(bytes) / Math.log(1024)),
+		units.length - 1,
+	);
+	const value = bytes / 1024 ** unit;
+	return `${value % 1 === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}`;
+}