diff --git a/scripts/jobs-worker.ts b/scripts/jobs-worker.ts index 1c190fe0..c0df4f42 100644 --- a/scripts/jobs-worker.ts +++ b/scripts/jobs-worker.ts @@ -11,8 +11,13 @@ import { } from "../src/lib/db"; import { logger } from "../src/lib/logger"; import { redis } from "../src/lib/redis"; -import { emulatorOffline, healthDegraded } from "../src/lib/services/alert"; +import { + diskPressure, + emulatorOffline, + healthDegraded, +} from "../src/lib/services/alert"; import { runCatalogExport } from "../src/lib/services/catalog-git-export"; +import { diskLevel, parseDfOutput } from "../src/lib/services/disk-usage"; import { invalidateNewsCache } from "../src/lib/services/news-cache"; import { rcon } from "../src/lib/services/rcon"; @@ -88,6 +93,49 @@ async function checkOpsHealth(): Promise { } } +/** + * Host-side: probe filesystem fill levels via `df -P -B1` and raise a + * diskPressure() alert per mount once it crosses 85/90/95% (cooldown-gated per + * mount+level so a stuck fill level does not spam Discord/email). + */ +async function checkDiskUsage(): Promise { + const { execFile } = await import("node:child_process"); + + let dfOut: string; + try { + dfOut = await new Promise((resolve, reject) => { + execFile( + "df", + ["-P", "-B1"], + { + maxBuffer: 4 * 1024 * 1024, + timeout: 15_000, + }, + (err, stdout) => (err ? reject(err) : resolve(stdout)), + ); + }); + } catch (err) { + captureWorkerError(err, "Disk probe failed (is df available?)"); + return; + } + + const mounts = parseDfOutput(dfOut); + if (mounts.length === 0) return; + + for (const usage of mounts) { + const level = diskLevel(usage.percent); + if (level === 0) continue; + if (canAlert(`disk-${usage.mount}-${level}`)) { + logger.warn("Raising disk usage alert", { + module: "jobs", + mount: usage.mount, + percent: usage.percent, + }); + await diskPressure(usage); + } + } +} + async function backupEmulatorJar(): Promise { if (!env.EMULATOR_JAR_PATH || !env.EMULATOR_BACKUP_DIR) return; @@ -351,6 +399,11 @@ async function main() { }); logger.info("Scheduled: ops health probe (every 5 min)", { module: "jobs" }); + new Cron("*/5 * * * *", () => { + checkDiskUsage().catch((e) => captureWorkerError(e, "Disk check error")); + }); + logger.info("Scheduled: disk usage probe (every 5 min)", { module: "jobs" }); + new Cron("* * * * *", () => { publishScheduledArticles().catch((e) => captureWorkerError(e, "ScheduledArticlePublish"), @@ -369,6 +422,7 @@ async function main() { cleanupOldLogs(), cleanupOldSessions(), checkOpsHealth(), + checkDiskUsage(), ]); } diff --git a/src/lib/services/alert.test.ts b/src/lib/services/alert.test.ts index 63e94055..adad4cbf 100644 --- a/src/lib/services/alert.test.ts +++ b/src/lib/services/alert.test.ts @@ -24,7 +24,7 @@ vi.mock("@/env", () => ({ })); import { sendMail } from "@/lib/services/email"; -import { sendAlert } from "./alert"; +import { diskPressure, sendAlert } from "./alert"; beforeEach(() => { insertValues.mockReset(); @@ -64,3 +64,45 @@ describe("sendAlert", () => { expect(sendMail).not.toHaveBeenCalled(); }); }); + +describe("diskPressure", () => { + const usage = (percent: number) => ({ + filesystem: "/dev/vda1", + mount: "/", + totalBytes: 100000000000, + usedBytes: (percent / 100) * 100000000000, + availableBytes: 100000000000 - (percent / 100) * 100000000000, + percent, + }); + + it("raises a warning at 87% fill", async () => { + await diskPressure(usage(87)); + expect(insertValues).toHaveBeenCalledWith( + expect.objectContaining({ + type: "disk", + severity: "warning", + }), + ); + }); + + it("raises an error at 92% fill", async () => { + await diskPressure(usage(92)); + expect(insertValues).toHaveBeenCalledWith( + expect.objectContaining({ + type: "disk", + severity: "error", + }), + ); + }); + + it("raises a critical alert at 96% fill", async () => { + await diskPressure(usage(96)); + expect(insertValues).toHaveBeenCalledWith( + expect.objectContaining({ + type: "disk", + severity: "critical", + context: expect.stringContaining('"mount":"/"'), + }), + ); + }); +}); diff --git a/src/lib/services/alert.ts b/src/lib/services/alert.ts index 38319bd4..5361ce03 100644 --- a/src/lib/services/alert.ts +++ b/src/lib/services/alert.ts @@ -2,6 +2,7 @@ import { env } from "@/env"; import { AlertLogs, db } from "@/lib/db"; import { logger } from "@/lib/logger"; import { sendMail } from "@/lib/services/email"; +import { diskLevel, formatBytes } from "./disk-usage"; // === Alert service (AtomCMS → Next.js) =========================================== // @@ -274,3 +275,38 @@ export function ddosDetected( context: { ip, count }, }); } + +/** + * Raise a WARNING/ERROR/CRITICAL alert when a filesystem crosses a fill + * threshold (see DISK_THRESHOLDS in disk-usage.ts): 85% warning, 90% error, + * 95% critical. Includes the offending mount and a pointer to the Docker prune. + */ +export function diskPressure(usage: { + filesystem: string; + mount: string; + totalBytes: number; + usedBytes: number; + availableBytes: number; + percent: number; +}): Promise { + const level = diskLevel(usage.percent); + const severity: AlertSeverity = + level >= 3 ? "critical" : level === 2 ? "error" : "warning"; + return sendAlert({ + type: "disk", + severity, + message: + `Disk ${level >= 3 ? "CRITICAL" : level === 2 ? "degraded" : "high"}: ` + + `${usage.mount} is ${usage.percent}% full ` + + `(${formatBytes(usage.usedBytes)} of ${formatBytes(usage.totalBytes)} used).`, + context: { + mount: usage.mount, + filesystem: usage.filesystem, + percentUsed: usage.percent, + used: formatBytes(usage.usedBytes), + total: formatBytes(usage.totalBytes), + available: formatBytes(usage.availableBytes), + hint: "Docker prune runs nightly; run scripts/docker-prune.sh to reclaim cache sooner.", + }, + }); +} diff --git a/src/lib/services/disk-usage.test.ts b/src/lib/services/disk-usage.test.ts new file mode 100644 index 00000000..9d1e9424 --- /dev/null +++ b/src/lib/services/disk-usage.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { + DISK_THRESHOLDS, + diskLevel, + formatBytes, + parseDfOutput, +} from "./disk-usage"; + +const SAMPLE = `Filesystem 1-blocks Used Available Capacity Mounted on +/dev/vda1 10240000000 5120000000 5120000000 50% / +/dev/vda2 20480000000 19456000000 1024000000 95% /var/www +overlay 10240000000 5120000000 5120000000 50% /home/me +tmpfs 10240000000 5120000000 5120000000 50% /dev/shm +/dev/vdb1 51200000000 43929600000 7270400000 86% /var/lib/docker +/dev/vdc1 10240000000 5120000000 5120000000 50% /mnt/with\\040space`; + +describe("parseDfOutput", () => { + it("parses real mounts and percent from the Capacity column", () => { + const mounts = parseDfOutput(SAMPLE); + expect(mounts).toHaveLength(4); + expect(mounts[0]).toEqual({ + filesystem: "/dev/vda1", + mount: "/", + totalBytes: 10240000000, + usedBytes: 5120000000, + availableBytes: 5120000000, + percent: 50, + }); + expect(mounts[1].mount).toBe("/var/www"); + expect(mounts[1].percent).toBe(95); + expect(mounts[2].mount).toBe("/var/lib/docker"); + expect(mounts[2].percent).toBe(86); + }); + + it("skips overlay and tmpfs pseudo filesystems", () => { + const mounts = parseDfOutput(SAMPLE); + expect(mounts.some((m) => m.filesystem === "overlay")).toBe(false); + expect(mounts.some((m) => m.filesystem === "tmpfs")).toBe(false); + }); + + it("unescapes \\040 mount points", () => { + const mounts = parseDfOutput(SAMPLE); + expect(mounts.some((m) => m.mount === "/mnt/with space")).toBe(true); + }); + + it("returns empty on header-only or empty input", () => { + expect( + parseDfOutput("Filesystem 1-blocks Used Available Capacity Mounted on\n"), + ).toHaveLength(0); + expect(parseDfOutput("")).toHaveLength(0); + }); + + it("derives percent from bytes when the Capacity column is missing", () => { + const out = parseDfOutput("/dev/xd1 100000 75000 25000 - /data"); + expect(out).toHaveLength(1); + expect(out[0].percent).toBe(75); + }); +}); + +describe("diskLevel", () => { + it("mapps fill levels to the shared thresholds", () => { + expect(diskLevel(0)).toBe(0); + expect(diskLevel(50)).toBe(0); + expect(diskLevel(DISK_THRESHOLDS.warning - 1)).toBe(0); + expect(diskLevel(DISK_THRESHOLDS.warning)).toBe(1); + expect(diskLevel(DISK_THRESHOLDS.error - 1)).toBe(1); + expect(diskLevel(DISK_THRESHOLDS.error)).toBe(2); + expect(diskLevel(DISK_THRESHOLDS.critical - 1)).toBe(2); + expect(diskLevel(DISK_THRESHOLDS.critical)).toBe(3); + expect(diskLevel(99)).toBe(3); + expect(diskLevel(100)).toBe(3); + }); + + it("clamps out-of-range input", () => { + expect(diskLevel(-1)).toBe(0); + expect(diskLevel(101)).toBe(3); + }); +}); + +describe("formatBytes", () => { + it("formats byte values with binary units", () => { + expect(formatBytes(0)).toBe("0 B"); + expect(formatBytes(512)).toBe("512 B"); + expect(formatBytes(1536)).toBe("1.5 KiB"); + expect(formatBytes(2048)).toBe("2 KiB"); + expect(formatBytes(1073741824)).toBe("1 GiB"); + expect(formatBytes(-5)).toBe("0 B"); + }); +}); diff --git a/src/lib/services/disk-usage.ts b/src/lib/services/disk-usage.ts new file mode 100644 index 00000000..db5d470c --- /dev/null +++ b/src/lib/services/disk-usage.ts @@ -0,0 +1,110 @@ +// === Disk usage probing (pure helpers) =========================================== +// +// Parses `df -P -B1` output into per-mount DiskUsage records and classifies the +// fill level against shared thresholds. Pure (no imports, no side effects) so the +// parsing and threshold boundaries are trivially unit-testable; the host-side +// jobs-worker calls them via execFile and raises diskPressure() alerts. + +export interface DiskUsage { + filesystem: string; + mount: string; + totalBytes: number; + usedBytes: number; + availableBytes: number; + /** Fill percentage (0-100), taken from the `Capacity` column of `df -P`. */ + percent: number; +} + +export type DiskLevel = 0 | 1 | 2 | 3; + +/** Fill thresholds that decide when an alert fires (see DiskLevel). */ +export const DISK_THRESHOLDS = { + warning: 85, + error: 90, + critical: 95, +} as const; + +/** + * Pseudo/synthetic filesystems that mirror a backing store rather than owning + * disk space of their own; alerting on them is noise (overlay mirrors /, + * tmpfs/shm are RAM-backed, etc.). + */ +const IGNORED_FILESYSTEMS = + /^(?:overlay|tmpfs|shm|devtmpfs|devfs|proc|sysfs|cgroup|cgroup2|debugfs|devpts|mqueue|squashfs|none|udev|efivarfs|pstore|securityfs|configfs|bpf|tracefs|hugetlbfs|rpc_pipefs|nsfs|binfmt_misc)$/; + +/** Header column of `df -P` (first field) used to skip the title line. */ +const DF_HEADER = "Filesystem"; + +/** + * Parse `df -P -B1` stdout into real filesystem records. Lines whose mount is a + * synthetic filesystem (overlay, tmpfs, ...) are skipped; `\040` escapes for + * spaces in mount points are unescaped. Malformed or zero-size rows are ignored. + */ +export function parseDfOutput(output: string): DiskUsage[] { + const result: DiskUsage[] = []; + for (const rawLine of output.split("\n")) { + const line = rawLine.trimEnd(); + if (!line) continue; + + const fields = line.split(/\s+/); + if (fields.length < 5) continue; + + const [filesystem, totalStr, usedStr, availStr, capacityStr, ...rest] = + fields; + if ( + !filesystem || + filesystem === DF_HEADER || + !totalStr || + !usedStr || + !availStr || + !capacityStr + ) { + continue; + } + if (IGNORED_FILESYSTEMS.test(filesystem)) continue; + + const totalBytes = Number(totalStr); + const usedBytes = Number(usedStr); + const availableBytes = Number(availStr); + if (!Number.isFinite(totalBytes) || totalBytes <= 0) continue; + + const capacity = capacityStr.endsWith("%") + ? Number(capacityStr.slice(0, -1)) + : Number(capacityStr); + const percent = Number.isFinite(capacity) + ? capacity + : Math.round((usedBytes / totalBytes) * 100); + + const mount = rest.join(" ").replace(/\\040/g, " "); + result.push({ + filesystem, + mount, + totalBytes, + usedBytes, + availableBytes, + percent, + }); + } + return result; +} + +/** Classify a fill percentage against DISK_THRESHOLDS (0 = fine, 3 = critical). */ +export function diskLevel(percent: number): DiskLevel { + if (percent >= DISK_THRESHOLDS.critical) return 3; + if (percent >= DISK_THRESHOLDS.error) return 2; + if (percent >= DISK_THRESHOLDS.warning) return 1; + return 0; +} + +/** Human-friendly byte formatting (KiB/MiB/GiB/…) for alert messages. */ +export function formatBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes < 0) return "0 B"; + if (bytes === 0) return "0 B"; + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + const unit = Math.min( + Math.floor(Math.log(bytes) / Math.log(1024)), + units.length - 1, + ); + const value = bytes / 1024 ** unit; + return `${value % 1 === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}`; +}