feat(ops): alert on filesystem fill levels from the host worker
Add a pure df parser (disk-usage.ts) with 85/90/95% threshold classification, a diskPressure() alert (Discord/email/alert_logs, severity escalates with fill), and a 5-minute host-side probe in jobs-worker.ts that raises one alert per crossing mount, cooldown-gated per mount+level. Real mounts only: overlay/tmpfs pseudo filesystems are ignored.
This commit is contained in:
1 parent
a0d7f42227
commit
fe26ca3ff3
5 files changed
+333
-2
No files matched your search
@@ -24,7 +24,7 @@ vi.mock("@/env", () => ({
|
||||
}));
|
||||
|
||||
import { sendMail } from "@/lib/services/email";
|
||||
import { sendAlert } from "./alert";
|
||||
import { diskPressure, sendAlert } from "./alert";
|
||||
|
||||
beforeEach(() => {
|
||||
insertValues.mockReset();
|
||||
@@ -64,3 +64,45 @@ describe("sendAlert", () => {
|
||||
expect(sendMail).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("diskPressure", () => {
|
||||
const usage = (percent: number) => ({
|
||||
filesystem: "/dev/vda1",
|
||||
mount: "/",
|
||||
totalBytes: 100000000000,
|
||||
usedBytes: (percent / 100) * 100000000000,
|
||||
availableBytes: 100000000000 - (percent / 100) * 100000000000,
|
||||
percent,
|
||||
});
|
||||
|
||||
it("raises a warning at 87% fill", async () => {
|
||||
await diskPressure(usage(87));
|
||||
expect(insertValues).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: "disk",
|
||||
severity: "warning",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("raises an error at 92% fill", async () => {
|
||||
await diskPressure(usage(92));
|
||||
expect(insertValues).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: "disk",
|
||||
severity: "error",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("raises a critical alert at 96% fill", async () => {
|
||||
await diskPressure(usage(96));
|
||||
expect(insertValues).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: "disk",
|
||||
severity: "critical",
|
||||
context: expect.stringContaining('"mount":"/"'),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user