feat(ops): add integrity-checked backups and isolated restore drills

scripts/backup/cli.mjs

@@ -0,0 +1,95 @@
+import { lstat, readFile } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { parseArgs } from "node:util";
+import { safeDirectory } from "./core.mjs";
+import { createBackup, drillBackup, validateDatabase } from "./database.mjs";
+
+export async function loadConfig(file) {
+	if (!path.isAbsolute(file)) throw Error("Use an absolute configuration path");
+	await safeDirectory(path.dirname(file));
+	const stat = await lstat(file);
+	if (
+		!stat.isFile() ||
+		stat.isSymbolicLink() ||
+		stat.nlink !== 1 ||
+		(process.platform !== "win32" && stat.mode & 0o077)
+	)
+		throw Error("Configuration must be a private regular file");
+	const config = JSON.parse(await readFile(file, "utf8"));
+	if (
+		Object.keys(config).some((key) => !["database", "roots"].includes(key)) ||
+		!config.roots
+	)
+		throw Error("Invalid backup configuration");
+	validateDatabase(config.database);
+	for (const root of Object.values(config.roots)) {
+		if (typeof root !== "string" || !path.isAbsolute(root))
+			throw Error("Use absolute source paths");
+		const relative = path.relative(path.resolve(root), file);
+		if (
+			!relative ||
+			(!relative.startsWith(`..${path.sep}`) &&
+				relative !== ".." &&
+				!path.isAbsolute(relative))
+		)
+			throw Error("Keep the configuration outside every source root");
+	}
+	return config;
+}
+
+export async function main(args = process.argv.slice(2)) {
+	const [command, ...options] = args;
+	if (!["create", "drill"].includes(command))
+		throw Error("Choose create or drill");
+	const schema =
+		command === "create"
+			? {
+					config: { type: "string" },
+					output: { type: "string" },
+					"writers-quiesced": { type: "boolean" },
+				}
+			: { artifact: { type: "string" } };
+	const { values } = parseArgs({
+		args: options,
+		options: schema,
+		strict: true,
+		allowPositionals: false,
+	});
+	if (command === "create") {
+		if (
+			!values.config ||
+			!values.output ||
+			values["writers-quiesced"] !== true ||
+			process.platform !== "linux"
+		)
+			throw Error("Create requires Linux, explicit paths and paused writers");
+		const config = await loadConfig(values.config);
+		await createBackup({
+			...config,
+			output: values.output,
+			writersQuiesced: true,
+		});
+		process.stdout.write(
+			"Backup created and checksums verified. Run the isolated drill before relying on it.\n",
+		);
+	} else {
+		if (!values.artifact) throw Error("Provide an artifact");
+		await drillBackup({ artifact: values.artifact });
+		process.stdout.write(
+			"Isolated restore verified; disposable database and files removed.\n",
+		);
+	}
+}
+
+if (
+	process.argv[1] &&
+	import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
+) {
+	main().catch(() => {
+		process.stderr.write(
+			"Backup command failed. Check private configuration, prerequisites, paused writers and artifact integrity. No server output was logged.\n",
+		);
+		process.exitCode = 1;
+	});
+}

scripts/backup/cli.test.mjs

@@ -0,0 +1,51 @@
+import { spawnSync } from "node:child_process";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { loadConfig } from "./cli.mjs";
+
+it("does not provide a production restore command or target option", () => {
+	for (const args of [
+		["restore", "--target", "production"],
+		["drill", "--artifact", "/missing", "--target", "production"],
+	]) {
+		const result = spawnSync(
+			process.execPath,
+			["scripts/backup/cli.mjs", ...args],
+			{ encoding: "utf8" },
+		);
+		expect(result.status).toBe(1);
+		expect(result.stderr).not.toContain("production");
+	}
+});
+
+it("loads only explicit configuration and refuses to include that configuration in file roots", async () => {
+	const directory = await mkdtemp(path.join(tmpdir(), "cms-backup-cli-"));
+	try {
+		const root = path.join(directory, "storage");
+		await mkdir(root);
+		const config = {
+			database: {
+				host: "127.0.0.1",
+				port: 3306,
+				user: "fixture",
+				password: "secret-value",
+				database: "fixture",
+			},
+			roots: {
+				storage: root,
+				nitro: path.join(directory, "nitro"),
+				swf: path.join(directory, "swf"),
+			},
+		};
+		const file = path.join(directory, "private.json");
+		await writeFile(file, JSON.stringify(config), { mode: 0o600 });
+		expect(await loadConfig(file)).toEqual(config);
+		const unsafe = path.join(root, "custom-settings.json");
+		await writeFile(unsafe, JSON.stringify(config), { mode: 0o600 });
+		await expect(loadConfig(unsafe)).rejects.toThrow();
+	} finally {
+		await rm(directory, { recursive: true, force: true });
+	}
+});

scripts/backup/core.mjs

@@ -0,0 +1,316 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import {
+	lstat,
+	mkdir,
+	open,
+	readdir,
+	readFile,
+	rm,
+	writeFile,
+} from "node:fs/promises";
+import path from "node:path";
+
+const requiredRoots = ["storage", "nitro", "swf"];
+const allowedRoots = [...requiredRoots, "gamedata"];
+const fail = () => {
+	throw Error("Backup file validation failed");
+};
+const sortedEntries = (entries) =>
+	[...entries].sort((left, right) =>
+		left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
+	);
+const inside = (parent, child) => {
+	const relative = path.relative(parent, child);
+	return (
+		!relative ||
+		(!relative.startsWith(`..${path.sep}`) &&
+			relative !== ".." &&
+			!path.isAbsolute(relative))
+	);
+};
+
+export async function safeDirectory(value) {
+	if (
+		typeof value !== "string" ||
+		!path.isAbsolute(value) ||
+		value.split(/[\\/]/).includes("..")
+	)
+		fail();
+	const resolved = path.resolve(value);
+	if (resolved === path.parse(resolved).root) fail();
+	for (
+		let current = resolved;
+		current !== path.dirname(current);
+		current = path.dirname(current)
+	) {
+		const stat = await lstat(current);
+		if (!stat.isDirectory() || stat.isSymbolicLink()) fail();
+	}
+	return resolved;
+}
+
+function validEntry(entry, roots) {
+	const name = entry.path;
+	if (
+		typeof name !== "string" ||
+		/[\\:]/.test(name) ||
+		[...name].some((character) => character.charCodeAt(0) < 32) ||
+		name.split("/").some((part) => !part || part === "." || part === "..")
+	)
+		fail();
+	if (
+		name !== "database.sql" &&
+		name !== "files" &&
+		!roots.some(
+			(root) => name === `files/${root}` || name.startsWith(`files/${root}/`),
+		)
+	)
+		fail();
+	if (!["file", "directory"].includes(entry.type)) fail();
+	if (
+		entry.type === "file" &&
+		(!Number.isSafeInteger(entry.bytes) ||
+			entry.bytes < 0 ||
+			!/^[a-f0-9]{64}$/.test(entry.sha256))
+	)
+		fail();
+}
+
+async function transfer(source, target) {
+	const before = await lstat(source);
+	if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1) fail();
+	const input = await open(
+		source,
+		constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
+	);
+	let output;
+	try {
+		const opened = await input.stat();
+		if (opened.dev !== before.dev || opened.ino !== before.ino) fail();
+		if (target) output = await open(target, "wx", 0o600);
+		const hash = createHash("sha256");
+		let bytes = 0;
+		for await (const chunk of input.createReadStream({ autoClose: false })) {
+			hash.update(chunk);
+			bytes += chunk.length;
+			if (output) await output.writeFile(chunk);
+		}
+		const after = await input.stat();
+		if (
+			before.size !== bytes ||
+			after.size !== before.size ||
+			after.mtimeMs !== before.mtimeMs ||
+			after.ctimeMs !== before.ctimeMs
+		)
+			fail();
+		return { bytes, sha256: hash.digest("hex") };
+	} finally {
+		await input.close();
+		await output?.close();
+	}
+}
+
+async function walk(directory, prefix, destination, rejectSecrets = false) {
+	await safeDirectory(directory);
+	const entries = [{ path: prefix, type: "directory" }];
+	for (const name of (await readdir(directory)).sort()) {
+		if (
+			rejectSecrets &&
+			(/^\.env(?:\.|$)/i.test(name) ||
+				[".docker-install", "persistent.path", "backup.config.json"].includes(
+					name,
+				))
+		)
+			fail();
+		const source = path.join(directory, name);
+		const relative = `${prefix}/${name}`;
+		const stat = await lstat(source);
+		if (stat.isSymbolicLink()) fail();
+		if (stat.isDirectory()) {
+			if (destination)
+				await mkdir(path.join(destination, name), { mode: 0o700 });
+			entries.push(
+				...(await walk(
+					source,
+					relative,
+					destination && path.join(destination, name),
+					rejectSecrets,
+				)),
+			);
+		} else {
+			const content = await transfer(
+				source,
+				destination && path.join(destination, name),
+			);
+			entries.push({ path: relative, type: "file", ...content });
+		}
+	}
+	return entries;
+}
+
+export async function createArtifact({ roots, output }, writeDatabase) {
+	if (
+		!roots ||
+		requiredRoots.some((key) => !roots[key]) ||
+		Object.keys(roots).some((key) => !allowedRoots.includes(key))
+	)
+		fail();
+	const sources = await Promise.all(Object.values(roots).map(safeDirectory));
+	if (!path.isAbsolute(output) || output.split(/[\\/]/).includes("..")) fail();
+	output = path.join(
+		await safeDirectory(path.dirname(output)),
+		path.basename(output),
+	);
+	for (let index = 0; index < sources.length; index++) {
+		if (
+			inside(sources[index], output) ||
+			inside(output, sources[index]) ||
+			sources.some(
+				(source, other) =>
+					other !== index &&
+					(inside(source, sources[index]) || inside(sources[index], source)),
+			)
+		)
+			fail();
+	}
+	await mkdir(output, { mode: 0o700 }); // Exclusive reservation; never replace an existing artifact.
+	try {
+		await mkdir(path.join(output, "files"), { mode: 0o700 });
+		const entries = [{ path: "files", type: "directory" }];
+		for (const key of Object.keys(roots).sort()) {
+			const destination = path.join(output, "files", key);
+			await mkdir(destination, { mode: 0o700 });
+			entries.push(
+				...(await walk(roots[key], `files/${key}`, destination, true)),
+			);
+		}
+		const database = await writeDatabase(path.join(output, "database.sql"));
+		const sql = await transfer(path.join(output, "database.sql"));
+		if (!sql.bytes) fail();
+		entries.push({ path: "database.sql", type: "file", ...sql });
+		// Detect source changes across the database dump and the file copy interval.
+		for (const key of Object.keys(roots)) {
+			const current = await walk(roots[key], `files/${key}`, undefined, true);
+			if (
+				JSON.stringify(current) !==
+				JSON.stringify(
+					entries.filter(
+						(entry) =>
+							entry.path === `files/${key}` ||
+							entry.path.startsWith(`files/${key}/`),
+					),
+				)
+			)
+				fail();
+		}
+		entries.sort((left, right) =>
+			left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
+		);
+		const manifest = {
+			format: 1,
+			complete: true,
+			createdAt: new Date().toISOString(),
+			roots: Object.keys(roots).sort(),
+			database,
+			entries,
+		};
+		for (const entry of entries) validEntry(entry, manifest.roots);
+		await writeFile(
+			path.join(output, "manifest.json"),
+			`${JSON.stringify(manifest, null, 2)}\n`,
+			{ flag: "wx", mode: 0o600 },
+		);
+		await verifyArtifact(output);
+		return manifest;
+	} catch (error) {
+		await rm(output, { recursive: true, force: true });
+		throw error;
+	}
+}
+
+export async function verifyArtifact(directory) {
+	await safeDirectory(directory);
+	const manifestPath = path.join(directory, "manifest.json");
+	await transfer(manifestPath); // Reject links before parsing the manifest.
+	const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+	if (
+		manifest.format !== 1 ||
+		manifest.complete !== true ||
+		!Array.isArray(manifest.roots) ||
+		requiredRoots.some((root) => !manifest.roots.includes(root)) ||
+		manifest.roots.some((root) => !allowedRoots.includes(root)) ||
+		!Array.isArray(manifest.entries)
+	)
+		fail();
+	const names = new Set();
+	for (const entry of manifest.entries) {
+		validEntry(entry, manifest.roots);
+		if (names.has(entry.path)) fail();
+		names.add(entry.path);
+	}
+	if (
+		!manifest.entries.some(
+			(entry) =>
+				entry.path === "database.sql" &&
+				entry.type === "file" &&
+				entry.bytes > 0,
+		)
+	)
+		fail();
+	const actual = (await walk(directory, "artifact"))
+		.filter(
+			(entry) =>
+				entry.path !== "artifact" && entry.path !== "artifact/manifest.json",
+		)
+		.map((entry) => ({ ...entry, path: entry.path.slice(9) }));
+	if (
+		JSON.stringify(sortedEntries(actual)) !==
+		JSON.stringify(sortedEntries(manifest.entries))
+	)
+		fail();
+	for (const root of manifest.roots)
+		if (
+			!actual.some(
+				(entry) => entry.path === `files/${root}` && entry.type === "directory",
+			)
+		)
+			fail();
+	return manifest;
+}
+
+export async function restoreFiles(artifact, destination) {
+	const manifest = await verifyArtifact(artifact);
+	if (
+		!path.isAbsolute(destination) ||
+		inside(path.resolve(artifact), path.resolve(destination))
+	)
+		fail();
+	await safeDirectory(path.dirname(destination));
+	await mkdir(destination, { mode: 0o700 });
+	try {
+		for (const root of manifest.roots) {
+			const target = path.join(destination, root);
+			await mkdir(target, { mode: 0o700 });
+			const entries = await walk(
+				path.join(artifact, "files", root),
+				`files/${root}`,
+				target,
+			);
+			const expectedEntries = manifest.entries.filter(
+				(entry) =>
+					entry.path === `files/${root}` ||
+					entry.path.startsWith(`files/${root}/`),
+			);
+			if (
+				JSON.stringify(sortedEntries(entries)) !==
+				JSON.stringify(sortedEntries(expectedEntries))
+			)
+				fail();
+		}
+	} catch (error) {
+		await rm(destination, { recursive: true, force: true });
+		throw error;
+	}
+	return manifest;
+}

scripts/backup/core.test.mjs

@@ -0,0 +1,167 @@
+import {
+	mkdir,
+	mkdtemp,
+	readdir,
+	readFile,
+	rm,
+	symlink,
+	writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, expect, it } from "vitest";
+import { createArtifact, restoreFiles, verifyArtifact } from "./core.mjs";
+
+let directory;
+let roots;
+let output;
+const database = { database: "fixture", objects: [], tables: [] };
+const dump = async (target) => {
+	await writeFile(target, "CREATE DATABASE fixture;\n");
+	return database;
+};
+beforeEach(async () => {
+	directory = await mkdtemp(path.join(tmpdir(), "cms-backup-test-"));
+	roots = Object.fromEntries(
+		["storage", "nitro", "swf"].map((key) => [key, path.join(directory, key)]),
+	);
+	for (const root of Object.values(roots)) await mkdir(root);
+	await mkdir(path.join(roots.storage, "empty"));
+	await writeFile(
+		path.join(roots.storage, "image.bin"),
+		Buffer.from([0, 255, 128, 1]),
+	);
+	await writeFile(
+		path.join(roots.nitro, "furniture.json"),
+		'{"caption":"Caffè 🏨"}',
+	);
+	output = path.join(directory, "artifact");
+});
+afterEach(async () => {
+	await rm(directory, { recursive: true, force: true });
+});
+
+it("copies exact bytes and empty directories, records hashes without source paths, and restores only into a new directory", async () => {
+	const manifest = await createArtifact({ roots, output }, dump);
+	expect(manifest.complete).toBe(true);
+	expect(JSON.stringify(manifest)).not.toContain(directory);
+	expect(
+		manifest.entries.find((entry) => entry.path === "files/storage/image.bin")
+			.sha256,
+	).toMatch(/^[a-f0-9]{64}$/);
+	expect(await verifyArtifact(output)).toEqual(manifest);
+	const restored = path.join(directory, "restored");
+	await restoreFiles(output, restored);
+	expect(await readFile(path.join(restored, "storage/image.bin"))).toEqual(
+		Buffer.from([0, 255, 128, 1]),
+	);
+	expect(await readdir(path.join(restored, "storage/empty"))).toEqual([]);
+	await expect(restoreFiles(output, restored)).rejects.toThrow();
+});
+
+it.each(["files/storage/image.bin", "database.sql"])(
+	"rejects modified %s before restoring files",
+	async (entry) => {
+		await createArtifact({ roots, output }, dump);
+		await writeFile(path.join(output, entry), "tampered");
+		const restored = path.join(directory, "restored");
+		await expect(restoreFiles(output, restored)).rejects.toThrow();
+		await expect(readdir(restored)).rejects.toThrow();
+	},
+);
+
+it("rejects unlisted files and malicious manifest traversal", async () => {
+	await createArtifact({ roots, output }, dump);
+	const extra = path.join(output, "unexpected.txt");
+	await writeFile(extra, "extra");
+	await expect(verifyArtifact(output)).rejects.toThrow();
+	await rm(extra);
+	const manifest = JSON.parse(
+		await readFile(path.join(output, "manifest.json"), "utf8"),
+	);
+	manifest.entries[0].path = "../escape";
+	await writeFile(path.join(output, "manifest.json"), JSON.stringify(manifest));
+	await expect(verifyArtifact(output)).rejects.toThrow();
+});
+
+it("rejects an incomplete artifact and leaves no final artifact after dump failure", async () => {
+	await expect(
+		createArtifact({ roots, output }, async () => {
+			throw Error("database failed");
+		}),
+	).rejects.toThrow();
+	await expect(readdir(output)).rejects.toThrow();
+	await mkdir(output);
+	await writeFile(path.join(output, "database.sql"), "partial");
+	await expect(verifyArtifact(output)).rejects.toThrow();
+});
+
+it("refuses overwrites and output inside a source root", async () => {
+	await mkdir(output);
+	await writeFile(path.join(output, "keep"), "original");
+	await expect(createArtifact({ roots, output }, dump)).rejects.toThrow();
+	expect(await readFile(path.join(output, "keep"), "utf8")).toBe("original");
+	await expect(
+		createArtifact({ roots, output: path.join(roots.storage, "backup") }, dump),
+	).rejects.toThrow();
+});
+
+it("rejects omitted roots, overlapping roots, and secret configuration files", async () => {
+	await expect(
+		createArtifact({ roots: { storage: roots.storage }, output }, dump),
+	).rejects.toThrow();
+	await expect(
+		createArtifact({ roots: { ...roots, nitro: roots.storage }, output }, dump),
+	).rejects.toThrow();
+	await writeFile(path.join(roots.storage, ".env"), "DATABASE_URL=secret");
+	await expect(createArtifact({ roots, output }, dump)).rejects.toThrow();
+});
+
+it("rejects symbolic links in source trees and artifact trees", async () => {
+	const outside = path.join(directory, "outside");
+	await mkdir(outside);
+	await writeFile(path.join(outside, "secret"), "private");
+	const link = path.join(roots.storage, "linked");
+	await symlink(
+		outside,
+		link,
+		process.platform === "win32" ? "junction" : "dir",
+	);
+	await expect(createArtifact({ roots, output }, dump)).rejects.toThrow();
+	await rm(link);
+	await createArtifact({ roots, output }, dump);
+	await symlink(
+		outside,
+		path.join(output, "files/storage/linked"),
+		process.platform === "win32" ? "junction" : "dir",
+	);
+	await expect(verifyArtifact(output)).rejects.toThrow();
+});
+
+it("rejects files changed while the database is being dumped", async () => {
+	await expect(
+		createArtifact({ roots, output }, async (target) => {
+			await writeFile(
+				path.join(roots.storage, "image.bin"),
+				"changed during backup",
+			);
+			return dump(target);
+		}),
+	).rejects.toThrow();
+	await expect(readdir(output)).rejects.toThrow();
+});
+
+it("restores a directory whose prefix also appears in a sibling filename", async () => {
+	await mkdir(path.join(roots.storage, "foo"));
+	await writeFile(path.join(roots.storage, "foo/file"), "nested");
+	await writeFile(path.join(roots.storage, "foo.json"), "sibling");
+	await createArtifact({ roots, output }, dump);
+	const destination = path.join(directory, "restored");
+	await restoreFiles(output, destination);
+	expect(
+		await readFile(path.join(destination, "storage/foo/file"), "utf8"),
+	).toBe("nested");
+	expect(
+		await readFile(path.join(destination, "storage/foo.json"), "utf8"),
+	).toBe("sibling");
+});

scripts/backup/database.mjs

@@ -0,0 +1,362 @@
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { chmod, mkdtemp, open, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { pipeline } from "node:stream/promises";
+import { setTimeout as delay } from "node:timers/promises";
+import { createArtifact, restoreFiles, verifyArtifact } from "./core.mjs";
+
+export const IMAGE = "mariadb:11.4.5";
+const failure = () =>
+	Error(
+		"MariaDB backup operation failed; no credentials or server output were logged",
+	);
+const quote = (value) =>
+	`"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
+const literal = (value) => `'${value.replaceAll("'", "''")}'`;
+const identifier = (value) => {
+	if (
+		typeof value !== "string" ||
+		[...value].some((character) => character.charCodeAt(0) < 32)
+	)
+		throw failure();
+	return `\`${value.replaceAll("`", "``")}\``;
+};
+
+export function validateDatabase(config) {
+	if (
+		!config ||
+		Object.keys(config).some(
+			(key) => !["host", "port", "user", "password", "database"].includes(key),
+		) ||
+		!Number.isInteger(config.port) ||
+		config.port < 1 ||
+		config.port > 65535
+	)
+		throw failure();
+	for (const key of ["host", "user", "password", "database"])
+		if (
+			typeof config[key] !== "string" ||
+			!config[key] ||
+			[...config[key]].some((character) => character.charCodeAt(0) < 32)
+		)
+			throw failure();
+	if (
+		!/^[a-zA-Z0-9_]+$/.test(config.database) ||
+		["mysql", "sys", "information_schema", "performance_schema"].includes(
+			config.database.toLowerCase(),
+		)
+	)
+		throw failure();
+	return config;
+}
+
+export function credentialOptions(config) {
+	validateDatabase(config);
+	return `[client]\nhost=${quote(config.host)}\nport=${config.port}\nuser=${quote(config.user)}\npassword=${quote(config.password)}\nprotocol=tcp\ndefault-character-set=utf8mb4\n`;
+}
+
+export function dockerEnvironment(source = process.env) {
+	const keys = [
+		"PATH",
+		"Path",
+		"SystemRoot",
+		"SystemDrive",
+		"TEMP",
+		"TMP",
+		"HOME",
+		"USERPROFILE",
+		"DOCKER_HOST",
+		"DOCKER_CONTEXT",
+		"DOCKER_CONFIG",
+		"DOCKER_TLS_VERIFY",
+		"DOCKER_CERT_PATH",
+	];
+	return Object.fromEntries(
+		keys
+			.filter((key) => source[key] !== undefined)
+			.map((key) => [key, source[key]]),
+	);
+}
+
+async function docker(args, { input, output, timeout = 1_800_000 } = {}) {
+	const file = output ? await open(output, "wx", 0o600) : undefined;
+	try {
+		const child = spawn("docker", args, {
+			env: dockerEnvironment(),
+			windowsHide: true,
+			stdio: [input ? "pipe" : "ignore", file ? file.fd : "pipe", "ignore"],
+		});
+		let stdout = "";
+		if (!file)
+			child.stdout.on("data", (chunk) => {
+				stdout += chunk.toString("utf8");
+				if (stdout.length > 16 * 1024 * 1024) child.kill();
+			});
+		const timer = setTimeout(() => child.kill(), timeout);
+		const completion = new Promise((resolve, reject) => {
+			child.once("error", () => reject(failure()));
+			child.once("close", (code) =>
+				code === 0 ? resolve(stdout) : reject(failure()),
+			);
+		});
+		try {
+			await Promise.all([
+				completion,
+				input
+					? pipeline(createReadStream(input), child.stdin)
+					: Promise.resolve(),
+			]);
+			return stdout;
+		} catch {
+			child.kill();
+			throw failure();
+		} finally {
+			clearTimeout(timer);
+		}
+	} finally {
+		await file?.close();
+	}
+}
+
+async function credentials(config, use) {
+	const directory = await mkdtemp(path.join(tmpdir(), "cms-backup-private-"));
+	try {
+		await chmod(directory, 0o700);
+		await writeFile(
+			path.join(directory, "client.cnf"),
+			credentialOptions(config),
+			{ flag: "wx", mode: 0o600 },
+		);
+		return await use(directory);
+	} finally {
+		await rm(directory, { recursive: true, force: true });
+	}
+}
+
+async function sourceClient(directory, program, args, options) {
+	const name = `cms-backup-client-${randomUUID()}`;
+	try {
+		return await docker(
+			[
+				"run",
+				"--rm",
+				"--name",
+				name,
+				"--network",
+				"host",
+				"--mount",
+				`type=bind,src=${directory},dst=/run/backup,readonly`,
+				"--entrypoint",
+				program,
+				IMAGE,
+				"--defaults-file=/run/backup/client.cnf",
+				...args,
+			],
+			options,
+		);
+	} finally {
+		await docker(["rm", "-f", "-v", name], { timeout: 15_000 }).catch(() => {});
+	}
+}
+
+async function inventory(query, database) {
+	const schema = literal(database);
+	const records = await query(
+		`SELECT JSON_OBJECT('kind', TABLE_TYPE, 'name', TABLE_NAME, 'engine', ENGINE) FROM information_schema.TABLES WHERE TABLE_SCHEMA=${schema} UNION ALL SELECT JSON_OBJECT('kind', ROUTINE_TYPE, 'name', ROUTINE_NAME, 'engine', NULL) FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=${schema} UNION ALL SELECT JSON_OBJECT('kind', 'TRIGGER', 'name', TRIGGER_NAME, 'engine', NULL) FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=${schema} UNION ALL SELECT JSON_OBJECT('kind', 'EVENT', 'name', EVENT_NAME, 'engine', NULL) FROM information_schema.EVENTS WHERE EVENT_SCHEMA=${schema}`,
+	);
+	const objects = records
+		.trim()
+		.split("\n")
+		.filter(Boolean)
+		.map((line) => JSON.parse(line));
+	if (
+		objects.some(
+			(object) => object.kind === "BASE TABLE" && object.engine !== "InnoDB",
+		)
+	)
+		throw Error("Only InnoDB tables are supported by this snapshot workflow");
+	objects.sort((a, b) =>
+		`${a.kind}:${a.name}`.localeCompare(`${b.kind}:${b.name}`, "en"),
+	);
+	const tables = [];
+	for (const table of objects.filter(
+		(object) => object.kind === "BASE TABLE",
+	)) {
+		const qualified = `${identifier(database)}.${identifier(table.name)}`;
+		const result = (
+			await query(
+				`SELECT CAST(COUNT(*) AS CHAR) FROM ${qualified}; CHECKSUM TABLE ${qualified} EXTENDED;`,
+			)
+		)
+			.trim()
+			.split("\n");
+		const checksum = result[1]?.split("\t").at(-1)?.trim();
+		if (!/^\d+$/.test(result[0]) || !/^\d+$/.test(checksum ?? ""))
+			throw failure();
+		tables.push({ name: table.name, rows: result[0], checksum });
+	}
+	if (!tables.length)
+		throw Error("The backup database must contain at least one InnoDB table");
+	return {
+		database,
+		objects: objects.map(({ kind, name }) => ({ kind, name })),
+		tables,
+	};
+}
+
+const queryArgs = (sql) => [
+	"--batch",
+	"--raw",
+	"--skip-column-names",
+	"--execute",
+	sql,
+];
+
+export async function createBackup({
+	database,
+	roots,
+	output,
+	writersQuiesced,
+}) {
+	if (writersQuiesced !== true)
+		throw Error(
+			"Pause all database and file writers, then pass --writers-quiesced",
+		);
+	validateDatabase(database);
+	return credentials(database, (directory) =>
+		createArtifact({ roots, output }, async (target) => {
+			const query = (sql) => sourceClient(directory, "mariadb", queryArgs(sql));
+			const before = await inventory(query, database.database);
+			await sourceClient(
+				directory,
+				"mariadb-dump",
+				[
+					"--single-transaction",
+					"--quick",
+					"--skip-lock-tables",
+					"--routines",
+					"--events",
+					"--triggers",
+					"--hex-blob",
+					"--tz-utc",
+					"--skip-comments",
+					"--max-allowed-packet=512M",
+					"--databases",
+					database.database,
+				],
+				{ output: target },
+			);
+			const after = await inventory(query, database.database);
+			if (JSON.stringify(before) !== JSON.stringify(after))
+				throw Error(
+					"Database changed during backup; keep every writer paused and retry",
+				);
+			return before;
+		}),
+	);
+}
+
+export async function drillBackup({ artifact }) {
+	const expected = await verifyArtifact(artifact);
+	const database = expected.database?.database;
+	const password = randomUUID();
+	const config = validateDatabase({
+		host: "127.0.0.1",
+		port: 3306,
+		user: "root",
+		password,
+		database,
+	});
+	return credentials(config, async (directory) => {
+		const name = `cms-backup-drill-${randomUUID()}`;
+		const query = (sql) =>
+			docker([
+				"exec",
+				name,
+				"mariadb",
+				"--defaults-file=/run/backup/client.cnf",
+				...queryArgs(sql),
+			]);
+		let started = false;
+		try {
+			await restoreFiles(artifact, path.join(directory, "restored"));
+			await writeFile(path.join(directory, "password"), password, {
+				flag: "wx",
+				mode: 0o600,
+			});
+			await docker(
+				[
+					"run",
+					"-d",
+					"--name",
+					name,
+					"--label",
+					"cms.backup-drill=true",
+					"--network",
+					"none",
+					"--mount",
+					`type=bind,src=${directory},dst=/run/backup,readonly`,
+					"-e",
+					"MARIADB_ROOT_PASSWORD_FILE=/run/backup/password",
+					IMAGE,
+					"--character-set-server=utf8mb4",
+					"--collation-server=utf8mb4_unicode_ci",
+					"--event-scheduler=OFF",
+					"--max-allowed-packet=512M",
+				],
+				{ timeout: 120_000 },
+			);
+			started = true;
+			let ready = false;
+			for (let attempt = 0; attempt < 90; attempt++) {
+				try {
+					await docker(
+						[
+							"exec",
+							name,
+							"mariadb-admin",
+							"--defaults-file=/run/backup/client.cnf",
+							"ping",
+							"--silent",
+						],
+						{ timeout: 5_000 },
+					);
+					ready = true;
+					break;
+				} catch {
+					await delay(1000);
+				}
+			}
+			if (!ready) throw failure();
+			// Import over stdin; no network, published port or production filesystem mount.
+			await docker(
+				[
+					"exec",
+					"-i",
+					name,
+					"mariadb",
+					"--defaults-file=/run/backup/client.cnf",
+					"--binary-mode",
+				],
+				{ input: path.join(artifact, "database.sql") },
+			);
+			const actual = await inventory(query, database);
+			if (JSON.stringify(actual) !== JSON.stringify(expected.database))
+				throw Error("Restored database inventory does not match the backup");
+			return {
+				verified: true,
+				database: actual,
+				files: expected.entries.filter(
+					(entry) => entry.type === "file" && entry.path.startsWith("files/"),
+				).length,
+			};
+		} finally {
+			const cleanup = docker(["rm", "-f", "-v", name], { timeout: 30_000 });
+			if (started) await cleanup;
+			else await cleanup.catch(() => {});
+		}
+	});
+}

scripts/backup/database.test.mjs

@@ -0,0 +1,43 @@
+import { expect, it } from "vitest";
+import {
+	credentialOptions,
+	dockerEnvironment,
+	validateDatabase,
+} from "./database.mjs";
+
+it("quotes credentials without permitting option-file injection", () => {
+	const config = {
+		host: "127.0.0.1",
+		port: 3306,
+		user: "backup",
+		password: 'quote"slash\\secret',
+		database: "cms",
+	};
+	expect(credentialOptions(config)).toContain(
+		'password="quote\\"slash\\\\secret"',
+	);
+	expect(() =>
+		credentialOptions({ ...config, password: "secret\n[client]" }),
+	).toThrow();
+	expect(() => validateDatabase({ ...config, database: "mysql" })).toThrow();
+	expect(() =>
+		validateDatabase({ ...config, database: "cms; DROP DATABASE other" }),
+	).toThrow();
+});
+
+it("passes only explicit Docker runtime variables to subprocesses", () => {
+	expect(
+		dockerEnvironment({
+			PATH: "/bin",
+			DOCKER_HOST: "unix:///run/docker.sock",
+			DATABASE_URL: "private",
+			MARIADB_PWD: "secret",
+			NODE_OPTIONS: "--inspect",
+			HOME: "/operator",
+		}),
+	).toEqual({
+		PATH: "/bin",
+		DOCKER_HOST: "unix:///run/docker.sock",
+		HOME: "/operator",
+	});
+});