feat(catalog): verify deterministic release manifests before Git export

src/lib/services/catalog-git-core.ts

@@ -4,6 +4,16 @@ import { promises as fs } from "node:fs";
 import { hostname } from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
+import {
+	digestFile,
+	type FileDigest,
+	verifyStagedRelease,
+} from "@/features/catalog/releases/files";
+import {
+	CATALOG_RELEASE_MANIFEST_PATH,
+	type ReleaseManifest,
+	validateReleaseManifest,
+} from "@/features/catalog/releases/manifest";
 import { gitProcessEnvironment } from "./git-process-environment";
 
 const exec = promisify(execFile);
@@ -115,14 +125,14 @@ export async function publishCatalogFiles(options: {
 		const { stdout } = await exec("git", args, {
 			cwd: checkout,
 			timeout: 120_000,
-			maxBuffer: 16 * 1024 * 1024,
+			maxBuffer: (args.includes("ls-files") ? 128 : 16) * 1024 * 1024,
 			env: {
 				...gitProcessEnvironment(),
 				...options.env,
 				GIT_TERMINAL_PROMPT: "0",
 			},
 		});
-		return stdout.trim();
+		return args.includes("-z") ? stdout : stdout.trim();
 	};
 	for (const file of files) catalogTarget(checkout, file.target);
 	if ((await git("remote", "get-url", "origin")) !== remote)
@@ -172,6 +182,33 @@ export async function publishCatalogFiles(options: {
 				...files.slice(i, i + 50).map((f) => f.target),
 			);
 		}
+		// Validate the actual index objects, including transformations by Git filters.
+		const manifestFile = files.find(
+			(file) => file.target === CATALOG_RELEASE_MANIFEST_PATH,
+		);
+		if (manifestFile) {
+			const manifest: ReleaseManifest = JSON.parse(
+				await fs.readFile(manifestFile.source, "utf8"),
+			);
+			const expected = new Map<string, FileDigest>();
+			for (const file of files) {
+				if (expected.has(file.target))
+					throw new Error("Duplicate catalog target");
+				expected.set(file.target, await digestFile(file.source));
+			}
+			if (manifest.files.length !== files.length - 1)
+				throw new Error("Catalog release file set mismatch");
+			await validateReleaseManifest(manifest, async (target) => {
+				const digest = expected.get(target);
+				if (!digest) throw new Error("Catalog release file set mismatch");
+				return digest;
+			});
+			verifyStagedRelease(
+				await git("ls-files", "--stage", "-z"),
+				await git("rev-parse", "--show-object-format"),
+				expected,
+			);
+		}
 		if (await git("diff", "--cached", "--name-only")) {
 			await git(
 				"-c",

src/lib/services/catalog-git-release.test.ts

@@ -0,0 +1,95 @@
+import { execFileSync } from "node:child_process";
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { digestFile } from "@/features/catalog/releases/files";
+import {
+	CATALOG_RELEASE_MANIFEST_PATH,
+	createReleaseManifest,
+} from "@/features/catalog/releases/manifest";
+import { publishCatalogFiles } from "./catalog-git-core";
+
+async function fixture() {
+	const root = await mkdtemp(
+		path.join(os.tmpdir(), "catalog-release-publish-"),
+	);
+	const remote = path.join(root, "remote.git");
+	const checkout = path.join(root, "checkout");
+	const git = (...args: string[]) =>
+		execFileSync("git", args, {
+			cwd: checkout,
+			encoding: "utf8",
+			stdio: ["ignore", "pipe", "pipe"],
+		}).trim();
+	execFileSync("git", ["init", "--bare", remote], { stdio: "ignore" });
+	await mkdir(checkout);
+	git("init", "-b", "catalog");
+	git("config", "core.autocrlf", "false");
+	git("config", "user.name", "Test");
+	git("config", "user.email", "[email protected]");
+	await writeFile(path.join(checkout, "README.md"), "fixture\n");
+	git("add", ".");
+	git("commit", "-m", "Initialize");
+	git("remote", "add", "origin", remote);
+	git("push", "-u", "origin", "catalog");
+	const source = path.join(root, "asset");
+	await writeFile(source, "asset\r\n");
+	const { bytes, sha256 } = await digestFile(source);
+	const manifest = createReleaseManifest([
+		{ path: "Gamedata/icons/chair.png", bytes, sha256 },
+	]);
+	const manifestSource = path.join(root, "manifest.json");
+	await writeFile(manifestSource, JSON.stringify(manifest));
+	const options = {
+		checkout,
+		remote,
+		branch: "catalog",
+		files: [
+			{ source, target: "Gamedata/icons/chair.png" },
+			{ source: manifestSource, target: CATALOG_RELEASE_MANIFEST_PATH },
+		],
+	};
+	return { options, git, manifestSource, source };
+}
+
+describe("catalog release Git publication", () => {
+	it("publishes exact files and does not create another commit for identical data", async () => {
+		const { options, git } = await fixture();
+		const first = await publishCatalogFiles(options);
+		expect(await publishCatalogFiles(options)).toBe(first);
+		expect(git("rev-list", "--count", "HEAD")).toBe("2");
+		expect(
+			await readFile(
+				path.join(options.checkout, "Gamedata/icons/chair.png"),
+				"utf8",
+			),
+		).toBe("asset\r\n");
+	}, 30_000);
+	it("rejects a snapshot changed after manifest creation and restores a clean checkout", async () => {
+		const { options, source, git } = await fixture();
+		const head = git("rev-parse", "HEAD");
+		await writeFile(source, "tampered");
+		await expect(publishCatalogFiles(options)).rejects.toThrow(
+			"bytes mismatch",
+		);
+		expect(git("rev-parse", "HEAD")).toBe(head);
+		expect(git("status", "--porcelain")).toBe("");
+	}, 30_000);
+	it("rejects Git newline normalization before creating a commit", async () => {
+		const { options, git } = await fixture();
+		await writeFile(
+			path.join(options.checkout, ".gitattributes"),
+			"Gamedata/** text eol=lf\n",
+		);
+		git("add", ".gitattributes");
+		git("commit", "-m", "Normalize newlines");
+		git("push");
+		const head = git("rev-parse", "HEAD");
+		await expect(publishCatalogFiles(options)).rejects.toThrow(
+			"Staged catalog bytes mismatch",
+		);
+		expect(git("rev-parse", "HEAD")).toBe(head);
+		expect(git("status", "--porcelain")).toBe("");
+	}, 30_000);
+});

src/lib/services/catalog-git-snapshot.test.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 vi.mock("@/lib/services/figuredata", () => ({
 	getFigureDataPath: async () =>
 		path.join(await mocks.gamedata(), "custom/FigureData.json"),
@@ -125,6 +127,26 @@ describe("catalog snapshot", () => {
 		const root = await fixture();
 		const files = await createCatalogSnapshot(path.join(root, "output"));
 		const targets = files.map((f) => f.target);
+		const manifestFile = files.find(
+			(file) => file.target === "Gamedata/catalog-release.json",
+		);
+		expect(manifestFile).toBeDefined();
+		if (!manifestFile) throw new Error("Missing manifest");
+		const manifest = JSON.parse(await readFile(manifestFile.source, "utf8"));
+		expect(manifest.files).toHaveLength(files.length - 1);
+		expect(
+			manifest.files.some((file: { path: string }) =>
+				file.path.endsWith("items_base.sql"),
+			),
+		).toBe(true);
+		for (const file of files.filter((file) => file !== manifestFile)) {
+			const bytes = await readFile(file.source);
+			expect(manifest.files).toContainEqual({
+				path: file.target,
+				bytes: bytes.length,
+				sha256: createHash("sha256").update(bytes).digest("hex"),
+			});
+		}
 		expect(targets).toContain("Gamedata/bundled/furniture/chair.nitro");
 		expect(targets).toContain("Gamedata/icons/chair_icon.png");
 		expect(targets).toContain("Gamedata/c_images/catalogue/icon_1.png");
@@ -163,4 +185,41 @@ describe("catalog snapshot", () => {
 			createCatalogSnapshot(path.join(root, "output")),
 		).rejects.toThrow();
 	});
+	it("uses one consistent SQL transaction and rejects sources changed during capture", async () => {
+		const root = await fixture();
+		const original = mocks.query.getMockImplementation();
+		if (!original) throw new Error("Missing query fixture");
+		mocks.query.mockImplementation(async (sql: string) => {
+			if (sql.startsWith("SELECT") && sql.includes("items_base")) {
+				await writeFile(
+					path.join(root, "icons/chair_icon.png"),
+					"changed image",
+				);
+			}
+			return original(sql);
+		});
+		await expect(
+			createCatalogSnapshot(path.join(root, "output")),
+		).rejects.toThrow("Assets changed");
+		const queries = mocks.query.mock.calls.map(([sql]) => sql);
+		expect(
+			queries.filter((sql) => sql.startsWith("START TRANSACTION")),
+		).toEqual(["START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"]);
+		expect(queries.filter((sql) => sql.startsWith("SELECT"))).toHaveLength(3);
+	});
+	it("rejects nontransactional tables instead of advertising a consistent database snapshot", async () => {
+		const root = await fixture();
+		const original = mocks.query.getMockImplementation();
+		if (!original) throw new Error("Missing query fixture");
+		mocks.query.mockImplementation(async (sql: string) =>
+			sql.startsWith("SHOW CREATE")
+				? [[{ "Create Table": "CREATE TABLE `test` (`id` int) ENGINE=MyISAM" }]]
+				: original(sql),
+		);
+		await expect(
+			createCatalogSnapshot(path.join(root, "output")),
+		).rejects.toThrow("InnoDB");
+		expect(mocks.rollback).toHaveBeenCalledOnce();
+		expect(mocks.release).toHaveBeenCalledOnce();
+	});
 });

src/lib/services/catalog-git-snapshot.ts

@@ -1,6 +1,14 @@
 import { promises as fs } from "node:fs";
 import path from "node:path";
 import type { RowDataPacket } from "mysql2";
+import {
+	digestFile,
+	sourceFingerprint,
+} from "@/features/catalog/releases/files";
+import {
+	CATALOG_RELEASE_MANIFEST_PATH,
+	createReleaseManifest,
+} from "@/features/catalog/releases/manifest";
 import { db } from "@/lib/db";
 import { getEffectMapPath } from "@/lib/services/effectmap";
 import { getFigureDataPath } from "@/lib/services/figuredata";
@@ -133,16 +141,13 @@ export async function createCatalogSnapshot(
 	}
 	const files: CatalogFile[] = [];
 	const fingerprints = new Map<string, string>();
-	const fingerprint = async (source: string) => {
-		const stat = await fs.lstat(source);
-		if (!stat.isFile() || stat.isSymbolicLink())
-			throw new Error("Invalid catalog source");
-		return `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`;
-	};
+	const copiedHashes = new Map<string, string>();
+	const fingerprint = sourceFingerprint;
 	for (const [target, source] of sources) {
 		fingerprints.set(source, await fingerprint(source));
 		const copy = path.join(directory, String(files.length));
 		await fs.copyFile(source, copy);
+		copiedHashes.set(source, (await digestFile(copy)).sha256);
 		if (target.endsWith(".json")) JSON.parse(await fs.readFile(copy, "utf8"));
 		files.push({ source: copy, target });
 	}
@@ -210,8 +215,22 @@ export async function createCatalogSnapshot(
 		connection.release();
 	}
 	for (const [source, stamp] of fingerprints) {
-		if ((await fingerprint(source)) !== stamp)
+		if (
+			(await fingerprint(source)) !== stamp ||
+			(await digestFile(source)).sha256 !== copiedHashes.get(source)
+		)
 			throw new Error("Assets changed during export; retry required");
 	}
+	const entries = [];
+	for (const file of files) {
+		const { bytes, sha256 } = await digestFile(file.source);
+		entries.push({ path: file.target, bytes, sha256 });
+	}
+	const manifest = createReleaseManifest(entries);
+	const manifestSource = path.join(directory, "catalog-release.json");
+	await fs.writeFile(manifestSource, `${JSON.stringify(manifest, null, 2)}\n`, {
+		flag: "wx",
+	});
+	files.push({ source: manifestSource, target: CATALOG_RELEASE_MANIFEST_PATH });
 	return files;
 }