feat(catalog): verify deterministic release manifests before Git export

src/features/catalog/releases/README.md

@@ -0,0 +1,27 @@
+# Catalog release manifests
+
+The Git export writes `Gamedata/catalog-release.json` alongside the existing SQL,
+furniture data, icons and Nitro bundles. Its SHA-256 content identity includes a
+sorted list of repository-relative paths, byte lengths and SHA-256 hashes. There
+is no generated timestamp or local source path. Identical exported bytes produce
+identical manifests.
+
+The manifest describes the files captured by this export, not every file already
+in the destination repository. Existing behavior is preserved: missing optional
+sources do not delete previously exported files; FurnitureData remains required;
+SQL updates existing rows and does not delete absent rows.
+
+SQL tables retain the existing single InnoDB repeatable-read consistent snapshot.
+Asset copies are checked against source metadata and content before publication.
+This is not a transaction spanning MySQL and the filesystem: uncoordinated writes
+or newly created files outside the export queue can require another export.
+
+Before committing, the exporter verifies captured files against the manifest and
+compares Git index object identities with raw captured bytes, including the
+manifest. Git filters and newline conversions that alter staged bytes cause the
+export to fail and remain queued for retry.
+
+This manifest provides provenance for an exported snapshot. It does not activate
+a live generation. Live atomic switching additionally requires immutable complete
+generations, a shared activation pointer respected by readers, concurrency and
+failure handling, and retention/rollback rules.

src/features/catalog/releases/files.test.ts

@@ -0,0 +1,43 @@
+import { execFileSync } from "node:child_process";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { digestFile, verifyStagedRelease } from "./files";
+
+describe("release file integrity", () => {
+	it("matches Git raw blob identity and rejects staged bytes changed by filters", async () => {
+		const root = await mkdtemp(path.join(os.tmpdir(), "release-index-"));
+		const git = (...args: string[]) =>
+			execFileSync("git", args, { cwd: root, encoding: "utf8" });
+		git("init");
+		git("config", "core.autocrlf", "false");
+		await writeFile(path.join(root, "payload"), "first\n");
+		const expected = await digestFile(path.join(root, "payload"));
+		expect(expected.gitSha1).toBe(
+			git("hash-object", "--no-filters", "payload").trim(),
+		);
+		git("add", "payload");
+		const files = new Map([["payload", expected]]);
+		expect(() =>
+			verifyStagedRelease(git("ls-files", "--stage", "-z"), "sha1", files),
+		).not.toThrow();
+		await writeFile(path.join(root, "payload"), "other\n");
+		git("add", "payload");
+		expect(() =>
+			verifyStagedRelease(git("ls-files", "--stage", "-z"), "sha1", files),
+		).toThrow("Staged catalog bytes mismatch");
+	});
+	it("rejects missing and nonregular staged entries", () => {
+		const digest = { bytes: 0, sha256: "", gitSha1: "abc", gitSha256: "def" };
+		const files = new Map([["Gamedata/icons/a.png", digest]]);
+		expect(() => verifyStagedRelease("", "sha1", files)).toThrow();
+		expect(() =>
+			verifyStagedRelease(
+				"120000 abc 0\tGamedata/icons/a.png\0",
+				"sha1",
+				files,
+			),
+		).toThrow();
+	});
+});

src/features/catalog/releases/files.ts

@@ -0,0 +1,67 @@
+import { createHash } from "node:crypto";
+import { createReadStream, promises as fs } from "node:fs";
+
+export interface FileDigest {
+	bytes: number;
+	sha256: string;
+	gitSha1: string;
+	gitSha256: string;
+}
+export async function sourceFingerprint(source: string): Promise<string> {
+	const stat = await fs.lstat(source, { bigint: true });
+	if (!stat.isFile() || stat.isSymbolicLink())
+		throw new Error("Invalid catalog source");
+	return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
+}
+export async function digestFile(source: string): Promise<FileDigest> {
+	const before = await sourceFingerprint(source);
+	const stat = await fs.stat(source);
+	const sha256 = createHash("sha256");
+	const header = `blob ${stat.size}\0`;
+	const gitSha1 = createHash("sha1").update(header);
+	const gitSha256 = createHash("sha256").update(header);
+	let bytes = 0;
+	for await (const chunk of createReadStream(source)) {
+		bytes += chunk.length;
+		sha256.update(chunk);
+		gitSha1.update(chunk);
+		gitSha256.update(chunk);
+	}
+	if (bytes !== stat.size || before !== (await sourceFingerprint(source)))
+		throw new Error("Assets changed during export; retry required");
+	return {
+		bytes,
+		sha256: sha256.digest("hex"),
+		gitSha1: gitSha1.digest("hex"),
+		gitSha256: gitSha256.digest("hex"),
+	};
+}
+
+export function verifyStagedRelease(
+	index: string,
+	format: string,
+	expected: Map<string, FileDigest>,
+): void {
+	if (format !== "sha1" && format !== "sha256")
+		throw new Error("Unsupported Git object format");
+	const staged = new Map(
+		index
+			.split("\0")
+			.filter(Boolean)
+			.map((entry) => {
+				const tab = entry.indexOf("\t");
+				return [entry.slice(tab + 1), entry.slice(0, tab).split(" ")] as const;
+			}),
+	);
+	for (const [target, digest] of expected) {
+		const entry = staged.get(target);
+		const hash = format === "sha1" ? digest.gitSha1 : digest.gitSha256;
+		if (
+			!entry ||
+			!["100644", "100755"].includes(entry[0]) ||
+			entry[1] !== hash ||
+			entry[2] !== "0"
+		)
+			throw new Error(`Staged catalog bytes mismatch: ${target}`);
+	}
+}

src/features/catalog/releases/manifest.test.ts

@@ -0,0 +1,56 @@
+import { createHash } from "node:crypto";
+import { describe, expect, it } from "vitest";
+import { createReleaseManifest, validateReleaseManifest } from "./manifest";
+
+const digest = (value: string) => ({
+	bytes: Buffer.byteLength(value),
+	sha256: createHash("sha256").update(value).digest("hex"),
+});
+const entry = (path: string, value = "bytes") => ({ path, ...digest(value) });
+
+describe("catalog release manifest", () => {
+	it("has stable identity regardless of enumeration order and changes when bytes change", () => {
+		const a = entry("Gamedata/icons/chair.png");
+		const b = entry("Gamedata/config/FurnitureData.json", "{}");
+		const first = createReleaseManifest([a, b]);
+		expect(first).toEqual(createReleaseManifest([b, a]));
+		expect(first.releaseId).toMatch(/^sha256:[a-f0-9]{64}$/);
+		expect(createReleaseManifest([a, entry(b.path, "[]")]).releaseId).not.toBe(
+			first.releaseId,
+		);
+		expect(JSON.stringify(first)).not.toContain("createdAt");
+	});
+	it.each([
+		"../escape",
+		"Gamedata/../escape",
+		"/absolute",
+		"Gamedata\\icons\\chair.png",
+		"Gamedata//chair.png",
+		"Gamedata/.git/config",
+	])("rejects invalid path %s", (target) => {
+		expect(() => createReleaseManifest([entry(target)])).toThrow();
+	});
+	it("rejects duplicate paths and invalid hashes", () => {
+		expect(() =>
+			createReleaseManifest([entry("Gamedata/a"), entry("Gamedata/a")]),
+		).toThrow();
+		expect(() =>
+			createReleaseManifest([{ ...entry("Gamedata/a"), sha256: "bad" }]),
+		).toThrow();
+	});
+	it("validates content hashes and manifest identity", async () => {
+		const manifest = createReleaseManifest([entry("Gamedata/a")]);
+		await expect(
+			validateReleaseManifest(manifest, async () => digest("bytes")),
+		).resolves.toBeUndefined();
+		await expect(
+			validateReleaseManifest(manifest, async () => digest("other")),
+		).rejects.toThrow("mismatch");
+		await expect(
+			validateReleaseManifest(
+				{ ...manifest, releaseId: "sha256:bad" },
+				async () => digest("bytes"),
+			),
+		).rejects.toThrow("identity");
+	});
+});

src/features/catalog/releases/manifest.ts

@@ -0,0 +1,79 @@
+import { createHash } from "node:crypto";
+
+export const CATALOG_RELEASE_MANIFEST_PATH = "Gamedata/catalog-release.json";
+export interface ReleaseFile {
+	path: string;
+	bytes: number;
+	sha256: string;
+}
+export interface ReleaseManifest {
+	schemaVersion: 1;
+	scope: "exported-files";
+	absentFiles: "preserve";
+	releaseId: string;
+	files: ReleaseFile[];
+}
+
+export function createReleaseManifest(entries: ReleaseFile[]): ReleaseManifest {
+	const seen = new Set<string>();
+	const files = entries
+		.map(({ path, bytes, sha256 }) => {
+			if (
+				typeof path !== "string" ||
+				!path ||
+				path.startsWith("/") ||
+				/[\\:]/.test(path) ||
+				Array.from(path).some((character) => character.charCodeAt(0) < 32) ||
+				path
+					.split("/")
+					.some(
+						(part) =>
+							!part ||
+							part === "." ||
+							part === ".." ||
+							part.toLowerCase() === ".git",
+					) ||
+				path === CATALOG_RELEASE_MANIFEST_PATH ||
+				seen.has(path)
+			)
+				throw new Error("Invalid release path");
+			if (
+				!Number.isSafeInteger(bytes) ||
+				bytes < 0 ||
+				!/^[a-f0-9]{64}$/.test(sha256)
+			)
+				throw new Error("Invalid release digest");
+			seen.add(path);
+			return { path, bytes, sha256 };
+		})
+		.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+	const body = {
+		schemaVersion: 1 as const,
+		scope: "exported-files" as const,
+		absentFiles: "preserve" as const,
+		files,
+	};
+	return {
+		...body,
+		releaseId: `sha256:${createHash("sha256").update(JSON.stringify(body)).digest("hex")}`,
+	};
+}
+
+export async function validateReleaseManifest(
+	manifest: ReleaseManifest,
+	readDigest: (path: string) => Promise<{ bytes: number; sha256: string }>,
+): Promise<void> {
+	const expected = createReleaseManifest(manifest.files);
+	if (
+		manifest.schemaVersion !== 1 ||
+		manifest.scope !== expected.scope ||
+		manifest.absentFiles !== expected.absentFiles ||
+		manifest.releaseId !== expected.releaseId
+	)
+		throw new Error("Catalog release identity mismatch");
+	for (const file of expected.files) {
+		const actual = await readDigest(file.path);
+		if (actual.bytes !== file.bytes || actual.sha256 !== file.sha256)
+			throw new Error(`Catalog release bytes mismatch: ${file.path}`);
+	}
+}

src/lib/services/catalog-git-core.ts

@@ -4,6 +4,16 @@ import { promises as fs } from "node:fs";
 import { hostname } from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
+import {
+	digestFile,
+	type FileDigest,
+	verifyStagedRelease,
+} from "@/features/catalog/releases/files";
+import {
+	CATALOG_RELEASE_MANIFEST_PATH,
+	type ReleaseManifest,
+	validateReleaseManifest,
+} from "@/features/catalog/releases/manifest";
 import { gitProcessEnvironment } from "./git-process-environment";
 
 const exec = promisify(execFile);
@@ -115,14 +125,14 @@ export async function publishCatalogFiles(options: {
 		const { stdout } = await exec("git", args, {
 			cwd: checkout,
 			timeout: 120_000,
-			maxBuffer: 16 * 1024 * 1024,
+			maxBuffer: (args.includes("ls-files") ? 128 : 16) * 1024 * 1024,
 			env: {
 				...gitProcessEnvironment(),
 				...options.env,
 				GIT_TERMINAL_PROMPT: "0",
 			},
 		});
-		return stdout.trim();
+		return args.includes("-z") ? stdout : stdout.trim();
 	};
 	for (const file of files) catalogTarget(checkout, file.target);
 	if ((await git("remote", "get-url", "origin")) !== remote)
@@ -172,6 +182,33 @@ export async function publishCatalogFiles(options: {
 				...files.slice(i, i + 50).map((f) => f.target),
 			);
 		}
+		// Validate the actual index objects, including transformations by Git filters.
+		const manifestFile = files.find(
+			(file) => file.target === CATALOG_RELEASE_MANIFEST_PATH,
+		);
+		if (manifestFile) {
+			const manifest: ReleaseManifest = JSON.parse(
+				await fs.readFile(manifestFile.source, "utf8"),
+			);
+			const expected = new Map<string, FileDigest>();
+			for (const file of files) {
+				if (expected.has(file.target))
+					throw new Error("Duplicate catalog target");
+				expected.set(file.target, await digestFile(file.source));
+			}
+			if (manifest.files.length !== files.length - 1)
+				throw new Error("Catalog release file set mismatch");
+			await validateReleaseManifest(manifest, async (target) => {
+				const digest = expected.get(target);
+				if (!digest) throw new Error("Catalog release file set mismatch");
+				return digest;
+			});
+			verifyStagedRelease(
+				await git("ls-files", "--stage", "-z"),
+				await git("rev-parse", "--show-object-format"),
+				expected,
+			);
+		}
 		if (await git("diff", "--cached", "--name-only")) {
 			await git(
 				"-c",

src/lib/services/catalog-git-release.test.ts

@@ -0,0 +1,95 @@
+import { execFileSync } from "node:child_process";
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { digestFile } from "@/features/catalog/releases/files";
+import {
+	CATALOG_RELEASE_MANIFEST_PATH,
+	createReleaseManifest,
+} from "@/features/catalog/releases/manifest";
+import { publishCatalogFiles } from "./catalog-git-core";
+
+async function fixture() {
+	const root = await mkdtemp(
+		path.join(os.tmpdir(), "catalog-release-publish-"),
+	);
+	const remote = path.join(root, "remote.git");
+	const checkout = path.join(root, "checkout");
+	const git = (...args: string[]) =>
+		execFileSync("git", args, {
+			cwd: checkout,
+			encoding: "utf8",
+			stdio: ["ignore", "pipe", "pipe"],
+		}).trim();
+	execFileSync("git", ["init", "--bare", remote], { stdio: "ignore" });
+	await mkdir(checkout);
+	git("init", "-b", "catalog");
+	git("config", "core.autocrlf", "false");
+	git("config", "user.name", "Test");
+	git("config", "user.email", "[email protected]");
+	await writeFile(path.join(checkout, "README.md"), "fixture\n");
+	git("add", ".");
+	git("commit", "-m", "Initialize");
+	git("remote", "add", "origin", remote);
+	git("push", "-u", "origin", "catalog");
+	const source = path.join(root, "asset");
+	await writeFile(source, "asset\r\n");
+	const { bytes, sha256 } = await digestFile(source);
+	const manifest = createReleaseManifest([
+		{ path: "Gamedata/icons/chair.png", bytes, sha256 },
+	]);
+	const manifestSource = path.join(root, "manifest.json");
+	await writeFile(manifestSource, JSON.stringify(manifest));
+	const options = {
+		checkout,
+		remote,
+		branch: "catalog",
+		files: [
+			{ source, target: "Gamedata/icons/chair.png" },
+			{ source: manifestSource, target: CATALOG_RELEASE_MANIFEST_PATH },
+		],
+	};
+	return { options, git, manifestSource, source };
+}
+
+describe("catalog release Git publication", () => {
+	it("publishes exact files and does not create another commit for identical data", async () => {
+		const { options, git } = await fixture();
+		const first = await publishCatalogFiles(options);
+		expect(await publishCatalogFiles(options)).toBe(first);
+		expect(git("rev-list", "--count", "HEAD")).toBe("2");
+		expect(
+			await readFile(
+				path.join(options.checkout, "Gamedata/icons/chair.png"),
+				"utf8",
+			),
+		).toBe("asset\r\n");
+	}, 30_000);
+	it("rejects a snapshot changed after manifest creation and restores a clean checkout", async () => {
+		const { options, source, git } = await fixture();
+		const head = git("rev-parse", "HEAD");
+		await writeFile(source, "tampered");
+		await expect(publishCatalogFiles(options)).rejects.toThrow(
+			"bytes mismatch",
+		);
+		expect(git("rev-parse", "HEAD")).toBe(head);
+		expect(git("status", "--porcelain")).toBe("");
+	}, 30_000);
+	it("rejects Git newline normalization before creating a commit", async () => {
+		const { options, git } = await fixture();
+		await writeFile(
+			path.join(options.checkout, ".gitattributes"),
+			"Gamedata/** text eol=lf\n",
+		);
+		git("add", ".gitattributes");
+		git("commit", "-m", "Normalize newlines");
+		git("push");
+		const head = git("rev-parse", "HEAD");
+		await expect(publishCatalogFiles(options)).rejects.toThrow(
+			"Staged catalog bytes mismatch",
+		);
+		expect(git("rev-parse", "HEAD")).toBe(head);
+		expect(git("status", "--porcelain")).toBe("");
+	}, 30_000);
+});

src/lib/services/catalog-git-snapshot.test.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 vi.mock("@/lib/services/figuredata", () => ({
 	getFigureDataPath: async () =>
 		path.join(await mocks.gamedata(), "custom/FigureData.json"),
@@ -125,6 +127,26 @@ describe("catalog snapshot", () => {
 		const root = await fixture();
 		const files = await createCatalogSnapshot(path.join(root, "output"));
 		const targets = files.map((f) => f.target);
+		const manifestFile = files.find(
+			(file) => file.target === "Gamedata/catalog-release.json",
+		);
+		expect(manifestFile).toBeDefined();
+		if (!manifestFile) throw new Error("Missing manifest");
+		const manifest = JSON.parse(await readFile(manifestFile.source, "utf8"));
+		expect(manifest.files).toHaveLength(files.length - 1);
+		expect(
+			manifest.files.some((file: { path: string }) =>
+				file.path.endsWith("items_base.sql"),
+			),
+		).toBe(true);
+		for (const file of files.filter((file) => file !== manifestFile)) {
+			const bytes = await readFile(file.source);
+			expect(manifest.files).toContainEqual({
+				path: file.target,
+				bytes: bytes.length,
+				sha256: createHash("sha256").update(bytes).digest("hex"),
+			});
+		}
 		expect(targets).toContain("Gamedata/bundled/furniture/chair.nitro");
 		expect(targets).toContain("Gamedata/icons/chair_icon.png");
 		expect(targets).toContain("Gamedata/c_images/catalogue/icon_1.png");
@@ -163,4 +185,41 @@ describe("catalog snapshot", () => {
 			createCatalogSnapshot(path.join(root, "output")),
 		).rejects.toThrow();
 	});
+	it("uses one consistent SQL transaction and rejects sources changed during capture", async () => {
+		const root = await fixture();
+		const original = mocks.query.getMockImplementation();
+		if (!original) throw new Error("Missing query fixture");
+		mocks.query.mockImplementation(async (sql: string) => {
+			if (sql.startsWith("SELECT") && sql.includes("items_base")) {
+				await writeFile(
+					path.join(root, "icons/chair_icon.png"),
+					"changed image",
+				);
+			}
+			return original(sql);
+		});
+		await expect(
+			createCatalogSnapshot(path.join(root, "output")),
+		).rejects.toThrow("Assets changed");
+		const queries = mocks.query.mock.calls.map(([sql]) => sql);
+		expect(
+			queries.filter((sql) => sql.startsWith("START TRANSACTION")),
+		).toEqual(["START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"]);
+		expect(queries.filter((sql) => sql.startsWith("SELECT"))).toHaveLength(3);
+	});
+	it("rejects nontransactional tables instead of advertising a consistent database snapshot", async () => {
+		const root = await fixture();
+		const original = mocks.query.getMockImplementation();
+		if (!original) throw new Error("Missing query fixture");
+		mocks.query.mockImplementation(async (sql: string) =>
+			sql.startsWith("SHOW CREATE")
+				? [[{ "Create Table": "CREATE TABLE `test` (`id` int) ENGINE=MyISAM" }]]
+				: original(sql),
+		);
+		await expect(
+			createCatalogSnapshot(path.join(root, "output")),
+		).rejects.toThrow("InnoDB");
+		expect(mocks.rollback).toHaveBeenCalledOnce();
+		expect(mocks.release).toHaveBeenCalledOnce();
+	});
 });

src/lib/services/catalog-git-snapshot.ts

@@ -1,6 +1,14 @@
 import { promises as fs } from "node:fs";
 import path from "node:path";
 import type { RowDataPacket } from "mysql2";
+import {
+	digestFile,
+	sourceFingerprint,
+} from "@/features/catalog/releases/files";
+import {
+	CATALOG_RELEASE_MANIFEST_PATH,
+	createReleaseManifest,
+} from "@/features/catalog/releases/manifest";
 import { db } from "@/lib/db";
 import { getEffectMapPath } from "@/lib/services/effectmap";
 import { getFigureDataPath } from "@/lib/services/figuredata";
@@ -133,16 +141,13 @@ export async function createCatalogSnapshot(
 	}
 	const files: CatalogFile[] = [];
 	const fingerprints = new Map<string, string>();
-	const fingerprint = async (source: string) => {
-		const stat = await fs.lstat(source);
-		if (!stat.isFile() || stat.isSymbolicLink())
-			throw new Error("Invalid catalog source");
-		return `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`;
-	};
+	const copiedHashes = new Map<string, string>();
+	const fingerprint = sourceFingerprint;
 	for (const [target, source] of sources) {
 		fingerprints.set(source, await fingerprint(source));
 		const copy = path.join(directory, String(files.length));
 		await fs.copyFile(source, copy);
+		copiedHashes.set(source, (await digestFile(copy)).sha256);
 		if (target.endsWith(".json")) JSON.parse(await fs.readFile(copy, "utf8"));
 		files.push({ source: copy, target });
 	}
@@ -210,8 +215,22 @@ export async function createCatalogSnapshot(
 		connection.release();
 	}
 	for (const [source, stamp] of fingerprints) {
-		if ((await fingerprint(source)) !== stamp)
+		if (
+			(await fingerprint(source)) !== stamp ||
+			(await digestFile(source)).sha256 !== copiedHashes.get(source)
+		)
 			throw new Error("Assets changed during export; retry required");
 	}
+	const entries = [];
+	for (const file of files) {
+		const { bytes, sha256 } = await digestFile(file.source);
+		entries.push({ path: file.target, bytes, sha256 });
+	}
+	const manifest = createReleaseManifest(entries);
+	const manifestSource = path.join(directory, "catalog-release.json");
+	await fs.writeFile(manifestSource, `${JSON.stringify(manifest, null, 2)}\n`, {
+		flag: "wx",
+	});
+	files.push({ source: manifestSource, target: CATALOG_RELEASE_MANIFEST_PATH });
 	return files;
 }