From a320e47c293ea22243385a31c3260067580aee4a Mon Sep 17 00:00:00 2001 From: simoleo89 Date: Sun, 13 Sep 2026 17:48:23 +0200 Subject: [PATCH] feat(catalog): verify deterministic release manifests before Git export --- src/features/catalog/releases/README.md | 27 ++++++ src/features/catalog/releases/files.test.ts | 43 +++++++++ src/features/catalog/releases/files.ts | 67 +++++++++++++ .../catalog/releases/manifest.test.ts | 56 +++++++++++ src/features/catalog/releases/manifest.ts | 79 +++++++++++++++ src/lib/services/catalog-git-core.ts | 41 +++++++- src/lib/services/catalog-git-release.test.ts | 95 +++++++++++++++++++ src/lib/services/catalog-git-snapshot.test.ts | 59 ++++++++++++ src/lib/services/catalog-git-snapshot.ts | 33 +++++-- 9 files changed, 491 insertions(+), 9 deletions(-) create mode 100644 src/features/catalog/releases/README.md create mode 100644 src/features/catalog/releases/files.test.ts create mode 100644 src/features/catalog/releases/files.ts create mode 100644 src/features/catalog/releases/manifest.test.ts create mode 100644 src/features/catalog/releases/manifest.ts create mode 100644 src/lib/services/catalog-git-release.test.ts diff --git a/src/features/catalog/releases/README.md b/src/features/catalog/releases/README.md new file mode 100644 index 00000000..6cc3639d --- /dev/null +++ b/src/features/catalog/releases/README.md @@ -0,0 +1,27 @@ +# Catalog release manifests + +The Git export writes `Gamedata/catalog-release.json` alongside the existing SQL, +furniture data, icons and Nitro bundles. Its SHA-256 content identity includes a +sorted list of repository-relative paths, byte lengths and SHA-256 hashes. There +is no generated timestamp or local source path. Identical exported bytes produce +identical manifests. + +The manifest describes the files captured by this export, not every file already +in the destination repository. Existing behavior is preserved: missing optional +sources do not delete previously exported files; FurnitureData remains required; +SQL updates existing rows and does not delete absent rows. + +SQL tables retain the existing single InnoDB repeatable-read consistent snapshot. +Asset copies are checked against source metadata and content before publication. +This is not a transaction spanning MySQL and the filesystem: uncoordinated writes +or newly created files outside the export queue can require another export. + +Before committing, the exporter verifies captured files against the manifest and +compares Git index object identities with raw captured bytes, including the +manifest. Git filters and newline conversions that alter staged bytes cause the +export to fail and remain queued for retry. + +This manifest provides provenance for an exported snapshot. It does not activate +a live generation. Live atomic switching additionally requires immutable complete +generations, a shared activation pointer respected by readers, concurrency and +failure handling, and retention/rollback rules. diff --git a/src/features/catalog/releases/files.test.ts b/src/features/catalog/releases/files.test.ts new file mode 100644 index 00000000..cdab59a9 --- /dev/null +++ b/src/features/catalog/releases/files.test.ts @@ -0,0 +1,43 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { digestFile, verifyStagedRelease } from "./files"; + +describe("release file integrity", () => { + it("matches Git raw blob identity and rejects staged bytes changed by filters", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "release-index-")); + const git = (...args: string[]) => + execFileSync("git", args, { cwd: root, encoding: "utf8" }); + git("init"); + git("config", "core.autocrlf", "false"); + await writeFile(path.join(root, "payload"), "first\n"); + const expected = await digestFile(path.join(root, "payload")); + expect(expected.gitSha1).toBe( + git("hash-object", "--no-filters", "payload").trim(), + ); + git("add", "payload"); + const files = new Map([["payload", expected]]); + expect(() => + verifyStagedRelease(git("ls-files", "--stage", "-z"), "sha1", files), + ).not.toThrow(); + await writeFile(path.join(root, "payload"), "other\n"); + git("add", "payload"); + expect(() => + verifyStagedRelease(git("ls-files", "--stage", "-z"), "sha1", files), + ).toThrow("Staged catalog bytes mismatch"); + }); + it("rejects missing and nonregular staged entries", () => { + const digest = { bytes: 0, sha256: "", gitSha1: "abc", gitSha256: "def" }; + const files = new Map([["Gamedata/icons/a.png", digest]]); + expect(() => verifyStagedRelease("", "sha1", files)).toThrow(); + expect(() => + verifyStagedRelease( + "120000 abc 0\tGamedata/icons/a.png\0", + "sha1", + files, + ), + ).toThrow(); + }); +}); diff --git a/src/features/catalog/releases/files.ts b/src/features/catalog/releases/files.ts new file mode 100644 index 00000000..b0957f02 --- /dev/null +++ b/src/features/catalog/releases/files.ts @@ -0,0 +1,67 @@ +import { createHash } from "node:crypto"; +import { createReadStream, promises as fs } from "node:fs"; + +export interface FileDigest { + bytes: number; + sha256: string; + gitSha1: string; + gitSha256: string; +} +export async function sourceFingerprint(source: string): Promise { + const stat = await fs.lstat(source, { bigint: true }); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error("Invalid catalog source"); + return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`; +} +export async function digestFile(source: string): Promise { + const before = await sourceFingerprint(source); + const stat = await fs.stat(source); + const sha256 = createHash("sha256"); + const header = `blob ${stat.size}\0`; + const gitSha1 = createHash("sha1").update(header); + const gitSha256 = createHash("sha256").update(header); + let bytes = 0; + for await (const chunk of createReadStream(source)) { + bytes += chunk.length; + sha256.update(chunk); + gitSha1.update(chunk); + gitSha256.update(chunk); + } + if (bytes !== stat.size || before !== (await sourceFingerprint(source))) + throw new Error("Assets changed during export; retry required"); + return { + bytes, + sha256: sha256.digest("hex"), + gitSha1: gitSha1.digest("hex"), + gitSha256: gitSha256.digest("hex"), + }; +} + +export function verifyStagedRelease( + index: string, + format: string, + expected: Map, +): void { + if (format !== "sha1" && format !== "sha256") + throw new Error("Unsupported Git object format"); + const staged = new Map( + index + .split("\0") + .filter(Boolean) + .map((entry) => { + const tab = entry.indexOf("\t"); + return [entry.slice(tab + 1), entry.slice(0, tab).split(" ")] as const; + }), + ); + for (const [target, digest] of expected) { + const entry = staged.get(target); + const hash = format === "sha1" ? digest.gitSha1 : digest.gitSha256; + if ( + !entry || + !["100644", "100755"].includes(entry[0]) || + entry[1] !== hash || + entry[2] !== "0" + ) + throw new Error(`Staged catalog bytes mismatch: ${target}`); + } +} diff --git a/src/features/catalog/releases/manifest.test.ts b/src/features/catalog/releases/manifest.test.ts new file mode 100644 index 00000000..10391c2a --- /dev/null +++ b/src/features/catalog/releases/manifest.test.ts @@ -0,0 +1,56 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { createReleaseManifest, validateReleaseManifest } from "./manifest"; + +const digest = (value: string) => ({ + bytes: Buffer.byteLength(value), + sha256: createHash("sha256").update(value).digest("hex"), +}); +const entry = (path: string, value = "bytes") => ({ path, ...digest(value) }); + +describe("catalog release manifest", () => { + it("has stable identity regardless of enumeration order and changes when bytes change", () => { + const a = entry("Gamedata/icons/chair.png"); + const b = entry("Gamedata/config/FurnitureData.json", "{}"); + const first = createReleaseManifest([a, b]); + expect(first).toEqual(createReleaseManifest([b, a])); + expect(first.releaseId).toMatch(/^sha256:[a-f0-9]{64}$/); + expect(createReleaseManifest([a, entry(b.path, "[]")]).releaseId).not.toBe( + first.releaseId, + ); + expect(JSON.stringify(first)).not.toContain("createdAt"); + }); + it.each([ + "../escape", + "Gamedata/../escape", + "/absolute", + "Gamedata\\icons\\chair.png", + "Gamedata//chair.png", + "Gamedata/.git/config", + ])("rejects invalid path %s", (target) => { + expect(() => createReleaseManifest([entry(target)])).toThrow(); + }); + it("rejects duplicate paths and invalid hashes", () => { + expect(() => + createReleaseManifest([entry("Gamedata/a"), entry("Gamedata/a")]), + ).toThrow(); + expect(() => + createReleaseManifest([{ ...entry("Gamedata/a"), sha256: "bad" }]), + ).toThrow(); + }); + it("validates content hashes and manifest identity", async () => { + const manifest = createReleaseManifest([entry("Gamedata/a")]); + await expect( + validateReleaseManifest(manifest, async () => digest("bytes")), + ).resolves.toBeUndefined(); + await expect( + validateReleaseManifest(manifest, async () => digest("other")), + ).rejects.toThrow("mismatch"); + await expect( + validateReleaseManifest( + { ...manifest, releaseId: "sha256:bad" }, + async () => digest("bytes"), + ), + ).rejects.toThrow("identity"); + }); +}); diff --git a/src/features/catalog/releases/manifest.ts b/src/features/catalog/releases/manifest.ts new file mode 100644 index 00000000..0b8c132d --- /dev/null +++ b/src/features/catalog/releases/manifest.ts @@ -0,0 +1,79 @@ +import { createHash } from "node:crypto"; + +export const CATALOG_RELEASE_MANIFEST_PATH = "Gamedata/catalog-release.json"; +export interface ReleaseFile { + path: string; + bytes: number; + sha256: string; +} +export interface ReleaseManifest { + schemaVersion: 1; + scope: "exported-files"; + absentFiles: "preserve"; + releaseId: string; + files: ReleaseFile[]; +} + +export function createReleaseManifest(entries: ReleaseFile[]): ReleaseManifest { + const seen = new Set(); + const files = entries + .map(({ path, bytes, sha256 }) => { + if ( + typeof path !== "string" || + !path || + path.startsWith("/") || + /[\\:]/.test(path) || + Array.from(path).some((character) => character.charCodeAt(0) < 32) || + path + .split("/") + .some( + (part) => + !part || + part === "." || + part === ".." || + part.toLowerCase() === ".git", + ) || + path === CATALOG_RELEASE_MANIFEST_PATH || + seen.has(path) + ) + throw new Error("Invalid release path"); + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + !/^[a-f0-9]{64}$/.test(sha256) + ) + throw new Error("Invalid release digest"); + seen.add(path); + return { path, bytes, sha256 }; + }) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + const body = { + schemaVersion: 1 as const, + scope: "exported-files" as const, + absentFiles: "preserve" as const, + files, + }; + return { + ...body, + releaseId: `sha256:${createHash("sha256").update(JSON.stringify(body)).digest("hex")}`, + }; +} + +export async function validateReleaseManifest( + manifest: ReleaseManifest, + readDigest: (path: string) => Promise<{ bytes: number; sha256: string }>, +): Promise { + const expected = createReleaseManifest(manifest.files); + if ( + manifest.schemaVersion !== 1 || + manifest.scope !== expected.scope || + manifest.absentFiles !== expected.absentFiles || + manifest.releaseId !== expected.releaseId + ) + throw new Error("Catalog release identity mismatch"); + for (const file of expected.files) { + const actual = await readDigest(file.path); + if (actual.bytes !== file.bytes || actual.sha256 !== file.sha256) + throw new Error(`Catalog release bytes mismatch: ${file.path}`); + } +} diff --git a/src/lib/services/catalog-git-core.ts b/src/lib/services/catalog-git-core.ts index a8e70299..9e405b02 100644 --- a/src/lib/services/catalog-git-core.ts +++ b/src/lib/services/catalog-git-core.ts @@ -4,6 +4,16 @@ import { promises as fs } from "node:fs"; import { hostname } from "node:os"; import path from "node:path"; import { promisify } from "node:util"; +import { + digestFile, + type FileDigest, + verifyStagedRelease, +} from "@/features/catalog/releases/files"; +import { + CATALOG_RELEASE_MANIFEST_PATH, + type ReleaseManifest, + validateReleaseManifest, +} from "@/features/catalog/releases/manifest"; import { gitProcessEnvironment } from "./git-process-environment"; const exec = promisify(execFile); @@ -115,14 +125,14 @@ export async function publishCatalogFiles(options: { const { stdout } = await exec("git", args, { cwd: checkout, timeout: 120_000, - maxBuffer: 16 * 1024 * 1024, + maxBuffer: (args.includes("ls-files") ? 128 : 16) * 1024 * 1024, env: { ...gitProcessEnvironment(), ...options.env, GIT_TERMINAL_PROMPT: "0", }, }); - return stdout.trim(); + return args.includes("-z") ? stdout : stdout.trim(); }; for (const file of files) catalogTarget(checkout, file.target); if ((await git("remote", "get-url", "origin")) !== remote) @@ -172,6 +182,33 @@ export async function publishCatalogFiles(options: { ...files.slice(i, i + 50).map((f) => f.target), ); } + // Validate the actual index objects, including transformations by Git filters. + const manifestFile = files.find( + (file) => file.target === CATALOG_RELEASE_MANIFEST_PATH, + ); + if (manifestFile) { + const manifest: ReleaseManifest = JSON.parse( + await fs.readFile(manifestFile.source, "utf8"), + ); + const expected = new Map(); + for (const file of files) { + if (expected.has(file.target)) + throw new Error("Duplicate catalog target"); + expected.set(file.target, await digestFile(file.source)); + } + if (manifest.files.length !== files.length - 1) + throw new Error("Catalog release file set mismatch"); + await validateReleaseManifest(manifest, async (target) => { + const digest = expected.get(target); + if (!digest) throw new Error("Catalog release file set mismatch"); + return digest; + }); + verifyStagedRelease( + await git("ls-files", "--stage", "-z"), + await git("rev-parse", "--show-object-format"), + expected, + ); + } if (await git("diff", "--cached", "--name-only")) { await git( "-c", diff --git a/src/lib/services/catalog-git-release.test.ts b/src/lib/services/catalog-git-release.test.ts new file mode 100644 index 00000000..b474bb9b --- /dev/null +++ b/src/lib/services/catalog-git-release.test.ts @@ -0,0 +1,95 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { digestFile } from "@/features/catalog/releases/files"; +import { + CATALOG_RELEASE_MANIFEST_PATH, + createReleaseManifest, +} from "@/features/catalog/releases/manifest"; +import { publishCatalogFiles } from "./catalog-git-core"; + +async function fixture() { + const root = await mkdtemp( + path.join(os.tmpdir(), "catalog-release-publish-"), + ); + const remote = path.join(root, "remote.git"); + const checkout = path.join(root, "checkout"); + const git = (...args: string[]) => + execFileSync("git", args, { + cwd: checkout, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + execFileSync("git", ["init", "--bare", remote], { stdio: "ignore" }); + await mkdir(checkout); + git("init", "-b", "catalog"); + git("config", "core.autocrlf", "false"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.invalid"); + await writeFile(path.join(checkout, "README.md"), "fixture\n"); + git("add", "."); + git("commit", "-m", "Initialize"); + git("remote", "add", "origin", remote); + git("push", "-u", "origin", "catalog"); + const source = path.join(root, "asset"); + await writeFile(source, "asset\r\n"); + const { bytes, sha256 } = await digestFile(source); + const manifest = createReleaseManifest([ + { path: "Gamedata/icons/chair.png", bytes, sha256 }, + ]); + const manifestSource = path.join(root, "manifest.json"); + await writeFile(manifestSource, JSON.stringify(manifest)); + const options = { + checkout, + remote, + branch: "catalog", + files: [ + { source, target: "Gamedata/icons/chair.png" }, + { source: manifestSource, target: CATALOG_RELEASE_MANIFEST_PATH }, + ], + }; + return { options, git, manifestSource, source }; +} + +describe("catalog release Git publication", () => { + it("publishes exact files and does not create another commit for identical data", async () => { + const { options, git } = await fixture(); + const first = await publishCatalogFiles(options); + expect(await publishCatalogFiles(options)).toBe(first); + expect(git("rev-list", "--count", "HEAD")).toBe("2"); + expect( + await readFile( + path.join(options.checkout, "Gamedata/icons/chair.png"), + "utf8", + ), + ).toBe("asset\r\n"); + }, 30_000); + it("rejects a snapshot changed after manifest creation and restores a clean checkout", async () => { + const { options, source, git } = await fixture(); + const head = git("rev-parse", "HEAD"); + await writeFile(source, "tampered"); + await expect(publishCatalogFiles(options)).rejects.toThrow( + "bytes mismatch", + ); + expect(git("rev-parse", "HEAD")).toBe(head); + expect(git("status", "--porcelain")).toBe(""); + }, 30_000); + it("rejects Git newline normalization before creating a commit", async () => { + const { options, git } = await fixture(); + await writeFile( + path.join(options.checkout, ".gitattributes"), + "Gamedata/** text eol=lf\n", + ); + git("add", ".gitattributes"); + git("commit", "-m", "Normalize newlines"); + git("push"); + const head = git("rev-parse", "HEAD"); + await expect(publishCatalogFiles(options)).rejects.toThrow( + "Staged catalog bytes mismatch", + ); + expect(git("rev-parse", "HEAD")).toBe(head); + expect(git("status", "--porcelain")).toBe(""); + }, 30_000); +}); diff --git a/src/lib/services/catalog-git-snapshot.test.ts b/src/lib/services/catalog-git-snapshot.test.ts index b988a70e..c63ca6d6 100644 --- a/src/lib/services/catalog-git-snapshot.test.ts +++ b/src/lib/services/catalog-git-snapshot.test.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + vi.mock("@/lib/services/figuredata", () => ({ getFigureDataPath: async () => path.join(await mocks.gamedata(), "custom/FigureData.json"), @@ -125,6 +127,26 @@ describe("catalog snapshot", () => { const root = await fixture(); const files = await createCatalogSnapshot(path.join(root, "output")); const targets = files.map((f) => f.target); + const manifestFile = files.find( + (file) => file.target === "Gamedata/catalog-release.json", + ); + expect(manifestFile).toBeDefined(); + if (!manifestFile) throw new Error("Missing manifest"); + const manifest = JSON.parse(await readFile(manifestFile.source, "utf8")); + expect(manifest.files).toHaveLength(files.length - 1); + expect( + manifest.files.some((file: { path: string }) => + file.path.endsWith("items_base.sql"), + ), + ).toBe(true); + for (const file of files.filter((file) => file !== manifestFile)) { + const bytes = await readFile(file.source); + expect(manifest.files).toContainEqual({ + path: file.target, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + } expect(targets).toContain("Gamedata/bundled/furniture/chair.nitro"); expect(targets).toContain("Gamedata/icons/chair_icon.png"); expect(targets).toContain("Gamedata/c_images/catalogue/icon_1.png"); @@ -163,4 +185,41 @@ describe("catalog snapshot", () => { createCatalogSnapshot(path.join(root, "output")), ).rejects.toThrow(); }); + it("uses one consistent SQL transaction and rejects sources changed during capture", async () => { + const root = await fixture(); + const original = mocks.query.getMockImplementation(); + if (!original) throw new Error("Missing query fixture"); + mocks.query.mockImplementation(async (sql: string) => { + if (sql.startsWith("SELECT") && sql.includes("items_base")) { + await writeFile( + path.join(root, "icons/chair_icon.png"), + "changed image", + ); + } + return original(sql); + }); + await expect( + createCatalogSnapshot(path.join(root, "output")), + ).rejects.toThrow("Assets changed"); + const queries = mocks.query.mock.calls.map(([sql]) => sql); + expect( + queries.filter((sql) => sql.startsWith("START TRANSACTION")), + ).toEqual(["START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"]); + expect(queries.filter((sql) => sql.startsWith("SELECT"))).toHaveLength(3); + }); + it("rejects nontransactional tables instead of advertising a consistent database snapshot", async () => { + const root = await fixture(); + const original = mocks.query.getMockImplementation(); + if (!original) throw new Error("Missing query fixture"); + mocks.query.mockImplementation(async (sql: string) => + sql.startsWith("SHOW CREATE") + ? [[{ "Create Table": "CREATE TABLE `test` (`id` int) ENGINE=MyISAM" }]] + : original(sql), + ); + await expect( + createCatalogSnapshot(path.join(root, "output")), + ).rejects.toThrow("InnoDB"); + expect(mocks.rollback).toHaveBeenCalledOnce(); + expect(mocks.release).toHaveBeenCalledOnce(); + }); }); diff --git a/src/lib/services/catalog-git-snapshot.ts b/src/lib/services/catalog-git-snapshot.ts index 82d9a247..c77c844f 100644 --- a/src/lib/services/catalog-git-snapshot.ts +++ b/src/lib/services/catalog-git-snapshot.ts @@ -1,6 +1,14 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import type { RowDataPacket } from "mysql2"; +import { + digestFile, + sourceFingerprint, +} from "@/features/catalog/releases/files"; +import { + CATALOG_RELEASE_MANIFEST_PATH, + createReleaseManifest, +} from "@/features/catalog/releases/manifest"; import { db } from "@/lib/db"; import { getEffectMapPath } from "@/lib/services/effectmap"; import { getFigureDataPath } from "@/lib/services/figuredata"; @@ -133,16 +141,13 @@ export async function createCatalogSnapshot( } const files: CatalogFile[] = []; const fingerprints = new Map(); - const fingerprint = async (source: string) => { - const stat = await fs.lstat(source); - if (!stat.isFile() || stat.isSymbolicLink()) - throw new Error("Invalid catalog source"); - return `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; - }; + const copiedHashes = new Map(); + const fingerprint = sourceFingerprint; for (const [target, source] of sources) { fingerprints.set(source, await fingerprint(source)); const copy = path.join(directory, String(files.length)); await fs.copyFile(source, copy); + copiedHashes.set(source, (await digestFile(copy)).sha256); if (target.endsWith(".json")) JSON.parse(await fs.readFile(copy, "utf8")); files.push({ source: copy, target }); } @@ -210,8 +215,22 @@ export async function createCatalogSnapshot( connection.release(); } for (const [source, stamp] of fingerprints) { - if ((await fingerprint(source)) !== stamp) + if ( + (await fingerprint(source)) !== stamp || + (await digestFile(source)).sha256 !== copiedHashes.get(source) + ) throw new Error("Assets changed during export; retry required"); } + const entries = []; + for (const file of files) { + const { bytes, sha256 } = await digestFile(file.source); + entries.push({ path: file.target, bytes, sha256 }); + } + const manifest = createReleaseManifest(entries); + const manifestSource = path.join(directory, "catalog-release.json"); + await fs.writeFile(manifestSource, `${JSON.stringify(manifest, null, 2)}\n`, { + flag: "wx", + }); + files.push({ source: manifestSource, target: CATALOG_RELEASE_MANIFEST_PATH }); return files; }