fix(housekeeping): make rank synchronization recoverable

src/features/housekeeping/domains/system/services/rank-mutations-production.test.ts

@@ -9,24 +9,39 @@ import type { HousekeepingCapabilityContext } from "../../../foundation/contract
 
 const doubles = vi.hoisted(() => ({
 	createEmulatorRank: vi.fn(),
+	createEmulatorRankInTransaction: vi.fn(),
+	dbDelete: vi.fn(),
 	dbExecute: vi.fn(),
 	dbInsert: vi.fn(),
 	dbSelect: vi.fn(),
 	dbTransaction: vi.fn(),
 	dbUpdate: vi.fn(),
 	deleteEmulatorRank: vi.fn(),
+	deleteEmulatorRankInTransaction: vi.fn(),
+	logAudit: vi.fn(),
 	logStaffActivity: vi.fn(),
+	logStaffActivityInTransaction: vi.fn(),
+	prepareEmulatorRankCreation: vi.fn(),
 	rconSend: vi.fn(),
 	rconSetRank: vi.fn(),
 	updateEmulatorRank: vi.fn(),
 }));
 
+const transactionToken = {
+	delete: doubles.dbDelete,
+	execute: doubles.dbExecute,
+	insert: doubles.dbInsert,
+	select: doubles.dbSelect,
+	update: doubles.dbUpdate,
+};
+
 vi.mock("@/lib/db", async (importOriginal) => {
 	const actual = await importOriginal<typeof import("@/lib/db")>();
 	return {
 		...actual,
 		db: {
 			...actual.db,
+			delete: doubles.dbDelete,
 			execute: doubles.dbExecute,
 			insert: doubles.dbInsert,
 			select: doubles.dbSelect,
@@ -36,9 +51,16 @@ vi.mock("@/lib/db", async (importOriginal) => {
 	};
 });
 
+vi.mock("@/lib/services/audit", () => ({
+	logAudit: doubles.logAudit,
+}));
+
 vi.mock("@/lib/services/permission-ranks", () => ({
 	createEmulatorRank: doubles.createEmulatorRank,
+	createEmulatorRankInTransaction: doubles.createEmulatorRankInTransaction,
 	deleteEmulatorRank: doubles.deleteEmulatorRank,
+	deleteEmulatorRankInTransaction: doubles.deleteEmulatorRankInTransaction,
+	prepareEmulatorRankCreation: doubles.prepareEmulatorRankCreation,
 	updateEmulatorRank: doubles.updateEmulatorRank,
 }));
 
@@ -48,6 +70,7 @@ vi.mock("@/lib/services/rcon", () => ({
 
 vi.mock("@/lib/services/staff-activity", () => ({
 	logStaffActivity: doubles.logStaffActivity,
+	logStaffActivityInTransaction: doubles.logStaffActivityInTransaction,
 }));
 
 import { createSystemCommands } from "../commands/system-commands";
@@ -73,6 +96,7 @@ function serviceContext(permission: string) {
 	return {
 		capability: capabilityContext([permission]),
 		correlationId: "rank-mutation-correlation",
+		reason: "Approved rank lifecycle change",
 	};
 }
 
@@ -84,10 +108,21 @@ function insertChain() {
 	};
 }
 
+function deleteChain() {
+	return { where: vi.fn().mockResolvedValue(undefined) };
+}
+
+function updateChain(result: Promise<unknown> = Promise.resolve(undefined)) {
+	return { set: () => ({ where: () => result }) };
+}
+
 function limitedSelection(rows: readonly unknown[]) {
 	return {
 		from: () => ({
-			where: () => ({ limit: () => Promise.resolve(rows) }),
+			where: () => ({
+				for: () => Promise.resolve(rows),
+				limit: () => Promise.resolve(rows),
+			}),
 		}),
 	};
 }
@@ -101,14 +136,72 @@ function plainSelection(rows: readonly unknown[]) {
 beforeEach(() => {
 	for (const mock of Object.values(doubles)) mock.mockReset();
 	doubles.createEmulatorRank.mockResolvedValue(7);
+	doubles.prepareEmulatorRankCreation.mockResolvedValue(7);
+	doubles.createEmulatorRankInTransaction.mockResolvedValue(undefined);
 	doubles.deleteEmulatorRank.mockResolvedValue(undefined);
+	doubles.deleteEmulatorRankInTransaction.mockResolvedValue(undefined);
 	doubles.updateEmulatorRank.mockResolvedValue(undefined);
+	doubles.logAudit.mockResolvedValue(undefined);
 	doubles.logStaffActivity.mockResolvedValue(undefined);
+	doubles.logStaffActivityInTransaction.mockResolvedValue(undefined);
+	doubles.dbDelete.mockImplementation(deleteChain);
 	doubles.dbInsert.mockImplementation(insertChain);
+	doubles.dbTransaction.mockImplementation(async (run) =>
+		run(transactionToken),
+	);
+	doubles.dbUpdate.mockImplementation(() => updateChain());
 });
 
-describe("rank synchronization failures", () => {
-	it("returns failure and audits failure when create committed but updatepermissions returned false", async () => {
+function expectPendingSynchronization(
+	result: unknown,
+	operation:
+		| "access.rank.create"
+		| "access.rank.delete"
+		| "access.rank.update"
+		| "rcon.set-rank",
+) {
+	expect(result).toMatchObject({
+		ok: true,
+		data: {
+			operation,
+			recoveryId: expect.any(String),
+			completed: ["database", "audit"],
+		},
+		completion: {
+			status: "partial",
+			external: "failed",
+			audit: "persisted",
+		},
+	});
+	if (typeof result === "object" && result !== null && "ok" in result) {
+		const typed = result as {
+			ok: boolean;
+			correlationId?: string;
+			data?: { recoveryId?: string };
+		};
+		if (typed.ok) expect(typed.data?.recoveryId).toBe(typed.correlationId);
+	}
+}
+
+describe("durable rank synchronization", () => {
+	it("rolls back before RCON when the transaction-bound recovery intent cannot be audited", async () => {
+		doubles.logAudit.mockRejectedValueOnce(new Error("audit unavailable"));
+		doubles.rconSend.mockResolvedValue(true);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.PERMISSIONS_MANAGE),
+			"access.rank.create",
+			{ name: "Administrator", level: 7 },
+		);
+
+		expect(result).toMatchObject({
+			ok: false,
+			error: { code: "DEPENDENCY_UNAVAILABLE" },
+		});
+		expect(doubles.rconSend).not.toHaveBeenCalled();
+	});
+
+	it("commits create, ACL, staff audit, and recovery intent together before returning a retry key", async () => {
 		doubles.rconSend.mockResolvedValue(false);
 		const command = createSystemCommands(systemMutationService).find(
 			(entry) => entry.id === "system.access.rank.create",
@@ -137,19 +230,24 @@ describe("rank synchronization failures", () => {
 			},
 		);
 
-		expect(result).toMatchObject({
-			ok: false,
-			error: {
-				code: "DEPENDENCY_UNAVAILABLE",
-				messageKey: "errors.housekeeping.system.rankSynchronizationIncomplete",
-				fieldErrors: {
-					operation: ["access.rank.create"],
-					completed: ["database", "audit"],
-					pending: ["emulator-permission-cache"],
-				},
-			},
-		});
-		expect(outcomes).toEqual(["intent", "failure"]);
+		expectPendingSynchronization(result, "access.rank.create");
+		expect(outcomes).toEqual(["intent", "partial"]);
+		expect(doubles.createEmulatorRankInTransaction).toHaveBeenCalledWith(
+			transactionToken,
+			7,
+			{ rank_name: "Administrator", level: 7 },
+		);
+		expect(doubles.logStaffActivityInTransaction).toHaveBeenCalledWith(
+			expect.objectContaining({ action: "rank_create", targetId: 7 }),
+			transactionToken,
+		);
+		expect(doubles.logAudit).toHaveBeenCalledWith(
+			expect.objectContaining({
+				correlationId: expect.any(String),
+				outcome: "intent",
+			}),
+			transactionToken,
+		);
 	});
 
 	it.each([
@@ -161,11 +259,17 @@ describe("rank synchronization failures", () => {
 					.mockImplementationOnce(() => plainSelection([{ total: 0 }]))
 					.mockImplementationOnce(() => limitedSelection([]));
 			},
+			"deleteEmulatorRankInTransaction",
+		],
+		[
+			"access.rank.update",
+			{ id: 7, fields: { badge: "ADM" } },
+			() => {},
+			"updateEmulatorRank",
 		],
-		["access.rank.update", { id: 7, fields: { badge: "ADM" } }, () => {}],
 	] as const)(
-		"returns failure when %s committed but updatepermissions returned false",
-		async (operation, input, prepare) => {
+		"returns a recoverable partial after atomic %s commit when cache refresh fails",
+		async (operation, input, prepare, helperName) => {
 			prepare();
 			doubles.rconSend.mockResolvedValue(false);
 
@@ -175,33 +279,23 @@ describe("rank synchronization failures", () => {
 				input,
 			);
 
-			expect(result).toMatchObject({
-				ok: false,
-				error: {
-					code: "DEPENDENCY_UNAVAILABLE",
-					messageKey:
-						"errors.housekeeping.system.rankSynchronizationIncomplete",
-					fieldErrors: {
-						operation: [operation],
-						completed: ["database", "audit"],
-						pending: ["emulator-permission-cache"],
-					},
-				},
-			});
+			expectPendingSynchronization(result, operation);
+			expect(doubles[helperName]).toHaveBeenCalledWith(
+				transactionToken,
+				7,
+				...(operation === "access.rank.update" ? [{ badge: "ADM" }] : []),
+			);
 		},
 	);
 
-	it("reports external completion when set-rank RCON succeeded but DB persistence failed", async () => {
+	it("never calls RCON when set-rank database persistence rolls back", async () => {
 		doubles.dbSelect.mockImplementationOnce(() =>
 			limitedSelection([{ rank: 3 }]),
 		);
 		doubles.dbExecute.mockResolvedValue([[{ id: 4 }]]);
-		doubles.rconSetRank.mockResolvedValue(true);
-		doubles.dbUpdate.mockReturnValue({
-			set: () => ({
-				where: () => Promise.reject(new Error("users update unavailable")),
-			}),
-		});
+		doubles.dbUpdate.mockImplementationOnce(() =>
+			updateChain(Promise.reject(new Error("users update unavailable"))),
+		);
 
 		const result = await systemMutationService.execute(
 			serviceContext(PERMS.RCON_EXECUTE),
@@ -211,15 +305,124 @@ describe("rank synchronization failures", () => {
 
 		expect(result).toMatchObject({
 			ok: false,
-			error: {
-				code: "DEPENDENCY_UNAVAILABLE",
-				messageKey: "errors.housekeeping.system.rankPersistenceIncomplete",
-				fieldErrors: {
-					operation: ["rcon.set-rank"],
-					completed: ["emulator"],
-					pending: ["database"],
+			error: { code: "DEPENDENCY_UNAVAILABLE" },
+		});
+		expect(doubles.rconSetRank).not.toHaveBeenCalled();
+	});
+
+	it("returns the durable recovery key when set-rank RCON fails after commit", async () => {
+		doubles.dbSelect.mockImplementationOnce(() =>
+			limitedSelection([{ rank: 3 }]),
+		);
+		doubles.dbExecute.mockResolvedValue([[{ id: 4 }]]);
+		doubles.rconSetRank.mockResolvedValue(false);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.RCON_EXECUTE),
+			"rcon.set-rank",
+			{ userId: 8, rank: 4 },
+		);
+
+		expectPendingSynchronization(result, "rcon.set-rank");
+		expect(doubles.dbUpdate).toHaveBeenCalledBefore(doubles.rconSetRank);
+	});
+
+	it("retries only the convergent cache refresh, never the original rank mutation", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([
+			[
+				{
+					after: JSON.stringify({
+						kind: "rank-permission-cache",
+						operation: "access.rank.create",
+						rankId: 7,
+					}),
+					outcome: "partial",
 				},
+			],
+		]);
+		doubles.rconSend.mockResolvedValue(true);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.PERMISSIONS_MANAGE),
+			"access.rank.sync.retry" as SystemMutationOperation,
+			{ recoveryId: "original-rank-correlation" },
+		);
+
+		expect(result).toMatchObject({
+			ok: true,
+			data: {
+				recoveryId: "original-rank-correlation",
+				synchronization: "completed",
 			},
 		});
+		expect(doubles.rconSend).toHaveBeenCalledWith("updatepermissions");
+		expect(doubles.createEmulatorRank).not.toHaveBeenCalled();
+		expect(doubles.createEmulatorRankInTransaction).not.toHaveBeenCalled();
+		expect(doubles.deleteEmulatorRank).not.toHaveBeenCalled();
+		expect(doubles.deleteEmulatorRankInTransaction).not.toHaveBeenCalled();
+		expect(doubles.updateEmulatorRank).not.toHaveBeenCalled();
+	});
+
+	it("retries set-rank from its committed desired-state intent without another database mutation", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([
+			[
+				{
+					after: JSON.stringify({
+						kind: "set-rank",
+						operation: "rcon.set-rank",
+						userId: 8,
+						rank: 4,
+					}),
+					outcome: "intent",
+				},
+			],
+		]);
+		doubles.rconSetRank.mockResolvedValue(true);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.RCON_EXECUTE),
+			"rcon.set-rank.retry" as SystemMutationOperation,
+			{ recoveryId: "original-set-rank-correlation" },
+		);
+
+		expect(result).toMatchObject({
+			ok: true,
+			data: {
+				recoveryId: "original-set-rank-correlation",
+				synchronization: "completed",
+			},
+		});
+		expect(doubles.rconSetRank).toHaveBeenCalledWith(8, 4);
+		expect(doubles.dbUpdate).not.toHaveBeenCalled();
+	});
+
+	it("does not repeat an already completed external synchronization", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([
+			[
+				{
+					after: JSON.stringify({
+						kind: "rank-permission-cache",
+						operation: "access.rank.update",
+						rankId: 7,
+					}),
+					outcome: "success",
+				},
+			],
+		]);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.PERMISSIONS_MANAGE),
+			"access.rank.sync.retry",
+			{ recoveryId: "completed-rank-correlation" },
+		);
+
+		expect(result).toMatchObject({
+			ok: true,
+			data: {
+				alreadyCompleted: true,
+				recoveryId: "completed-rank-correlation",
+			},
+		});
+		expect(doubles.rconSend).not.toHaveBeenCalled();
 	});
 });