fix(housekeeping): make rank synchronization recoverable

src/features/housekeeping/domains/system/commands/system-commands.test.ts

@@ -17,6 +17,7 @@ const expectedCommandIds = [
 	"system.access.rank.create",
 	"system.access.rank.delete",
 	"system.access.rank.update",
+	"system.access.rank.sync-retry",
 	"system.access.permissions.update",
 	"system.access.permissions.repair",
 	"system.configuration.settings.save",
@@ -40,6 +41,7 @@ const expectedCommandIds = [
 	"system.operations.rcon.give-badge",
 	"system.operations.rcon.set-motto",
 	"system.operations.rcon.set-rank",
+	"system.operations.rcon.set-rank-retry",
 	"system.operations.rcon.execute-command",
 	"system.operations.rcon.send-gift",
 	"system.operations.maintenance.update",
@@ -165,6 +167,7 @@ describe("SYSTEM_COMMANDS", () => {
 				capability: capabilityContext([PERMS.RCON_EXECUTE]),
 				correlationId: "command-correlation",
 				ipAddress: "198.51.100.8",
+				reason: "Approved credit correction",
 			},
 			parsed,
 		);
@@ -172,6 +175,7 @@ describe("SYSTEM_COMMANDS", () => {
 		expect(result).toMatchObject({
 			ok: true,
 			data: {
+				context: { reason: "Approved credit correction" },
 				operation: "rcon.give-credits",
 				input: { userId: 7, amount: 25 },
 			},

src/features/housekeeping/domains/system/commands/system-commands.ts

@@ -14,6 +14,7 @@ export const SYSTEM_COMMAND_IDS = [
 	"system.access.rank.create",
 	"system.access.rank.delete",
 	"system.access.rank.update",
+	"system.access.rank.sync-retry",
 	"system.access.permissions.update",
 	"system.access.permissions.repair",
 	"system.configuration.settings.save",
@@ -37,6 +38,7 @@ export const SYSTEM_COMMAND_IDS = [
 	"system.operations.rcon.give-badge",
 	"system.operations.rcon.set-motto",
 	"system.operations.rcon.set-rank",
+	"system.operations.rcon.set-rank-retry",
 	"system.operations.rcon.execute-command",
 	"system.operations.rcon.send-gift",
 	"system.operations.maintenance.update",
@@ -68,6 +70,7 @@ function systemCommand<I>(
 				{
 					capability: context.capability,
 					correlationId: context.correlationId,
+					reason: context.reason,
 				},
 				options.operation,
 				input,
@@ -111,6 +114,14 @@ export function createSystemCommands(
 			}),
 			requiresReason: true,
 		}),
+		systemCommand(service, {
+			id: "system.access.rank.sync-retry",
+			operation: "access.rank.sync.retry",
+			capability: PERMS.PERMISSIONS_MANAGE,
+			input: z.object({ recoveryId: requiredText(64) }),
+			requiresReason: true,
+			attempts: 5,
+		}),
 		systemCommand(service, {
 			id: "system.access.permissions.update",
 			operation: "access.permissions.update",
@@ -278,6 +289,14 @@ export function createSystemCommands(
 			requiresReason: true,
 			attempts: 5,
 		}),
+		systemCommand(service, {
+			id: "system.operations.rcon.set-rank-retry",
+			operation: "rcon.set-rank.retry",
+			capability: PERMS.RCON_EXECUTE,
+			input: z.object({ recoveryId: requiredText(64) }),
+			requiresReason: true,
+			attempts: 5,
+		}),
 		systemCommand(service, {
 			id: "system.operations.rcon.execute-command",
 			operation: "rcon.execute-command",

src/features/housekeeping/domains/system/services/mutations.ts

@@ -20,16 +20,21 @@ import {
 	normalizeHabboGamedataHotel,
 } from "@/lib/habbo-gamedata-hotel";
 import { PERMS } from "@/lib/permission-slugs";
+import { logAudit } from "@/lib/services/audit";
 import { clearOfficialHabboFurnidataCache } from "@/lib/services/habbo-furnidata-cache";
 import { clearBadgeCache } from "@/lib/services/habboassets";
 import {
-	createEmulatorRank,
+	createEmulatorRankInTransaction,
-	deleteEmulatorRank,
+	deleteEmulatorRankInTransaction,
+	prepareEmulatorRankCreation,
 	updateEmulatorRank,
 } from "@/lib/services/permission-ranks";
 import { rcon } from "@/lib/services/rcon";
 import { siteSettings } from "@/lib/services/site-settings";
-import { logStaffActivity } from "@/lib/services/staff-activity";
+import {
+	logStaffActivity,
+	logStaffActivityInTransaction,
+} from "@/lib/services/staff-activity";
 import { satisfiesCapability } from "../../../foundation/capability-context";
 import {
 	anyCapability,
@@ -44,6 +49,7 @@ export type SystemMutationOperation =
 	| "access.rank.create"
 	| "access.rank.delete"
 	| "access.rank.update"
+	| "access.rank.sync.retry"
 	| "access.permissions.update"
 	| "access.permissions.repair"
 	| "configuration.settings.save"
@@ -67,6 +73,7 @@ export type SystemMutationOperation =
 	| "rcon.give-badge"
 	| "rcon.set-motto"
 	| "rcon.set-rank"
+	| "rcon.set-rank.retry"
 	| "rcon.execute-command"
 	| "rcon.send-gift"
 	| "operations.maintenance.update";
@@ -74,6 +81,7 @@ export type SystemMutationOperation =
 export interface SystemMutationContext {
 	readonly capability: HousekeepingCapabilityContext;
 	readonly correlationId: string;
+	readonly reason?: string;
 }
 
 export interface SystemMutationAdapter {
@@ -103,6 +111,19 @@ class SystemMutationFailure extends Error {
 	}
 }
 
+class SystemCommittedExternalFailure extends Error {
+	constructor(
+		readonly data: unknown,
+		readonly external: "completed" | "failed",
+		readonly audit: "persisted" | "unavailable",
+	) {
+		super(
+			"System database change committed but external synchronization is incomplete",
+		);
+		this.name = "SystemCommittedExternalFailure";
+	}
+}
+
 function operationCapability(operation: SystemMutationOperation) {
 	if (operation.startsWith("access.")) {
 		return anyCapability(PERMS.PERMISSIONS_MANAGE);
@@ -143,6 +164,13 @@ export function createSystemMutationService(
 					context.correlationId,
 				);
 			} catch (error) {
+				if (error instanceof SystemCommittedExternalFailure) {
+					return ok(error.data, context.correlationId, {
+						status: "partial",
+						external: error.external,
+						audit: error.audit,
+					});
+				}
 				if (error instanceof SystemMutationFailure) {
 					return fail(
 						error.code,
@@ -208,23 +236,225 @@ async function requireRcon(result: boolean): Promise<void> {
 	}
 }
 
-async function requireRankPermissionSynchronization(
+const SYSTEM_EXTERNAL_SYNC_ACTION = "system.external-sync";
-	operation: "access.rank.create" | "access.rank.delete" | "access.rank.update",
+
-): Promise<void> {
+type RankLifecycleOperation =
+	| "access.rank.create"
+	| "access.rank.delete"
+	| "access.rank.update";
+
+type SystemExternalSyncIntent =
+	| {
+			readonly kind: "rank-permission-cache";
+			readonly operation: RankLifecycleOperation;
+			readonly rankId: number;
+	  }
+	| {
+			readonly kind: "set-rank";
+			readonly operation: "rcon.set-rank";
+			readonly userId: number;
+			readonly rank: number;
+	  };
+
+function syncTargetId(intent: SystemExternalSyncIntent): number {
+	return intent.kind === "set-rank" ? intent.userId : intent.rankId;
+}
+
+function pendingExternal(intent: SystemExternalSyncIntent): string {
+	return intent.kind === "set-rank"
+		? "emulator-user-rank"
+		: "emulator-permission-cache";
+}
+
+function syncAuditEntry(
+	intent: SystemExternalSyncIntent,
+	context: SystemMutationContext,
+	recoveryId: string,
+	outcome: "intent" | "success" | "partial",
+) {
+	return {
+		userId: context.capability.actor.id,
+		action: SYSTEM_EXTERNAL_SYNC_ACTION,
+		target: intent.operation,
+		targetId: syncTargetId(intent),
+		correlationId: recoveryId,
+		outcome,
+		reason: context.reason,
+		domain: "system" as const,
+		after: { ...intent },
+	};
+}
+
+function synchronizationData(
+	intent: SystemExternalSyncIntent,
+	recoveryId: string,
+	synchronization: "completed" | "pending",
+	extra: Readonly<Record<string, unknown>> = {},
+) {
+	return {
+		...extra,
+		operation: intent.operation,
+		recoveryId,
+		synchronization,
+		completed:
+			synchronization === "completed"
+				? ["database", "audit", pendingExternal(intent)]
+				: ["database", "audit"],
+		pending: synchronization === "completed" ? [] : [pendingExternal(intent)],
+	};
+}
+
+async function executeExternalSynchronization(
+	intent: SystemExternalSyncIntent,
+): Promise<boolean> {
+	return intent.kind === "set-rank"
+		? rcon.setRank(intent.userId, intent.rank)
+		: rcon.send("updatepermissions");
+}
+
+async function completeExternalSynchronization(
+	intent: SystemExternalSyncIntent,
+	context: SystemMutationContext,
+	recoveryId = context.correlationId,
+	extra: Readonly<Record<string, unknown>> = {},
+): Promise<unknown> {
+	let synchronized = false;
 	try {
-		if (await rcon.send("updatepermissions")) return;
+		synchronized = await executeExternalSynchronization(intent);
 	} catch {
-		// Report the already committed local changes through the typed envelope.
+		// The committed intent below remains the retry source of truth.
 	}
-	throw new SystemMutationFailure(
+
-		"DEPENDENCY_UNAVAILABLE",
+	if (!synchronized) {
-		"errors.housekeeping.system.rankSynchronizationIncomplete",
+		try {
-		{
+			await logAudit(syncAuditEntry(intent, context, recoveryId, "partial"));
-			operation: [operation],
+		} catch {
-			completed: ["database", "audit"],
+			// The transactionally persisted intent is sufficient for recovery.
-			pending: ["emulator-permission-cache"],
+		}
-		},
+		throw new SystemCommittedExternalFailure(
-	);
+			synchronizationData(intent, recoveryId, "pending", extra),
+			"failed",
+			"persisted",
+		);
+	}
+
+	try {
+		await logAudit(syncAuditEntry(intent, context, recoveryId, "success"));
+	} catch {
+		throw new SystemCommittedExternalFailure(
+			{
+				...synchronizationData(intent, recoveryId, "completed", extra),
+				pending: ["completion-audit"],
+			},
+			"completed",
+			"unavailable",
+		);
+	}
+
+	return synchronizationData(intent, recoveryId, "completed", extra);
+}
+
+function parseExternalSyncIntent(
+	value: unknown,
+): SystemExternalSyncIntent | null {
+	let parsed: unknown = value;
+	if (typeof value === "string") {
+		try {
+			parsed = JSON.parse(value);
+		} catch {
+			return null;
+		}
+	}
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+		return null;
+	}
+	const candidate = parsed as Record<string, unknown>;
+	if (
+		candidate.kind === "rank-permission-cache" &&
+		(candidate.operation === "access.rank.create" ||
+			candidate.operation === "access.rank.delete" ||
+			candidate.operation === "access.rank.update") &&
+		Number.isInteger(candidate.rankId) &&
+		Number(candidate.rankId) > 0
+	) {
+		return {
+			kind: candidate.kind,
+			operation: candidate.operation,
+			rankId: Number(candidate.rankId),
+		};
+	}
+	if (
+		candidate.kind === "set-rank" &&
+		candidate.operation === "rcon.set-rank" &&
+		Number.isInteger(candidate.userId) &&
+		Number(candidate.userId) > 0 &&
+		Number.isInteger(candidate.rank) &&
+		Number(candidate.rank) > 0
+	) {
+		return {
+			kind: candidate.kind,
+			operation: candidate.operation,
+			userId: Number(candidate.userId),
+			rank: Number(candidate.rank),
+		};
+	}
+	return null;
+}
+
+async function loadExternalSyncIntent(recoveryId: string): Promise<{
+	readonly intent: SystemExternalSyncIntent;
+	readonly completed: boolean;
+} | null> {
+	const [result] = await db.execute(sql`
+		SELECT after, outcome
+		FROM admin_audit_log
+		WHERE correlation_id = ${recoveryId}
+			AND action = ${SYSTEM_EXTERNAL_SYNC_ACTION}
+		ORDER BY id DESC
+		LIMIT 1
+	`);
+	const row = (result as unknown as { after: unknown; outcome: string }[])[0];
+	const intent = parseExternalSyncIntent(row?.after);
+	return intent ? { intent, completed: row.outcome === "success" } : null;
+}
+
+function recoveryIdentifier(value: unknown): string {
+	const recoveryId = text(value, 64);
+	if (!recoveryId) {
+		throw new SystemMutationFailure(
+			"VALIDATION",
+			"errors.housekeeping.validation",
+		);
+	}
+	return recoveryId;
+}
+
+async function retryExternalSynchronization(
+	expectedKind: SystemExternalSyncIntent["kind"],
+	input: Record<string, unknown>,
+	context: SystemMutationContext,
+): Promise<unknown> {
+	const recoveryId = recoveryIdentifier(input.recoveryId);
+	const recovered = await loadExternalSyncIntent(recoveryId);
+	if (!recovered) {
+		throw new SystemMutationFailure(
+			"NOT_FOUND",
+			"errors.housekeeping.notFound",
+		);
+	}
+	if (recovered.intent.kind !== expectedKind) {
+		throw new SystemMutationFailure(
+			"CONFLICT",
+			"errors.housekeeping.validation",
+		);
+	}
+	if (recovered.completed) {
+		return {
+			...synchronizationData(recovered.intent, recoveryId, "completed"),
+			alreadyCompleted: true,
+		};
+	}
+	return completeExternalSynchronization(recovered.intent, context, recoveryId);
 }
 
 async function upsertWebsiteSetting(
@@ -244,51 +474,76 @@ async function executeAccessMutation(
 	context: SystemMutationContext,
 ): Promise<unknown> {
 	const data = record(input);
+	if (operation === "access.rank.sync.retry") {
+		return retryExternalSynchronization("rank-permission-cache", data, context);
+	}
 	if (operation === "access.rank.create") {
 		const name = text(data.name, 25);
 		const level = positiveInteger(data.level);
-		const id = await createEmulatorRank(db, { rank_name: name, level });
+		const id = await prepareEmulatorRankCreation(db);
-		await db
+		const intent = {
-			.insert(AclRole)
+			kind: "rank-permission-cache",
-			.values({
+			operation,
-				slug: `rank_${id}`,
+			rankId: id,
-				title: name,
+		} as const satisfies SystemExternalSyncIntent;
-				description: "CMS role synchronized from permission_ranks",
+		await db.transaction(async (tx) => {
-			})
+			await createEmulatorRankInTransaction(tx, id, {
-			.onDuplicateKeyUpdate({ set: { title: name } });
+				rank_name: name,
-		await logStaffActivity({
+				level,
-			staffId: context.capability.actor.id,
+			});
-			action: "rank_create",
+			await tx
-			description: `Created rank #${id}`,
+				.insert(AclRole)
-			targetType: "rank",
+				.values({
-			targetId: id,
+					slug: `rank_${id}`,
+					title: name,
+					description: "CMS role synchronized from permission_ranks",
+				})
+				.onDuplicateKeyUpdate({ set: { title: name } });
+			await logStaffActivityInTransaction(
+				{
+					staffId: context.capability.actor.id,
+					action: "rank_create",
+					description: `Created rank #${id}`,
+					targetType: "rank",
+					targetId: id,
+				},
+				tx,
+			);
+			await logAudit(
+				syncAuditEntry(intent, context, context.correlationId, "intent"),
+				tx,
+			);
 		});
-		await requireRankPermissionSynchronization("access.rank.create");
+		return completeExternalSynchronization(intent, context, undefined, { id });
-		return { id };
 	}
 
 	const id = positiveInteger(data.id);
 	if (operation === "access.rank.delete") {
-		const [userCount] = await db
+		const intent = {
-			.select({ total: count() })
+			kind: "rank-permission-cache",
-			.from(User)
+			operation,
-			.where(eq(User.rank, id));
+			rankId: id,
-		const users = Number(userCount?.total ?? 0);
+		} as const satisfies SystemExternalSyncIntent;
-		if (users > 0) {
+		await db.transaction(async (tx) => {
-			throw new SystemMutationFailure(
+			const [userCount] = await tx
-				"CONFLICT",
+				.select({ total: count() })
-				"errors.housekeeping.system.rankInUse",
+				.from(User)
-				{ rank: [String(users)] },
+				.where(eq(User.rank, id));
-			);
+			const users = Number(userCount?.total ?? 0);
-		}
+			if (users > 0) {
-		const [role] = await db
+				throw new SystemMutationFailure(
-			.select({ id: AclRole.id })
+					"CONFLICT",
-			.from(AclRole)
+					"errors.housekeeping.system.rankInUse",
-			.where(eq(AclRole.slug, `rank_${id}`))
+					{ rank: [String(users)] },
-			.limit(1);
+				);
-		await deleteEmulatorRank(db, id);
+			}
-		if (role) {
+			const [role] = await tx
-			await db.transaction(async (tx) => {
+				.select({ id: AclRole.id })
+				.from(AclRole)
+				.where(eq(AclRole.slug, `rank_${id}`))
+				.limit(1);
+			await deleteEmulatorRankInTransaction(tx, id);
+			if (role) {
 				await tx
 					.delete(AclModelPermission)
 					.where(
@@ -299,17 +554,23 @@ async function executeAccessMutation(
 					);
 				await tx.delete(AclModelRole).where(eq(AclModelRole.roleId, role.id));
 				await tx.delete(AclRole).where(eq(AclRole.id, role.id));
-			});
+			}
-		}
+			await logStaffActivityInTransaction(
-		await logStaffActivity({
+				{
-			staffId: context.capability.actor.id,
+					staffId: context.capability.actor.id,
-			action: "rank_delete",
+					action: "rank_delete",
-			description: `Deleted rank #${id}`,
+					description: `Deleted rank #${id}`,
-			targetType: "rank",
+					targetType: "rank",
-			targetId: id,
+					targetId: id,
+				},
+				tx,
+			);
+			await logAudit(
+				syncAuditEntry(intent, context, context.correlationId, "intent"),
+				tx,
+			);
 		});
-		await requireRankPermissionSynchronization("access.rank.delete");
+		return completeExternalSynchronization(intent, context, undefined, { id });
-		return null;
 	}
 
 	if (operation === "access.rank.update") {
@@ -321,22 +582,35 @@ async function executeAccessMutation(
 					: [],
 			),
 		);
-		await updateEmulatorRank(db, id, normalizedFields);
+		const intent = {
-		if (typeof normalizedFields.rank_name === "string") {
+			kind: "rank-permission-cache",
-			await db
+			operation,
-				.update(AclRole)
+			rankId: id,
-				.set({ title: normalizedFields.rank_name })
+		} as const satisfies SystemExternalSyncIntent;
-				.where(eq(AclRole.slug, `rank_${id}`));
+		await db.transaction(async (tx) => {
-		}
+			await updateEmulatorRank(tx, id, normalizedFields);
-		await logStaffActivity({
+			if (typeof normalizedFields.rank_name === "string") {
-			staffId: context.capability.actor.id,
+				await tx
-			action: "rank_update",
+					.update(AclRole)
-			description: `Updated rank #${id}`,
+					.set({ title: normalizedFields.rank_name })
-			targetType: "rank",
+					.where(eq(AclRole.slug, `rank_${id}`));
-			targetId: id,
+			}
+			await logStaffActivityInTransaction(
+				{
+					staffId: context.capability.actor.id,
+					action: "rank_update",
+					description: `Updated rank #${id}`,
+					targetType: "rank",
+					targetId: id,
+				},
+				tx,
+			);
+			await logAudit(
+				syncAuditEntry(intent, context, context.correlationId, "intent"),
+				tx,
+			);
 		});
-		await requireRankPermissionSynchronization("access.rank.update");
+		return completeExternalSynchronization(intent, context, undefined, { id });
-		return null;
 	}
 
 	if (operation === "access.permissions.update") {
@@ -632,65 +906,79 @@ async function executeRconMutation(
 				),
 			);
 			break;
+		case "rcon.set-rank.retry":
+			return retryExternalSynchronization("set-rank", data, context);
 		case "rcon.set-rank": {
 			const userId = positiveInteger(data.userId);
 			const rank = positiveInteger(data.rank);
-			const [target] = await db
+			const intent = {
-				.select({ rank: User.rank })
+				kind: "set-rank",
-				.from(User)
+				operation,
-				.where(eq(User.id, userId))
+				userId,
-				.limit(1);
+				rank,
-			if (!target) {
+			} as const satisfies SystemExternalSyncIntent;
-				throw new SystemMutationFailure(
+			await db.transaction(async (tx) => {
-					"NOT_FOUND",
+				const [target] = await tx
-					"errors.housekeeping.system.userNotFound",
+					.select({ rank: User.rank })
-				);
+					.from(User)
-			}
+					.where(eq(User.id, userId))
-			let rankRows: { id: number }[] = [];
+					.limit(1);
-			try {
+				if (!target) {
-				const [rows] = await db.execute(
-					sql`SELECT id FROM permission_ranks WHERE id = ${rank} LIMIT 1`,
-				);
-				rankRows = rows as unknown as { id: number }[];
-			} catch {
-				rankRows = [];
-			}
-			if (rankRows.length === 0) {
-				throw new SystemMutationFailure(
-					"NOT_FOUND",
-					"errors.housekeeping.system.rankNotFound",
-				);
-			}
-			if (!context.capability.isSuperAdmin) {
-				const actorRank = context.capability.actor.rank;
-				if (target.rank >= actorRank) {
 					throw new SystemMutationFailure(
-						"FORBIDDEN",
+						"NOT_FOUND",
-						"errors.housekeeping.system.cannotChangePeerRank",
+						"errors.housekeeping.system.userNotFound",
 					);
 				}
-				if (rank >= actorRank) {
+				let rankRows: { id: number }[] = [];
+				try {
+					const [rows] = await tx.execute(
+						sql`SELECT id FROM permission_ranks WHERE id = ${rank} LIMIT 1 FOR UPDATE`,
+					);
+					rankRows = rows as unknown as { id: number }[];
+				} catch {
+					rankRows = [];
+				}
+				if (rankRows.length === 0) {
 					throw new SystemMutationFailure(
-						"FORBIDDEN",
+						"NOT_FOUND",
-						"errors.housekeeping.system.cannotAssignPeerRank",
+						"errors.housekeeping.system.rankNotFound",
 					);
 				}
-			}
+				if (!context.capability.isSuperAdmin) {
-			await requireRcon(await rcon.setRank(userId, rank));
+					const actorRank = context.capability.actor.rank;
-			try {
+					if (target.rank >= actorRank) {
-				await db.update(User).set({ rank }).where(eq(User.id, userId));
+						throw new SystemMutationFailure(
-			} catch {
+							"FORBIDDEN",
-				throw new SystemMutationFailure(
+							"errors.housekeeping.system.cannotChangePeerRank",
-					"DEPENDENCY_UNAVAILABLE",
+						);
-					"errors.housekeeping.system.rankPersistenceIncomplete",
+					}
+					if (rank >= actorRank) {
+						throw new SystemMutationFailure(
+							"FORBIDDEN",
+							"errors.housekeeping.system.cannotAssignPeerRank",
+						);
+					}
+				}
+				await tx.update(User).set({ rank }).where(eq(User.id, userId));
+				await logStaffActivityInTransaction(
 					{
-						operation: ["rcon.set-rank"],
+						staffId: context.capability.actor.id,
-						completed: ["emulator"],
+						action: "rank_assign",
-						pending: ["database"],
+						description: `Assigned rank #${rank} to user #${userId}`,
+						targetType: "user",
+						targetId: userId,
 					},
+					tx,
 				);
-			}
+				await logAudit(
-			break;
+					syncAuditEntry(intent, context, context.correlationId, "intent"),
+					tx,
+				);
+			});
+			return completeExternalSynchronization(intent, context, undefined, {
+				userId,
+				rank,
+			});
 		}
 		case "rcon.execute-command":
 			await requireRcon(

src/features/housekeeping/domains/system/services/rank-mutations-production.test.ts

@@ -9,24 +9,39 @@ import type { HousekeepingCapabilityContext } from "../../../foundation/contract
 
 const doubles = vi.hoisted(() => ({
 	createEmulatorRank: vi.fn(),
+	createEmulatorRankInTransaction: vi.fn(),
+	dbDelete: vi.fn(),
 	dbExecute: vi.fn(),
 	dbInsert: vi.fn(),
 	dbSelect: vi.fn(),
 	dbTransaction: vi.fn(),
 	dbUpdate: vi.fn(),
 	deleteEmulatorRank: vi.fn(),
+	deleteEmulatorRankInTransaction: vi.fn(),
+	logAudit: vi.fn(),
 	logStaffActivity: vi.fn(),
+	logStaffActivityInTransaction: vi.fn(),
+	prepareEmulatorRankCreation: vi.fn(),
 	rconSend: vi.fn(),
 	rconSetRank: vi.fn(),
 	updateEmulatorRank: vi.fn(),
 }));
 
+const transactionToken = {
+	delete: doubles.dbDelete,
+	execute: doubles.dbExecute,
+	insert: doubles.dbInsert,
+	select: doubles.dbSelect,
+	update: doubles.dbUpdate,
+};
+
 vi.mock("@/lib/db", async (importOriginal) => {
 	const actual = await importOriginal<typeof import("@/lib/db")>();
 	return {
 		...actual,
 		db: {
 			...actual.db,
+			delete: doubles.dbDelete,
 			execute: doubles.dbExecute,
 			insert: doubles.dbInsert,
 			select: doubles.dbSelect,
@@ -36,9 +51,16 @@ vi.mock("@/lib/db", async (importOriginal) => {
 	};
 });
 
+vi.mock("@/lib/services/audit", () => ({
+	logAudit: doubles.logAudit,
+}));
+
 vi.mock("@/lib/services/permission-ranks", () => ({
 	createEmulatorRank: doubles.createEmulatorRank,
+	createEmulatorRankInTransaction: doubles.createEmulatorRankInTransaction,
 	deleteEmulatorRank: doubles.deleteEmulatorRank,
+	deleteEmulatorRankInTransaction: doubles.deleteEmulatorRankInTransaction,
+	prepareEmulatorRankCreation: doubles.prepareEmulatorRankCreation,
 	updateEmulatorRank: doubles.updateEmulatorRank,
 }));
 
@@ -48,6 +70,7 @@ vi.mock("@/lib/services/rcon", () => ({
 
 vi.mock("@/lib/services/staff-activity", () => ({
 	logStaffActivity: doubles.logStaffActivity,
+	logStaffActivityInTransaction: doubles.logStaffActivityInTransaction,
 }));
 
 import { createSystemCommands } from "../commands/system-commands";
@@ -73,6 +96,7 @@ function serviceContext(permission: string) {
 	return {
 		capability: capabilityContext([permission]),
 		correlationId: "rank-mutation-correlation",
+		reason: "Approved rank lifecycle change",
 	};
 }
 
@@ -84,10 +108,21 @@ function insertChain() {
 	};
 }
 
+function deleteChain() {
+	return { where: vi.fn().mockResolvedValue(undefined) };
+}
+
+function updateChain(result: Promise<unknown> = Promise.resolve(undefined)) {
+	return { set: () => ({ where: () => result }) };
+}
+
 function limitedSelection(rows: readonly unknown[]) {
 	return {
 		from: () => ({
-			where: () => ({ limit: () => Promise.resolve(rows) }),
+			where: () => ({
+				for: () => Promise.resolve(rows),
+				limit: () => Promise.resolve(rows),
+			}),
 		}),
 	};
 }
@@ -101,14 +136,72 @@ function plainSelection(rows: readonly unknown[]) {
 beforeEach(() => {
 	for (const mock of Object.values(doubles)) mock.mockReset();
 	doubles.createEmulatorRank.mockResolvedValue(7);
+	doubles.prepareEmulatorRankCreation.mockResolvedValue(7);
+	doubles.createEmulatorRankInTransaction.mockResolvedValue(undefined);
 	doubles.deleteEmulatorRank.mockResolvedValue(undefined);
+	doubles.deleteEmulatorRankInTransaction.mockResolvedValue(undefined);
 	doubles.updateEmulatorRank.mockResolvedValue(undefined);
+	doubles.logAudit.mockResolvedValue(undefined);
 	doubles.logStaffActivity.mockResolvedValue(undefined);
+	doubles.logStaffActivityInTransaction.mockResolvedValue(undefined);
+	doubles.dbDelete.mockImplementation(deleteChain);
 	doubles.dbInsert.mockImplementation(insertChain);
+	doubles.dbTransaction.mockImplementation(async (run) =>
+		run(transactionToken),
+	);
+	doubles.dbUpdate.mockImplementation(() => updateChain());
 });
 
-describe("rank synchronization failures", () => {
+function expectPendingSynchronization(
-	it("returns failure and audits failure when create committed but updatepermissions returned false", async () => {
+	result: unknown,
+	operation:
+		| "access.rank.create"
+		| "access.rank.delete"
+		| "access.rank.update"
+		| "rcon.set-rank",
+) {
+	expect(result).toMatchObject({
+		ok: true,
+		data: {
+			operation,
+			recoveryId: expect.any(String),
+			completed: ["database", "audit"],
+		},
+		completion: {
+			status: "partial",
+			external: "failed",
+			audit: "persisted",
+		},
+	});
+	if (typeof result === "object" && result !== null && "ok" in result) {
+		const typed = result as {
+			ok: boolean;
+			correlationId?: string;
+			data?: { recoveryId?: string };
+		};
+		if (typed.ok) expect(typed.data?.recoveryId).toBe(typed.correlationId);
+	}
+}
+
+describe("durable rank synchronization", () => {
+	it("rolls back before RCON when the transaction-bound recovery intent cannot be audited", async () => {
+		doubles.logAudit.mockRejectedValueOnce(new Error("audit unavailable"));
+		doubles.rconSend.mockResolvedValue(true);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.PERMISSIONS_MANAGE),
+			"access.rank.create",
+			{ name: "Administrator", level: 7 },
+		);
+
+		expect(result).toMatchObject({
+			ok: false,
+			error: { code: "DEPENDENCY_UNAVAILABLE" },
+		});
+		expect(doubles.rconSend).not.toHaveBeenCalled();
+	});
+
+	it("commits create, ACL, staff audit, and recovery intent together before returning a retry key", async () => {
 		doubles.rconSend.mockResolvedValue(false);
 		const command = createSystemCommands(systemMutationService).find(
 			(entry) => entry.id === "system.access.rank.create",
@@ -137,19 +230,24 @@ describe("rank synchronization failures", () => {
 			},
 		);
 
-		expect(result).toMatchObject({
+		expectPendingSynchronization(result, "access.rank.create");
-			ok: false,
+		expect(outcomes).toEqual(["intent", "partial"]);
-			error: {
+		expect(doubles.createEmulatorRankInTransaction).toHaveBeenCalledWith(
-				code: "DEPENDENCY_UNAVAILABLE",
+			transactionToken,
-				messageKey: "errors.housekeeping.system.rankSynchronizationIncomplete",
+			7,
-				fieldErrors: {
+			{ rank_name: "Administrator", level: 7 },
-					operation: ["access.rank.create"],
+		);
-					completed: ["database", "audit"],
+		expect(doubles.logStaffActivityInTransaction).toHaveBeenCalledWith(
-					pending: ["emulator-permission-cache"],
+			expect.objectContaining({ action: "rank_create", targetId: 7 }),
-				},
+			transactionToken,
-			},
+		);
-		});
+		expect(doubles.logAudit).toHaveBeenCalledWith(
-		expect(outcomes).toEqual(["intent", "failure"]);
+			expect.objectContaining({
+				correlationId: expect.any(String),
+				outcome: "intent",
+			}),
+			transactionToken,
+		);
 	});
 
 	it.each([
@@ -161,11 +259,17 @@ describe("rank synchronization failures", () => {
 					.mockImplementationOnce(() => plainSelection([{ total: 0 }]))
 					.mockImplementationOnce(() => limitedSelection([]));
 			},
+			"deleteEmulatorRankInTransaction",
+		],
+		[
+			"access.rank.update",
+			{ id: 7, fields: { badge: "ADM" } },
+			() => {},
+			"updateEmulatorRank",
 		],
-		["access.rank.update", { id: 7, fields: { badge: "ADM" } }, () => {}],
 	] as const)(
-		"returns failure when %s committed but updatepermissions returned false",
+		"returns a recoverable partial after atomic %s commit when cache refresh fails",
-		async (operation, input, prepare) => {
+		async (operation, input, prepare, helperName) => {
 			prepare();
 			doubles.rconSend.mockResolvedValue(false);
 
@@ -175,33 +279,23 @@ describe("rank synchronization failures", () => {
 				input,
 			);
 
-			expect(result).toMatchObject({
+			expectPendingSynchronization(result, operation);
-				ok: false,
+			expect(doubles[helperName]).toHaveBeenCalledWith(
-				error: {
+				transactionToken,
-					code: "DEPENDENCY_UNAVAILABLE",
+				7,
-					messageKey:
+				...(operation === "access.rank.update" ? [{ badge: "ADM" }] : []),
-						"errors.housekeeping.system.rankSynchronizationIncomplete",
+			);
-					fieldErrors: {
-						operation: [operation],
-						completed: ["database", "audit"],
-						pending: ["emulator-permission-cache"],
-					},
-				},
-			});
 		},
 	);
 
-	it("reports external completion when set-rank RCON succeeded but DB persistence failed", async () => {
+	it("never calls RCON when set-rank database persistence rolls back", async () => {
 		doubles.dbSelect.mockImplementationOnce(() =>
 			limitedSelection([{ rank: 3 }]),
 		);
 		doubles.dbExecute.mockResolvedValue([[{ id: 4 }]]);
-		doubles.rconSetRank.mockResolvedValue(true);
+		doubles.dbUpdate.mockImplementationOnce(() =>
-		doubles.dbUpdate.mockReturnValue({
+			updateChain(Promise.reject(new Error("users update unavailable"))),
-			set: () => ({
+		);
-				where: () => Promise.reject(new Error("users update unavailable")),
-			}),
-		});
 
 		const result = await systemMutationService.execute(
 			serviceContext(PERMS.RCON_EXECUTE),
@@ -211,15 +305,124 @@ describe("rank synchronization failures", () => {
 
 		expect(result).toMatchObject({
 			ok: false,
-			error: {
+			error: { code: "DEPENDENCY_UNAVAILABLE" },
-				code: "DEPENDENCY_UNAVAILABLE",
+		});
-				messageKey: "errors.housekeeping.system.rankPersistenceIncomplete",
+		expect(doubles.rconSetRank).not.toHaveBeenCalled();
-				fieldErrors: {
+	});
-					operation: ["rcon.set-rank"],
+
-					completed: ["emulator"],
+	it("returns the durable recovery key when set-rank RCON fails after commit", async () => {
-					pending: ["database"],
+		doubles.dbSelect.mockImplementationOnce(() =>
+			limitedSelection([{ rank: 3 }]),
+		);
+		doubles.dbExecute.mockResolvedValue([[{ id: 4 }]]);
+		doubles.rconSetRank.mockResolvedValue(false);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.RCON_EXECUTE),
+			"rcon.set-rank",
+			{ userId: 8, rank: 4 },
+		);
+
+		expectPendingSynchronization(result, "rcon.set-rank");
+		expect(doubles.dbUpdate).toHaveBeenCalledBefore(doubles.rconSetRank);
+	});
+
+	it("retries only the convergent cache refresh, never the original rank mutation", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([
+			[
+				{
+					after: JSON.stringify({
+						kind: "rank-permission-cache",
+						operation: "access.rank.create",
+						rankId: 7,
+					}),
+					outcome: "partial",
 				},
+			],
+		]);
+		doubles.rconSend.mockResolvedValue(true);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.PERMISSIONS_MANAGE),
+			"access.rank.sync.retry" as SystemMutationOperation,
+			{ recoveryId: "original-rank-correlation" },
+		);
+
+		expect(result).toMatchObject({
+			ok: true,
+			data: {
+				recoveryId: "original-rank-correlation",
+				synchronization: "completed",
 			},
 		});
+		expect(doubles.rconSend).toHaveBeenCalledWith("updatepermissions");
+		expect(doubles.createEmulatorRank).not.toHaveBeenCalled();
+		expect(doubles.createEmulatorRankInTransaction).not.toHaveBeenCalled();
+		expect(doubles.deleteEmulatorRank).not.toHaveBeenCalled();
+		expect(doubles.deleteEmulatorRankInTransaction).not.toHaveBeenCalled();
+		expect(doubles.updateEmulatorRank).not.toHaveBeenCalled();
+	});
+
+	it("retries set-rank from its committed desired-state intent without another database mutation", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([
+			[
+				{
+					after: JSON.stringify({
+						kind: "set-rank",
+						operation: "rcon.set-rank",
+						userId: 8,
+						rank: 4,
+					}),
+					outcome: "intent",
+				},
+			],
+		]);
+		doubles.rconSetRank.mockResolvedValue(true);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.RCON_EXECUTE),
+			"rcon.set-rank.retry" as SystemMutationOperation,
+			{ recoveryId: "original-set-rank-correlation" },
+		);
+
+		expect(result).toMatchObject({
+			ok: true,
+			data: {
+				recoveryId: "original-set-rank-correlation",
+				synchronization: "completed",
+			},
+		});
+		expect(doubles.rconSetRank).toHaveBeenCalledWith(8, 4);
+		expect(doubles.dbUpdate).not.toHaveBeenCalled();
+	});
+
+	it("does not repeat an already completed external synchronization", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([
+			[
+				{
+					after: JSON.stringify({
+						kind: "rank-permission-cache",
+						operation: "access.rank.update",
+						rankId: 7,
+					}),
+					outcome: "success",
+				},
+			],
+		]);
+
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.PERMISSIONS_MANAGE),
+			"access.rank.sync.retry",
+			{ recoveryId: "completed-rank-correlation" },
+		);
+
+		expect(result).toMatchObject({
+			ok: true,
+			data: {
+				alreadyCompleted: true,
+				recoveryId: "completed-rank-correlation",
+			},
+		});
+		expect(doubles.rconSend).not.toHaveBeenCalled();
 	});
 });

src/lib/services/permission-ranks.test.ts

@@ -3,6 +3,8 @@ import { MySqlDialect } from "drizzle-orm/mysql-core";
 import { describe, expect, it } from "vitest";
 
 import {
+	createEmulatorRankInTransaction,
+	deleteEmulatorRankInTransaction,
 	fetchEmulatorRankForEdit,
 	getRankPermissionColumn,
 	RANK_GENERAL_FIELDS,
@@ -86,6 +88,31 @@ describe("RANK_GENERAL_FIELDS", () => {
 	});
 });
 
+describe("transaction-aware rank lifecycle helpers", () => {
+	it("runs coupled create and delete statements through the supplied executor", async () => {
+		const statements: string[] = [];
+		const executor = {
+			execute: async (query: SQL | string) => {
+				statements.push(queryText(query));
+				return [{ affectedRows: 1 }, []];
+			},
+		};
+
+		await createEmulatorRankInTransaction(executor as never, 7, {
+			rank_name: "Administrator",
+			level: 7,
+		});
+		await deleteEmulatorRankInTransaction(executor as never, 7);
+
+		expect(statements).toHaveLength(5);
+		expect(statements[0]).toContain("INSERT INTO permission_ranks");
+		expect(statements[1]).toContain("INSERT INTO permissions");
+		expect(statements[2]).toContain("UPDATE permission_definitions");
+		expect(statements[3]).toContain("DELETE FROM permission_ranks");
+		expect(statements[4]).toContain("DELETE FROM permissions");
+	});
+});
+
 describe("fetchEmulatorRankForEdit", () => {
 	it("loads permissions from the legacy table when permission_definitions is absent", async () => {
 		const db = {

src/lib/services/permission-ranks.ts

@@ -43,15 +43,16 @@ export interface EmulatorRankForEdit extends EmulatorRankSummary {
 	permissionMaxValues: Record<string, number>;
 }
 
-/** Surface of `Db` used by these helpers (plain raw-SQL exec + transactions). */
+/** Surfaces used by rank helpers, including database transaction handles. */
+type SqlExecutor = Pick<Db, "execute">;
 type RawDb = Pick<Db, "execute" | "transaction">;
 
-async function rawRows<T>(db: RawDb, query: SQL | string): Promise<T> {
+async function rawRows<T>(db: SqlExecutor, query: SQL | string): Promise<T> {
 	const [rows] = await db.execute(query);
 	return rows as T;
 }
 
-async function rawExec(db: RawDb, query: SQL | string): Promise<number> {
+async function rawExec(db: SqlExecutor, query: SQL | string): Promise<number> {
 	const [result] = await db.execute(query);
 	return (result as ResultSetHeader).affectedRows;
 }
@@ -195,6 +196,19 @@ export async function fetchEmulatorRankForEdit(
 export async function createEmulatorRank(
 	db: RawDb,
 	data: { rank_name: string; level: number },
+): Promise<number> {
+	const id = await prepareEmulatorRankCreation(db);
+	await db.transaction((tx) => createEmulatorRankInTransaction(tx, id, data));
+	return id;
+}
+
+/**
+ * Prepare the dynamic permission column before opening the coupled DML
+ * transaction. MySQL ALTER TABLE commits implicitly, so this schema-only step
+ * is deliberately kept outside the atomic rank/ACL/audit unit.
+ */
+export async function prepareEmulatorRankCreation(
+	db: SqlExecutor,
 ): Promise<number> {
 	const rows = await rawRows<{ next_id: number }[]>(
 		db,
@@ -203,38 +217,46 @@ export async function createEmulatorRank(
 	const id = Number(rows[0]?.next_id ?? 1);
 	await ensurePermissionRankColumn(db, id);
 	await resetPermissionRankColumn(db, id);
+	return id;
+}
 
-	await db.transaction(async (tx) => {
+export async function createEmulatorRankInTransaction(
-		await rawExec(
+	db: SqlExecutor,
-			tx,
+	id: number,
-			sql`
+	data: { rank_name: string; level: number },
+): Promise<void> {
+	await rawExec(
+		db,
+		sql`
         INSERT INTO permission_ranks (id, rank_name, level)
         VALUES (${id}, ${data.rank_name}, ${data.level})
       `,
-		);
+	);
-		await rawExec(
+	await rawExec(
-			tx,
+		db,
-			sql`
+		sql`
         INSERT INTO permissions (id, rank_name, level)
         VALUES (${id}, ${data.rank_name}, ${data.level})
         ON DUPLICATE KEY UPDATE rank_name = VALUES(rank_name), level = VALUES(level)
       `,
-		);
+	);
-	});
-
-	return id;
 }
 
 export async function deleteEmulatorRank(db: RawDb, rankId: number) {
+	await db.transaction((tx) => deleteEmulatorRankInTransaction(tx, rankId));
+}
+
+export async function deleteEmulatorRankInTransaction(
+	db: SqlExecutor,
+	rankId: number,
+): Promise<void> {
 	await resetPermissionRankColumn(db, rankId);
-	await db.transaction(async (tx) => {
+	await rawExec(db, sql`DELETE FROM permission_ranks WHERE id = ${rankId}`);
-		await rawExec(tx, sql`DELETE FROM permission_ranks WHERE id = ${rankId}`);
+	await rawExec(db, sql`DELETE FROM permissions WHERE id = ${rankId}`);
-		await rawExec(tx, sql`DELETE FROM permissions WHERE id = ${rankId}`);
-	});
 }
 
 export async function updateEmulatorRank(
-	db: RawDb,
+	db: SqlExecutor,
 	rankId: number,
 	fields: Record<string, string | number>,
 ) {
@@ -252,7 +274,7 @@ export async function updateEmulatorRank(
 }
 
 async function updateTableFields(
-	db: RawDb,
+	db: SqlExecutor,
 	table: string,
 	rankId: number,
 	fields: Record<string, string | number>,
@@ -271,7 +293,7 @@ async function updateTableFields(
 }
 
 async function updatePermissionDefinitionValues(
-	db: RawDb,
+	db: SqlExecutor,
 	rankId: number,
 	permissionFields: Record<string, number>,
 ) {
@@ -297,7 +319,7 @@ async function updatePermissionDefinitionValues(
 }
 
 async function updateLegacyPermissionFields(
-	db: RawDb,
+	db: SqlExecutor,
 	rankId: number,
 	fields: Record<string, string | number>,
 ) {
@@ -309,7 +331,7 @@ async function updateLegacyPermissionFields(
 	return updateTableFields(db, "permissions", rankId, existingFields);
 }
 
-async function ensurePermissionRankColumn(db: RawDb, rankId: number) {
+async function ensurePermissionRankColumn(db: SqlExecutor, rankId: number) {
 	const rankColumn = getRankPermissionColumn(rankId);
 	const exists = await rawRows<{ c: number }[]>(
 		db,
@@ -332,7 +354,7 @@ async function ensurePermissionRankColumn(db: RawDb, rankId: number) {
 	);
 }
 
-async function resetPermissionRankColumn(db: RawDb, rankId: number) {
+async function resetPermissionRankColumn(db: SqlExecutor, rankId: number) {
 	const rankColumn = getRankPermissionColumn(rankId);
 	await rawExec(
 		db,
@@ -343,7 +365,7 @@ async function resetPermissionRankColumn(db: RawDb, rankId: number) {
 	);
 }
 
-async function getTableColumns(db: RawDb, table: string) {
+async function getTableColumns(db: SqlExecutor, table: string) {
 	const rows = await rawRows<{ column_name: string }[]>(
 		db,
 		sql`

src/lib/services/staff-activity.ts

@@ -1,37 +1,54 @@
 import { headers } from "next/headers";
-import { db, StaffActivities } from "@/lib/db";
+import { type Db, db, StaffActivities } from "@/lib/db";
 
-/**
+interface StaffActivityOptions {
- * Append a staff-action audit entry (AtomCMS StaffActivity). Never throws —
- * logging must not block the action it records.
- */
-export async function logStaffActivity(opts: {
 	staffId: number;
 	action: string;
 	description: string;
 	targetType?: string;
 	targetId?: number;
-}): Promise<void> {
+}
+
+type StaffActivityWriter = Pick<Db, "insert">;
+
+async function resolveStaffIp(): Promise<string | null> {
 	try {
-		let ip: string | null = null;
+		const h = await headers();
-		try {
+		return (
-			const h = await headers();
+			h.get("x-real-client-ip") ??
-			ip =
+			h.get("x-forwarded-for")?.split(",")[0]?.trim() ??
-				h.get("x-real-client-ip") ??
+			null
-				h.get("x-forwarded-for")?.split(",")[0]?.trim() ??
+		);
-				null;
+	} catch {
-		} catch {
+		return null;
-			ip = null;
+	}
-		}
+}
-		await db.insert(StaffActivities).values({
+
-			userId: BigInt(opts.staffId),
+/** Write an audit row through the caller's transaction and propagate failure. */
-			action: opts.action.slice(0, 50),
+export async function logStaffActivityInTransaction(
-			description: opts.description,
+	opts: StaffActivityOptions,
-			targetType: opts.targetType ?? null,
+	transaction: StaffActivityWriter,
-			targetId: opts.targetId != null ? BigInt(opts.targetId) : null,
+): Promise<void> {
-			ipAddress: ip,
+	await transaction.insert(StaffActivities).values({
-			createdAt: new Date(),
+		userId: BigInt(opts.staffId),
-		});
+		action: opts.action.slice(0, 50),
+		description: opts.description,
+		targetType: opts.targetType ?? null,
+		targetId: opts.targetId != null ? BigInt(opts.targetId) : null,
+		ipAddress: await resolveStaffIp(),
+		createdAt: new Date(),
+	});
+}
+
+/**
+ * Append a staff-action audit entry (AtomCMS StaffActivity). Never throws —
+ * logging must not block the action it records.
+ */
+export async function logStaffActivity(
+	opts: StaffActivityOptions,
+): Promise<void> {
+	try {
+		await logStaffActivityInTransaction(opts, db);
 	} catch {
 		// swallow — audit logging is best-effort
 	}