docs/superpowers/evidence/2026-09-05-housekeeping-rank-convergence.md

@@ -0,0 +1,52 @@
+# Housekeeping rank synchronization
+
+## Behavior
+
+Rank assignments commit the configured rank, user update, and audit intent before
+attempting emulator synchronization. Assignment and rank deletion acquire the
+configured-rank row lock first. Delivery acquires the user row lock, reads the
+current database rank, and holds that lock until the transport settles.
+
+Retrying an old recovery reference therefore dispatches the current committed
+rank rather than replaying the rank stored in the old audit record. A user deleted
+after persistence produces an audited superseded result. SQL lock errors remain
+dependency failures rather than being reported as missing ranks.
+
+The coordinated paths cover System operations, People user editing, the legacy
+command centre, legacy user editing, the user-actions API, legacy rank deletion,
+and shop rank upgrades. Administrative user creation also locks the selected rank.
+Shop upgrades compare the locked current rank so they cannot overwrite a newer
+staff promotion. A failed post-purchase rank delivery leaves a recovery intent
+without turning the completed purchase into another charge.
+
+People and legacy responses preserve partial-completion information and recovery
+references. Failure of the completion audit alone does not misreport successful
+transport delivery as a transport failure.
+
+## Verification
+
+- Focused rank, legacy-entrypoint, shop, System and People tests: 94 passed.
+- Full suite: 280 files passed, 3 skipped; 1,861 tests passed, 5 skipped.
+- Next.js 16.3.4 production build passed and generated all 245 pages.
+- TypeScript and canonical Knip checks passed.
+- Biome passed on all 13 changed source/test files.
+- Project-source lint without formatting passed on 1,412 files, with one existing
+  Catalog Studio warning. The full Windows checkout check also includes local
+  untracked brainstorm HTML and reports CRLF/LF formatting differences; those
+  local files were preserved and are not part of this change.
+- Migration matrix: 138 historical rows valid; 138 legacy pages retained;
+  runtime discovered/mapped/verified 138/138/138, with 2 intentional removals.
+- Independent read-only review found no remaining Important or Critical issues.
+
+## Validation boundary
+
+Tests exercise the parameterized locking SQL and controlled transaction/transport
+ordering. No live two-connection MariaDB race test or production emulator
+acceptance test was performed.
+
+TCP RCON success means the socket write completed. CMS dispatch is serialized,
+but the current protocol does not acknowledge emulator processing or enforce its
+processing order. End-to-end confirmation would require an emulator protocol
+change. The existing transport timeout and retry policy bounds the delivery wait.
+
+The PR remains draft; these checks are not a deployment or preview-cutover claim.

src/actions/commandocentrum.ts

@@ -1,9 +1,10 @@
 "use server";
 
-import { eq, sql } from "drizzle-orm";
+import crypto from "node:crypto";
 import { revalidatePath } from "next/cache";
 import { z } from "zod";
-import { db, User } from "@/lib/db";
+import { systemMutationService } from "@/features/housekeeping/domains/system/services/mutations";
+import { createHousekeepingCapabilityContext } from "@/features/housekeeping/foundation/capability-context";
 import { PERMS } from "@/lib/permissions";
 import { adminAction } from "@/lib/safe-action";
 import { ActionError, actionOk } from "@/lib/safe-action-shared";
@@ -198,44 +199,28 @@ const setRankSchema = z.object({
 export const setRank = adminAction(
 	{ permission: PERMS.RCON_EXECUTE, schema: setRankSchema },
 	async (ctx) => {
-		const staffRank = Number(ctx.session.user.rank);
-		const isSuper = ctx.permissions.isSuperAdmin;
-		const [target] = await db
-			.select({ rank: User.rank })
-			.from(User)
-			.where(eq(User.id, ctx.data.userId))
-			.limit(1);
-		if (!target) throw new ActionError("User not found");
-
-		let rankExists: { id: number }[] = [];
-		try {
-			const [rows] = await db.execute(
-				sql`SELECT id FROM permission_ranks WHERE id = ${ctx.data.rank} LIMIT 1`,
+		const result = await systemMutationService.execute(
+			{
+				capability: createHousekeepingCapabilityContext(
+					{
+						id: Number(ctx.session.user.id),
+						username: ctx.session.user.username,
+						rank: Number(ctx.session.user.rank),
+					},
+					ctx.permissions,
+				),
+				correlationId: crypto.randomUUID(),
+			},
+			"rcon.set-rank",
+			ctx.data,
 		);
-			rankExists = rows as unknown as { id: number }[];
-		} catch {
-			rankExists = [];
-		}
-		if (rankExists.length === 0) throw new ActionError("Rank does not exist");
-
-		if (!isSuper) {
-			if (target.rank >= staffRank) {
-				throw new ActionError(
-					"Cannot change rank of a user at or above your rank",
-				);
-			}
-			if (ctx.data.rank >= staffRank) {
-				throw new ActionError("Cannot set a rank equal to or above your own");
-			}
-		}
-
-		await requireRconOk(await rcon.setRank(ctx.data.userId, ctx.data.rank));
-		await db
-			.update(User)
-			.set({ rank: ctx.data.rank })
-			.where(eq(User.id, ctx.data.userId));
+		if (!result.ok) throw new ActionError(result.error.messageKey);
 		revalidatePath(PATH);
-		return actionOk();
+		if (result.completion)
+			throw new ActionError(
+				`Rank saved; emulator synchronization is pending. ASE recovery reference: ${result.correlationId}`,
+			);
+		return actionOk(result.data as Record<string, unknown>);
 	},
 );
 

src/actions/permissions.ts

@@ -1,23 +1,18 @@
 "use server";
 
-import { and, count, eq, inArray, sql } from "drizzle-orm";
+import crypto from "node:crypto";
+import { and, eq, inArray, sql } from "drizzle-orm";
 import type { ResultSetHeader } from "mysql2";
 import { revalidateTag } from "next/cache";
 import { z } from "zod";
-import {
-	AclModelPermission,
-	AclModelRole,
-	AclPermission,
-	AclRole,
-	db,
-	User,
-} from "@/lib/db";
+import { systemMutationService } from "@/features/housekeeping/domains/system/services/mutations";
+import { createHousekeepingCapabilityContext } from "@/features/housekeeping/foundation/capability-context";
+import { AclModelPermission, AclPermission, AclRole, db } from "@/lib/db";
 import { PERMS } from "@/lib/permission-slugs";
 import { adminAction } from "@/lib/safe-action";
 import { ActionError, actionOk } from "@/lib/safe-action-shared";
 import {
 	createEmulatorRank,
-	deleteEmulatorRank,
 	updateEmulatorRank,
 } from "@/lib/services/permission-ranks";
 import { rcon } from "@/lib/services/rcon";
@@ -58,42 +53,27 @@ const deleteRankSchema = z.object({ id: z.coerce.number().int().positive() });
 export const deleteRank = adminAction(
 	{ schema: deleteRankSchema, permission: PERMS.PERMISSIONS_MANAGE },
 	async (ctx) => {
-		const [userCount] = await db
-			.select({ total: count() })
-			.from(User)
-			.where(eq(User.rank, ctx.data.id));
-		const users = userCount?.total ?? 0;
-		if (users > 0)
-			throw new ActionError(`Cannot delete: ${users} users have this rank`);
-		const [role] = await db
-			.select({ id: AclRole.id })
-			.from(AclRole)
-			.where(eq(AclRole.slug, `rank_${ctx.data.id}`))
-			.limit(1);
-		await deleteEmulatorRank(db, ctx.data.id);
-		if (role) {
-			await db.transaction(async (tx) => {
-				await tx
-					.delete(AclModelPermission)
-					.where(
-						and(
-							eq(AclModelPermission.modelId, role.id),
-							eq(AclModelPermission.modelType, "Role"),
+		const result = await systemMutationService.execute(
+			{
+				capability: createHousekeepingCapabilityContext(
+					{
+						id: Number(ctx.session.user.id),
+						username: ctx.session.user.username,
+						rank: Number(ctx.session.user.rank),
+					},
+					ctx.permissions,
 				),
+				correlationId: crypto.randomUUID(),
+			},
+			"access.rank.delete",
+			ctx.data,
 		);
-				await tx.delete(AclModelRole).where(eq(AclModelRole.roleId, role.id));
-				await tx.delete(AclRole).where(eq(AclRole.id, role.id));
-			});
-		}
-		await logStaffActivity({
-			staffId: ctx.session.user.id,
-			action: "rank_delete",
-			description: `Deleted rank #${ctx.data.id}`,
-			targetType: "rank",
-			targetId: ctx.data.id,
-		});
-		await rcon.send("updatepermissions");
+		if (!result.ok) throw new ActionError(result.error.messageKey);
 		revalidateTag("permissions", { expire: 0 });
+		if (result.completion)
+			throw new ActionError(
+				`Rank deleted; emulator synchronization is pending. ASE recovery reference: ${result.correlationId}`,
+			);
 		return actionOk();
 	},
 );

src/actions/shop.ts

@@ -1,5 +1,6 @@
 "use server";
 
+import crypto from "node:crypto";
 import { and, eq, max, sql } from "drizzle-orm";
 import { revalidatePath } from "next/cache";
 import { redirect } from "next/navigation";
@@ -7,7 +8,12 @@ import { auth } from "@/lib/auth";
 import { db, User, UsersBadges, WebsiteShopArticles } from "@/lib/db";
 import { clientIp, rateLimit } from "@/lib/rate-limit";
 import { logServerError } from "@/lib/server-log";
+import { logAudit } from "@/lib/services/audit";
 import { creditsPerUnit } from "@/lib/services/paypal";
+import {
+	lockConfiguredRank,
+	rankAssignmentCoordinator,
+} from "@/lib/services/rank-assignment";
 import { rcon } from "@/lib/services/rcon";
 import { currencyDb, sendCurrency } from "@/lib/services/send-currency";
 
@@ -118,8 +124,38 @@ export async function buyShopArticle(formData: FormData): Promise<void> {
 						outcome = "credits";
 					} else {
 						const badgeCodes = parseBadgeCodes(article.badges);
+						let rankChanged = false;
+						const rankRecoveryId = crypto.randomUUID();
+						const rankAudit = {
+							userId,
+							action: "system.external-sync",
+							target: "rcon.set-rank",
+							targetId: userId,
+							correlationId: rankRecoveryId,
+							domain: "system" as const,
+							after: {
+								kind: "set-rank",
+								operation: "rcon.set-rank",
+								userId,
+								rank: article.giveRank,
+							},
+						};
 
 						await db.transaction(async (tx) => {
+							if (
+								article.giveRank != null &&
+								article.giveRank > 0 &&
+								!(await lockConfiguredRank(tx, article.giveRank))
+							) {
+								throw new Error("Package rank no longer exists");
+							}
+							const [lockedBuyer] = await tx
+								.select({ credits: User.credits, rank: User.rank })
+								.from(User)
+								.where(eq(User.id, userId))
+								.for("update");
+							if (!lockedBuyer || lockedBuyer.credits < price)
+								throw new Error("Insufficient credits");
 							if (price > 0) {
 								await tx
 									.update(User)
@@ -130,12 +166,14 @@ export async function buyShopArticle(formData: FormData): Promise<void> {
 							if (
 								article.giveRank != null &&
 								article.giveRank > 0 &&
-								article.giveRank > buyer.rank
+								article.giveRank > lockedBuyer.rank
 							) {
 								await tx
 									.update(User)
 									.set({ rank: article.giveRank })
 									.where(eq(User.id, userId));
+								rankChanged = true;
+								await logAudit({ ...rankAudit, outcome: "intent" }, tx);
 							}
 
 							for (const code of badgeCodes) {
@@ -164,6 +202,28 @@ export async function buyShopArticle(formData: FormData): Promise<void> {
 							}
 						});
 
+						if (rankChanged) {
+							try {
+								const delivery =
+									await rankAssignmentCoordinator.synchronize(userId);
+								await logAudit({
+									...rankAudit,
+									after: {
+										...rankAudit.after,
+										...(delivery.status === "superseded"
+											? { superseded: true }
+											: { rank: delivery.rank }),
+									},
+									outcome:
+										delivery.status === "pending" ? "partial" : "success",
+								});
+							} catch (error) {
+								logServerError("shop.rank_sync_pending", error, {
+									userId,
+									correlationId: rankRecoveryId,
+								});
+							}
+						}
 						await sendCurrency(
 							{ rcon, db: currencyDb },
 							userId,

src/features/housekeeping/domains/people/services/mutations-production-workflows.test.ts

@@ -22,6 +22,7 @@ const mocks = vi.hoisted(() => ({
 		kickAll: vi.fn(),
 		muteUser: vi.fn(),
 		setTradeLock: vi.fn(),
+		setRank: vi.fn(),
 		staffAlert: vi.fn(),
 		unmuteUser: vi.fn(),
 		updateWordFilter: vi.fn(),
@@ -46,6 +47,7 @@ function mutationPromise(
 function selectResult() {
 	const value = mocks.selectQueue.shift() ?? [];
 	return Object.assign(Promise.resolve(value), {
+		for: vi.fn(async () => value),
 		limit: vi.fn(async () => value),
 		orderBy: vi.fn(() =>
 			Object.assign(Promise.resolve(value), {
@@ -173,6 +175,36 @@ beforeEach(() => {
 });
 
 describe("People production workflow adapter", () => {
+	it("preserves completed rank delivery when only the synchronization audit fails", async () => {
+		mocks.selectQueue.push([target()], [target()], [target({ rank: 4 })]);
+		mocks.audit.mockImplementation(async (entry) => {
+			if (
+				entry.action === "system.external-sync" &&
+				entry.outcome === "success"
+			)
+				throw new Error("audit unavailable");
+		});
+		const result = await peopleMutationService.execute(
+			{ ...invocation, legacy: false },
+			"user.update",
+			{ userId: 7, fields: { rank: 4 } },
+		);
+		expect(result).toMatchObject({
+			ok: true,
+			completion: {
+				status: "partial",
+				external: "completed",
+				audit: "unavailable",
+			},
+			data: {
+				after: {
+					rankRecoveryId: "production-workflow",
+					rankCompletionAudit: "pending",
+				},
+			},
+		});
+		expect(mocks.rcon.setRank).toHaveBeenCalledWith(7, 4);
+	});
 	it("does not treat a high numeric rank as a super-admin hierarchy bypass", async () => {
 		mocks.resolveServerContext.mockResolvedValue({
 			...context(),
@@ -231,7 +263,8 @@ describe("People production workflow adapter", () => {
 				fieldErrors: { rank: ["errors.validation.invalid"] },
 			},
 		});
-		expect(mocks.transaction).not.toHaveBeenCalled();
+		expect(mocks.transaction).toHaveBeenCalledTimes(1);
+		expect(mocks.updateSet).not.toHaveBeenCalled();
 	});
 
 	it("assigns an existing configured rank above 7 for a super-admin", async () => {
@@ -242,6 +275,7 @@ describe("People production workflow adapter", () => {
 		});
 		mocks.selectQueue.push([target({ rank: 2 })]);
 		mocks.execute.mockResolvedValueOnce([[{ id: 9 }], []]);
+		mocks.selectQueue.push([target({ rank: 2 })], [target({ rank: 9 })]);
 
 		const result = await peopleMutationService.execute(
 			invocation,
@@ -253,7 +287,8 @@ describe("People production workflow adapter", () => {
 		expect(mocks.updateSet).toHaveBeenCalledWith(
 			expect.objectContaining({ rank: 9 }),
 		);
-		expect(mocks.transaction).toHaveBeenCalledTimes(1);
+		expect(mocks.transaction).toHaveBeenCalledTimes(2);
+		expect(mocks.rcon.setRank).toHaveBeenCalledWith(7, 9);
 	});
 
 	it.each([

src/features/housekeeping/domains/system/services/rank-mutations-production.test.ts

@@ -186,6 +186,47 @@ function expectPendingSynchronization(
 }
 
 describe("durable rank synchronization", () => {
+	it.each([
+		["access.rank.delete", PERMS.PERMISSIONS_MANAGE, { id: 7 }],
+		["rcon.set-rank", PERMS.RCON_EXECUTE, { userId: 8, rank: 4 }],
+	] as const)(
+		"reports SQL lock failure for %s as dependency unavailable",
+		async (operation, permission, input) => {
+			doubles.dbExecute.mockRejectedValueOnce(new Error("lock wait timeout"));
+			const result = await systemMutationService.execute(
+				serviceContext(permission),
+				operation,
+				input,
+			);
+			expect(result).toMatchObject({
+				ok: false,
+				error: { code: "DEPENDENCY_UNAVAILABLE" },
+			});
+			expect(doubles.dbUpdate).not.toHaveBeenCalled();
+			expect(doubles.rconSetRank).not.toHaveBeenCalled();
+			expect(doubles.deleteEmulatorRankInTransaction).not.toHaveBeenCalled();
+		},
+	);
+
+	it("audits a committed assignment as superseded when the user was subsequently deleted", async () => {
+		doubles.dbExecute.mockResolvedValueOnce([[{ id: 4 }]]);
+		doubles.dbSelect
+			.mockImplementationOnce(() => limitedSelection([{ rank: 3 }]))
+			.mockImplementationOnce(() => limitedSelection([]));
+		const result = await systemMutationService.execute(
+			serviceContext(PERMS.RCON_EXECUTE),
+			"rcon.set-rank",
+			{ userId: 8, rank: 4 },
+		);
+		expect(result).toMatchObject({
+			ok: true,
+			data: { superseded: true, recoveryId: "rank-mutation-correlation" },
+		});
+		expect(doubles.rconSetRank).not.toHaveBeenCalled();
+		expect(doubles.logAudit).toHaveBeenLastCalledWith(
+			expect.objectContaining({ outcome: "success" }),
+		);
+	});
 	it("rolls back before RCON when the transaction-bound recovery intent cannot be audited", async () => {
 		doubles.logAudit.mockRejectedValueOnce(new Error("audit unavailable"));
 		doubles.rconSend.mockResolvedValue(true);