feat(catalog): persist idempotent bulk operations and retryable deliveries

src/features/operations/README.md

@@ -0,0 +1,9 @@
+# Durable catalog operations
+
+Bulk offer apply and undo accept a request UUID. The authenticated actor, operation kind and UUID identify one mutation. A canonical payload hash rejects reuse for different changes. The result and outbox entries commit in the same database transaction as the offers and audit history; replay returns the stored result.
+
+The jobs worker claims pending effects under a database lock with a 120-second lease. A claim token protects completion from stale workers. Failed delivery uses exponential retry, stopping after eight attempts. `/admin/devops/deliveries` shows the latest 100 effects to DEVOPS_VIEW; DEVOPS_EDIT can retry failed effects without replaying the catalog mutation.
+
+Delivery is at least once: a crash after sending and before acknowledgement may repeat an effect. Catalog refresh is repeatable; export delivery means a request entered the existing export queue, not that Git publication or client refresh completed. Export disabled by configuration remains a no-op. This first adapter covers bulk offer apply/undo, not every CMS mutation.
+
+Migration0031 is required before the updated worker starts. Unit tests cover payload identity, dispatch limits, retries and authorization; the separate Docker integration suite covers concurrent SQL requests, transaction rollback and claim ownership. No production database was used for local tests.

src/features/operations/dispatcher.test.ts

@@ -0,0 +1,51 @@
+import { expect, it, vi } from "vitest";
+import { dispatchEffects } from "./dispatcher";
+
+it("settles only after successful delivery and preserves the claim token", async () => {
+	const effect = {
+		id: "a",
+		token: "lease1",
+		topic: "catalog.refresh" as const,
+		attempts: 1,
+	};
+	const repo = {
+		claim: vi.fn().mockResolvedValueOnce(effect).mockResolvedValue(null),
+		complete: vi.fn(),
+		fail: vi.fn(),
+	};
+	const deliver = vi.fn();
+	await dispatchEffects(repo, deliver);
+	expect(deliver).toHaveBeenCalledWith(effect);
+	expect(repo.complete).toHaveBeenCalledWith(effect);
+	expect(repo.fail).not.toHaveBeenCalled();
+});
+it("persists failures and continues to other pending work", async () => {
+	const effect = {
+		id: "a",
+		token: "lease1",
+		topic: "catalog.refresh" as const,
+		attempts: 1,
+	};
+	const repo = {
+		claim: vi.fn().mockResolvedValueOnce(effect).mockResolvedValue(null),
+		complete: vi.fn(),
+		fail: vi.fn(),
+	};
+	await dispatchEffects(repo, vi.fn().mockRejectedValue(Error("private")));
+	expect(repo.fail).toHaveBeenCalledWith(effect);
+	expect(repo.complete).not.toHaveBeenCalled();
+});
+it("bounds work per tick instead of draining indefinitely", async () => {
+	const repo = {
+		claim: vi.fn().mockResolvedValue({
+			id: "a",
+			token: "t",
+			topic: "catalog.refresh",
+			attempts: 1,
+		}),
+		complete: vi.fn(),
+		fail: vi.fn(),
+	};
+	await dispatchEffects(repo, async () => {});
+	expect(repo.claim).toHaveBeenCalledTimes(20);
+});

src/features/operations/dispatcher.ts

@@ -0,0 +1,23 @@
+import type { EffectClaim } from "./model";
+export interface EffectRepository {
+	claim(): Promise<EffectClaim | null>;
+	complete(claim: EffectClaim): Promise<unknown>;
+	fail(claim: EffectClaim): Promise<unknown>;
+}
+/** At-least-once delivery: handlers must tolerate replay after an expired claim. */
+export async function dispatchEffects(
+	repo: EffectRepository,
+	deliver: (claim: EffectClaim) => Promise<unknown>,
+) {
+	for (let i = 0; i < 20; i++) {
+		const claim = await repo.claim();
+		if (!claim) return;
+		try {
+			await deliver(claim);
+		} catch {
+			await repo.fail(claim);
+			continue;
+		}
+		await repo.complete(claim);
+	}
+}

src/features/operations/model.test.ts

@@ -0,0 +1,31 @@
+import { expect, it } from "vitest";
+import { operationHash, retryDelay, validateOperation } from "./model";
+
+it("hashes equivalent object payloads equally without ignoring array order", () => {
+	expect(operationHash({ b: 2, a: { y: 1, x: 0 } })).toBe(
+		operationHash({ a: { x: 0, y: 1 }, b: 2 }),
+	);
+	expect(operationHash([1, 2])).not.toBe(operationHash([2, 1]));
+});
+it("rejects unsupported or oversized input rather than hashing ambiguous values", () => {
+	expect(() => operationHash({ amount: NaN })).toThrow();
+	expect(() => operationHash({ value: undefined })).toThrow();
+	expect(() => operationHash("x".repeat(100000))).toThrow();
+});
+it("validates actor and request identity before database access", () => {
+	const valid = {
+		actorId: 1,
+		kind: "catalog.bulk.apply",
+		key: "123e4567-e89b-42d3-a456-426614174000",
+		input: {},
+	};
+	expect(() => validateOperation(valid)).not.toThrow();
+	expect(() => validateOperation({ ...valid, actorId: 0 })).toThrow();
+	expect(() => validateOperation({ ...valid, key: "shared" })).toThrow();
+	expect(() => validateOperation({ ...valid, kind: "bad kind" })).toThrow();
+});
+it("backs off boundedly for repeated delivery failures", () => {
+	expect(retryDelay(1)).toBe(60);
+	expect(retryDelay(2)).toBe(120);
+	expect(retryDelay(20)).toBe(3600);
+});

src/features/operations/model.ts

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+export interface OperationInput {
+	actorId: number;
+	kind: string;
+	key: string;
+	input: unknown;
+}
+export class OperationConflict extends Error {
+	constructor() {
+		super(
+			"This request key was already used for different changes. Refresh the preview.",
+		);
+	}
+}
+export function validateOperation(value: OperationInput) {
+	if (
+		!Number.isSafeInteger(value.actorId) ||
+		value.actorId < 1 ||
+		!/^[a-z][a-z0-9.-]{2,63}$/.test(value.kind) ||
+		!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(
+			value.key,
+		)
+	)
+		throw new Error("Invalid operation identity");
+}
+function canonical(value: unknown, depth = 0): unknown {
+	if (depth > 30) throw new Error("Operation input is too deep");
+	if (value === null || typeof value === "string" || typeof value === "boolean")
+		return value;
+	if (typeof value === "number" && Number.isFinite(value)) return value;
+	if (Array.isArray(value)) return value.map((v) => canonical(v, depth + 1));
+	if (
+		value &&
+		typeof value === "object" &&
+		Object.getPrototypeOf(value) === Object.prototype
+	)
+		return Object.fromEntries(
+			Object.keys(value)
+				.sort()
+				.map((key) => [
+					key,
+					canonical((value as Record<string, unknown>)[key], depth + 1),
+				]),
+		);
+	throw new Error("Unsupported operation input");
+}
+export function operationHash(value: unknown) {
+	const serialized = JSON.stringify(canonical(value));
+	if (Buffer.byteLength(serialized) > 65536)
+		throw new Error("Operation input is too large");
+	return createHash("sha256").update(serialized).digest("hex");
+}
+export function retryDelay(attempt: number) {
+	return Math.min(3600, 60 * 2 ** Math.max(0, attempt - 1));
+}
+export type EffectTopic = "catalog.refresh" | "catalog.export.request";
+export interface EffectClaim {
+	id: string;
+	token: string;
+	topic: EffectTopic;
+	attempts: number;
+}

src/features/operations/retry-action.test.ts

@@ -0,0 +1,51 @@
+import { beforeEach, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+	permission: vi.fn(),
+	retry: vi.fn(),
+	audit: vi.fn(),
+	refresh: vi.fn(),
+	error: vi.fn(),
+}));
+vi.mock("next/cache", () => ({ revalidatePath: mocks.refresh }));
+vi.mock("@/lib/admin/guard", () => ({ requirePermission: mocks.permission }));
+vi.mock("@/lib/logger", () => ({ logger: { error: mocks.error } }));
+vi.mock("@/lib/services/audit", () => ({ logAudit: mocks.audit }));
+vi.mock("./server", () => ({ retryEffect: mocks.retry }));
+
+import { retryDelivery } from "./retry-action";
+
+const id = "840f4023-04cd-4d7b-92c5-047aa620029c";
+beforeEach(() => {
+	vi.resetAllMocks();
+	mocks.permission.mockResolvedValue({ id: 7 });
+	mocks.retry.mockResolvedValue(undefined);
+	mocks.audit.mockResolvedValue(undefined);
+});
+it("rejects unauthorized access before touching the delivery", async () => {
+	mocks.permission.mockRejectedValue(new Error("Forbidden"));
+	await expect(retryDelivery(id)).rejects.toThrow("Forbidden");
+	expect(mocks.retry).not.toHaveBeenCalled();
+	expect(mocks.audit).not.toHaveBeenCalled();
+});
+it("reports a rejected retry without recording a successful action", async () => {
+	mocks.retry.mockRejectedValue(new Error("Database unavailable"));
+	expect(await retryDelivery(id)).toMatchObject({ ok: false });
+	expect(mocks.audit).not.toHaveBeenCalled();
+	expect(mocks.refresh).not.toHaveBeenCalled();
+});
+it("records the delivery UUID as audit metadata", async () => {
+	expect(await retryDelivery(id)).toMatchObject({ ok: true });
+	expect(mocks.audit).toHaveBeenCalledWith(
+		expect.objectContaining({ userId: 7, after: { deliveryId: id } }),
+	);
+});
+it("does not report a committed retry as failed when audit and refresh fail", async () => {
+	mocks.audit.mockRejectedValue(new Error("Audit unavailable"));
+	mocks.refresh.mockImplementation(() => {
+		throw new Error("Refresh unavailable");
+	});
+	expect(await retryDelivery(id)).toMatchObject({ ok: true });
+	expect(mocks.retry).toHaveBeenCalledTimes(1);
+	expect(mocks.error).toHaveBeenCalled();
+});

src/features/operations/retry-action.ts

@@ -0,0 +1,35 @@
+"use server";
+import { revalidatePath } from "next/cache";
+import { requirePermission } from "@/lib/admin/guard";
+import { logger } from "@/lib/logger";
+import { PERMS } from "@/lib/permission-slugs";
+import { logAudit } from "@/lib/services/audit";
+import { retryEffect } from "./server";
+export async function retryDelivery(id: string) {
+	const staff = await requirePermission(PERMS.DEVOPS_EDIT);
+	try {
+		await retryEffect(id);
+	} catch {
+		return {
+			ok: false as const,
+			error: "Delivery could not be retried. Refresh and try again.",
+		};
+	}
+	await logAudit({
+		userId: staff.id,
+		action: "operation_delivery_retry",
+		target: "operations",
+		after: { deliveryId: id },
+	}).catch((error) =>
+		logger.error("Delivery retried; audit failed", {
+			module: "operations",
+			error,
+		}),
+	);
+	try {
+		revalidatePath("/admin/devops/deliveries");
+	} catch {
+		/* Delivery retry remains saved. */
+	}
+	return { ok: true as const, data: {} };
+}

src/features/operations/retry-button.tsx

@@ -0,0 +1,23 @@
+"use client";
+import { useTranslations } from "next-intl";
+import { Button } from "@/components/ui/button";
+import { useServerAction } from "@/hooks/use-server-action";
+import { retryDelivery } from "./retry-action";
+export function DeliveryRetry({ id }: { id: string }) {
+	const t = useTranslations("pages.admin.deliveries");
+	const { run, isPending } = useServerAction();
+	return (
+		<Button
+			variant="outline"
+			disabled={isPending}
+			onClick={() =>
+				run(() => retryDelivery(id), {
+					successMessage: t("retried"),
+					errorMessage: t("retryError"),
+				})
+			}
+		>
+			{t("retry")}
+		</Button>
+	);
+}

src/features/operations/retry-effect.test.ts

@@ -0,0 +1,23 @@
+import { beforeEach, expect, it, vi } from "vitest";
+
+const execute = vi.hoisted(() => vi.fn());
+vi.mock("@/lib/db", () => ({ db: { execute } }));
+
+import { retryEffect } from "./server";
+
+const id = "840f4023-04cd-4d7b-92c5-047aa620029c";
+beforeEach(() => vi.resetAllMocks());
+it("does not report a missing or no-longer-failed delivery as retried", async () => {
+	execute.mockResolvedValue([{ affectedRows: 0 }]);
+	await expect(retryEffect(id)).rejects.toThrow("no longer available");
+});
+it("accepts a delivery that was actually moved back to pending", async () => {
+	execute.mockResolvedValue([{ affectedRows: 1 }]);
+	await expect(retryEffect(id)).resolves.toBeUndefined();
+});
+it("rejects malformed identifiers before a database query", async () => {
+	await expect(
+		retryEffect("------------------------------------"),
+	).rejects.toThrow("Invalid delivery");
+	expect(execute).not.toHaveBeenCalled();
+});

src/features/operations/server.ts

@@ -0,0 +1,117 @@
+import "server-only";
+import { randomUUID } from "node:crypto";
+import { sql } from "drizzle-orm";
+import { db } from "@/lib/db";
+import {
+	type EffectClaim,
+	type EffectTopic,
+	OperationConflict,
+	type OperationInput,
+	operationHash,
+	retryDelay,
+	validateOperation,
+} from "./model";
+export type OperationTransaction = Parameters<
+	Parameters<typeof db.transaction>[0]
+>[0];
+export async function runOperation<T>(
+	input: OperationInput,
+	work: (tx: OperationTransaction, operationId: string) => Promise<T>,
+): Promise<T> {
+	validateOperation(input);
+	const hash = operationHash(input.input);
+	return db.transaction(async (tx) => {
+		await tx.execute(
+			sql`INSERT INTO cms_operations (id,actor_id,kind,request_key,request_hash) VALUES (${randomUUID()},${input.actorId},${input.kind},${input.key},${hash}) ON DUPLICATE KEY UPDATE id=id`,
+		);
+		const [rows] = await tx.execute(
+			sql`SELECT id,request_hash AS requestHash,result_json AS resultJson FROM cms_operations WHERE actor_id=${input.actorId} AND kind=${input.kind} AND request_key=${input.key} FOR UPDATE`,
+		);
+		const row = (
+			rows as unknown as Array<{
+				id: string;
+				requestHash: string;
+				resultJson: string | null;
+			}>
+		)[0];
+		if (!row) throw new Error("Operation unavailable");
+		if (row.requestHash !== hash) throw new OperationConflict();
+		if (row.resultJson !== null) return JSON.parse(row.resultJson) as T;
+		const result = await work(tx, row.id);
+		const serialized = JSON.stringify(result);
+		if (!serialized || Buffer.byteLength(serialized) > 1048576)
+			throw new Error("Operation result exceeds storage limit");
+		await tx.execute(
+			sql`UPDATE cms_operations SET result_json=${serialized} WHERE id=${row.id}`,
+		);
+		return result;
+	});
+}
+export async function enqueueEffect(
+	tx: OperationTransaction,
+	operationId: string,
+	topic: EffectTopic,
+) {
+	await tx.execute(
+		sql`INSERT INTO cms_outbox (id,operation_id,topic) VALUES (${randomUUID()},${operationId},${topic}) ON DUPLICATE KEY UPDATE id=id`,
+	);
+}
+export const effectRepository = {
+	async claim(): Promise<EffectClaim | null> {
+		return db.transaction(async (tx) => {
+			const [rows] = await tx.execute(
+				sql`SELECT id,topic,attempts FROM cms_outbox WHERE (status='pending' AND available_at<=UTC_TIMESTAMP(3)) OR (status='running' AND lease_until<UTC_TIMESTAMP(3)) ORDER BY available_at,id LIMIT 1 FOR UPDATE`,
+			);
+			const row = (
+				rows as unknown as Array<{
+					id: string;
+					topic: EffectTopic;
+					attempts: number;
+				}>
+			)[0];
+			if (!row) return null;
+			const token = randomUUID();
+			await tx.execute(
+				sql`UPDATE cms_outbox SET status='running',lease_token=${token},lease_until=DATE_ADD(UTC_TIMESTAMP(3),INTERVAL 120 SECOND),attempts=attempts+1 WHERE id=${row.id}`,
+			);
+			return { ...row, token, attempts: Number(row.attempts) + 1 };
+		});
+	},
+	async complete(claim: EffectClaim) {
+		await db.execute(
+			sql`UPDATE cms_outbox SET status='done',lease_token=NULL,lease_until=NULL,last_error=NULL WHERE id=${claim.id} AND status='running' AND lease_token=${claim.token}`,
+		);
+	},
+	async fail(claim: EffectClaim) {
+		await db.execute(
+			sql`UPDATE cms_outbox SET status=${claim.attempts >= 8 ? "failed" : "pending"},available_at=DATE_ADD(UTC_TIMESTAMP(3),INTERVAL ${retryDelay(claim.attempts)} SECOND),lease_token=NULL,lease_until=NULL,last_error='Delivery failed; inspect the linked operation and service diagnostics.' WHERE id=${claim.id} AND status='running' AND lease_token=${claim.token}`,
+		);
+	},
+};
+export async function listEffects() {
+	const [rows] = await db.execute(
+		sql`SELECT e.id,e.operation_id AS operationId,e.topic,e.status,e.attempts,e.last_error AS lastError,e.created_at AS createdAt,o.actor_id AS actorId,o.kind FROM cms_outbox e JOIN cms_operations o ON o.id=e.operation_id ORDER BY e.created_at DESC,e.id DESC LIMIT 100`,
+	);
+	return rows as unknown as Array<{
+		id: string;
+		operationId: string;
+		topic: EffectTopic;
+		status: string;
+		attempts: number;
+		lastError: string | null;
+		createdAt: string;
+		actorId: number;
+		kind: string;
+	}>;
+}
+export async function retryEffect(id: string) {
+	if (
+		!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(id)
+	)
+		throw new Error("Invalid delivery");
+	const [result] = await db.execute(
+		sql`UPDATE cms_outbox SET status='pending',attempts=0,available_at=UTC_TIMESTAMP(3),last_error=NULL WHERE id=${id} AND status='failed'`,
+	);
+	if ((result as unknown as { affectedRows: number }).affectedRows !== 1)
+		throw new Error("Delivery is no longer available for retry");
+}

src/features/operations/worker.ts

@@ -0,0 +1,32 @@
+import "server-only";
+import { sendCatalogUpdate } from "@/features/catalog/server/sync-status";
+import { logger } from "@/lib/logger";
+import {
+	catalogExportEnabled,
+	catalogExportQueue,
+} from "@/lib/services/catalog-git-queue";
+import { dispatchEffects } from "./dispatcher";
+import { effectRepository } from "./server";
+
+let running = false;
+export async function drainOperationEffects() {
+	if (running) return;
+	running = true;
+	try {
+		await dispatchEffects(effectRepository, async (claim) => {
+			if (claim.topic === "catalog.refresh") {
+				if (!(await sendCatalogUpdate()).sent)
+					throw new Error("Hotel update not delivered");
+			} else if (claim.topic === "catalog.export.request") {
+				if (catalogExportEnabled()) await catalogExportQueue().request();
+			} else throw new Error("Unknown delivery topic");
+		});
+	} catch (error) {
+		logger.error("Operation delivery tick failed", {
+			module: "operations",
+			error,
+		});
+	} finally {
+		running = false;
+	}
+}