feat(import): run catalog synchronizations as durable jobs

scripts/jobs-worker.ts

@@ -1,3 +1,4 @@
+import { drainFurnitureImports } from "../src/lib/services/furni-job-worker";
 import "./load-env";
 import { Cron } from "croner";
 import { and, eq, lt, lte, sql } from "drizzle-orm";
@@ -268,6 +269,10 @@ async function reportWorkerHeartbeat(): Promise<void> {
 }
 
 async function main() {
+	new Cron("* * * * *", () => {
+		void drainFurnitureImports();
+	});
+	void drainFurnitureImports();
 	new Cron("* * * * *", () => {
 		runCatalogExport().catch((e) =>
 			captureWorkerError(e, "Catalog export failed"),

src/app/api/admin/import/official/sync-all/route.ts

@@ -1,97 +1,8 @@
 import { withAdmin } from "@/lib/api-handler";
-import { db, ItemsBase } from "@/lib/db";
 import { PERMS } from "@/lib/permissions";
-import { importSingleFurni } from "@/lib/services/furni-import";
+import { enqueueFurnitureSync } from "@/lib/services/furni-sync-queue";
-import { getOfficialHabboFurnidata } from "@/lib/services/habbo-furnidata-cache";
-import { runSseBatch } from "@/lib/services/import/core/sse-batch";
-
 export const POST = withAdmin(
 	{ permission: PERMS.ASSETS_IMPORT },
-	async (request) => {
+	(request, ctx) =>
-		// Haal alle officiële furnidata
+		enqueueFurnitureSync(request, ctx.session.user.id, "official"),
-		const furnidata = await getOfficialHabboFurnidata();
-
-		// Haal alle bestaande classnames uit de database
-		const existingClassnames = new Set(
-			await db
-				.select({ itemName: ItemsBase.itemName })
-				.from(ItemsBase)
-				.then((rows) => rows.map((r) => r.itemName)),
-		);
-
-		// Filter naar alleen items die niet in de database staan
-		const missingItems = Array.from(furnidata.entries())
-			.filter(([, entry]) => !existingClassnames.has(entry.classname))
-			.map(([, entry]) => ({
-				classname: entry.classname,
-				name: entry.name,
-				description: entry.description ?? "",
-				type: entry.category === "wallitem" ? "wallitem" : "flooritem",
-				revision: entry.revision ?? 0,
-				category: entry.category ?? "unknown",
-			}));
-
-		if (missingItems.length === 0) {
-			return new Response(
-				JSON.stringify({
-					success: true,
-					message: "All items are already in the database",
-					imported: 0,
-				}),
-				{
-					headers: { "Content-Type": "application/json" },
-				},
-			);
-		}
-
-		if (missingItems.length > 500) {
-			return new Response(
-				JSON.stringify({
-					success: false,
-					message: `Too many missing items (${missingItems.length}). Consider importing in smaller batches.`,
-				}),
-				{
-					headers: { "Content-Type": "application/json" },
-					status: 400,
-				},
-			);
-		}
-
-		// Importeren via SSE batch
-		return runSseBatch({
-			items: missingItems,
-			concurrency: 3,
-			signal: request.signal,
-			labelOf: (it) => it.classname,
-			worker: async (it, _index, report) => {
-				const result = await importSingleFurni({
-					id: 0,
-					classname: it.classname,
-					name: it.name,
-					description: it.description,
-					type: it.type,
-					revision: it.revision,
-					category: it.category,
-					onProgress: report,
-				});
-
-				if (result.ok) {
-					return {
-						ok: true,
-						warnings: result.warnings,
-						error: result.error,
-					};
-				}
-
-				return {
-					ok: false,
-					warnings: result.warnings,
-					error: result.error,
-				};
-			},
-			flush: async () => {
-				// FurnitureData.json wordt automatisch geschreven door importSingleFurni
-			},
-		});
-	},
 );

src/app/api/admin/studio/import-jobs/route.ts

@@ -2,6 +2,7 @@ import { after } from "next/server";
 import { z } from "zod";
 import { apiError, apiOk } from "@/lib/api";
 import { withAdmin } from "@/lib/api-handler";
+import { getRequestId } from "@/lib/foundation/request-context";
 import { validateClassnames } from "@/lib/furni/studio-inspection";
 import { PERMS } from "@/lib/permission-slugs";
 import { redis } from "@/lib/redis";
@@ -92,6 +93,7 @@ export const POST = withAdmin(
 		const job = await store.create({
 			...body,
 			userId: ctx.session.user.id,
+			operationId: getRequestId(),
 			createdAt,
 			updatedAt: createdAt,
 			state: "queued",

src/components/admin/studio/furniture-sync-queue.tsx

@@ -0,0 +1,73 @@
+"use client";
+import { useTranslations } from "next-intl";
+import { useRef, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { adminFetch } from "@/lib/admin-fetch";
+import { FurnitureJobHistory } from "./furniture-jobs";
+import { useFurnitureJobs } from "./use-furniture-jobs";
+
+export function FurnitureSyncQueue({ kind }: { kind: "official" | "clone" }) {
+	const t = useTranslations("pages.admin.syncQueue");
+	const history = useFurnitureJobs(() => {}, true);
+	const [busy, setBusy] = useState(false);
+	const [error, setError] = useState("");
+	const [queued, setQueued] = useState(false);
+	const requestId = useRef<string | null>(null);
+	const submitting = useRef(false);
+	async function start() {
+		if (submitting.current) return;
+		submitting.current = true;
+		setBusy(true);
+		setError("");
+		setQueued(false);
+		// Keep the ID after an uncertain response, including a browser reload.
+		const key = `furniture-sync-request:${kind}`;
+		try {
+			if (!requestId.current) {
+				try {
+					requestId.current = sessionStorage.getItem(key);
+				} catch {}
+				requestId.current ||= crypto.randomUUID();
+			}
+			try {
+				sessionStorage.setItem(key, requestId.current);
+			} catch {}
+			const response = await adminFetch(`/api/admin/import/${kind}/sync-all`, {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({ id: requestId.current }),
+			});
+			const result = await response.json().catch(() => ({}));
+			if (response.status >= 400 && response.status < 500) {
+				requestId.current = null;
+				try {
+					sessionStorage.removeItem(key);
+				} catch {}
+			}
+			if (!response.ok) throw Error(result.error || t("failed"));
+			if (result.job?.id !== requestId.current) throw Error(t("failed"));
+			try {
+				sessionStorage.removeItem(key);
+			} catch {}
+			requestId.current = null;
+			setQueued(true);
+			await history.refresh();
+		} catch (error) {
+			setError(error instanceof Error ? error.message : t("failed"));
+		} finally {
+			submitting.current = false;
+			setBusy(false);
+		}
+	}
+	return (
+		<div className="space-y-4">
+			<p className="text-sm text-muted-foreground">{t("description")}</p>
+			<Button disabled={busy} onClick={() => void start()}>
+				{busy ? t("preparing") : t("start")}
+			</Button>
+			{error && <p role="alert">{error}</p>}
+			{queued && <p role="status">{t("queued")}</p>}
+			<FurnitureJobHistory state={history} expanded />
+		</div>
+	);
+}

src/lib/furni/import-job.ts

@@ -1,3 +1,4 @@
+import type { SourceFurni } from "@/lib/services/clone-import";
 export interface ImportJobItem {
 	id: number;
 	classname: string;
@@ -7,9 +8,13 @@ export interface ImportJobItem {
 	revision: number;
 	category: string;
 	attachmentId?: string;
+	cloneSourceId?: string;
+	cloneEntry?: SourceFurni;
 }
 export interface ImportJob {
 	mode?: "repair";
+	syncKind?: "official" | "clone";
+	operationId?: string;
 	retryOf?: string;
 	id: string;
 	userId: number;

src/lib/services/furni-job-worker.test.ts

@@ -16,7 +16,7 @@ const mocks = vi.hoisted(() => ({
 }));
 vi.mock("@/lib/redis", () => ({ redis: { set: mocks.set, eval: mocks.eval } }));
 vi.mock("@/lib/server-log", () => ({ logServerError: vi.fn() }));
-vi.mock("./audit", () => ({ logAudit: vi.fn() }));
+vi.mock("./audit", () => ({ logAudit: vi.fn().mockResolvedValue(undefined) }));
 vi.mock("./catalog-git-queue", () => ({ withCatalogExport: mocks.export }));
 vi.mock("./clone-sources", () => ({ getSource: async () => null }));
 vi.mock("./furni-job-store", () => ({
@@ -42,6 +42,8 @@ vi.mock("./rcon", () => ({
 
 vi.mock("./catalog-git-export", () => ({ runCatalogExport: vi.fn() }));
 
+vi.mock("./furni-sync-item", () => ({ runSyncJobItem: mocks.import }));
+
 import { drainFurnitureImports } from "./furni-job-worker";
 
 let job: ImportJob;
@@ -223,3 +225,27 @@ it("does not save a translated outcome after ownership is lost during translatio
 	expect(snapshots.at(-1)?.items[0].state).toBe("running");
 	expect(mocks.updateCatalog).not.toHaveBeenCalled();
 });
+
+it("restores the original operation context while running a durable job", async () => {
+	const { getOperationContext } = await import(
+		"@/lib/foundation/request-context"
+	);
+	job.operationId = "original-operation";
+	mocks.import.mockImplementation(async () => {
+		expect(getOperationContext()).toEqual({
+			operationId: "original-operation",
+			userId: 5,
+		});
+		return { ok: true, itemId: 900, warnings: [] };
+	});
+	await drainFurnitureImports();
+	expect(getOperationContext()).toEqual({});
+});
+
+it("keeps successful imports complete if their audit write fails", async () => {
+	const { logAudit } = await import("./audit");
+	vi.mocked(logAudit).mockRejectedValueOnce(new Error("Audit unavailable"));
+	await drainFurnitureImports();
+	expect(job.items[0].state).toBe("done");
+	expect(job.items[0].warnings).toContain("Audit record could not be saved");
+});

src/lib/services/furni-job-worker.ts

@@ -1,4 +1,6 @@
 import { randomUUID } from "node:crypto";
+import { createStore, runWithStore } from "@/lib/foundation/request-context";
+import type { IpAddress, RequestId, UserId } from "@/lib/foundation/types";
 import type { ImportJob } from "@/lib/furni/import-job";
 import { redis } from "@/lib/redis";
 import { logServerError } from "@/lib/server-log";
@@ -10,6 +12,7 @@ import { readFurnitureAttachment } from "./furni-attachment";
 import { patchLocalizedFurniDataEntries } from "./furni-data-i18n";
 import { ensureDirectories, importSingleFurni } from "./furni-import";
 import { ImportJobStore } from "./furni-job-store";
+import { runSyncJobItem } from "./furni-sync-item";
 import { repairFurniture } from "./furniture-repair";
 import { rcon } from "./rcon";
 
@@ -100,7 +103,11 @@ async function drain() {
 			}
 			job.state = "running";
 			await saveOwned(job);
-			await withCatalogExport(async () => {
+			const operation = createStore("worker" as IpAddress);
+			operation.requestId = (job.operationId ?? job.id) as RequestId;
+			operation.userId = job.userId as UserId;
+			await runWithStore(operation, () =>
+				withCatalogExport(async () => {
 					await ensureDirectories();
 					const source = job.sourceId ? await getSource(job.sourceId) : null;
 					for (const item of job.items) {
@@ -108,7 +115,8 @@ async function drain() {
 						if (item.state !== "pending") continue;
 						if (await store.isCancellationRequested(job.id)) {
 							for (const remaining of job.items)
-							if (remaining.state === "pending") remaining.state = "cancelled";
+								if (remaining.state === "pending")
+									remaining.state = "cancelled";
 							break;
 						}
 						item.state = "running";
@@ -124,7 +132,9 @@ async function drain() {
 									)
 								: undefined;
 							await requireLease();
-						const result = await (job.mode === "repair"
+							const result = job.syncKind
+								? await runSyncJobItem(item)
+								: await (job.mode === "repair"
 										? repairFurniture
 										: importSingleFurni)({
 										...item,
@@ -139,7 +149,7 @@ async function drain() {
 							item.itemId = result.itemId;
 							if (!result.ok) throw Error(result.error || "Import failed");
 							item.state = "done";
-						logAudit({
+							await logAudit({
 								userId: job.userId,
 								action: "furni_import",
 								target: "ItemsBase",
@@ -147,8 +157,17 @@ async function drain() {
 								after: {
 									classname: item.classname,
 									jobId: job.id,
+									operationId: job.operationId,
+									syncKind: job.syncKind,
 									repairExisting: true,
 								},
+							}).catch((auditError) => {
+								item.warnings ??= [];
+								item.warnings.push("Audit record could not be saved");
+								logServerError("furni.audit_failed", auditError, {
+									jobId: job.id,
+									classname: item.classname,
+								});
 							});
 							if (job.translate && job.mode !== "repair")
 								try {
@@ -161,6 +180,10 @@ async function drain() {
 						} catch (error) {
 							if (error instanceof ImportLeaseLostError) throw error;
 							await requireLease();
+							logServerError("furni.item_failed", error, {
+								jobId: job.id,
+								classname: item.classname,
+							});
 							item.state = "failed";
 							item.error =
 								error instanceof Error ? error.message : "Import failed";
@@ -180,7 +203,8 @@ async function drain() {
 								item.warnings.push("Game cache refresh failed");
 							}
 					}
-			});
+				}),
+			);
 			await requireLease();
 			job.state = job.items.some((item) => item.state === "cancelled")
 				? "cancelled"

src/lib/services/furni-sync-item.test.ts

@@ -0,0 +1,75 @@
+import { beforeEach, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+	existing: vi.fn(),
+	clone: vi.fn(),
+	source: vi.fn(),
+	import: vi.fn(),
+}));
+vi.mock("@/lib/db", async () => ({
+	...(await import("@/db/schema")),
+	db: {
+		select: () => ({
+			from: () => ({ where: () => ({ limit: mocks.existing }) }),
+		}),
+	},
+}));
+vi.mock("./clone-import", () => ({ cloneSingleFurni: mocks.clone }));
+vi.mock("./clone-sources", () => ({ getSource: mocks.source }));
+vi.mock("./furni-import", () => ({ importSingleFurni: mocks.import }));
+
+import { runSyncJobItem } from "./furni-sync-item";
+
+const item = {
+	id: 0,
+	classname: "chair",
+	name: "Chair",
+	description: "",
+	type: "flooritem",
+	revision: 1,
+	category: "other",
+};
+beforeEach(() => {
+	vi.clearAllMocks();
+	mocks.existing.mockResolvedValue([]);
+	mocks.import.mockResolvedValue({ ok: true, warnings: [] });
+});
+it("skips an item another queued request has already imported", async () => {
+	mocks.existing.mockResolvedValue([{ id: 3 }]);
+	expect(await runSyncJobItem(item)).toMatchObject({ ok: true, itemId: 3 });
+	expect(mocks.import).not.toHaveBeenCalled();
+	expect(mocks.clone).not.toHaveBeenCalled();
+});
+it("never repairs existing furniture during official sync", async () => {
+	await runSyncJobItem(item);
+	expect(mocks.import).toHaveBeenCalledWith({ ...item, repairExisting: false });
+});
+it("fails a deleted clone source without substituting official assets", async () => {
+	mocks.source.mockResolvedValue(null);
+	await expect(
+		runSyncJobItem({ ...item, cloneSourceId: "gone" }),
+	).rejects.toThrow("no longer available");
+	expect(mocks.import).not.toHaveBeenCalled();
+});
+
+it("returns the created clone item ID for audit correlation", async () => {
+	mocks.existing.mockResolvedValueOnce([]).mockResolvedValueOnce([{ id: 71 }]);
+	mocks.source.mockResolvedValue({ id: "source" });
+	mocks.clone.mockResolvedValue({ ok: true, warnings: [] });
+	const cloneEntry = {
+		id: 1,
+		classname: "chair",
+		name: "Chair",
+		description: "",
+		xdim: 1,
+		ydim: 1,
+		canstandon: false,
+		cansiton: true,
+		canlayon: false,
+		customparams: "",
+		itemType: "s" as const,
+	};
+	expect(
+		await runSyncJobItem({ ...item, cloneSourceId: "source", cloneEntry }),
+	).toMatchObject({ ok: true, itemId: 71 });
+});

src/lib/services/furni-sync-item.ts

@@ -0,0 +1,36 @@
+import { eq } from "drizzle-orm";
+import { db, ItemsBase } from "@/lib/db";
+import type { ImportJobItem } from "@/lib/furni/import-job";
+import { cloneSingleFurni } from "./clone-import";
+import { getSource } from "./clone-sources";
+import { importSingleFurni } from "./furni-import";
+
+/** Called only inside the shared import lease and catalog export lock. */
+export async function runSyncJobItem(item: ImportJobItem) {
+	const [existing] = await db
+		.select({ id: ItemsBase.id })
+		.from(ItemsBase)
+		.where(eq(ItemsBase.itemName, item.classname))
+		.limit(1);
+	if (existing)
+		return {
+			ok: true,
+			itemId: existing.id,
+			warnings: ["Already present; synchronization skipped this item."],
+		};
+	if (item.cloneSourceId) {
+		const source = await getSource(item.cloneSourceId);
+		if (!source || !item.cloneEntry)
+			throw Error("Synchronization source no longer available");
+		const result = await cloneSingleFurni({ source, entry: item.cloneEntry });
+		const [created] = result.ok
+			? await db
+					.select({ id: ItemsBase.id })
+					.from(ItemsBase)
+					.where(eq(ItemsBase.itemName, item.classname))
+					.limit(1)
+			: [];
+		return { ...result, itemId: created?.id };
+	}
+	return importSingleFurni({ ...item, repairExisting: false });
+}