perf: offload FurnitureData translation build to a worker thread

The translation build's CPU-heavy work (deep clone of the 100k+ entry
master, the two full scans and JSON.stringify per language) now runs in a
worker thread, so "Alles vertalen" no longer blocks the main event loop.
Network/DB parts (official Habbo fetch, LibreTranslate, file writes) stay
on the main thread. The pure helpers were extracted to furni-data-i18n-core
(no server-only deps) so the worker can import them. Falls back to the same
helpers on the main thread when no worker is available or in tests.

src/lib/services/furni-data-i18n-core.ts

@@ -0,0 +1,155 @@
+/**
+ * Pure, side-effect-free helpers for building localized FurnitureData files.
+ *
+ * This module deliberately imports NOTHING from site-settings / server-only /
+ * network so it can run safely inside a worker thread. All I/O (fetching
+ * official Habbo translations, calling LibreTranslate, writing files) stays on
+ * the main thread; this module only does the CPU-heavy clone + scan + stringify.
+ */
+import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel";
+
+export type Translation = { name: string; description: string };
+
+export type FurniDataEntry = Record<string, unknown>;
+export interface FurniDataSection {
+	furnitype: FurniDataEntry[];
+}
+export type FurniDataMaster = Record<string, FurniDataSection>;
+
+export function cloneFurniData(data: Record<string, unknown>): FurniDataMaster {
+	// Deep clone via JSON — structure is pure JSON and 100k+ entries; this is the
+	// simplest safe copy so each language build gets its own mutable copy.
+	return JSON.parse(JSON.stringify(data)) as FurniDataMaster;
+}
+
+export function officialIsRealTranslation(
+	lang: string,
+	hotel: HabboGamedataHotel,
+): boolean {
+	// A language mapped to hotel "com" only carries English names, so it must NOT
+	// be treated as a real translation — fall through to LibreTranslate for those.
+	return lang === "en" || hotel !== "com";
+}
+
+export function lookupTranslation(
+	map: Map<string, Translation>,
+	classname: string,
+): Translation | undefined {
+	const exact = map.get(classname);
+	if (exact) return exact;
+	if (classname.includes("*")) {
+		const base = classname.split("*")[0];
+		const v0 = map.get(`${base}*0`);
+		if (v0) return v0;
+		const baseExact = map.get(base);
+		if (baseExact) return baseExact;
+	}
+	if (!classname.includes("*")) {
+		const v0 = map.get(`${classname}*0`);
+		if (v0) return v0;
+	}
+	return undefined;
+}
+
+export interface PendingRef {
+	entry: FurniDataEntry;
+	field: "name" | "description";
+	section: "roomitemtypes" | "wallitemtypes";
+}
+
+export interface PreparedLang {
+	cloned: FurniDataMaster;
+	pendingRefs: PendingRef[];
+	pendingTexts: string[];
+}
+
+/**
+ * Pass 1: deep-clone the master, apply official translations directly, and
+ * collect the texts that still need LibreTranslate. Returns the cloned object
+ * plus the pending refs/texts so the (network) translation step can run and the
+ * result be applied later without re-scanning.
+ */
+export function prepareLangFile(
+	master: FurniDataMaster,
+	lang: string,
+	hotel: HabboGamedataHotel,
+	officialMap: Map<string, Translation>,
+): PreparedLang {
+	const cloned = cloneFurniData(master);
+	const officialIsReal = officialIsRealTranslation(lang, hotel);
+	const pendingRefs: PendingRef[] = [];
+	const pendingTexts = new Set<string>();
+
+	for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
+		for (const entry of cloned[section]?.furnitype ?? []) {
+			const cn = typeof entry.classname === "string" ? entry.classname : "";
+			if (!cn) continue;
+			const t = lookupTranslation(officialMap, cn);
+			if (officialIsReal && t?.name) {
+				entry.name = t.name;
+			} else if (
+				lang !== "en" &&
+				typeof entry.name === "string" &&
+				entry.name.trim()
+			) {
+				pendingTexts.add(entry.name.trim());
+				pendingRefs.push({ entry, field: "name", section });
+			}
+			if (officialIsReal && t?.description) {
+				entry.description = t.description;
+			} else if (
+				lang !== "en" &&
+				typeof entry.description === "string" &&
+				entry.description.trim()
+			) {
+				pendingTexts.add(entry.description.trim());
+				pendingRefs.push({ entry, field: "description", section });
+			}
+		}
+	}
+
+	return { cloned, pendingRefs, pendingTexts: [...pendingTexts] };
+}
+
+export interface FinalizedLang {
+	json: string;
+	translatedRoom: number;
+	translatedWall: number;
+	total: number;
+}
+
+/**
+ * Pass 2: apply the LibreTranslate results to the prepared clone and serialize
+ * it. Kept separate from prepareLangFile so the (async, network) translation
+ * step can run in between on the main thread.
+ */
+export function finalizeLangFile(
+	cloned: FurniDataMaster,
+	pendingRefs: PendingRef[],
+	translations: Map<string, string>,
+): FinalizedLang {
+	const counted = new Set<FurniDataEntry>();
+	let translatedRoom = 0;
+	let translatedWall = 0;
+	for (const ref of pendingRefs) {
+		const original = ref.entry[ref.field];
+		if (typeof original !== "string") continue;
+		const translated = translations.get(original.trim());
+		if (translated === undefined || translated === original) continue;
+		ref.entry[ref.field] = translated;
+		if (!counted.has(ref.entry)) {
+			counted.add(ref.entry);
+			if (ref.section === "roomitemtypes") translatedRoom++;
+			else translatedWall++;
+		}
+	}
+	const total =
+		(cloned.roomitemtypes?.furnitype?.length ?? 0) +
+		(cloned.wallitemtypes?.furnitype?.length ?? 0);
+	return {
+		json: JSON.stringify(cloned),
+		translatedRoom,
+		translatedWall,
+		total,
+	};
+}

src/lib/services/furni-data-i18n.ts

@@ -22,6 +22,7 @@ import {
 } from "@/lib/services/furni-data";
 import { browserHeaders } from "@/lib/services/import/core/browser-headers";
 import { siteSettings } from "@/lib/services/site-settings";
+import { getTranslationWorker } from "@/lib/services/translation-pool";
 
 export const FURNIDATA_LANGUAGES = [
 	{ lang: "nl", hotel: "nl" as HabboGamedataHotel },
@@ -68,7 +69,7 @@ export interface LanguageFileResult {
 	error?: string;
 }
 
-type Translation = { name: string; description: string };
+import type { Translation } from "./furni-data-i18n-core";
 
 const LANG_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours — translations rarely change
 
@@ -256,38 +257,31 @@ async function translateManyWithLibre(
 	return result;
 }
 
-function lookupTranslation(
+import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core";
-	map: Map<string, Translation>,
-	classname: string,
-): Translation | undefined {
-	const exact = map.get(classname);
-	if (exact) return exact;
-	if (classname.includes("*")) {
-		const base = classname.split("*")[0];
-		const v0 = map.get(`${base}*0`);
-		if (v0) return v0;
-		const baseExact = map.get(base);
-		if (baseExact) return baseExact;
-	}
-	if (!classname.includes("*")) {
-		const v0 = map.get(`${classname}*0`);
-		if (v0) return v0;
-	}
-	return undefined;
-}
-
 /**
- * A language mapped to hotel "com" only carries English names, so it must
+ * The pure clone / scan / stringify helpers live in furni-data-i18n-core so they
- * NOT be treated as a real translation — fall through to LibreTranslate for
+ * can run inside a worker thread (that module has no server-only / network deps).
- * those (pl/ru/ar/ja). Shared by the full build and the single-import patch
+ * They're imported here for the main-thread call paths and re-exported for
- * so both paths stay in sync.
+ * backward compatibility.
  */
-function officialIsRealTranslation(
+import {
-	lang: string,
+	cloneFurniData,
-	hotel: HabboGamedataHotel,
+	finalizeLangFile,
-): boolean {
+	lookupTranslation,
-	return lang === "en" || hotel !== "com";
+	officialIsRealTranslation,
-}
+	prepareLangFile,
+} from "./furni-data-i18n-core";
+
+export type {
+	FurniDataMaster,
+	PreparedLang,
+	Translation,
+} from "./furni-data-i18n-core";
+export {
+	cloneFurniData,
+	lookupTranslation,
+	officialIsRealTranslation,
+} from "./furni-data-i18n-core";
 
 async function fetchTranslationsForHotel(
 	hotel: HabboGamedataHotel,
@@ -377,22 +371,14 @@ async function getLocalizedWritePaths(lang: string): Promise<string[]> {
 	});
 }
 
-function cloneFurniData(
-	data: Record<string, unknown>,
-): Record<string, { furnitype: Array<Record<string, unknown>> }> {
-	// Deep clone via JSON — structure is pure JSON and 100k+ entries, this is
-	// the simplest safe copy. Each language build gets its own clone.
-	return JSON.parse(JSON.stringify(data)) as Record<
-		string,
-		{ furnitype: Array<Record<string, unknown>> }
-	>;
-}
-
 /**
  * Build all language files from the current master FurnitureData.json.
  * Each file is a full copy with name/description overridden where an official
- * translation exists for that classname. Runs under the furnidata lock so it
+ * translation exists for that classname. The CPU-heavy clone + 100k-entry scan
- * never interleaves with concurrent imports.
+ * + stringify is offloaded to a worker thread (see translation-pool); only the
+ * network/DB parts (fetching official translations, LibreTranslate, writing
+ * files) stay on the main thread. Falls back to running the same pure helpers
+ * on the main thread when no worker is available.
  */
 export async function buildLocalizedFurniDataFiles(
 	force = false,
@@ -414,107 +400,79 @@ export async function buildLocalizedFurniDataFiles(
 			{ furnitype: Array<Record<string, unknown>> }
 		>;
 
-		// Fetch all language maps in parallel (uses per-hotel memory cache).
+		// Fetch each hotel's official translations once (network + per-hotel cache).
-		const maps = await Promise.all(
+		const hotels = [...new Set(langsToBuild.map((l) => l.hotel))];
-			langsToBuild.map(async ({ lang, hotel }) => {
+		const hotelMaps: Record<string, Map<string, Translation>> = {};
-				const map = await fetchTranslationsForHotel(hotel);
+		for (const hotel of hotels) {
-				return { lang, hotel, map };
+			hotelMaps[hotel] = await fetchTranslationsForHotel(hotel);
-			}),
+		}
-		);
+
+		const worker = getTranslationWorker();
+		if (worker) {
+			await worker.init(JSON.stringify(master), hotelMaps);
+		}
+		// Main-thread fallback store, used only when no worker is available.
+		const localPrepared = new Map<string, PreparedLang>();
 
 		const results: LanguageFileResult[] = [];
 
-		for (const { lang, hotel, map } of maps) {
+		for (const { lang, hotel } of langsToBuild) {
 			try {
 				// Yield before each language (except the first) so a long build
 				// never pins the CPU continuously and the server stays responsive.
 				if (results.length > 0) {
-					await new Promise((r) => setImmediate(r));
+					await yieldToEventLoop();
 					await sleep(BUILD_YIELD_MS);
 				}
-				const cloned = cloneFurniData(
-					master as unknown as Record<string, unknown>,
-				);
-				const officialIsReal = officialIsRealTranslation(lang, hotel);
 
-				// Pass 1 — apply official translations directly and collect the
+				let pendingTexts: string[];
-				// texts that need LibreTranslate (customs, or languages whose
+				let preparedKey: string | null = null;
-				// hotel only provides English) so they can be batch-translated.
+				if (worker) {
-				const pendingRefs: Array<{
+					pendingTexts = await worker.prepare(lang, hotel);
-					entry: Record<string, unknown>;
+				} else {
-					field: "name" | "description";
+					const r = prepareLangFile(
-					section: "roomitemtypes" | "wallitemtypes";
+						master as unknown as FurniDataMaster,
-				}> = [];
+						lang,
-				const pendingTexts = new Set<string>();
+						hotel,
-				const sections = ["roomitemtypes", "wallitemtypes"] as const;
+						hotelMaps[hotel] ?? new Map(),
-				for (let si = 0; si < sections.length; si++) {
+					);
-					const section = sections[si];
+					pendingTexts = r.pendingTexts;
-					const arr = cloned[section]?.furnitype;
+					preparedKey = lang;
-					if (Array.isArray(arr)) {
+					localPrepared.set(preparedKey, r);
-						for (const entry of arr) {
-							const cn =
-								typeof entry.classname === "string" ? entry.classname : "";
-							if (!cn) continue;
-							const t = lookupTranslation(map, cn);
-							if (officialIsReal && t?.name) {
-								entry.name = t.name;
-							} else if (
-								lang !== "en" &&
-								typeof entry.name === "string" &&
-								entry.name.trim()
-							) {
-								pendingTexts.add(entry.name.trim());
-								pendingRefs.push({ entry, field: "name", section });
-							}
-							if (officialIsReal && t?.description) {
-								entry.description = t.description;
-							} else if (
-								lang !== "en" &&
-								typeof entry.description === "string" &&
-								entry.description.trim()
-							) {
-								pendingTexts.add(entry.description.trim());
-								pendingRefs.push({ entry, field: "description", section });
-							}
-						}
-					}
-					// Yield between the two 100k+ scans so a single language never
-					// blocks the event loop for the whole pass at once.
-					if (si === 0) await yieldToEventLoop();
 				}
 
-				// Pass 2 — batch-translate everything pending in a handful of
+				// Pass 2 — batch-translate everything pending (network, async, so
-				// parallel requests instead of one HTTP call per text.
+				// this part never blocks the main thread regardless).
-				const translations = await translateManyWithLibre(pendingTexts, lang);
+				const translations = await translateManyWithLibre(
+					new Set(pendingTexts),
+					lang,
+				);
 
+				let json: string;
 				let translatedRoom = 0;
 				let translatedWall = 0;
-				if (pendingRefs.length > 0) {
+				let total = 0;
-					const counted = new Set<Record<string, unknown>>();
+				if (worker) {
-					for (const ref of pendingRefs) {
+					const res = await worker.finalize(lang, translations);
-						const original = ref.entry[ref.field];
+					json = res.json;
-						if (typeof original !== "string") continue;
+					translatedRoom = res.translatedRoom;
-						const translated = translations.get(original.trim());
+					translatedWall = res.translatedWall;
-						if (translated === undefined || translated === original) continue;
+					total = res.total;
-						ref.entry[ref.field] = translated;
+				} else {
-						if (!counted.has(ref.entry)) {
+					const r = localPrepared.get(preparedKey!)!;
-							counted.add(ref.entry);
+					const res = finalizeLangFile(r.cloned, r.pendingRefs, translations);
-							if (ref.section === "roomitemtypes") translatedRoom++;
+					json = res.json;
-							else translatedWall++;
+					translatedRoom = res.translatedRoom;
-						}
+					translatedWall = res.translatedWall;
-					}
+					total = res.total;
+					localPrepared.delete(preparedKey!);
 				}
 
-				const total =
-					(cloned.roomitemtypes?.furnitype?.length ?? 0) +
-					(cloned.wallitemtypes?.furnitype?.length ?? 0);
-
-				const payload = (await yieldToEventLoop(), JSON.stringify(cloned));
 				const targets = await getLocalizedWritePaths(lang);
 				for (const target of targets) {
 					await fs.mkdir(path.dirname(target), { recursive: true });
 					const tmp = `${target}.tmp`;
-					await fs.writeFile(tmp, payload, "utf-8");
+					await fs.writeFile(tmp, json, "utf-8");
 					await fs.rename(tmp, target);
 				}
 

src/lib/services/translation-pool.ts

@@ -0,0 +1,138 @@
+import { Worker } from "node:worker_threads";
+import type { Translation } from "./furni-data-i18n-core";
+
+/**
+ * Runs the CPU-heavy FurnitureData localization (deep clone + 100k-entry scans +
+ * JSON.stringify) in a single worker thread so the main Node event loop stays
+ * free during a "translate everything" build. Network/DB work (fetching official
+ * Habbo translations, calling LibreTranslate, writing files) stays on the main
+ * thread; only the synchronous CPU is offloaded.
+ *
+ * Degrades gracefully: in the test environment (or if the worker can't be
+ * created) getTranslationWorker() returns null and callers run the same pure
+ * helpers on the main thread instead.
+ */
+interface InitReq {
+	id: number;
+	type: "init";
+	masterJson: string;
+	hotelMaps: Record<string, Map<string, Translation>>;
+}
+interface PrepareReq {
+	id: number;
+	type: "prepare";
+	lang: string;
+	hotel: string;
+}
+interface FinalizeReq {
+	id: number;
+	type: "finalize";
+	lang: string;
+	translations: Map<string, string>;
+}
+type Req = InitReq | PrepareReq | FinalizeReq;
+type ReqWithoutId =
+	| Omit<InitReq, "id">
+	| Omit<PrepareReq, "id">
+	| Omit<FinalizeReq, "id">;
+
+type Res =
+	| {
+			id: number;
+			ok: true;
+			pendingTexts?: string[];
+			json?: string;
+			translatedRoom?: number;
+			translatedWall?: number;
+			total?: number;
+	  }
+	| { id: number; ok: false; error: string };
+
+let worker: Worker | null = null;
+let broken = false;
+let nextId = 1;
+const pending = new Map<
+	number,
+	{ resolve: (r: Res) => void; reject: (e: Error) => void }
+>();
+
+function isTestEnv(): boolean {
+	return process.env.NODE_ENV === "test" || process.env.VITEST !== undefined;
+}
+
+function spawn(): Worker | null {
+	try {
+		const w = new Worker(new URL("./translation-worker.ts", import.meta.url));
+		w.on("message", (res: Res) => {
+			const p = pending.get(res.id);
+			if (!p) return;
+			pending.delete(res.id);
+			if (res.ok) p.resolve(res);
+			else p.reject(new Error(res.error));
+		});
+		w.on("error", (err: Error) => {
+			broken = true;
+			for (const [, p] of pending) p.reject(err);
+			pending.clear();
+		});
+		return w;
+	} catch {
+		broken = true;
+		return null;
+	}
+}
+
+function call(req: ReqWithoutId): Promise<Res> {
+	if (!worker) return Promise.reject(new Error("no translation worker"));
+	return new Promise<Res>((resolve, reject) => {
+		const id = nextId++;
+		pending.set(id, { resolve, reject });
+		worker!.postMessage({ ...req, id } as Req);
+	});
+}
+
+export interface TranslationWorker {
+	init(
+		masterJson: string,
+		hotelMaps: Record<string, Map<string, Translation>>,
+	): Promise<void>;
+	prepare(lang: string, hotel: string): Promise<string[]>;
+	finalize(
+		lang: string,
+		translations: Map<string, string>,
+	): Promise<{
+		json: string;
+		translatedRoom: number;
+		translatedWall: number;
+		total: number;
+	}>;
+}
+
+export function getTranslationWorker(): TranslationWorker | null {
+	if (isTestEnv() || broken) return null;
+	if (!worker) worker = spawn();
+	if (!worker) return null;
+	return {
+		async init(masterJson, hotelMaps) {
+			const res = await call({ type: "init", masterJson, hotelMaps });
+			if (!res.ok) throw new Error(res.error);
+		},
+		async prepare(lang, hotel) {
+			const res = await call({ type: "prepare", lang, hotel });
+			if (!res.ok || !res.pendingTexts)
+				throw new Error(res.ok ? "missing pendingTexts" : res.error);
+			return res.pendingTexts;
+		},
+		async finalize(lang, translations) {
+			const res = await call({ type: "finalize", lang, translations });
+			if (!res.ok || res.json === undefined)
+				throw new Error(res.ok ? "missing json" : res.error);
+			return {
+				json: res.json,
+				translatedRoom: res.translatedRoom ?? 0,
+				translatedWall: res.translatedWall ?? 0,
+				total: res.total ?? 0,
+			};
+		},
+	};
+}

src/lib/services/translation-worker.ts

@@ -0,0 +1,76 @@
+import { parentPort } from "node:worker_threads";
+import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel";
+import {
+	type FurniDataMaster,
+	finalizeLangFile,
+	type PreparedLang,
+	prepareLangFile,
+	type Translation,
+} from "./furni-data-i18n-core";
+
+interface InitMsg {
+	id: number;
+	type: "init";
+	masterJson: string;
+	hotelMaps: Record<string, Map<string, Translation>>;
+}
+interface PrepareMsg {
+	id: number;
+	type: "prepare";
+	lang: string;
+	hotel: string;
+}
+interface FinalizeMsg {
+	id: number;
+	type: "finalize";
+	lang: string;
+	translations: Map<string, string>;
+}
+type Msg = InitMsg | PrepareMsg | FinalizeMsg;
+
+let master: FurniDataMaster | null = null;
+let hotelMaps: Record<string, Map<string, Translation>> = {};
+const prepared = new Map<string, PreparedLang>();
+
+if (parentPort) {
+	parentPort.on("message", (msg: Msg) => {
+		try {
+			if (msg.type === "init") {
+				master = JSON.parse(msg.masterJson);
+				hotelMaps = msg.hotelMaps ?? {};
+				parentPort!.postMessage({ id: msg.id, ok: true });
+				return;
+			}
+			if (msg.type === "prepare") {
+				if (!master) throw new Error("translation worker not initialized");
+				const officialMap =
+					hotelMaps[msg.hotel] ?? new Map<string, Translation>();
+				const r = prepareLangFile(
+					master,
+					msg.lang,
+					msg.hotel as HabboGamedataHotel,
+					officialMap,
+				);
+				prepared.set(msg.lang, r);
+				parentPort!.postMessage({
+					id: msg.id,
+					ok: true,
+					pendingTexts: r.pendingTexts,
+				});
+				return;
+			}
+			// finalize
+			const r = prepared.get(msg.lang);
+			if (!r) throw new Error(`no prepared data for ${msg.lang}`);
+			const res = finalizeLangFile(r.cloned, r.pendingRefs, msg.translations);
+			prepared.delete(msg.lang);
+			parentPort!.postMessage({ id: msg.id, ok: true, ...res });
+		} catch (err) {
+			parentPort!.postMessage({
+				id: msg.id,
+				ok: false,
+				error: (err as Error).message,
+			});
+		}
+	});
+}