From f94246e067cf9aab7d82af94baf5825458918a97 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 24 Aug 2026 19:10:33 +0200 Subject: [PATCH] perf: offload FurnitureData translation build to a worker thread The translation build's CPU-heavy work (deep clone of the 100k+ entry master, the two full scans and JSON.stringify per language) now runs in a worker thread, so "Alles vertalen" no longer blocks the main event loop. Network/DB parts (official Habbo fetch, LibreTranslate, file writes) stay on the main thread. The pure helpers were extracted to furni-data-i18n-core (no server-only deps) so the worker can import them. Falls back to the same helpers on the main thread when no worker is available or in tests. --- src/lib/services/furni-data-i18n-core.ts | 155 +++++++++++++++++ src/lib/services/furni-data-i18n.ts | 206 +++++++++-------------- src/lib/services/translation-pool.ts | 138 +++++++++++++++ src/lib/services/translation-worker.ts | 76 +++++++++ 4 files changed, 451 insertions(+), 124 deletions(-) create mode 100644 src/lib/services/furni-data-i18n-core.ts create mode 100644 src/lib/services/translation-pool.ts create mode 100644 src/lib/services/translation-worker.ts diff --git a/src/lib/services/furni-data-i18n-core.ts b/src/lib/services/furni-data-i18n-core.ts new file mode 100644 index 00000000..4674ac85 --- /dev/null +++ b/src/lib/services/furni-data-i18n-core.ts @@ -0,0 +1,155 @@ +/** + * Pure, side-effect-free helpers for building localized FurnitureData files. + * + * This module deliberately imports NOTHING from site-settings / server-only / + * network so it can run safely inside a worker thread. All I/O (fetching + * official Habbo translations, calling LibreTranslate, writing files) stays on + * the main thread; this module only does the CPU-heavy clone + scan + stringify. + */ +import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel"; + +export type Translation = { name: string; description: string }; + +export type FurniDataEntry = Record; +export interface FurniDataSection { + furnitype: FurniDataEntry[]; +} +export type FurniDataMaster = Record; + +export function cloneFurniData(data: Record): FurniDataMaster { + // Deep clone via JSON — structure is pure JSON and 100k+ entries; this is the + // simplest safe copy so each language build gets its own mutable copy. + return JSON.parse(JSON.stringify(data)) as FurniDataMaster; +} + +export function officialIsRealTranslation( + lang: string, + hotel: HabboGamedataHotel, +): boolean { + // A language mapped to hotel "com" only carries English names, so it must NOT + // be treated as a real translation — fall through to LibreTranslate for those. + return lang === "en" || hotel !== "com"; +} + +export function lookupTranslation( + map: Map, + classname: string, +): Translation | undefined { + const exact = map.get(classname); + if (exact) return exact; + if (classname.includes("*")) { + const base = classname.split("*")[0]; + const v0 = map.get(`${base}*0`); + if (v0) return v0; + const baseExact = map.get(base); + if (baseExact) return baseExact; + } + if (!classname.includes("*")) { + const v0 = map.get(`${classname}*0`); + if (v0) return v0; + } + return undefined; +} + +export interface PendingRef { + entry: FurniDataEntry; + field: "name" | "description"; + section: "roomitemtypes" | "wallitemtypes"; +} + +export interface PreparedLang { + cloned: FurniDataMaster; + pendingRefs: PendingRef[]; + pendingTexts: string[]; +} + +/** + * Pass 1: deep-clone the master, apply official translations directly, and + * collect the texts that still need LibreTranslate. Returns the cloned object + * plus the pending refs/texts so the (network) translation step can run and the + * result be applied later without re-scanning. + */ +export function prepareLangFile( + master: FurniDataMaster, + lang: string, + hotel: HabboGamedataHotel, + officialMap: Map, +): PreparedLang { + const cloned = cloneFurniData(master); + const officialIsReal = officialIsRealTranslation(lang, hotel); + const pendingRefs: PendingRef[] = []; + const pendingTexts = new Set(); + + for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + for (const entry of cloned[section]?.furnitype ?? []) { + const cn = typeof entry.classname === "string" ? entry.classname : ""; + if (!cn) continue; + const t = lookupTranslation(officialMap, cn); + if (officialIsReal && t?.name) { + entry.name = t.name; + } else if ( + lang !== "en" && + typeof entry.name === "string" && + entry.name.trim() + ) { + pendingTexts.add(entry.name.trim()); + pendingRefs.push({ entry, field: "name", section }); + } + if (officialIsReal && t?.description) { + entry.description = t.description; + } else if ( + lang !== "en" && + typeof entry.description === "string" && + entry.description.trim() + ) { + pendingTexts.add(entry.description.trim()); + pendingRefs.push({ entry, field: "description", section }); + } + } + } + + return { cloned, pendingRefs, pendingTexts: [...pendingTexts] }; +} + +export interface FinalizedLang { + json: string; + translatedRoom: number; + translatedWall: number; + total: number; +} + +/** + * Pass 2: apply the LibreTranslate results to the prepared clone and serialize + * it. Kept separate from prepareLangFile so the (async, network) translation + * step can run in between on the main thread. + */ +export function finalizeLangFile( + cloned: FurniDataMaster, + pendingRefs: PendingRef[], + translations: Map, +): FinalizedLang { + const counted = new Set(); + let translatedRoom = 0; + let translatedWall = 0; + for (const ref of pendingRefs) { + const original = ref.entry[ref.field]; + if (typeof original !== "string") continue; + const translated = translations.get(original.trim()); + if (translated === undefined || translated === original) continue; + ref.entry[ref.field] = translated; + if (!counted.has(ref.entry)) { + counted.add(ref.entry); + if (ref.section === "roomitemtypes") translatedRoom++; + else translatedWall++; + } + } + const total = + (cloned.roomitemtypes?.furnitype?.length ?? 0) + + (cloned.wallitemtypes?.furnitype?.length ?? 0); + return { + json: JSON.stringify(cloned), + translatedRoom, + translatedWall, + total, + }; +} diff --git a/src/lib/services/furni-data-i18n.ts b/src/lib/services/furni-data-i18n.ts index e9c87187..906b7d20 100644 --- a/src/lib/services/furni-data-i18n.ts +++ b/src/lib/services/furni-data-i18n.ts @@ -22,6 +22,7 @@ import { } from "@/lib/services/furni-data"; import { browserHeaders } from "@/lib/services/import/core/browser-headers"; import { siteSettings } from "@/lib/services/site-settings"; +import { getTranslationWorker } from "@/lib/services/translation-pool"; export const FURNIDATA_LANGUAGES = [ { lang: "nl", hotel: "nl" as HabboGamedataHotel }, @@ -68,7 +69,7 @@ export interface LanguageFileResult { error?: string; } -type Translation = { name: string; description: string }; +import type { Translation } from "./furni-data-i18n-core"; const LANG_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours — translations rarely change @@ -256,38 +257,31 @@ async function translateManyWithLibre( return result; } -function lookupTranslation( - map: Map, - classname: string, -): Translation | undefined { - const exact = map.get(classname); - if (exact) return exact; - if (classname.includes("*")) { - const base = classname.split("*")[0]; - const v0 = map.get(`${base}*0`); - if (v0) return v0; - const baseExact = map.get(base); - if (baseExact) return baseExact; - } - if (!classname.includes("*")) { - const v0 = map.get(`${classname}*0`); - if (v0) return v0; - } - return undefined; -} - +import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core"; /** - * A language mapped to hotel "com" only carries English names, so it must - * NOT be treated as a real translation — fall through to LibreTranslate for - * those (pl/ru/ar/ja). Shared by the full build and the single-import patch - * so both paths stay in sync. + * The pure clone / scan / stringify helpers live in furni-data-i18n-core so they + * can run inside a worker thread (that module has no server-only / network deps). + * They're imported here for the main-thread call paths and re-exported for + * backward compatibility. */ -function officialIsRealTranslation( - lang: string, - hotel: HabboGamedataHotel, -): boolean { - return lang === "en" || hotel !== "com"; -} +import { + cloneFurniData, + finalizeLangFile, + lookupTranslation, + officialIsRealTranslation, + prepareLangFile, +} from "./furni-data-i18n-core"; + +export type { + FurniDataMaster, + PreparedLang, + Translation, +} from "./furni-data-i18n-core"; +export { + cloneFurniData, + lookupTranslation, + officialIsRealTranslation, +} from "./furni-data-i18n-core"; async function fetchTranslationsForHotel( hotel: HabboGamedataHotel, @@ -377,22 +371,14 @@ async function getLocalizedWritePaths(lang: string): Promise { }); } -function cloneFurniData( - data: Record, -): Record> }> { - // Deep clone via JSON — structure is pure JSON and 100k+ entries, this is - // the simplest safe copy. Each language build gets its own clone. - return JSON.parse(JSON.stringify(data)) as Record< - string, - { furnitype: Array> } - >; -} - /** * Build all language files from the current master FurnitureData.json. * Each file is a full copy with name/description overridden where an official - * translation exists for that classname. Runs under the furnidata lock so it - * never interleaves with concurrent imports. + * translation exists for that classname. The CPU-heavy clone + 100k-entry scan + * + stringify is offloaded to a worker thread (see translation-pool); only the + * network/DB parts (fetching official translations, LibreTranslate, writing + * files) stay on the main thread. Falls back to running the same pure helpers + * on the main thread when no worker is available. */ export async function buildLocalizedFurniDataFiles( force = false, @@ -414,107 +400,79 @@ export async function buildLocalizedFurniDataFiles( { furnitype: Array> } >; - // Fetch all language maps in parallel (uses per-hotel memory cache). - const maps = await Promise.all( - langsToBuild.map(async ({ lang, hotel }) => { - const map = await fetchTranslationsForHotel(hotel); - return { lang, hotel, map }; - }), - ); + // Fetch each hotel's official translations once (network + per-hotel cache). + const hotels = [...new Set(langsToBuild.map((l) => l.hotel))]; + const hotelMaps: Record> = {}; + for (const hotel of hotels) { + hotelMaps[hotel] = await fetchTranslationsForHotel(hotel); + } + + const worker = getTranslationWorker(); + if (worker) { + await worker.init(JSON.stringify(master), hotelMaps); + } + // Main-thread fallback store, used only when no worker is available. + const localPrepared = new Map(); const results: LanguageFileResult[] = []; - for (const { lang, hotel, map } of maps) { + for (const { lang, hotel } of langsToBuild) { try { // Yield before each language (except the first) so a long build // never pins the CPU continuously and the server stays responsive. if (results.length > 0) { - await new Promise((r) => setImmediate(r)); + await yieldToEventLoop(); await sleep(BUILD_YIELD_MS); } - const cloned = cloneFurniData( - master as unknown as Record, - ); - const officialIsReal = officialIsRealTranslation(lang, hotel); - // Pass 1 — apply official translations directly and collect the - // texts that need LibreTranslate (customs, or languages whose - // hotel only provides English) so they can be batch-translated. - const pendingRefs: Array<{ - entry: Record; - field: "name" | "description"; - section: "roomitemtypes" | "wallitemtypes"; - }> = []; - const pendingTexts = new Set(); - const sections = ["roomitemtypes", "wallitemtypes"] as const; - for (let si = 0; si < sections.length; si++) { - const section = sections[si]; - const arr = cloned[section]?.furnitype; - if (Array.isArray(arr)) { - for (const entry of arr) { - const cn = - typeof entry.classname === "string" ? entry.classname : ""; - if (!cn) continue; - const t = lookupTranslation(map, cn); - if (officialIsReal && t?.name) { - entry.name = t.name; - } else if ( - lang !== "en" && - typeof entry.name === "string" && - entry.name.trim() - ) { - pendingTexts.add(entry.name.trim()); - pendingRefs.push({ entry, field: "name", section }); - } - if (officialIsReal && t?.description) { - entry.description = t.description; - } else if ( - lang !== "en" && - typeof entry.description === "string" && - entry.description.trim() - ) { - pendingTexts.add(entry.description.trim()); - pendingRefs.push({ entry, field: "description", section }); - } - } - } - // Yield between the two 100k+ scans so a single language never - // blocks the event loop for the whole pass at once. - if (si === 0) await yieldToEventLoop(); + let pendingTexts: string[]; + let preparedKey: string | null = null; + if (worker) { + pendingTexts = await worker.prepare(lang, hotel); + } else { + const r = prepareLangFile( + master as unknown as FurniDataMaster, + lang, + hotel, + hotelMaps[hotel] ?? new Map(), + ); + pendingTexts = r.pendingTexts; + preparedKey = lang; + localPrepared.set(preparedKey, r); } - // Pass 2 — batch-translate everything pending in a handful of - // parallel requests instead of one HTTP call per text. - const translations = await translateManyWithLibre(pendingTexts, lang); + // Pass 2 — batch-translate everything pending (network, async, so + // this part never blocks the main thread regardless). + const translations = await translateManyWithLibre( + new Set(pendingTexts), + lang, + ); + let json: string; let translatedRoom = 0; let translatedWall = 0; - if (pendingRefs.length > 0) { - const counted = new Set>(); - for (const ref of pendingRefs) { - const original = ref.entry[ref.field]; - if (typeof original !== "string") continue; - const translated = translations.get(original.trim()); - if (translated === undefined || translated === original) continue; - ref.entry[ref.field] = translated; - if (!counted.has(ref.entry)) { - counted.add(ref.entry); - if (ref.section === "roomitemtypes") translatedRoom++; - else translatedWall++; - } - } + let total = 0; + if (worker) { + const res = await worker.finalize(lang, translations); + json = res.json; + translatedRoom = res.translatedRoom; + translatedWall = res.translatedWall; + total = res.total; + } else { + const r = localPrepared.get(preparedKey!)!; + const res = finalizeLangFile(r.cloned, r.pendingRefs, translations); + json = res.json; + translatedRoom = res.translatedRoom; + translatedWall = res.translatedWall; + total = res.total; + localPrepared.delete(preparedKey!); } - const total = - (cloned.roomitemtypes?.furnitype?.length ?? 0) + - (cloned.wallitemtypes?.furnitype?.length ?? 0); - - const payload = (await yieldToEventLoop(), JSON.stringify(cloned)); const targets = await getLocalizedWritePaths(lang); for (const target of targets) { await fs.mkdir(path.dirname(target), { recursive: true }); const tmp = `${target}.tmp`; - await fs.writeFile(tmp, payload, "utf-8"); + await fs.writeFile(tmp, json, "utf-8"); await fs.rename(tmp, target); } diff --git a/src/lib/services/translation-pool.ts b/src/lib/services/translation-pool.ts new file mode 100644 index 00000000..8b9b7d76 --- /dev/null +++ b/src/lib/services/translation-pool.ts @@ -0,0 +1,138 @@ +import { Worker } from "node:worker_threads"; +import type { Translation } from "./furni-data-i18n-core"; + +/** + * Runs the CPU-heavy FurnitureData localization (deep clone + 100k-entry scans + + * JSON.stringify) in a single worker thread so the main Node event loop stays + * free during a "translate everything" build. Network/DB work (fetching official + * Habbo translations, calling LibreTranslate, writing files) stays on the main + * thread; only the synchronous CPU is offloaded. + * + * Degrades gracefully: in the test environment (or if the worker can't be + * created) getTranslationWorker() returns null and callers run the same pure + * helpers on the main thread instead. + */ +interface InitReq { + id: number; + type: "init"; + masterJson: string; + hotelMaps: Record>; +} +interface PrepareReq { + id: number; + type: "prepare"; + lang: string; + hotel: string; +} +interface FinalizeReq { + id: number; + type: "finalize"; + lang: string; + translations: Map; +} +type Req = InitReq | PrepareReq | FinalizeReq; +type ReqWithoutId = + | Omit + | Omit + | Omit; + +type Res = + | { + id: number; + ok: true; + pendingTexts?: string[]; + json?: string; + translatedRoom?: number; + translatedWall?: number; + total?: number; + } + | { id: number; ok: false; error: string }; + +let worker: Worker | null = null; +let broken = false; +let nextId = 1; +const pending = new Map< + number, + { resolve: (r: Res) => void; reject: (e: Error) => void } +>(); + +function isTestEnv(): boolean { + return process.env.NODE_ENV === "test" || process.env.VITEST !== undefined; +} + +function spawn(): Worker | null { + try { + const w = new Worker(new URL("./translation-worker.ts", import.meta.url)); + w.on("message", (res: Res) => { + const p = pending.get(res.id); + if (!p) return; + pending.delete(res.id); + if (res.ok) p.resolve(res); + else p.reject(new Error(res.error)); + }); + w.on("error", (err: Error) => { + broken = true; + for (const [, p] of pending) p.reject(err); + pending.clear(); + }); + return w; + } catch { + broken = true; + return null; + } +} + +function call(req: ReqWithoutId): Promise { + if (!worker) return Promise.reject(new Error("no translation worker")); + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + worker!.postMessage({ ...req, id } as Req); + }); +} + +export interface TranslationWorker { + init( + masterJson: string, + hotelMaps: Record>, + ): Promise; + prepare(lang: string, hotel: string): Promise; + finalize( + lang: string, + translations: Map, + ): Promise<{ + json: string; + translatedRoom: number; + translatedWall: number; + total: number; + }>; +} + +export function getTranslationWorker(): TranslationWorker | null { + if (isTestEnv() || broken) return null; + if (!worker) worker = spawn(); + if (!worker) return null; + return { + async init(masterJson, hotelMaps) { + const res = await call({ type: "init", masterJson, hotelMaps }); + if (!res.ok) throw new Error(res.error); + }, + async prepare(lang, hotel) { + const res = await call({ type: "prepare", lang, hotel }); + if (!res.ok || !res.pendingTexts) + throw new Error(res.ok ? "missing pendingTexts" : res.error); + return res.pendingTexts; + }, + async finalize(lang, translations) { + const res = await call({ type: "finalize", lang, translations }); + if (!res.ok || res.json === undefined) + throw new Error(res.ok ? "missing json" : res.error); + return { + json: res.json, + translatedRoom: res.translatedRoom ?? 0, + translatedWall: res.translatedWall ?? 0, + total: res.total ?? 0, + }; + }, + }; +} diff --git a/src/lib/services/translation-worker.ts b/src/lib/services/translation-worker.ts new file mode 100644 index 00000000..2a653a95 --- /dev/null +++ b/src/lib/services/translation-worker.ts @@ -0,0 +1,76 @@ +import { parentPort } from "node:worker_threads"; +import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel"; +import { + type FurniDataMaster, + finalizeLangFile, + type PreparedLang, + prepareLangFile, + type Translation, +} from "./furni-data-i18n-core"; + +interface InitMsg { + id: number; + type: "init"; + masterJson: string; + hotelMaps: Record>; +} +interface PrepareMsg { + id: number; + type: "prepare"; + lang: string; + hotel: string; +} +interface FinalizeMsg { + id: number; + type: "finalize"; + lang: string; + translations: Map; +} +type Msg = InitMsg | PrepareMsg | FinalizeMsg; + +let master: FurniDataMaster | null = null; +let hotelMaps: Record> = {}; +const prepared = new Map(); + +if (parentPort) { + parentPort.on("message", (msg: Msg) => { + try { + if (msg.type === "init") { + master = JSON.parse(msg.masterJson); + hotelMaps = msg.hotelMaps ?? {}; + parentPort!.postMessage({ id: msg.id, ok: true }); + return; + } + if (msg.type === "prepare") { + if (!master) throw new Error("translation worker not initialized"); + const officialMap = + hotelMaps[msg.hotel] ?? new Map(); + const r = prepareLangFile( + master, + msg.lang, + msg.hotel as HabboGamedataHotel, + officialMap, + ); + prepared.set(msg.lang, r); + parentPort!.postMessage({ + id: msg.id, + ok: true, + pendingTexts: r.pendingTexts, + }); + return; + } + // finalize + const r = prepared.get(msg.lang); + if (!r) throw new Error(`no prepared data for ${msg.lang}`); + const res = finalizeLangFile(r.cloned, r.pendingRefs, msg.translations); + prepared.delete(msg.lang); + parentPort!.postMessage({ id: msg.id, ok: true, ...res }); + } catch (err) { + parentPort!.postMessage({ + id: msg.id, + ok: false, + error: (err as Error).message, + }); + } + }); +}