perf: offload FurnitureData translation build to a worker thread
The translation build's CPU-heavy work (deep clone of the 100k+ entry master, the two full scans and JSON.stringify per language) now runs in a worker thread, so "Alles vertalen" no longer blocks the main event loop. Network/DB parts (official Habbo fetch, LibreTranslate, file writes) stay on the main thread. The pure helpers were extracted to furni-data-i18n-core (no server-only deps) so the worker can import them. Falls back to the same helpers on the main thread when no worker is available or in tests.
This commit is contained in:
openhands committed
1 parent
8d1b09f151
commit
f94246e067
4 files changed
+451
-124
No files matched your search
@@ -22,6 +22,7 @@ import {
|
||||
} from "@/lib/services/furni-data";
|
||||
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
||||
import { siteSettings } from "@/lib/services/site-settings";
|
||||
import { getTranslationWorker } from "@/lib/services/translation-pool";
|
||||
|
||||
export const FURNIDATA_LANGUAGES = [
|
||||
{ lang: "nl", hotel: "nl" as HabboGamedataHotel },
|
||||
@@ -68,7 +69,7 @@ export interface LanguageFileResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
type Translation = { name: string; description: string };
|
||||
import type { Translation } from "./furni-data-i18n-core";
|
||||
|
||||
const LANG_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours — translations rarely change
|
||||
|
||||
@@ -256,38 +257,31 @@ async function translateManyWithLibre(
|
||||
return result;
|
||||
}
|
||||
|
||||
function lookupTranslation(
|
||||
map: Map<string, Translation>,
|
||||
classname: string,
|
||||
): Translation | undefined {
|
||||
const exact = map.get(classname);
|
||||
if (exact) return exact;
|
||||
if (classname.includes("*")) {
|
||||
const base = classname.split("*")[0];
|
||||
const v0 = map.get(`${base}*0`);
|
||||
if (v0) return v0;
|
||||
const baseExact = map.get(base);
|
||||
if (baseExact) return baseExact;
|
||||
}
|
||||
if (!classname.includes("*")) {
|
||||
const v0 = map.get(`${classname}*0`);
|
||||
if (v0) return v0;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core";
|
||||
/**
|
||||
* A language mapped to hotel "com" only carries English names, so it must
|
||||
* NOT be treated as a real translation — fall through to LibreTranslate for
|
||||
* those (pl/ru/ar/ja). Shared by the full build and the single-import patch
|
||||
* so both paths stay in sync.
|
||||
* The pure clone / scan / stringify helpers live in furni-data-i18n-core so they
|
||||
* can run inside a worker thread (that module has no server-only / network deps).
|
||||
* They're imported here for the main-thread call paths and re-exported for
|
||||
* backward compatibility.
|
||||
*/
|
||||
function officialIsRealTranslation(
|
||||
lang: string,
|
||||
hotel: HabboGamedataHotel,
|
||||
): boolean {
|
||||
return lang === "en" || hotel !== "com";
|
||||
}
|
||||
import {
|
||||
cloneFurniData,
|
||||
finalizeLangFile,
|
||||
lookupTranslation,
|
||||
officialIsRealTranslation,
|
||||
prepareLangFile,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
export type {
|
||||
FurniDataMaster,
|
||||
PreparedLang,
|
||||
Translation,
|
||||
} from "./furni-data-i18n-core";
|
||||
export {
|
||||
cloneFurniData,
|
||||
lookupTranslation,
|
||||
officialIsRealTranslation,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
async function fetchTranslationsForHotel(
|
||||
hotel: HabboGamedataHotel,
|
||||
@@ -377,22 +371,14 @@ async function getLocalizedWritePaths(lang: string): Promise<string[]> {
|
||||
});
|
||||
}
|
||||
|
||||
function cloneFurniData(
|
||||
data: Record<string, unknown>,
|
||||
): Record<string, { furnitype: Array<Record<string, unknown>> }> {
|
||||
// Deep clone via JSON — structure is pure JSON and 100k+ entries, this is
|
||||
// the simplest safe copy. Each language build gets its own clone.
|
||||
return JSON.parse(JSON.stringify(data)) as Record<
|
||||
string,
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build all language files from the current master FurnitureData.json.
|
||||
* Each file is a full copy with name/description overridden where an official
|
||||
* translation exists for that classname. Runs under the furnidata lock so it
|
||||
* never interleaves with concurrent imports.
|
||||
* translation exists for that classname. The CPU-heavy clone + 100k-entry scan
|
||||
* + stringify is offloaded to a worker thread (see translation-pool); only the
|
||||
* network/DB parts (fetching official translations, LibreTranslate, writing
|
||||
* files) stay on the main thread. Falls back to running the same pure helpers
|
||||
* on the main thread when no worker is available.
|
||||
*/
|
||||
export async function buildLocalizedFurniDataFiles(
|
||||
force = false,
|
||||
@@ -414,107 +400,79 @@ export async function buildLocalizedFurniDataFiles(
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
|
||||
// Fetch all language maps in parallel (uses per-hotel memory cache).
|
||||
const maps = await Promise.all(
|
||||
langsToBuild.map(async ({ lang, hotel }) => {
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
return { lang, hotel, map };
|
||||
}),
|
||||
);
|
||||
// Fetch each hotel's official translations once (network + per-hotel cache).
|
||||
const hotels = [...new Set(langsToBuild.map((l) => l.hotel))];
|
||||
const hotelMaps: Record<string, Map<string, Translation>> = {};
|
||||
for (const hotel of hotels) {
|
||||
hotelMaps[hotel] = await fetchTranslationsForHotel(hotel);
|
||||
}
|
||||
|
||||
const worker = getTranslationWorker();
|
||||
if (worker) {
|
||||
await worker.init(JSON.stringify(master), hotelMaps);
|
||||
}
|
||||
// Main-thread fallback store, used only when no worker is available.
|
||||
const localPrepared = new Map<string, PreparedLang>();
|
||||
|
||||
const results: LanguageFileResult[] = [];
|
||||
|
||||
for (const { lang, hotel, map } of maps) {
|
||||
for (const { lang, hotel } of langsToBuild) {
|
||||
try {
|
||||
// Yield before each language (except the first) so a long build
|
||||
// never pins the CPU continuously and the server stays responsive.
|
||||
if (results.length > 0) {
|
||||
await new Promise((r) => setImmediate(r));
|
||||
await yieldToEventLoop();
|
||||
await sleep(BUILD_YIELD_MS);
|
||||
}
|
||||
const cloned = cloneFurniData(
|
||||
master as unknown as Record<string, unknown>,
|
||||
);
|
||||
const officialIsReal = officialIsRealTranslation(lang, hotel);
|
||||
|
||||
// Pass 1 — apply official translations directly and collect the
|
||||
// texts that need LibreTranslate (customs, or languages whose
|
||||
// hotel only provides English) so they can be batch-translated.
|
||||
const pendingRefs: Array<{
|
||||
entry: Record<string, unknown>;
|
||||
field: "name" | "description";
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
}> = [];
|
||||
const pendingTexts = new Set<string>();
|
||||
const sections = ["roomitemtypes", "wallitemtypes"] as const;
|
||||
for (let si = 0; si < sections.length; si++) {
|
||||
const section = sections[si];
|
||||
const arr = cloned[section]?.furnitype;
|
||||
if (Array.isArray(arr)) {
|
||||
for (const entry of arr) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
if (officialIsReal && t?.name) {
|
||||
entry.name = t.name;
|
||||
} else if (
|
||||
lang !== "en" &&
|
||||
typeof entry.name === "string" &&
|
||||
entry.name.trim()
|
||||
) {
|
||||
pendingTexts.add(entry.name.trim());
|
||||
pendingRefs.push({ entry, field: "name", section });
|
||||
}
|
||||
if (officialIsReal && t?.description) {
|
||||
entry.description = t.description;
|
||||
} else if (
|
||||
lang !== "en" &&
|
||||
typeof entry.description === "string" &&
|
||||
entry.description.trim()
|
||||
) {
|
||||
pendingTexts.add(entry.description.trim());
|
||||
pendingRefs.push({ entry, field: "description", section });
|
||||
}
|
||||
}
|
||||
}
|
||||
// Yield between the two 100k+ scans so a single language never
|
||||
// blocks the event loop for the whole pass at once.
|
||||
if (si === 0) await yieldToEventLoop();
|
||||
let pendingTexts: string[];
|
||||
let preparedKey: string | null = null;
|
||||
if (worker) {
|
||||
pendingTexts = await worker.prepare(lang, hotel);
|
||||
} else {
|
||||
const r = prepareLangFile(
|
||||
master as unknown as FurniDataMaster,
|
||||
lang,
|
||||
hotel,
|
||||
hotelMaps[hotel] ?? new Map(),
|
||||
);
|
||||
pendingTexts = r.pendingTexts;
|
||||
preparedKey = lang;
|
||||
localPrepared.set(preparedKey, r);
|
||||
}
|
||||
|
||||
// Pass 2 — batch-translate everything pending in a handful of
|
||||
// parallel requests instead of one HTTP call per text.
|
||||
const translations = await translateManyWithLibre(pendingTexts, lang);
|
||||
// Pass 2 — batch-translate everything pending (network, async, so
|
||||
// this part never blocks the main thread regardless).
|
||||
const translations = await translateManyWithLibre(
|
||||
new Set(pendingTexts),
|
||||
lang,
|
||||
);
|
||||
|
||||
let json: string;
|
||||
let translatedRoom = 0;
|
||||
let translatedWall = 0;
|
||||
if (pendingRefs.length > 0) {
|
||||
const counted = new Set<Record<string, unknown>>();
|
||||
for (const ref of pendingRefs) {
|
||||
const original = ref.entry[ref.field];
|
||||
if (typeof original !== "string") continue;
|
||||
const translated = translations.get(original.trim());
|
||||
if (translated === undefined || translated === original) continue;
|
||||
ref.entry[ref.field] = translated;
|
||||
if (!counted.has(ref.entry)) {
|
||||
counted.add(ref.entry);
|
||||
if (ref.section === "roomitemtypes") translatedRoom++;
|
||||
else translatedWall++;
|
||||
}
|
||||
}
|
||||
let total = 0;
|
||||
if (worker) {
|
||||
const res = await worker.finalize(lang, translations);
|
||||
json = res.json;
|
||||
translatedRoom = res.translatedRoom;
|
||||
translatedWall = res.translatedWall;
|
||||
total = res.total;
|
||||
} else {
|
||||
const r = localPrepared.get(preparedKey!)!;
|
||||
const res = finalizeLangFile(r.cloned, r.pendingRefs, translations);
|
||||
json = res.json;
|
||||
translatedRoom = res.translatedRoom;
|
||||
translatedWall = res.translatedWall;
|
||||
total = res.total;
|
||||
localPrepared.delete(preparedKey!);
|
||||
}
|
||||
|
||||
const total =
|
||||
(cloned.roomitemtypes?.furnitype?.length ?? 0) +
|
||||
(cloned.wallitemtypes?.furnitype?.length ?? 0);
|
||||
|
||||
const payload = (await yieldToEventLoop(), JSON.stringify(cloned));
|
||||
const targets = await getLocalizedWritePaths(lang);
|
||||
for (const target of targets) {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
const tmp = `${target}.tmp`;
|
||||
await fs.writeFile(tmp, payload, "utf-8");
|
||||
await fs.writeFile(tmp, json, "utf-8");
|
||||
await fs.rename(tmp, target);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user