perf: offload per-import localized patch to worker + cache FurnitureData parse
Per single furni import, patchLocalizedFurniDataEntries ran for every language and did a full 100k-entry JSON.parse + scan + JSON.stringify on the main thread. That CPU spike is now offloaded to the translation worker (init/prepare/finalize reuses the same pure core helpers), so importing never blocks the event loop. The main thread only does async file I/O and the network LibreTranslate calls. Falls back to the same helpers on the main thread if no worker is available. Also cache readFurniData() by file mtime+size so the 100k-entry JSON is parsed once per change instead of on every import/fix/reconcile read (invalidated on write).
This commit is contained in:
openhands committed
1 parent
6ed1b03e24
commit
6ae7c09682
5 files changed
+502
-226
No files matched your search
@@ -69,8 +69,6 @@ export interface LanguageFileResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
import type { Translation } from "./furni-data-i18n-core";
|
||||
|
||||
const LANG_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours — translations rarely change
|
||||
|
||||
interface LangCacheEntry {
|
||||
@@ -257,7 +255,12 @@ async function translateManyWithLibre(
|
||||
return result;
|
||||
}
|
||||
|
||||
import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core";
|
||||
import type {
|
||||
FurniDataMaster,
|
||||
PreparedLang,
|
||||
PreparedPatch,
|
||||
Translation,
|
||||
} from "./furni-data-i18n-core";
|
||||
/**
|
||||
* The pure clone / scan / stringify helpers live in furni-data-i18n-core so they
|
||||
* can run inside a worker thread (that module has no server-only / network deps).
|
||||
@@ -265,25 +268,26 @@ import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core";
|
||||
* backward compatibility.
|
||||
*/
|
||||
import {
|
||||
cloneFurniData,
|
||||
finalizeLangFile,
|
||||
finalizePatch,
|
||||
lookupTranslation,
|
||||
officialIsRealTranslation,
|
||||
prepareLangFile,
|
||||
preparePatch,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
export type {
|
||||
FurniDataMaster,
|
||||
PatchEntry,
|
||||
PreparedLang,
|
||||
PreparedPatch,
|
||||
Translation,
|
||||
} from "./furni-data-i18n-core";
|
||||
export {
|
||||
cloneFurniData,
|
||||
lookupTranslation,
|
||||
officialIsRealTranslation,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
async function fetchTranslationsForHotel(
|
||||
export async function fetchTranslationsForHotel(
|
||||
hotel: HabboGamedataHotel,
|
||||
): Promise<Map<string, Translation>> {
|
||||
const cached = langCache.get(hotel);
|
||||
@@ -400,16 +404,12 @@ export async function buildLocalizedFurniDataFiles(
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
|
||||
// Fetch each hotel's official translations once (network + per-hotel cache).
|
||||
const hotels = [...new Set(langsToBuild.map((l) => l.hotel))];
|
||||
const hotelMaps: Record<string, Map<string, Translation>> = {};
|
||||
for (const hotel of hotels) {
|
||||
hotelMaps[hotel] = await fetchTranslationsForHotel(hotel);
|
||||
}
|
||||
|
||||
const worker = getTranslationWorker();
|
||||
if (worker) {
|
||||
await worker.init(JSON.stringify(master), hotelMaps);
|
||||
// Reads the master + official maps once on the main thread (cached
|
||||
// per process) and ships them to the worker so the per-language
|
||||
// clone/scan/stringify runs off the main thread.
|
||||
await worker.ensureInit();
|
||||
}
|
||||
// Main-thread fallback store, used only when no worker is available.
|
||||
const localPrepared = new Map<string, PreparedLang>();
|
||||
@@ -430,11 +430,12 @@ export async function buildLocalizedFurniDataFiles(
|
||||
if (worker) {
|
||||
pendingTexts = await worker.prepare(lang, hotel);
|
||||
} else {
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
const r = prepareLangFile(
|
||||
master as unknown as FurniDataMaster,
|
||||
lang,
|
||||
hotel,
|
||||
hotelMaps[hotel] ?? new Map(),
|
||||
map,
|
||||
);
|
||||
pendingTexts = r.pendingTexts;
|
||||
preparedKey = lang;
|
||||
@@ -531,220 +532,95 @@ export async function patchLocalizedFurniDataEntries(
|
||||
: FURNIDATA_LANGUAGES;
|
||||
|
||||
return withFurniDataLock(async () => {
|
||||
// Read the master once and reuse it for the fallback clone and for
|
||||
// looking up brand-new classnames — avoids parsing FurnitureData twice.
|
||||
const master = (await readFurniData()) as Record<
|
||||
string,
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
const maps = await Promise.all(
|
||||
langsToBuild.map(async ({ lang, hotel }) => {
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
return { lang, hotel, map };
|
||||
}),
|
||||
);
|
||||
|
||||
const wanted = new Map<string, { name?: string; description?: string }>();
|
||||
for (const e of entries) wanted.set(e.classname, e);
|
||||
const wantedSet = new Set(wanted.keys());
|
||||
const worker = getTranslationWorker();
|
||||
// Fallback store for when there is no worker (test env / spawn failure).
|
||||
const localPrepared = new Map<string, PreparedPatch>();
|
||||
|
||||
const results: LanguageFileResult[] = [];
|
||||
|
||||
for (const { lang, hotel, map } of maps) {
|
||||
for (const { lang, hotel } of langsToBuild) {
|
||||
try {
|
||||
const officialIsReal = officialIsRealTranslation(lang, hotel);
|
||||
const targets = await getLocalizedWritePaths(lang);
|
||||
let patched = 0;
|
||||
|
||||
// Load the existing language file if present, otherwise build it
|
||||
// from master (official translations only — never LibreTranslate,
|
||||
// to avoid 100k+ network calls on first creation).
|
||||
let data: Record<string, { furnitype: Array<Record<string, unknown>> }>;
|
||||
// Read the existing language file as raw text. The expensive
|
||||
// parse + scan + stringify all happen in the worker, so the main
|
||||
// thread only does async file I/O here.
|
||||
let fileContent: string | null = null;
|
||||
try {
|
||||
const raw = await fs.readFile(targets[0], "utf-8");
|
||||
data = JSON.parse(raw);
|
||||
fileContent = await fs.readFile(targets[0], "utf-8");
|
||||
} catch {
|
||||
data = cloneFurniData(master);
|
||||
if (officialIsReal) {
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
if (t?.name) entry.name = t.name;
|
||||
if (t?.description) entry.description = t.description;
|
||||
}
|
||||
}
|
||||
}
|
||||
fileContent = null;
|
||||
}
|
||||
|
||||
// Pass 1 — touch ONLY the wanted classnames (most patches are a
|
||||
// single import), collecting the texts that need LibreTranslate.
|
||||
const refs: Array<{
|
||||
entry: Record<string, unknown>;
|
||||
field: "name" | "description";
|
||||
}> = [];
|
||||
const pendingTexts = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn || !wantedSet.has(cn)) continue;
|
||||
const wantedEntry = wanted.get(cn);
|
||||
if (!wantedEntry) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
let didPatch = false;
|
||||
const origName =
|
||||
wantedEntry.name ??
|
||||
(typeof entry.name === "string" ? entry.name : "");
|
||||
const origDesc =
|
||||
wantedEntry.description ??
|
||||
(typeof entry.description === "string" ? entry.description : "");
|
||||
if (officialIsReal && t?.name) {
|
||||
entry.name = t.name;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origName) {
|
||||
pendingTexts.add(origName);
|
||||
refs.push({ entry, field: "name" });
|
||||
}
|
||||
if (officialIsReal && t?.description) {
|
||||
entry.description = t.description;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origDesc) {
|
||||
pendingTexts.add(origDesc);
|
||||
refs.push({ entry, field: "description" });
|
||||
}
|
||||
if (didPatch) patched++;
|
||||
}
|
||||
let pendingTexts: string[];
|
||||
let localKey: string | null = null;
|
||||
try {
|
||||
if (!worker) throw new Error("no translation worker");
|
||||
pendingTexts = await worker.patchPrepare(
|
||||
fileContent,
|
||||
lang,
|
||||
hotel,
|
||||
entries,
|
||||
);
|
||||
} catch {
|
||||
// Main-thread fallback: run the exact same pure helpers here.
|
||||
const master = (await readFurniData()) as unknown as FurniDataMaster;
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
const r = preparePatch(
|
||||
fileContent,
|
||||
master,
|
||||
lang,
|
||||
hotel,
|
||||
map,
|
||||
entries,
|
||||
);
|
||||
localPrepared.set(lang, r);
|
||||
pendingTexts = [...r.pendingTexts];
|
||||
localKey = lang;
|
||||
}
|
||||
|
||||
// Batch-translate (dedup + concurrency-capped) instead of one
|
||||
// HTTP call per text.
|
||||
const translations = await translateManyWithLibre(pendingTexts, lang);
|
||||
const counted = new Set<Record<string, unknown>>();
|
||||
for (const ref of refs) {
|
||||
const original = ref.entry[ref.field];
|
||||
if (typeof original !== "string") continue;
|
||||
const translated = translations.get(original.trim());
|
||||
if (translated === undefined || translated === original) continue;
|
||||
ref.entry[ref.field] = translated;
|
||||
if (!counted.has(ref.entry)) {
|
||||
counted.add(ref.entry);
|
||||
patched++;
|
||||
}
|
||||
// Batch-translate (network, async — never blocks the main thread).
|
||||
const translations = await translateManyWithLibre(
|
||||
new Set(pendingTexts),
|
||||
lang,
|
||||
);
|
||||
|
||||
let json: string;
|
||||
let patched = 0;
|
||||
let added = 0;
|
||||
let total = 0;
|
||||
if (worker && localKey === null) {
|
||||
const res = await worker.patchFinalize(lang, translations);
|
||||
json = res.json;
|
||||
patched = res.patched;
|
||||
added = res.added;
|
||||
total = res.total;
|
||||
} else {
|
||||
const r = localPrepared.get(localKey!)!;
|
||||
const res = finalizePatch(r, translations);
|
||||
json = res.json;
|
||||
patched = res.patched;
|
||||
added = res.added;
|
||||
total = res.total;
|
||||
localPrepared.delete(localKey!);
|
||||
}
|
||||
|
||||
// New classnames not yet in the file: copy from master with
|
||||
// translation applied (only for the wanted set).
|
||||
const existingClassnames = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const e of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof e?.classname === "string" ? e.classname : "";
|
||||
if (cn) existingClassnames.add(cn);
|
||||
}
|
||||
}
|
||||
const newItems: Array<{
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
found: Record<string, unknown>;
|
||||
name?: string;
|
||||
description?: string;
|
||||
translateName: boolean;
|
||||
translateDesc: boolean;
|
||||
}> = [];
|
||||
const newTexts = new Set<string>();
|
||||
for (const { classname, name, description } of entries) {
|
||||
if (existingClassnames.has(classname)) continue;
|
||||
let found: Record<string, unknown> | undefined;
|
||||
let foundSection: "roomitemtypes" | "wallitemtypes" | undefined;
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
const hit = (
|
||||
master[section]?.furnitype as
|
||||
| Array<Record<string, unknown>>
|
||||
| undefined
|
||||
)?.find((e) => e.classname === classname) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (hit) {
|
||||
found = hit;
|
||||
foundSection = section;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found || !foundSection) continue;
|
||||
const t = lookupTranslation(map, classname);
|
||||
const srcName =
|
||||
name ?? (typeof found.name === "string" ? found.name : "");
|
||||
const srcDesc =
|
||||
description ??
|
||||
(typeof found.description === "string" ? found.description : "");
|
||||
let finalName: string | undefined;
|
||||
let finalDesc: string | undefined;
|
||||
let translateName = false;
|
||||
let translateDesc = false;
|
||||
if (officialIsReal && t?.name) finalName = t.name;
|
||||
else if (lang !== "en" && srcName) {
|
||||
finalName = srcName;
|
||||
translateName = true;
|
||||
newTexts.add(srcName);
|
||||
} else finalName = srcName;
|
||||
if (officialIsReal && t?.description) finalDesc = t.description;
|
||||
else if (lang !== "en" && srcDesc) {
|
||||
finalDesc = srcDesc;
|
||||
translateDesc = true;
|
||||
newTexts.add(srcDesc);
|
||||
} else finalDesc = srcDesc;
|
||||
newItems.push({
|
||||
section: foundSection,
|
||||
found,
|
||||
name: finalName,
|
||||
description: finalDesc,
|
||||
translateName,
|
||||
translateDesc,
|
||||
});
|
||||
}
|
||||
const newTranslations = await translateManyWithLibre(newTexts, lang);
|
||||
for (const item of newItems) {
|
||||
const clone = { ...item.found } as Record<string, unknown>;
|
||||
if (item.name !== undefined) {
|
||||
if (item.translateName) {
|
||||
const tr = newTranslations.get(item.name.trim());
|
||||
clone.name = tr && tr !== item.name ? tr : item.name;
|
||||
} else {
|
||||
clone.name = item.name;
|
||||
}
|
||||
}
|
||||
if (item.description !== undefined) {
|
||||
if (item.translateDesc) {
|
||||
const tr = newTranslations.get(item.description.trim());
|
||||
clone.description =
|
||||
tr && tr !== item.description ? tr : item.description;
|
||||
} else {
|
||||
clone.description = item.description;
|
||||
}
|
||||
}
|
||||
if (!data[item.section]) data[item.section] = { furnitype: [] };
|
||||
data[item.section].furnitype.push(clone);
|
||||
patched++;
|
||||
}
|
||||
|
||||
if (lang !== "en") await saveLibreCache();
|
||||
|
||||
const payload = JSON.stringify(data);
|
||||
for (const target of targets) {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
const tmp = `${target}.tmp`;
|
||||
await fs.writeFile(tmp, payload, "utf-8");
|
||||
await fs.writeFile(tmp, json, "utf-8");
|
||||
await fs.rename(tmp, target);
|
||||
}
|
||||
|
||||
if (lang !== "en") await saveLibreCache();
|
||||
|
||||
results.push({
|
||||
lang,
|
||||
hotel,
|
||||
file: targets[0] ?? `FurnitureData_${lang}.json`,
|
||||
translatedRoom: patched,
|
||||
translatedWall: 0,
|
||||
total: patched,
|
||||
translatedWall: added,
|
||||
total,
|
||||
ok: true,
|
||||
});
|
||||
} catch (err) {
|
||||
|
||||
Reference in new issue
Block a user