perf: offload per-import localized patch to worker + cache FurnitureData parse
Per single furni import, patchLocalizedFurniDataEntries ran for every language and did a full 100k-entry JSON.parse + scan + JSON.stringify on the main thread. That CPU spike is now offloaded to the translation worker (init/prepare/finalize reuses the same pure core helpers), so importing never blocks the event loop. The main thread only does async file I/O and the network LibreTranslate calls. Falls back to the same helpers on the main thread if no worker is available. Also cache readFurniData() by file mtime+size so the 100k-entry JSON is parsed once per change instead of on every import/fix/reconcile read (invalidated on write).
This commit is contained in:
openhands committed
1 parent
6ed1b03e24
commit
6ae7c09682
5 files changed
+502
-226
No files matched your search
@@ -153,3 +153,242 @@ export function finalizeLangFile(
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Localized patch (single-item import path) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface PatchEntry {
|
||||
classname: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface PatchRef {
|
||||
entry: FurniDataEntry;
|
||||
field: "name" | "description";
|
||||
}
|
||||
|
||||
interface PatchNewItem {
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
found: FurniDataEntry;
|
||||
name?: string;
|
||||
description?: string;
|
||||
translateName: boolean;
|
||||
translateDesc: boolean;
|
||||
}
|
||||
|
||||
export interface PreparedPatch {
|
||||
data: FurniDataMaster;
|
||||
refs: PatchRef[];
|
||||
newItems: PatchNewItem[];
|
||||
pendingTexts: Set<string>;
|
||||
officialPatched: number;
|
||||
}
|
||||
|
||||
function findEntryInFurniData(
|
||||
data: FurniDataMaster,
|
||||
classname: string,
|
||||
): {
|
||||
entry: FurniDataEntry;
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
} | null {
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
const hit = data[section]?.furnitype.find((e) => e.classname === classname);
|
||||
if (hit) return { entry: hit, section };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 1 of a localized patch: load the existing language file (or clone the
|
||||
* master when it does not exist yet), touch ONLY the wanted classnames, and
|
||||
* collect the texts that still need LibreTranslate. The official-vs-Libre
|
||||
* decision, master clone and the full file scan all run here (CPU-heavy) so the
|
||||
* network translation step can run afterwards on the main thread.
|
||||
*/
|
||||
export function preparePatch(
|
||||
fileContent: string | null,
|
||||
master: FurniDataMaster,
|
||||
lang: string,
|
||||
hotel: HabboGamedataHotel,
|
||||
officialMap: Map<string, Translation>,
|
||||
entries: PatchEntry[],
|
||||
): PreparedPatch {
|
||||
const officialIsReal = officialIsRealTranslation(lang, hotel);
|
||||
let data: FurniDataMaster;
|
||||
try {
|
||||
data = fileContent
|
||||
? (JSON.parse(fileContent) as FurniDataMaster)
|
||||
: cloneFurniData(master);
|
||||
if (!fileContent && officialIsReal) {
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn) continue;
|
||||
const t = lookupTranslation(officialMap, cn);
|
||||
if (t?.name) entry.name = t.name;
|
||||
if (t?.description) entry.description = t.description;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
data = cloneFurniData(master);
|
||||
}
|
||||
|
||||
const wanted = new Map(entries.map((e) => [e.classname, e]));
|
||||
const wantedSet = new Set(wanted.keys());
|
||||
const refs: PatchRef[] = [];
|
||||
const pendingTexts = new Set<string>();
|
||||
let officialPatched = 0;
|
||||
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn || !wantedSet.has(cn)) continue;
|
||||
const wantedEntry = wanted.get(cn);
|
||||
if (!wantedEntry) continue;
|
||||
const t = lookupTranslation(officialMap, cn);
|
||||
const origName =
|
||||
wantedEntry.name ?? (typeof entry.name === "string" ? entry.name : "");
|
||||
const origDesc =
|
||||
wantedEntry.description ??
|
||||
(typeof entry.description === "string" ? entry.description : "");
|
||||
let didPatch = false;
|
||||
if (officialIsReal && t?.name) {
|
||||
entry.name = t.name;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origName) {
|
||||
pendingTexts.add(origName);
|
||||
refs.push({ entry, field: "name" });
|
||||
}
|
||||
if (officialIsReal && t?.description) {
|
||||
entry.description = t.description;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origDesc) {
|
||||
pendingTexts.add(origDesc);
|
||||
refs.push({ entry, field: "description" });
|
||||
}
|
||||
if (didPatch) officialPatched++;
|
||||
}
|
||||
}
|
||||
|
||||
// New classnames not yet in the file: copy from master with translation applied.
|
||||
const existingClassnames = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const e of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof e?.classname === "string" ? e.classname : "";
|
||||
if (cn) existingClassnames.add(cn);
|
||||
}
|
||||
}
|
||||
const newItems: PatchNewItem[] = [];
|
||||
const newTexts = new Set<string>();
|
||||
for (const { classname, name, description } of entries) {
|
||||
if (existingClassnames.has(classname)) continue;
|
||||
const found = findEntryInFurniData(master, classname);
|
||||
if (!found) continue;
|
||||
const t = lookupTranslation(officialMap, classname);
|
||||
const srcName =
|
||||
name ?? (typeof found.entry.name === "string" ? found.entry.name : "");
|
||||
const srcDesc =
|
||||
description ??
|
||||
(typeof found.entry.description === "string"
|
||||
? found.entry.description
|
||||
: "");
|
||||
let finalName: string | undefined;
|
||||
let finalDesc: string | undefined;
|
||||
let translateName = false;
|
||||
let translateDesc = false;
|
||||
if (officialIsReal && t?.name) finalName = t.name;
|
||||
else if (lang !== "en" && srcName) {
|
||||
finalName = srcName;
|
||||
translateName = true;
|
||||
newTexts.add(srcName);
|
||||
} else finalName = srcName;
|
||||
if (officialIsReal && t?.description) finalDesc = t.description;
|
||||
else if (lang !== "en" && srcDesc) {
|
||||
finalDesc = srcDesc;
|
||||
translateDesc = true;
|
||||
newTexts.add(srcDesc);
|
||||
} else finalDesc = srcDesc;
|
||||
newItems.push({
|
||||
section: found.section,
|
||||
found: found.entry,
|
||||
name: finalName,
|
||||
description: finalDesc,
|
||||
translateName,
|
||||
translateDesc,
|
||||
});
|
||||
}
|
||||
for (const tx of newTexts) pendingTexts.add(tx);
|
||||
|
||||
return { data, refs, newItems, pendingTexts, officialPatched };
|
||||
}
|
||||
|
||||
export interface FinalizedPatch {
|
||||
json: string;
|
||||
patched: number;
|
||||
added: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 2 of a localized patch: apply the (network) LibreTranslate results to the
|
||||
* prepared clone and serialize it. New items are appended from the master copy.
|
||||
*/
|
||||
export function finalizePatch(
|
||||
prepared: PreparedPatch,
|
||||
translations: Map<string, string>,
|
||||
): FinalizedPatch {
|
||||
const { data, refs, newItems, officialPatched } = prepared;
|
||||
|
||||
const counted = new Set<FurniDataEntry>();
|
||||
let librePatched = 0;
|
||||
for (const ref of refs) {
|
||||
const original = ref.entry[ref.field];
|
||||
if (typeof original !== "string") continue;
|
||||
const translated = translations.get(original.trim());
|
||||
if (translated === undefined || translated === original) continue;
|
||||
ref.entry[ref.field] = translated;
|
||||
if (!counted.has(ref.entry)) {
|
||||
counted.add(ref.entry);
|
||||
librePatched++;
|
||||
}
|
||||
}
|
||||
|
||||
let added = 0;
|
||||
for (const item of newItems) {
|
||||
const clone = { ...item.found } as FurniDataEntry;
|
||||
if (item.name !== undefined) {
|
||||
if (item.translateName) {
|
||||
const tr = translations.get(item.name.trim());
|
||||
clone.name = tr && tr !== item.name ? tr : item.name;
|
||||
} else {
|
||||
clone.name = item.name;
|
||||
}
|
||||
}
|
||||
if (item.description !== undefined) {
|
||||
if (item.translateDesc) {
|
||||
const tr = translations.get(item.description.trim());
|
||||
clone.description =
|
||||
tr && tr !== item.description ? tr : item.description;
|
||||
} else {
|
||||
clone.description = item.description;
|
||||
}
|
||||
}
|
||||
if (!data[item.section]) data[item.section] = { furnitype: [] };
|
||||
data[item.section].furnitype.push(clone);
|
||||
added++;
|
||||
}
|
||||
|
||||
const total =
|
||||
(data.roomitemtypes?.furnitype?.length ?? 0) +
|
||||
(data.wallitemtypes?.furnitype?.length ?? 0);
|
||||
|
||||
return {
|
||||
json: JSON.stringify(data),
|
||||
patched: officialPatched + librePatched + added,
|
||||
added,
|
||||
total,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user