perf: offload per-import localized patch to worker + cache FurnitureData parse
Per single furni import, patchLocalizedFurniDataEntries ran for every language and did a full 100k-entry JSON.parse + scan + JSON.stringify on the main thread. That CPU spike is now offloaded to the translation worker (init/prepare/finalize reuses the same pure core helpers), so importing never blocks the event loop. The main thread only does async file I/O and the network LibreTranslate calls. Falls back to the same helpers on the main thread if no worker is available. Also cache readFurniData() by file mtime+size so the 100k-entry JSON is parsed once per change instead of on every import/fix/reconcile read (invalidated on write).
This commit is contained in:
openhands committed
1 parent
6ed1b03e24
commit
6ae7c09682
5 files changed
+502
-226
No files matched your search
@@ -153,3 +153,242 @@ export function finalizeLangFile(
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Localized patch (single-item import path) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface PatchEntry {
|
||||
classname: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface PatchRef {
|
||||
entry: FurniDataEntry;
|
||||
field: "name" | "description";
|
||||
}
|
||||
|
||||
interface PatchNewItem {
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
found: FurniDataEntry;
|
||||
name?: string;
|
||||
description?: string;
|
||||
translateName: boolean;
|
||||
translateDesc: boolean;
|
||||
}
|
||||
|
||||
export interface PreparedPatch {
|
||||
data: FurniDataMaster;
|
||||
refs: PatchRef[];
|
||||
newItems: PatchNewItem[];
|
||||
pendingTexts: Set<string>;
|
||||
officialPatched: number;
|
||||
}
|
||||
|
||||
function findEntryInFurniData(
|
||||
data: FurniDataMaster,
|
||||
classname: string,
|
||||
): {
|
||||
entry: FurniDataEntry;
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
} | null {
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
const hit = data[section]?.furnitype.find((e) => e.classname === classname);
|
||||
if (hit) return { entry: hit, section };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 1 of a localized patch: load the existing language file (or clone the
|
||||
* master when it does not exist yet), touch ONLY the wanted classnames, and
|
||||
* collect the texts that still need LibreTranslate. The official-vs-Libre
|
||||
* decision, master clone and the full file scan all run here (CPU-heavy) so the
|
||||
* network translation step can run afterwards on the main thread.
|
||||
*/
|
||||
export function preparePatch(
|
||||
fileContent: string | null,
|
||||
master: FurniDataMaster,
|
||||
lang: string,
|
||||
hotel: HabboGamedataHotel,
|
||||
officialMap: Map<string, Translation>,
|
||||
entries: PatchEntry[],
|
||||
): PreparedPatch {
|
||||
const officialIsReal = officialIsRealTranslation(lang, hotel);
|
||||
let data: FurniDataMaster;
|
||||
try {
|
||||
data = fileContent
|
||||
? (JSON.parse(fileContent) as FurniDataMaster)
|
||||
: cloneFurniData(master);
|
||||
if (!fileContent && officialIsReal) {
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn) continue;
|
||||
const t = lookupTranslation(officialMap, cn);
|
||||
if (t?.name) entry.name = t.name;
|
||||
if (t?.description) entry.description = t.description;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
data = cloneFurniData(master);
|
||||
}
|
||||
|
||||
const wanted = new Map(entries.map((e) => [e.classname, e]));
|
||||
const wantedSet = new Set(wanted.keys());
|
||||
const refs: PatchRef[] = [];
|
||||
const pendingTexts = new Set<string>();
|
||||
let officialPatched = 0;
|
||||
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn || !wantedSet.has(cn)) continue;
|
||||
const wantedEntry = wanted.get(cn);
|
||||
if (!wantedEntry) continue;
|
||||
const t = lookupTranslation(officialMap, cn);
|
||||
const origName =
|
||||
wantedEntry.name ?? (typeof entry.name === "string" ? entry.name : "");
|
||||
const origDesc =
|
||||
wantedEntry.description ??
|
||||
(typeof entry.description === "string" ? entry.description : "");
|
||||
let didPatch = false;
|
||||
if (officialIsReal && t?.name) {
|
||||
entry.name = t.name;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origName) {
|
||||
pendingTexts.add(origName);
|
||||
refs.push({ entry, field: "name" });
|
||||
}
|
||||
if (officialIsReal && t?.description) {
|
||||
entry.description = t.description;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origDesc) {
|
||||
pendingTexts.add(origDesc);
|
||||
refs.push({ entry, field: "description" });
|
||||
}
|
||||
if (didPatch) officialPatched++;
|
||||
}
|
||||
}
|
||||
|
||||
// New classnames not yet in the file: copy from master with translation applied.
|
||||
const existingClassnames = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const e of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof e?.classname === "string" ? e.classname : "";
|
||||
if (cn) existingClassnames.add(cn);
|
||||
}
|
||||
}
|
||||
const newItems: PatchNewItem[] = [];
|
||||
const newTexts = new Set<string>();
|
||||
for (const { classname, name, description } of entries) {
|
||||
if (existingClassnames.has(classname)) continue;
|
||||
const found = findEntryInFurniData(master, classname);
|
||||
if (!found) continue;
|
||||
const t = lookupTranslation(officialMap, classname);
|
||||
const srcName =
|
||||
name ?? (typeof found.entry.name === "string" ? found.entry.name : "");
|
||||
const srcDesc =
|
||||
description ??
|
||||
(typeof found.entry.description === "string"
|
||||
? found.entry.description
|
||||
: "");
|
||||
let finalName: string | undefined;
|
||||
let finalDesc: string | undefined;
|
||||
let translateName = false;
|
||||
let translateDesc = false;
|
||||
if (officialIsReal && t?.name) finalName = t.name;
|
||||
else if (lang !== "en" && srcName) {
|
||||
finalName = srcName;
|
||||
translateName = true;
|
||||
newTexts.add(srcName);
|
||||
} else finalName = srcName;
|
||||
if (officialIsReal && t?.description) finalDesc = t.description;
|
||||
else if (lang !== "en" && srcDesc) {
|
||||
finalDesc = srcDesc;
|
||||
translateDesc = true;
|
||||
newTexts.add(srcDesc);
|
||||
} else finalDesc = srcDesc;
|
||||
newItems.push({
|
||||
section: found.section,
|
||||
found: found.entry,
|
||||
name: finalName,
|
||||
description: finalDesc,
|
||||
translateName,
|
||||
translateDesc,
|
||||
});
|
||||
}
|
||||
for (const tx of newTexts) pendingTexts.add(tx);
|
||||
|
||||
return { data, refs, newItems, pendingTexts, officialPatched };
|
||||
}
|
||||
|
||||
export interface FinalizedPatch {
|
||||
json: string;
|
||||
patched: number;
|
||||
added: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 2 of a localized patch: apply the (network) LibreTranslate results to the
|
||||
* prepared clone and serialize it. New items are appended from the master copy.
|
||||
*/
|
||||
export function finalizePatch(
|
||||
prepared: PreparedPatch,
|
||||
translations: Map<string, string>,
|
||||
): FinalizedPatch {
|
||||
const { data, refs, newItems, officialPatched } = prepared;
|
||||
|
||||
const counted = new Set<FurniDataEntry>();
|
||||
let librePatched = 0;
|
||||
for (const ref of refs) {
|
||||
const original = ref.entry[ref.field];
|
||||
if (typeof original !== "string") continue;
|
||||
const translated = translations.get(original.trim());
|
||||
if (translated === undefined || translated === original) continue;
|
||||
ref.entry[ref.field] = translated;
|
||||
if (!counted.has(ref.entry)) {
|
||||
counted.add(ref.entry);
|
||||
librePatched++;
|
||||
}
|
||||
}
|
||||
|
||||
let added = 0;
|
||||
for (const item of newItems) {
|
||||
const clone = { ...item.found } as FurniDataEntry;
|
||||
if (item.name !== undefined) {
|
||||
if (item.translateName) {
|
||||
const tr = translations.get(item.name.trim());
|
||||
clone.name = tr && tr !== item.name ? tr : item.name;
|
||||
} else {
|
||||
clone.name = item.name;
|
||||
}
|
||||
}
|
||||
if (item.description !== undefined) {
|
||||
if (item.translateDesc) {
|
||||
const tr = translations.get(item.description.trim());
|
||||
clone.description =
|
||||
tr && tr !== item.description ? tr : item.description;
|
||||
} else {
|
||||
clone.description = item.description;
|
||||
}
|
||||
}
|
||||
if (!data[item.section]) data[item.section] = { furnitype: [] };
|
||||
data[item.section].furnitype.push(clone);
|
||||
added++;
|
||||
}
|
||||
|
||||
const total =
|
||||
(data.roomitemtypes?.furnitype?.length ?? 0) +
|
||||
(data.wallitemtypes?.furnitype?.length ?? 0);
|
||||
|
||||
return {
|
||||
json: JSON.stringify(data),
|
||||
patched: officialPatched + librePatched + added,
|
||||
added,
|
||||
total,
|
||||
};
|
||||
}
|
||||
@@ -69,8 +69,6 @@ export interface LanguageFileResult {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
import type { Translation } from "./furni-data-i18n-core";
|
||||
|
||||
const LANG_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours — translations rarely change
|
||||
|
||||
interface LangCacheEntry {
|
||||
@@ -257,7 +255,12 @@ async function translateManyWithLibre(
|
||||
return result;
|
||||
}
|
||||
|
||||
import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core";
|
||||
import type {
|
||||
FurniDataMaster,
|
||||
PreparedLang,
|
||||
PreparedPatch,
|
||||
Translation,
|
||||
} from "./furni-data-i18n-core";
|
||||
/**
|
||||
* The pure clone / scan / stringify helpers live in furni-data-i18n-core so they
|
||||
* can run inside a worker thread (that module has no server-only / network deps).
|
||||
@@ -265,25 +268,26 @@ import type { FurniDataMaster, PreparedLang } from "./furni-data-i18n-core";
|
||||
* backward compatibility.
|
||||
*/
|
||||
import {
|
||||
cloneFurniData,
|
||||
finalizeLangFile,
|
||||
finalizePatch,
|
||||
lookupTranslation,
|
||||
officialIsRealTranslation,
|
||||
prepareLangFile,
|
||||
preparePatch,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
export type {
|
||||
FurniDataMaster,
|
||||
PatchEntry,
|
||||
PreparedLang,
|
||||
PreparedPatch,
|
||||
Translation,
|
||||
} from "./furni-data-i18n-core";
|
||||
export {
|
||||
cloneFurniData,
|
||||
lookupTranslation,
|
||||
officialIsRealTranslation,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
async function fetchTranslationsForHotel(
|
||||
export async function fetchTranslationsForHotel(
|
||||
hotel: HabboGamedataHotel,
|
||||
): Promise<Map<string, Translation>> {
|
||||
const cached = langCache.get(hotel);
|
||||
@@ -400,16 +404,12 @@ export async function buildLocalizedFurniDataFiles(
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
|
||||
// Fetch each hotel's official translations once (network + per-hotel cache).
|
||||
const hotels = [...new Set(langsToBuild.map((l) => l.hotel))];
|
||||
const hotelMaps: Record<string, Map<string, Translation>> = {};
|
||||
for (const hotel of hotels) {
|
||||
hotelMaps[hotel] = await fetchTranslationsForHotel(hotel);
|
||||
}
|
||||
|
||||
const worker = getTranslationWorker();
|
||||
if (worker) {
|
||||
await worker.init(JSON.stringify(master), hotelMaps);
|
||||
// Reads the master + official maps once on the main thread (cached
|
||||
// per process) and ships them to the worker so the per-language
|
||||
// clone/scan/stringify runs off the main thread.
|
||||
await worker.ensureInit();
|
||||
}
|
||||
// Main-thread fallback store, used only when no worker is available.
|
||||
const localPrepared = new Map<string, PreparedLang>();
|
||||
@@ -430,11 +430,12 @@ export async function buildLocalizedFurniDataFiles(
|
||||
if (worker) {
|
||||
pendingTexts = await worker.prepare(lang, hotel);
|
||||
} else {
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
const r = prepareLangFile(
|
||||
master as unknown as FurniDataMaster,
|
||||
lang,
|
||||
hotel,
|
||||
hotelMaps[hotel] ?? new Map(),
|
||||
map,
|
||||
);
|
||||
pendingTexts = r.pendingTexts;
|
||||
preparedKey = lang;
|
||||
@@ -531,220 +532,95 @@ export async function patchLocalizedFurniDataEntries(
|
||||
: FURNIDATA_LANGUAGES;
|
||||
|
||||
return withFurniDataLock(async () => {
|
||||
// Read the master once and reuse it for the fallback clone and for
|
||||
// looking up brand-new classnames — avoids parsing FurnitureData twice.
|
||||
const master = (await readFurniData()) as Record<
|
||||
string,
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
const maps = await Promise.all(
|
||||
langsToBuild.map(async ({ lang, hotel }) => {
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
return { lang, hotel, map };
|
||||
}),
|
||||
);
|
||||
|
||||
const wanted = new Map<string, { name?: string; description?: string }>();
|
||||
for (const e of entries) wanted.set(e.classname, e);
|
||||
const wantedSet = new Set(wanted.keys());
|
||||
const worker = getTranslationWorker();
|
||||
// Fallback store for when there is no worker (test env / spawn failure).
|
||||
const localPrepared = new Map<string, PreparedPatch>();
|
||||
|
||||
const results: LanguageFileResult[] = [];
|
||||
|
||||
for (const { lang, hotel, map } of maps) {
|
||||
for (const { lang, hotel } of langsToBuild) {
|
||||
try {
|
||||
const officialIsReal = officialIsRealTranslation(lang, hotel);
|
||||
const targets = await getLocalizedWritePaths(lang);
|
||||
let patched = 0;
|
||||
|
||||
// Load the existing language file if present, otherwise build it
|
||||
// from master (official translations only — never LibreTranslate,
|
||||
// to avoid 100k+ network calls on first creation).
|
||||
let data: Record<string, { furnitype: Array<Record<string, unknown>> }>;
|
||||
// Read the existing language file as raw text. The expensive
|
||||
// parse + scan + stringify all happen in the worker, so the main
|
||||
// thread only does async file I/O here.
|
||||
let fileContent: string | null = null;
|
||||
try {
|
||||
const raw = await fs.readFile(targets[0], "utf-8");
|
||||
data = JSON.parse(raw);
|
||||
fileContent = await fs.readFile(targets[0], "utf-8");
|
||||
} catch {
|
||||
data = cloneFurniData(master);
|
||||
if (officialIsReal) {
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
if (t?.name) entry.name = t.name;
|
||||
if (t?.description) entry.description = t.description;
|
||||
}
|
||||
}
|
||||
}
|
||||
fileContent = null;
|
||||
}
|
||||
|
||||
// Pass 1 — touch ONLY the wanted classnames (most patches are a
|
||||
// single import), collecting the texts that need LibreTranslate.
|
||||
const refs: Array<{
|
||||
entry: Record<string, unknown>;
|
||||
field: "name" | "description";
|
||||
}> = [];
|
||||
const pendingTexts = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn || !wantedSet.has(cn)) continue;
|
||||
const wantedEntry = wanted.get(cn);
|
||||
if (!wantedEntry) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
let didPatch = false;
|
||||
const origName =
|
||||
wantedEntry.name ??
|
||||
(typeof entry.name === "string" ? entry.name : "");
|
||||
const origDesc =
|
||||
wantedEntry.description ??
|
||||
(typeof entry.description === "string" ? entry.description : "");
|
||||
if (officialIsReal && t?.name) {
|
||||
entry.name = t.name;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origName) {
|
||||
pendingTexts.add(origName);
|
||||
refs.push({ entry, field: "name" });
|
||||
}
|
||||
if (officialIsReal && t?.description) {
|
||||
entry.description = t.description;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origDesc) {
|
||||
pendingTexts.add(origDesc);
|
||||
refs.push({ entry, field: "description" });
|
||||
}
|
||||
if (didPatch) patched++;
|
||||
}
|
||||
let pendingTexts: string[];
|
||||
let localKey: string | null = null;
|
||||
try {
|
||||
if (!worker) throw new Error("no translation worker");
|
||||
pendingTexts = await worker.patchPrepare(
|
||||
fileContent,
|
||||
lang,
|
||||
hotel,
|
||||
entries,
|
||||
);
|
||||
} catch {
|
||||
// Main-thread fallback: run the exact same pure helpers here.
|
||||
const master = (await readFurniData()) as unknown as FurniDataMaster;
|
||||
const map = await fetchTranslationsForHotel(hotel);
|
||||
const r = preparePatch(
|
||||
fileContent,
|
||||
master,
|
||||
lang,
|
||||
hotel,
|
||||
map,
|
||||
entries,
|
||||
);
|
||||
localPrepared.set(lang, r);
|
||||
pendingTexts = [...r.pendingTexts];
|
||||
localKey = lang;
|
||||
}
|
||||
|
||||
// Batch-translate (dedup + concurrency-capped) instead of one
|
||||
// HTTP call per text.
|
||||
const translations = await translateManyWithLibre(pendingTexts, lang);
|
||||
const counted = new Set<Record<string, unknown>>();
|
||||
for (const ref of refs) {
|
||||
const original = ref.entry[ref.field];
|
||||
if (typeof original !== "string") continue;
|
||||
const translated = translations.get(original.trim());
|
||||
if (translated === undefined || translated === original) continue;
|
||||
ref.entry[ref.field] = translated;
|
||||
if (!counted.has(ref.entry)) {
|
||||
counted.add(ref.entry);
|
||||
patched++;
|
||||
}
|
||||
// Batch-translate (network, async — never blocks the main thread).
|
||||
const translations = await translateManyWithLibre(
|
||||
new Set(pendingTexts),
|
||||
lang,
|
||||
);
|
||||
|
||||
let json: string;
|
||||
let patched = 0;
|
||||
let added = 0;
|
||||
let total = 0;
|
||||
if (worker && localKey === null) {
|
||||
const res = await worker.patchFinalize(lang, translations);
|
||||
json = res.json;
|
||||
patched = res.patched;
|
||||
added = res.added;
|
||||
total = res.total;
|
||||
} else {
|
||||
const r = localPrepared.get(localKey!)!;
|
||||
const res = finalizePatch(r, translations);
|
||||
json = res.json;
|
||||
patched = res.patched;
|
||||
added = res.added;
|
||||
total = res.total;
|
||||
localPrepared.delete(localKey!);
|
||||
}
|
||||
|
||||
// New classnames not yet in the file: copy from master with
|
||||
// translation applied (only for the wanted set).
|
||||
const existingClassnames = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const e of data[section]?.furnitype ?? []) {
|
||||
const cn = typeof e?.classname === "string" ? e.classname : "";
|
||||
if (cn) existingClassnames.add(cn);
|
||||
}
|
||||
}
|
||||
const newItems: Array<{
|
||||
section: "roomitemtypes" | "wallitemtypes";
|
||||
found: Record<string, unknown>;
|
||||
name?: string;
|
||||
description?: string;
|
||||
translateName: boolean;
|
||||
translateDesc: boolean;
|
||||
}> = [];
|
||||
const newTexts = new Set<string>();
|
||||
for (const { classname, name, description } of entries) {
|
||||
if (existingClassnames.has(classname)) continue;
|
||||
let found: Record<string, unknown> | undefined;
|
||||
let foundSection: "roomitemtypes" | "wallitemtypes" | undefined;
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
const hit = (
|
||||
master[section]?.furnitype as
|
||||
| Array<Record<string, unknown>>
|
||||
| undefined
|
||||
)?.find((e) => e.classname === classname) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (hit) {
|
||||
found = hit;
|
||||
foundSection = section;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found || !foundSection) continue;
|
||||
const t = lookupTranslation(map, classname);
|
||||
const srcName =
|
||||
name ?? (typeof found.name === "string" ? found.name : "");
|
||||
const srcDesc =
|
||||
description ??
|
||||
(typeof found.description === "string" ? found.description : "");
|
||||
let finalName: string | undefined;
|
||||
let finalDesc: string | undefined;
|
||||
let translateName = false;
|
||||
let translateDesc = false;
|
||||
if (officialIsReal && t?.name) finalName = t.name;
|
||||
else if (lang !== "en" && srcName) {
|
||||
finalName = srcName;
|
||||
translateName = true;
|
||||
newTexts.add(srcName);
|
||||
} else finalName = srcName;
|
||||
if (officialIsReal && t?.description) finalDesc = t.description;
|
||||
else if (lang !== "en" && srcDesc) {
|
||||
finalDesc = srcDesc;
|
||||
translateDesc = true;
|
||||
newTexts.add(srcDesc);
|
||||
} else finalDesc = srcDesc;
|
||||
newItems.push({
|
||||
section: foundSection,
|
||||
found,
|
||||
name: finalName,
|
||||
description: finalDesc,
|
||||
translateName,
|
||||
translateDesc,
|
||||
});
|
||||
}
|
||||
const newTranslations = await translateManyWithLibre(newTexts, lang);
|
||||
for (const item of newItems) {
|
||||
const clone = { ...item.found } as Record<string, unknown>;
|
||||
if (item.name !== undefined) {
|
||||
if (item.translateName) {
|
||||
const tr = newTranslations.get(item.name.trim());
|
||||
clone.name = tr && tr !== item.name ? tr : item.name;
|
||||
} else {
|
||||
clone.name = item.name;
|
||||
}
|
||||
}
|
||||
if (item.description !== undefined) {
|
||||
if (item.translateDesc) {
|
||||
const tr = newTranslations.get(item.description.trim());
|
||||
clone.description =
|
||||
tr && tr !== item.description ? tr : item.description;
|
||||
} else {
|
||||
clone.description = item.description;
|
||||
}
|
||||
}
|
||||
if (!data[item.section]) data[item.section] = { furnitype: [] };
|
||||
data[item.section].furnitype.push(clone);
|
||||
patched++;
|
||||
}
|
||||
|
||||
if (lang !== "en") await saveLibreCache();
|
||||
|
||||
const payload = JSON.stringify(data);
|
||||
for (const target of targets) {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
const tmp = `${target}.tmp`;
|
||||
await fs.writeFile(tmp, payload, "utf-8");
|
||||
await fs.writeFile(tmp, json, "utf-8");
|
||||
await fs.rename(tmp, target);
|
||||
}
|
||||
|
||||
if (lang !== "en") await saveLibreCache();
|
||||
|
||||
results.push({
|
||||
lang,
|
||||
hotel,
|
||||
file: targets[0] ?? `FurnitureData_${lang}.json`,
|
||||
translatedRoom: patched,
|
||||
translatedWall: 0,
|
||||
total: patched,
|
||||
translatedWall: added,
|
||||
total,
|
||||
ok: true,
|
||||
});
|
||||
} catch (err) {
|
||||
|
||||
@@ -199,10 +199,34 @@ export async function withFurniDataLock<T>(fn: () => Promise<T>): Promise<T> {
|
||||
}
|
||||
}
|
||||
|
||||
let cachedFurniData: {
|
||||
path: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
data: Record<string, unknown>;
|
||||
} | null = null;
|
||||
|
||||
export async function readFurniData(): Promise<Record<string, unknown>> {
|
||||
const target = await getFurnitureDataReadPath();
|
||||
const stat = await fs.stat(target);
|
||||
if (
|
||||
cachedFurniData &&
|
||||
cachedFurniData.path === target &&
|
||||
cachedFurniData.mtimeMs === stat.mtimeMs &&
|
||||
cachedFurniData.size === stat.size
|
||||
) {
|
||||
// File unchanged since last parse: skip the (expensive) full JSON.parse.
|
||||
return cachedFurniData.data;
|
||||
}
|
||||
const raw = await fs.readFile(/*turbopackIgnore: true*/ target, "utf-8");
|
||||
return JSON.parse(raw);
|
||||
const data = JSON.parse(raw) as Record<string, unknown>;
|
||||
cachedFurniData = {
|
||||
path: target,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
data,
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function writeFurniData(
|
||||
@@ -216,6 +240,9 @@ export async function writeFurniData(
|
||||
await fs.writeFile(/*turbopackIgnore: true*/ tmp, payload, "utf-8");
|
||||
await fs.rename(/*turbopackIgnore: true*/ tmp, target);
|
||||
}
|
||||
// The file just changed: force the next read to reparse instead of serving
|
||||
// a possibly-mutated cached object.
|
||||
cachedFurniData = null;
|
||||
}
|
||||
|
||||
export function buildFurniEntry(params: {
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { Worker } from "node:worker_threads";
|
||||
import type { Translation } from "./furni-data-i18n-core";
|
||||
import type { PatchEntry, Translation } from "./furni-data-i18n-core";
|
||||
|
||||
/**
|
||||
* Runs the CPU-heavy FurnitureData localization (deep clone + 100k-entry scans +
|
||||
* JSON.stringify) in a single worker thread so the main Node event loop stays
|
||||
* free during a "translate everything" build. Network/DB work (fetching official
|
||||
* Habbo translations, calling LibreTranslate, writing files) stays on the main
|
||||
* thread; only the synchronous CPU is offloaded.
|
||||
* free during a "translate everything" build and during per-import patches.
|
||||
* Network/DB work (fetching official Habbo translations, calling LibreTranslate,
|
||||
* writing files) stays on the main thread; only the synchronous CPU is offloaded.
|
||||
*
|
||||
* Degrades gracefully: in the test environment (or if the worker can't be
|
||||
* created) getTranslationWorker() returns null and callers run the same pure
|
||||
@@ -30,11 +30,32 @@ interface FinalizeReq {
|
||||
lang: string;
|
||||
translations: Map<string, string>;
|
||||
}
|
||||
type Req = InitReq | PrepareReq | FinalizeReq;
|
||||
interface PatchPrepareReq {
|
||||
id: number;
|
||||
type: "patchPrepare";
|
||||
lang: string;
|
||||
hotel: string;
|
||||
fileContent: string | null;
|
||||
entries: PatchEntry[];
|
||||
}
|
||||
interface PatchFinalizeReq {
|
||||
id: number;
|
||||
type: "patchFinalize";
|
||||
lang: string;
|
||||
translations: Map<string, string>;
|
||||
}
|
||||
type Req =
|
||||
| InitReq
|
||||
| PrepareReq
|
||||
| FinalizeReq
|
||||
| PatchPrepareReq
|
||||
| PatchFinalizeReq;
|
||||
type ReqWithoutId =
|
||||
| Omit<InitReq, "id">
|
||||
| Omit<PrepareReq, "id">
|
||||
| Omit<FinalizeReq, "id">;
|
||||
| Omit<FinalizeReq, "id">
|
||||
| Omit<PatchPrepareReq, "id">
|
||||
| Omit<PatchFinalizeReq, "id">;
|
||||
|
||||
type Res =
|
||||
| {
|
||||
@@ -44,12 +65,15 @@ type Res =
|
||||
json?: string;
|
||||
translatedRoom?: number;
|
||||
translatedWall?: number;
|
||||
patched?: number;
|
||||
added?: number;
|
||||
total?: number;
|
||||
}
|
||||
| { id: number; ok: false; error: string };
|
||||
|
||||
let worker: Worker | null = null;
|
||||
let broken = false;
|
||||
let initialized = false;
|
||||
let nextId = 1;
|
||||
const pending = new Map<
|
||||
number,
|
||||
@@ -84,18 +108,41 @@ function spawn(): Worker | null {
|
||||
|
||||
function call(req: ReqWithoutId): Promise<Res> {
|
||||
if (!worker) return Promise.reject(new Error("no translation worker"));
|
||||
const w = worker;
|
||||
return new Promise<Res>((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
worker!.postMessage({ ...req, id } as Req);
|
||||
w.postMessage({ ...req, id } as Req);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the master FurnitureData + each hotel's official translation map once and
|
||||
* initialize the worker. Idempotent — safe to call before every operation.
|
||||
* Reads happen on the main thread but only once per process, so the per-import
|
||||
* CPU cost of parsing the 100k-entry master is amortized away.
|
||||
*/
|
||||
async function ensureInit(): Promise<void> {
|
||||
if (!worker || initialized) return;
|
||||
const { readFurniData } = await import("@/lib/services/furni-data");
|
||||
const { fetchTranslationsForHotel, FURNIDATA_LANGUAGES } = await import(
|
||||
"@/lib/services/furni-data-i18n"
|
||||
);
|
||||
const master = (await readFurniData()) as Record<string, unknown>;
|
||||
const hotels = [...new Set(FURNIDATA_LANGUAGES.map((l) => l.hotel))];
|
||||
const hotelMaps: Record<string, Map<string, Translation>> = {};
|
||||
for (const h of hotels) hotelMaps[h] = await fetchTranslationsForHotel(h);
|
||||
const res = await call({
|
||||
type: "init",
|
||||
masterJson: JSON.stringify(master),
|
||||
hotelMaps,
|
||||
});
|
||||
if (!res.ok) throw new Error(res.error);
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
export interface TranslationWorker {
|
||||
init(
|
||||
masterJson: string,
|
||||
hotelMaps: Record<string, Map<string, Translation>>,
|
||||
): Promise<void>;
|
||||
ensureInit(): Promise<void>;
|
||||
prepare(lang: string, hotel: string): Promise<string[]>;
|
||||
finalize(
|
||||
lang: string,
|
||||
@@ -106,6 +153,16 @@ export interface TranslationWorker {
|
||||
translatedWall: number;
|
||||
total: number;
|
||||
}>;
|
||||
patchPrepare(
|
||||
fileContent: string | null,
|
||||
lang: string,
|
||||
hotel: string,
|
||||
entries: PatchEntry[],
|
||||
): Promise<string[]>;
|
||||
patchFinalize(
|
||||
lang: string,
|
||||
translations: Map<string, string>,
|
||||
): Promise<{ json: string; patched: number; added: number; total: number }>;
|
||||
}
|
||||
|
||||
export function getTranslationWorker(): TranslationWorker | null {
|
||||
@@ -113,11 +170,11 @@ export function getTranslationWorker(): TranslationWorker | null {
|
||||
if (!worker) worker = spawn();
|
||||
if (!worker) return null;
|
||||
return {
|
||||
async init(masterJson, hotelMaps) {
|
||||
const res = await call({ type: "init", masterJson, hotelMaps });
|
||||
if (!res.ok) throw new Error(res.error);
|
||||
async ensureInit() {
|
||||
await ensureInit();
|
||||
},
|
||||
async prepare(lang, hotel) {
|
||||
await ensureInit();
|
||||
const res = await call({ type: "prepare", lang, hotel });
|
||||
if (!res.ok || !res.pendingTexts)
|
||||
throw new Error(res.ok ? "missing pendingTexts" : res.error);
|
||||
@@ -134,5 +191,29 @@ export function getTranslationWorker(): TranslationWorker | null {
|
||||
total: res.total ?? 0,
|
||||
};
|
||||
},
|
||||
async patchPrepare(fileContent, lang, hotel, entries) {
|
||||
await ensureInit();
|
||||
const res = await call({
|
||||
type: "patchPrepare",
|
||||
lang,
|
||||
hotel,
|
||||
fileContent,
|
||||
entries,
|
||||
});
|
||||
if (!res.ok || !res.pendingTexts)
|
||||
throw new Error(res.ok ? "missing pendingTexts" : res.error);
|
||||
return res.pendingTexts;
|
||||
},
|
||||
async patchFinalize(lang, translations) {
|
||||
const res = await call({ type: "patchFinalize", lang, translations });
|
||||
if (!res.ok || res.json === undefined)
|
||||
throw new Error(res.ok ? "missing json" : res.error);
|
||||
return {
|
||||
json: res.json,
|
||||
patched: res.patched ?? 0,
|
||||
added: res.added ?? 0,
|
||||
total: res.total ?? 0,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -3,8 +3,12 @@ import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel";
|
||||
import {
|
||||
type FurniDataMaster,
|
||||
finalizeLangFile,
|
||||
finalizePatch,
|
||||
type PatchEntry,
|
||||
type PreparedLang,
|
||||
type PreparedPatch,
|
||||
prepareLangFile,
|
||||
preparePatch,
|
||||
type Translation,
|
||||
} from "./furni-data-i18n-core";
|
||||
|
||||
@@ -26,19 +30,40 @@ interface FinalizeMsg {
|
||||
lang: string;
|
||||
translations: Map<string, string>;
|
||||
}
|
||||
type Msg = InitMsg | PrepareMsg | FinalizeMsg;
|
||||
interface PatchPrepareMsg {
|
||||
id: number;
|
||||
type: "patchPrepare";
|
||||
lang: string;
|
||||
hotel: string;
|
||||
fileContent: string | null;
|
||||
entries: PatchEntry[];
|
||||
}
|
||||
interface PatchFinalizeMsg {
|
||||
id: number;
|
||||
type: "patchFinalize";
|
||||
lang: string;
|
||||
translations: Map<string, string>;
|
||||
}
|
||||
type Msg =
|
||||
| InitMsg
|
||||
| PrepareMsg
|
||||
| FinalizeMsg
|
||||
| PatchPrepareMsg
|
||||
| PatchFinalizeMsg;
|
||||
|
||||
let master: FurniDataMaster | null = null;
|
||||
let hotelMaps: Record<string, Map<string, Translation>> = {};
|
||||
const prepared = new Map<string, PreparedLang>();
|
||||
const preparedPatches = new Map<string, PreparedPatch>();
|
||||
|
||||
if (parentPort) {
|
||||
parentPort.on("message", (msg: Msg) => {
|
||||
const port = parentPort;
|
||||
port.on("message", (msg: Msg) => {
|
||||
try {
|
||||
if (msg.type === "init") {
|
||||
master = JSON.parse(msg.masterJson);
|
||||
hotelMaps = msg.hotelMaps ?? {};
|
||||
parentPort!.postMessage({ id: msg.id, ok: true });
|
||||
port.postMessage({ id: msg.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (msg.type === "prepare") {
|
||||
@@ -52,21 +77,49 @@ if (parentPort) {
|
||||
officialMap,
|
||||
);
|
||||
prepared.set(msg.lang, r);
|
||||
parentPort!.postMessage({
|
||||
port.postMessage({
|
||||
id: msg.id,
|
||||
ok: true,
|
||||
pendingTexts: r.pendingTexts,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (msg.type === "patchPrepare") {
|
||||
if (!master) throw new Error("translation worker not initialized");
|
||||
const officialMap =
|
||||
hotelMaps[msg.hotel] ?? new Map<string, Translation>();
|
||||
const r = preparePatch(
|
||||
msg.fileContent,
|
||||
master,
|
||||
msg.lang,
|
||||
msg.hotel as HabboGamedataHotel,
|
||||
officialMap,
|
||||
msg.entries,
|
||||
);
|
||||
preparedPatches.set(msg.lang, r);
|
||||
port.postMessage({
|
||||
id: msg.id,
|
||||
ok: true,
|
||||
pendingTexts: [...r.pendingTexts],
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (msg.type === "patchFinalize") {
|
||||
const r = preparedPatches.get(msg.lang);
|
||||
if (!r) throw new Error(`no prepared patch for ${msg.lang}`);
|
||||
const res = finalizePatch(r, msg.translations);
|
||||
preparedPatches.delete(msg.lang);
|
||||
port.postMessage({ id: msg.id, ok: true, ...res });
|
||||
return;
|
||||
}
|
||||
// finalize
|
||||
const r = prepared.get(msg.lang);
|
||||
if (!r) throw new Error(`no prepared data for ${msg.lang}`);
|
||||
const res = finalizeLangFile(r.cloned, r.pendingRefs, msg.translations);
|
||||
prepared.delete(msg.lang);
|
||||
parentPort!.postMessage({ id: msg.id, ok: true, ...res });
|
||||
port.postMessage({ id: msg.id, ok: true, ...res });
|
||||
} catch (err) {
|
||||
parentPort!.postMessage({
|
||||
port.postMessage({
|
||||
id: msg.id,
|
||||
ok: false,
|
||||
error: (err as Error).message,
|
||||
|
||||
Reference in new issue
Block a user