perf: offload per-import localized patch to worker + cache FurnitureData parse
Per single furni import, patchLocalizedFurniDataEntries ran for every language and did a full 100k-entry JSON.parse + scan + JSON.stringify on the main thread. That CPU spike is now offloaded to the translation worker (init/prepare/finalize reuses the same pure core helpers), so importing never blocks the event loop. The main thread only does async file I/O and the network LibreTranslate calls. Falls back to the same helpers on the main thread if no worker is available. Also cache readFurniData() by file mtime+size so the 100k-entry JSON is parsed once per change instead of on every import/fix/reconcile read (invalidated on write).
This commit is contained in:
1 parent
6ed1b03e24
commit
6ae7c09682
5 files changed
+502
-226
No files matched your search
@@ -1,12 +1,12 @@
|
||||
import { Worker } from "node:worker_threads";
|
||||
import type { Translation } from "./furni-data-i18n-core";
|
||||
import type { PatchEntry, Translation } from "./furni-data-i18n-core";
|
||||
|
||||
/**
|
||||
* Runs the CPU-heavy FurnitureData localization (deep clone + 100k-entry scans +
|
||||
* JSON.stringify) in a single worker thread so the main Node event loop stays
|
||||
* free during a "translate everything" build. Network/DB work (fetching official
|
||||
* Habbo translations, calling LibreTranslate, writing files) stays on the main
|
||||
* thread; only the synchronous CPU is offloaded.
|
||||
* free during a "translate everything" build and during per-import patches.
|
||||
* Network/DB work (fetching official Habbo translations, calling LibreTranslate,
|
||||
* writing files) stays on the main thread; only the synchronous CPU is offloaded.
|
||||
*
|
||||
* Degrades gracefully: in the test environment (or if the worker can't be
|
||||
* created) getTranslationWorker() returns null and callers run the same pure
|
||||
@@ -30,11 +30,32 @@ interface FinalizeReq {
|
||||
lang: string;
|
||||
translations: Map<string, string>;
|
||||
}
|
||||
type Req = InitReq | PrepareReq | FinalizeReq;
|
||||
interface PatchPrepareReq {
|
||||
id: number;
|
||||
type: "patchPrepare";
|
||||
lang: string;
|
||||
hotel: string;
|
||||
fileContent: string | null;
|
||||
entries: PatchEntry[];
|
||||
}
|
||||
interface PatchFinalizeReq {
|
||||
id: number;
|
||||
type: "patchFinalize";
|
||||
lang: string;
|
||||
translations: Map<string, string>;
|
||||
}
|
||||
type Req =
|
||||
| InitReq
|
||||
| PrepareReq
|
||||
| FinalizeReq
|
||||
| PatchPrepareReq
|
||||
| PatchFinalizeReq;
|
||||
type ReqWithoutId =
|
||||
| Omit<InitReq, "id">
|
||||
| Omit<PrepareReq, "id">
|
||||
| Omit<FinalizeReq, "id">;
|
||||
| Omit<FinalizeReq, "id">
|
||||
| Omit<PatchPrepareReq, "id">
|
||||
| Omit<PatchFinalizeReq, "id">;
|
||||
|
||||
type Res =
|
||||
| {
|
||||
@@ -44,12 +65,15 @@ type Res =
|
||||
json?: string;
|
||||
translatedRoom?: number;
|
||||
translatedWall?: number;
|
||||
patched?: number;
|
||||
added?: number;
|
||||
total?: number;
|
||||
}
|
||||
| { id: number; ok: false; error: string };
|
||||
|
||||
let worker: Worker | null = null;
|
||||
let broken = false;
|
||||
let initialized = false;
|
||||
let nextId = 1;
|
||||
const pending = new Map<
|
||||
number,
|
||||
@@ -84,18 +108,41 @@ function spawn(): Worker | null {
|
||||
|
||||
function call(req: ReqWithoutId): Promise<Res> {
|
||||
if (!worker) return Promise.reject(new Error("no translation worker"));
|
||||
const w = worker;
|
||||
return new Promise<Res>((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
worker!.postMessage({ ...req, id } as Req);
|
||||
w.postMessage({ ...req, id } as Req);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the master FurnitureData + each hotel's official translation map once and
|
||||
* initialize the worker. Idempotent — safe to call before every operation.
|
||||
* Reads happen on the main thread but only once per process, so the per-import
|
||||
* CPU cost of parsing the 100k-entry master is amortized away.
|
||||
*/
|
||||
async function ensureInit(): Promise<void> {
|
||||
if (!worker || initialized) return;
|
||||
const { readFurniData } = await import("@/lib/services/furni-data");
|
||||
const { fetchTranslationsForHotel, FURNIDATA_LANGUAGES } = await import(
|
||||
"@/lib/services/furni-data-i18n"
|
||||
);
|
||||
const master = (await readFurniData()) as Record<string, unknown>;
|
||||
const hotels = [...new Set(FURNIDATA_LANGUAGES.map((l) => l.hotel))];
|
||||
const hotelMaps: Record<string, Map<string, Translation>> = {};
|
||||
for (const h of hotels) hotelMaps[h] = await fetchTranslationsForHotel(h);
|
||||
const res = await call({
|
||||
type: "init",
|
||||
masterJson: JSON.stringify(master),
|
||||
hotelMaps,
|
||||
});
|
||||
if (!res.ok) throw new Error(res.error);
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
export interface TranslationWorker {
|
||||
init(
|
||||
masterJson: string,
|
||||
hotelMaps: Record<string, Map<string, Translation>>,
|
||||
): Promise<void>;
|
||||
ensureInit(): Promise<void>;
|
||||
prepare(lang: string, hotel: string): Promise<string[]>;
|
||||
finalize(
|
||||
lang: string,
|
||||
@@ -106,6 +153,16 @@ export interface TranslationWorker {
|
||||
translatedWall: number;
|
||||
total: number;
|
||||
}>;
|
||||
patchPrepare(
|
||||
fileContent: string | null,
|
||||
lang: string,
|
||||
hotel: string,
|
||||
entries: PatchEntry[],
|
||||
): Promise<string[]>;
|
||||
patchFinalize(
|
||||
lang: string,
|
||||
translations: Map<string, string>,
|
||||
): Promise<{ json: string; patched: number; added: number; total: number }>;
|
||||
}
|
||||
|
||||
export function getTranslationWorker(): TranslationWorker | null {
|
||||
@@ -113,11 +170,11 @@ export function getTranslationWorker(): TranslationWorker | null {
|
||||
if (!worker) worker = spawn();
|
||||
if (!worker) return null;
|
||||
return {
|
||||
async init(masterJson, hotelMaps) {
|
||||
const res = await call({ type: "init", masterJson, hotelMaps });
|
||||
if (!res.ok) throw new Error(res.error);
|
||||
async ensureInit() {
|
||||
await ensureInit();
|
||||
},
|
||||
async prepare(lang, hotel) {
|
||||
await ensureInit();
|
||||
const res = await call({ type: "prepare", lang, hotel });
|
||||
if (!res.ok || !res.pendingTexts)
|
||||
throw new Error(res.ok ? "missing pendingTexts" : res.error);
|
||||
@@ -134,5 +191,29 @@ export function getTranslationWorker(): TranslationWorker | null {
|
||||
total: res.total ?? 0,
|
||||
};
|
||||
},
|
||||
async patchPrepare(fileContent, lang, hotel, entries) {
|
||||
await ensureInit();
|
||||
const res = await call({
|
||||
type: "patchPrepare",
|
||||
lang,
|
||||
hotel,
|
||||
fileContent,
|
||||
entries,
|
||||
});
|
||||
if (!res.ok || !res.pendingTexts)
|
||||
throw new Error(res.ok ? "missing pendingTexts" : res.error);
|
||||
return res.pendingTexts;
|
||||
},
|
||||
async patchFinalize(lang, translations) {
|
||||
const res = await call({ type: "patchFinalize", lang, translations });
|
||||
if (!res.ok || res.json === undefined)
|
||||
throw new Error(res.ok ? "missing json" : res.error);
|
||||
return {
|
||||
json: res.json,
|
||||
patched: res.patched ?? 0,
|
||||
added: res.added ?? 0,
|
||||
total: res.total ?? 0,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user