remco/EpicNext-Cms
1
1 Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf: eliminate lag spikes and redundant work in furnidata i18n
CI / check (push) Successful in 28s
Details
CI / release (push) Skipped
Details
CI / deploy (push) Successful in 1m1s
Details

Browse Source 
- patchLocalizedFurniDataEntries now only touches the wanted classnames
  instead of scanning the whole 100k file, reads the master once (was
  twice), and uses the batched translator instead of one HTTP call per
  text.
- Remove the now-unused serial translateWithLibre.
- buildLocalizedFurniDataFiles yields between the two section scans and
  before stringify so a single language no longer blocks the event loop
  for the whole pass at once.
This commit is contained in:
openhands committed 2026-08-24 18:52:55 +02:00
1 parent 5efd004533
commit 7116f49e2b
1 file changed
+150 -141
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/services/furni-data-i18n.ts
+150 -141
View File
@@ -139,52 +139,6 @@ async function saveLibreCache(): Promise<void> {
} }
} }
async function translateWithLibre(
text: string,
targetLang: string,
): Promise<string> {
const trimmed = text.trim();
if (!trimmed || targetLang === "en") return text;
const cache = await loadLibreCache();
const key = `${targetLang}|${trimmed}`;
const cached = cache.get(key);
if (cached !== undefined) return cached;
const url = await getLibreTranslateUrl();
try {
const res = await fetch(`${url}/translate`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
q: trimmed,
source: "en",
target: targetLang,
format: "text",
}),
signal: AbortSignal.timeout(10_000),
});
if (!res.ok) {
logger.warn(
`[furni-i18n] LibreTranslate ${targetLang} failed: ${res.status}`,
);
return text;
}
const data = (await res.json()) as { translatedText?: string };
const out =
typeof data.translatedText === "string" ? data.translatedText : text;
cache.set(key, out);
// Best-effort persist, don't await per-call to avoid hammering disk — save
// in background throttled (every 50 new entries).
if (cache.size % 50 === 0) void saveLibreCache();
return out;
} catch (err) {
logger.warn(`[furni-i18n] LibreTranslate error (${targetLang})`, {
error: (err as Error).message,
});
return text;
}
}
const LIBRE_BATCH_SIZE = 48; const LIBRE_BATCH_SIZE = 48;
// Kept low on purpose: the build already pegs a core with the synchronous // Kept low on purpose: the build already pegs a core with the synchronous
// deep-clone + stringify of the 100k+ entry master per language, so we avoid // deep-clone + stringify of the 100k+ entry master per language, so we avoid
@@ -195,6 +149,7 @@ const LIBRE_CONCURRENCY = 2;
const BUILD_YIELD_MS = 150; const BUILD_YIELD_MS = 150;
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
const yieldToEventLoop = () => new Promise((r) => setImmediate(r));
/** /**
* Translate many texts in one go using LibreTranslate's array API * Translate many texts in one go using LibreTranslate's array API
@@ -491,35 +446,41 @@ export async function buildLocalizedFurniDataFiles(
section: "roomitemtypes" | "wallitemtypes"; section: "roomitemtypes" | "wallitemtypes";
}> = []; }> = [];
const pendingTexts = new Set<string>(); const pendingTexts = new Set<string>();
for (const section of ["roomitemtypes", "wallitemtypes"] as const) { const sections = ["roomitemtypes", "wallitemtypes"] as const;
for (let si = 0; si < sections.length; si++) {
const section = sections[si];
const arr = cloned[section]?.furnitype; const arr = cloned[section]?.furnitype;
if (!Array.isArray(arr)) continue; if (Array.isArray(arr)) {
for (const entry of arr) { for (const entry of arr) {
const cn = const cn =
typeof entry.classname === "string" ? entry.classname : ""; typeof entry.classname === "string" ? entry.classname : "";
if (!cn) continue; if (!cn) continue;
const t = lookupTranslation(map, cn); const t = lookupTranslation(map, cn);
if (officialIsReal && t?.name) { if (officialIsReal && t?.name) {
entry.name = t.name; entry.name = t.name;
} else if ( } else if (
lang !== "en" && lang !== "en" &&
typeof entry.name === "string" && typeof entry.name === "string" &&
entry.name.trim() entry.name.trim()
) { ) {
pendingTexts.add(entry.name.trim()); pendingTexts.add(entry.name.trim());
pendingRefs.push({ entry, field: "name", section }); pendingRefs.push({ entry, field: "name", section });
} }
if (officialIsReal && t?.description) { if (officialIsReal && t?.description) {
entry.description = t.description; entry.description = t.description;
} else if ( } else if (
lang !== "en" && lang !== "en" &&
typeof entry.description === "string" && typeof entry.description === "string" &&
entry.description.trim() entry.description.trim()
) { ) {
pendingTexts.add(entry.description.trim()); pendingTexts.add(entry.description.trim());
pendingRefs.push({ entry, field: "description", section }); pendingRefs.push({ entry, field: "description", section });
}
} }
} }
// Yield between the two 100k+ scans so a single language never
// blocks the event loop for the whole pass at once.
if (si === 0) await yieldToEventLoop();
} }
// Pass 2 — batch-translate everything pending in a handful of // Pass 2 — batch-translate everything pending in a handful of
@@ -548,7 +509,7 @@ export async function buildLocalizedFurniDataFiles(
(cloned.roomitemtypes?.furnitype?.length ?? 0) + (cloned.roomitemtypes?.furnitype?.length ?? 0) +
(cloned.wallitemtypes?.furnitype?.length ?? 0); (cloned.wallitemtypes?.furnitype?.length ?? 0);
const payload = JSON.stringify(cloned); const payload = (await yieldToEventLoop(), JSON.stringify(cloned));
const targets = await getLocalizedWritePaths(lang); const targets = await getLocalizedWritePaths(lang);
for (const target of targets) { for (const target of targets) {
await fs.mkdir(path.dirname(target), { recursive: true }); await fs.mkdir(path.dirname(target), { recursive: true });
@@ -612,6 +573,12 @@ export async function patchLocalizedFurniDataEntries(
: FURNIDATA_LANGUAGES; : FURNIDATA_LANGUAGES;
return withFurniDataLock(async () => { return withFurniDataLock(async () => {
// Read the master once and reuse it for the fallback clone and for
// looking up brand-new classnames — avoids parsing FurnitureData twice.
const master = (await readFurniData()) as Record<
string,
{ furnitype: Array<Record<string, unknown>> }
>;
const maps = await Promise.all( const maps = await Promise.all(
langsToBuild.map(async ({ lang, hotel }) => { langsToBuild.map(async ({ lang, hotel }) => {
const map = await fetchTranslationsForHotel(hotel); const map = await fetchTranslationsForHotel(hotel);
@@ -621,6 +588,7 @@ export async function patchLocalizedFurniDataEntries(
const wanted = new Map<string, { name?: string; description?: string }>(); const wanted = new Map<string, { name?: string; description?: string }>();
for (const e of entries) wanted.set(e.classname, e); for (const e of entries) wanted.set(e.classname, e);
const wantedSet = new Set(wanted.keys());
const results: LanguageFileResult[] = []; const results: LanguageFileResult[] = [];
@@ -628,41 +596,43 @@ export async function patchLocalizedFurniDataEntries(
try { try {
const officialIsReal = officialIsRealTranslation(lang, hotel); const officialIsReal = officialIsRealTranslation(lang, hotel);
const targets = await getLocalizedWritePaths(lang); const targets = await getLocalizedWritePaths(lang);
// Load the existing language file if present, otherwise clone master. let patched = 0;
// Load the existing language file if present, otherwise build it
// from master (official translations only — never LibreTranslate,
// to avoid 100k+ network calls on first creation).
let data: Record<string, { furnitype: Array<Record<string, unknown>> }>; let data: Record<string, { furnitype: Array<Record<string, unknown>> }>;
try { try {
const raw = await fs.readFile(targets[0], "utf-8"); const raw = await fs.readFile(targets[0], "utf-8");
data = JSON.parse(raw); data = JSON.parse(raw);
} catch { } catch {
const master = (await readFurniData()) as Record< data = cloneFurniData(master);
string, if (officialIsReal) {
{ furnitype: Array<Record<string, unknown>> } for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
>; for (const entry of data[section]?.furnitype ?? []) {
data = cloneFurniData(master as unknown as Record<string, unknown>); const cn =
// Apply all translations for the fallback path (same as full build typeof entry.classname === "string" ? entry.classname : "";
// but only for the file that didn't exist — official only to avoid if (!cn) continue;
// 100k+ LibreTranslate calls on first creation). const t = lookupTranslation(map, cn);
for (const section of ["roomitemtypes", "wallitemtypes"] as const) { if (t?.name) entry.name = t.name;
for (const entry of data[section]?.furnitype ?? []) { if (t?.description) entry.description = t.description;
const cn = }
typeof entry.classname === "string" ? entry.classname : "";
if (!cn) continue;
// Only real official translations count here — English names
// from a com-mapped hotel must not be written over the file
// as if they were localized.
if (!officialIsReal) continue;
const t = lookupTranslation(map, cn);
if (t?.name) entry.name = t.name;
if (t?.description) entry.description = t.description;
} }
} }
} }
let patched = 0; // Pass 1 — touch ONLY the wanted classnames (most patches are a
// single import), collecting the texts that need LibreTranslate.
const refs: Array<{
entry: Record<string, unknown>;
field: "name" | "description";
}> = [];
const pendingTexts = new Set<string>();
for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
for (const entry of data[section]?.furnitype ?? []) { for (const entry of data[section]?.furnitype ?? []) {
const cn = const cn =
typeof entry.classname === "string" ? entry.classname : ""; typeof entry.classname === "string" ? entry.classname : "";
if (!cn || !wantedSet.has(cn)) continue;
const wantedEntry = wanted.get(cn); const wantedEntry = wanted.get(cn);
if (!wantedEntry) continue; if (!wantedEntry) continue;
const t = lookupTranslation(map, cn); const t = lookupTranslation(map, cn);
@@ -677,28 +647,38 @@ export async function patchLocalizedFurniDataEntries(
entry.name = t.name; entry.name = t.name;
didPatch = true; didPatch = true;
} else if (lang !== "en" && origName) { } else if (lang !== "en" && origName) {
const translated = await translateWithLibre(origName, lang); pendingTexts.add(origName);
if (translated && translated !== origName) { refs.push({ entry, field: "name" });
entry.name = translated;
didPatch = true;
}
} }
if (officialIsReal && t?.description) { if (officialIsReal && t?.description) {
entry.description = t.description; entry.description = t.description;
didPatch = true; didPatch = true;
} else if (lang !== "en" && origDesc) { } else if (lang !== "en" && origDesc) {
const translated = await translateWithLibre(origDesc, lang); pendingTexts.add(origDesc);
if (translated && translated !== origDesc) { refs.push({ entry, field: "description" });
entry.description = translated;
didPatch = true;
}
} }
if (didPatch) patched++; if (didPatch) patched++;
} }
} }
// Also handle brand-new entries that weren't in the file yet (rare
// when a single import added a classname not previously in lang file // Batch-translate (dedup + concurrency-capped) instead of one
// because the file was stale). Append with translated names. // HTTP call per text.
const translations = await translateManyWithLibre(pendingTexts, lang);
const counted = new Set<Record<string, unknown>>();
for (const ref of refs) {
const original = ref.entry[ref.field];
if (typeof original !== "string") continue;
const translated = translations.get(original.trim());
if (translated === undefined || translated === original) continue;
ref.entry[ref.field] = translated;
if (!counted.has(ref.entry)) {
counted.add(ref.entry);
patched++;
}
}
// New classnames not yet in the file: copy from master with
// translation applied (only for the wanted set).
const existingClassnames = new Set<string>(); const existingClassnames = new Set<string>();
for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
for (const e of data[section]?.furnitype ?? []) { for (const e of data[section]?.furnitype ?? []) {
@@ -706,21 +686,25 @@ export async function patchLocalizedFurniDataEntries(
if (cn) existingClassnames.add(cn); if (cn) existingClassnames.add(cn);
} }
} }
// New classnames are already in the master; if our loaded data const newItems: Array<{
// came from an old lang file before the import, the entry won't section: "roomitemtypes" | "wallitemtypes";
// exist — copy it from master with translation applied. found: Record<string, unknown>;
const master = (await readFurniData()) as Record< name?: string;
string, description?: string;
{ furnitype: Array<Record<string, unknown>> } translateName: boolean;
>; translateDesc: boolean;
}> = [];
const newTexts = new Set<string>();
for (const { classname, name, description } of entries) { for (const { classname, name, description } of entries) {
if (existingClassnames.has(classname)) continue; if (existingClassnames.has(classname)) continue;
// Find in master.
let found: Record<string, unknown> | undefined; let found: Record<string, unknown> | undefined;
let foundSection: "roomitemtypes" | "wallitemtypes" | undefined; let foundSection: "roomitemtypes" | "wallitemtypes" | undefined;
for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
const arr = master[section]?.furnitype; const hit = (
const hit = arr?.find((e) => e.classname === classname) as master[section]?.furnitype as
| Array<Record<string, unknown>>
| undefined
)?.find((e) => e.classname === classname) as
| Record<string, unknown> | Record<string, unknown>
| undefined; | undefined;
if (hit) { if (hit) {
@@ -730,37 +714,62 @@ export async function patchLocalizedFurniDataEntries(
} }
} }
if (!found || !foundSection) continue; if (!found || !foundSection) continue;
const clone = { ...found };
const t = lookupTranslation(map, classname); const t = lookupTranslation(map, classname);
if (officialIsReal && t?.name) clone.name = t.name; const srcName =
else if (lang !== "en") { name ?? (typeof found.name === "string" ? found.name : "");
const src = const srcDesc =
name ?? (typeof found.name === "string" ? found.name : "") ?? ""; description ??
if (src) { (typeof found.description === "string" ? found.description : "");
const translated = await translateWithLibre(src, lang); let finalName: string | undefined;
if (translated !== src) clone.name = translated; let finalDesc: string | undefined;
let translateName = false;
let translateDesc = false;
if (officialIsReal && t?.name) finalName = t.name;
else if (lang !== "en" && srcName) {
finalName = srcName;
translateName = true;
newTexts.add(srcName);
} else finalName = srcName;
if (officialIsReal && t?.description) finalDesc = t.description;
else if (lang !== "en" && srcDesc) {
finalDesc = srcDesc;
translateDesc = true;
newTexts.add(srcDesc);
} else finalDesc = srcDesc;
newItems.push({
section: foundSection,
found,
name: finalName,
description: finalDesc,
translateName,
translateDesc,
});
}
const newTranslations = await translateManyWithLibre(newTexts, lang);
for (const item of newItems) {
const clone = { ...item.found } as Record<string, unknown>;
if (item.name !== undefined) {
if (item.translateName) {
const tr = newTranslations.get(item.name.trim());
clone.name = tr && tr !== item.name ? tr : item.name;
} else {
clone.name = item.name;
} }
} }
if (officialIsReal && t?.description) if (item.description !== undefined) {
clone.description = t.description; if (item.translateDesc) {
else if (lang !== "en") { const tr = newTranslations.get(item.description.trim());
const src = clone.description =
description ?? tr && tr !== item.description ? tr : item.description;
(typeof found.description === "string" } else {
? found.description clone.description = item.description;
: "") ??
"";
if (src) {
const translated = await translateWithLibre(src, lang);
if (translated !== src) clone.description = translated;
} }
} }
if (!data[foundSection]) data[foundSection] = { furnitype: [] }; if (!data[item.section]) data[item.section] = { furnitype: [] };
data[foundSection].furnitype.push(clone); data[item.section].furnitype.push(clone);
patched++; patched++;
} }
// Persist LibreTranslate cache once per language (batched).
if (lang !== "en") await saveLibreCache(); if (lang !== "en") await saveLibreCache();
const payload = JSON.stringify(data); const payload = JSON.stringify(data);