remco/Epicnabbo-Catalogus-Updated-Daily
1
0 Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf: eliminate lag spikes and redundant work in furnidata i18n

Browse Source 
- patchLocalizedFurniDataEntries now only touches the wanted classnames
  instead of scanning the whole 100k file, reads the master once (was
  twice), and uses the batched translator instead of one HTTP call per
  text.
- Remove the now-unused serial translateWithLibre.
- buildLocalizedFurniDataFiles yields between the two section scans and
  before stringify so a single language no longer blocks the event loop
  for the whole pass at once.
This commit is contained in:
openhands committed 2026-08-24 18:52:55 +02:00
1 parent 5efd004533
commit 7116f49e2b
1 file changed
+150 -141
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/services/furni-data-i18n.ts
+150 -141
View File
@@ -139,52 +139,6 @@ async function saveLibreCache(): Promise<void> {
}
}
async function translateWithLibre(
text: string,
targetLang: string,
): Promise<string> {
const trimmed = text.trim();
if (!trimmed || targetLang === "en") return text;
const cache = await loadLibreCache();
const key = `${targetLang}|${trimmed}`;
const cached = cache.get(key);
if (cached !== undefined) return cached;
const url = await getLibreTranslateUrl();
try {
const res = await fetch(`${url}/translate`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
q: trimmed,
source: "en",
target: targetLang,
format: "text",
}),
signal: AbortSignal.timeout(10_000),
});
if (!res.ok) {
logger.warn(
`[furni-i18n] LibreTranslate ${targetLang} failed: ${res.status}`,
);
return text;
}
const data = (await res.json()) as { translatedText?: string };
const out =
typeof data.translatedText === "string" ? data.translatedText : text;
cache.set(key, out);
// Best-effort persist, don't await per-call to avoid hammering disk — save
// in background throttled (every 50 new entries).
if (cache.size % 50 === 0) void saveLibreCache();
return out;
} catch (err) {
logger.warn(`[furni-i18n] LibreTranslate error (${targetLang})`, {
error: (err as Error).message,
});
return text;
}
}
const LIBRE_BATCH_SIZE = 48;
// Kept low on purpose: the build already pegs a core with the synchronous
// deep-clone + stringify of the 100k+ entry master per language, so we avoid
@@ -195,6 +149,7 @@ const LIBRE_CONCURRENCY = 2;
const BUILD_YIELD_MS = 150;
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
const yieldToEventLoop = () => new Promise((r) => setImmediate(r));
/**
* Translate many texts in one go using LibreTranslate's array API
@@ -491,35 +446,41 @@ export async function buildLocalizedFurniDataFiles(
section: "roomitemtypes" | "wallitemtypes";
}> = [];
const pendingTexts = new Set<string>();
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
const sections = ["roomitemtypes", "wallitemtypes"] as const;
for (let si = 0; si < sections.length; si++) {
const section = sections[si];
const arr = cloned[section]?.furnitype;
if (!Array.isArray(arr)) continue;
for (const entry of arr) {
const cn =
typeof entry.classname === "string" ? entry.classname : "";
if (!cn) continue;
const t = lookupTranslation(map, cn);
if (officialIsReal && t?.name) {
entry.name = t.name;
} else if (
lang !== "en" &&
typeof entry.name === "string" &&
entry.name.trim()
) {
pendingTexts.add(entry.name.trim());
pendingRefs.push({ entry, field: "name", section });
}
if (officialIsReal && t?.description) {
entry.description = t.description;
} else if (
lang !== "en" &&
typeof entry.description === "string" &&
entry.description.trim()
) {
pendingTexts.add(entry.description.trim());
pendingRefs.push({ entry, field: "description", section });
if (Array.isArray(arr)) {
for (const entry of arr) {
const cn =
typeof entry.classname === "string" ? entry.classname : "";
if (!cn) continue;
const t = lookupTranslation(map, cn);
if (officialIsReal && t?.name) {
entry.name = t.name;
} else if (
lang !== "en" &&
typeof entry.name === "string" &&
entry.name.trim()
) {
pendingTexts.add(entry.name.trim());
pendingRefs.push({ entry, field: "name", section });
}
if (officialIsReal && t?.description) {
entry.description = t.description;
} else if (
lang !== "en" &&
typeof entry.description === "string" &&
entry.description.trim()
) {
pendingTexts.add(entry.description.trim());
pendingRefs.push({ entry, field: "description", section });
}
}
}
// Yield between the two 100k+ scans so a single language never
// blocks the event loop for the whole pass at once.
if (si === 0) await yieldToEventLoop();
}
// Pass 2 — batch-translate everything pending in a handful of
@@ -548,7 +509,7 @@ export async function buildLocalizedFurniDataFiles(
(cloned.roomitemtypes?.furnitype?.length ?? 0) +
(cloned.wallitemtypes?.furnitype?.length ?? 0);
const payload = JSON.stringify(cloned);
const payload = (await yieldToEventLoop(), JSON.stringify(cloned));
const targets = await getLocalizedWritePaths(lang);
for (const target of targets) {
await fs.mkdir(path.dirname(target), { recursive: true });
@@ -612,6 +573,12 @@ export async function patchLocalizedFurniDataEntries(
: FURNIDATA_LANGUAGES;
return withFurniDataLock(async () => {
// Read the master once and reuse it for the fallback clone and for
// looking up brand-new classnames — avoids parsing FurnitureData twice.
const master = (await readFurniData()) as Record<
string,
{ furnitype: Array<Record<string, unknown>> }
>;
const maps = await Promise.all(
langsToBuild.map(async ({ lang, hotel }) => {
const map = await fetchTranslationsForHotel(hotel);
@@ -621,6 +588,7 @@ export async function patchLocalizedFurniDataEntries(
const wanted = new Map<string, { name?: string; description?: string }>();
for (const e of entries) wanted.set(e.classname, e);
const wantedSet = new Set(wanted.keys());
const results: LanguageFileResult[] = [];
@@ -628,41 +596,43 @@ export async function patchLocalizedFurniDataEntries(
try {
const officialIsReal = officialIsRealTranslation(lang, hotel);
const targets = await getLocalizedWritePaths(lang);
// Load the existing language file if present, otherwise clone master.
let patched = 0;
// Load the existing language file if present, otherwise build it
// from master (official translations only — never LibreTranslate,
// to avoid 100k+ network calls on first creation).
let data: Record<string, { furnitype: Array<Record<string, unknown>> }>;
try {
const raw = await fs.readFile(targets[0], "utf-8");
data = JSON.parse(raw);
} catch {
const master = (await readFurniData()) as Record<
string,
{ furnitype: Array<Record<string, unknown>> }
>;
data = cloneFurniData(master as unknown as Record<string, unknown>);
// Apply all translations for the fallback path (same as full build
// but only for the file that didn't exist — official only to avoid
// 100k+ LibreTranslate calls on first creation).
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
for (const entry of data[section]?.furnitype ?? []) {
const cn =
typeof entry.classname === "string" ? entry.classname : "";
if (!cn) continue;
// Only real official translations count here — English names
// from a com-mapped hotel must not be written over the file
// as if they were localized.
if (!officialIsReal) continue;
const t = lookupTranslation(map, cn);
if (t?.name) entry.name = t.name;
if (t?.description) entry.description = t.description;
data = cloneFurniData(master);
if (officialIsReal) {
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
for (const entry of data[section]?.furnitype ?? []) {
const cn =
typeof entry.classname === "string" ? entry.classname : "";
if (!cn) continue;
const t = lookupTranslation(map, cn);
if (t?.name) entry.name = t.name;
if (t?.description) entry.description = t.description;
}
}
}
}
let patched = 0;
// Pass 1 — touch ONLY the wanted classnames (most patches are a
// single import), collecting the texts that need LibreTranslate.
const refs: Array<{
entry: Record<string, unknown>;
field: "name" | "description";
}> = [];
const pendingTexts = new Set<string>();
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
for (const entry of data[section]?.furnitype ?? []) {
const cn =
typeof entry.classname === "string" ? entry.classname : "";
if (!cn || !wantedSet.has(cn)) continue;
const wantedEntry = wanted.get(cn);
if (!wantedEntry) continue;
const t = lookupTranslation(map, cn);
@@ -677,28 +647,38 @@ export async function patchLocalizedFurniDataEntries(
entry.name = t.name;
didPatch = true;
} else if (lang !== "en" && origName) {
const translated = await translateWithLibre(origName, lang);
if (translated && translated !== origName) {
entry.name = translated;
didPatch = true;
}
pendingTexts.add(origName);
refs.push({ entry, field: "name" });
}
if (officialIsReal && t?.description) {
entry.description = t.description;
didPatch = true;
} else if (lang !== "en" && origDesc) {
const translated = await translateWithLibre(origDesc, lang);
if (translated && translated !== origDesc) {
entry.description = translated;
didPatch = true;
}
pendingTexts.add(origDesc);
refs.push({ entry, field: "description" });
}
if (didPatch) patched++;
}
}
// Also handle brand-new entries that weren't in the file yet (rare
// when a single import added a classname not previously in lang file
// because the file was stale). Append with translated names.
// Batch-translate (dedup + concurrency-capped) instead of one
// HTTP call per text.
const translations = await translateManyWithLibre(pendingTexts, lang);
const counted = new Set<Record<string, unknown>>();
for (const ref of refs) {
const original = ref.entry[ref.field];
if (typeof original !== "string") continue;
const translated = translations.get(original.trim());
if (translated === undefined || translated === original) continue;
ref.entry[ref.field] = translated;
if (!counted.has(ref.entry)) {
counted.add(ref.entry);
patched++;
}
}
// New classnames not yet in the file: copy from master with
// translation applied (only for the wanted set).
const existingClassnames = new Set<string>();
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
for (const e of data[section]?.furnitype ?? []) {
@@ -706,21 +686,25 @@ export async function patchLocalizedFurniDataEntries(
if (cn) existingClassnames.add(cn);
}
}
// New classnames are already in the master; if our loaded data
// came from an old lang file before the import, the entry won't
// exist — copy it from master with translation applied.
const master = (await readFurniData()) as Record<
string,
{ furnitype: Array<Record<string, unknown>> }
>;
const newItems: Array<{
section: "roomitemtypes" | "wallitemtypes";
found: Record<string, unknown>;
name?: string;
description?: string;
translateName: boolean;
translateDesc: boolean;
}> = [];
const newTexts = new Set<string>();
for (const { classname, name, description } of entries) {
if (existingClassnames.has(classname)) continue;
// Find in master.
let found: Record<string, unknown> | undefined;
let foundSection: "roomitemtypes" | "wallitemtypes" | undefined;
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
const arr = master[section]?.furnitype;
const hit = arr?.find((e) => e.classname === classname) as
const hit = (
master[section]?.furnitype as
| Array<Record<string, unknown>>
| undefined
)?.find((e) => e.classname === classname) as
| Record<string, unknown>
| undefined;
if (hit) {
@@ -730,37 +714,62 @@ export async function patchLocalizedFurniDataEntries(
}
}
if (!found || !foundSection) continue;
const clone = { ...found };
const t = lookupTranslation(map, classname);
if (officialIsReal && t?.name) clone.name = t.name;
else if (lang !== "en") {
const src =
name ?? (typeof found.name === "string" ? found.name : "") ?? "";
if (src) {
const translated = await translateWithLibre(src, lang);
if (translated !== src) clone.name = translated;
const srcName =
name ?? (typeof found.name === "string" ? found.name : "");
const srcDesc =
description ??
(typeof found.description === "string" ? found.description : "");
let finalName: string | undefined;
let finalDesc: string | undefined;
let translateName = false;
let translateDesc = false;
if (officialIsReal && t?.name) finalName = t.name;
else if (lang !== "en" && srcName) {
finalName = srcName;
translateName = true;
newTexts.add(srcName);
} else finalName = srcName;
if (officialIsReal && t?.description) finalDesc = t.description;
else if (lang !== "en" && srcDesc) {
finalDesc = srcDesc;
translateDesc = true;
newTexts.add(srcDesc);
} else finalDesc = srcDesc;
newItems.push({
section: foundSection,
found,
name: finalName,
description: finalDesc,
translateName,
translateDesc,
});
}
const newTranslations = await translateManyWithLibre(newTexts, lang);
for (const item of newItems) {
const clone = { ...item.found } as Record<string, unknown>;
if (item.name !== undefined) {
if (item.translateName) {
const tr = newTranslations.get(item.name.trim());
clone.name = tr && tr !== item.name ? tr : item.name;
} else {
clone.name = item.name;
}
}
if (officialIsReal && t?.description)
clone.description = t.description;
else if (lang !== "en") {
const src =
description ??
(typeof found.description === "string"
? found.description
: "") ??
"";
if (src) {
const translated = await translateWithLibre(src, lang);
if (translated !== src) clone.description = translated;
if (item.description !== undefined) {
if (item.translateDesc) {
const tr = newTranslations.get(item.description.trim());
clone.description =
tr && tr !== item.description ? tr : item.description;
} else {
clone.description = item.description;
}
}
if (!data[foundSection]) data[foundSection] = { furnitype: [] };
data[foundSection].furnitype.push(clone);
if (!data[item.section]) data[item.section] = { furnitype: [] };
data[item.section].furnitype.push(clone);
patched++;
}
// Persist LibreTranslate cache once per language (batched).
if (lang !== "en") await saveLibreCache();
const payload = JSON.stringify(data);