diff --git a/src/lib/services/furni-data-i18n.ts b/src/lib/services/furni-data-i18n.ts index a36830556d..e9c871875d 100644 --- a/src/lib/services/furni-data-i18n.ts +++ b/src/lib/services/furni-data-i18n.ts @@ -139,52 +139,6 @@ async function saveLibreCache(): Promise { } } -async function translateWithLibre( - text: string, - targetLang: string, -): Promise { - const trimmed = text.trim(); - if (!trimmed || targetLang === "en") return text; - const cache = await loadLibreCache(); - const key = `${targetLang}|${trimmed}`; - const cached = cache.get(key); - if (cached !== undefined) return cached; - - const url = await getLibreTranslateUrl(); - try { - const res = await fetch(`${url}/translate`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - q: trimmed, - source: "en", - target: targetLang, - format: "text", - }), - signal: AbortSignal.timeout(10_000), - }); - if (!res.ok) { - logger.warn( - `[furni-i18n] LibreTranslate ${targetLang} failed: ${res.status}`, - ); - return text; - } - const data = (await res.json()) as { translatedText?: string }; - const out = - typeof data.translatedText === "string" ? data.translatedText : text; - cache.set(key, out); - // Best-effort persist, don't await per-call to avoid hammering disk — save - // in background throttled (every 50 new entries). - if (cache.size % 50 === 0) void saveLibreCache(); - return out; - } catch (err) { - logger.warn(`[furni-i18n] LibreTranslate error (${targetLang})`, { - error: (err as Error).message, - }); - return text; - } -} - const LIBRE_BATCH_SIZE = 48; // Kept low on purpose: the build already pegs a core with the synchronous // deep-clone + stringify of the 100k+ entry master per language, so we avoid @@ -195,6 +149,7 @@ const LIBRE_CONCURRENCY = 2; const BUILD_YIELD_MS = 150; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const yieldToEventLoop = () => new Promise((r) => setImmediate(r)); /** * Translate many texts in one go using LibreTranslate's array API @@ -491,35 +446,41 @@ export async function buildLocalizedFurniDataFiles( section: "roomitemtypes" | "wallitemtypes"; }> = []; const pendingTexts = new Set(); - for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + const sections = ["roomitemtypes", "wallitemtypes"] as const; + for (let si = 0; si < sections.length; si++) { + const section = sections[si]; const arr = cloned[section]?.furnitype; - if (!Array.isArray(arr)) continue; - for (const entry of arr) { - const cn = - typeof entry.classname === "string" ? entry.classname : ""; - if (!cn) continue; - const t = lookupTranslation(map, cn); - if (officialIsReal && t?.name) { - entry.name = t.name; - } else if ( - lang !== "en" && - typeof entry.name === "string" && - entry.name.trim() - ) { - pendingTexts.add(entry.name.trim()); - pendingRefs.push({ entry, field: "name", section }); - } - if (officialIsReal && t?.description) { - entry.description = t.description; - } else if ( - lang !== "en" && - typeof entry.description === "string" && - entry.description.trim() - ) { - pendingTexts.add(entry.description.trim()); - pendingRefs.push({ entry, field: "description", section }); + if (Array.isArray(arr)) { + for (const entry of arr) { + const cn = + typeof entry.classname === "string" ? entry.classname : ""; + if (!cn) continue; + const t = lookupTranslation(map, cn); + if (officialIsReal && t?.name) { + entry.name = t.name; + } else if ( + lang !== "en" && + typeof entry.name === "string" && + entry.name.trim() + ) { + pendingTexts.add(entry.name.trim()); + pendingRefs.push({ entry, field: "name", section }); + } + if (officialIsReal && t?.description) { + entry.description = t.description; + } else if ( + lang !== "en" && + typeof entry.description === "string" && + entry.description.trim() + ) { + pendingTexts.add(entry.description.trim()); + pendingRefs.push({ entry, field: "description", section }); + } } } + // Yield between the two 100k+ scans so a single language never + // blocks the event loop for the whole pass at once. + if (si === 0) await yieldToEventLoop(); } // Pass 2 — batch-translate everything pending in a handful of @@ -548,7 +509,7 @@ export async function buildLocalizedFurniDataFiles( (cloned.roomitemtypes?.furnitype?.length ?? 0) + (cloned.wallitemtypes?.furnitype?.length ?? 0); - const payload = JSON.stringify(cloned); + const payload = (await yieldToEventLoop(), JSON.stringify(cloned)); const targets = await getLocalizedWritePaths(lang); for (const target of targets) { await fs.mkdir(path.dirname(target), { recursive: true }); @@ -612,6 +573,12 @@ export async function patchLocalizedFurniDataEntries( : FURNIDATA_LANGUAGES; return withFurniDataLock(async () => { + // Read the master once and reuse it for the fallback clone and for + // looking up brand-new classnames — avoids parsing FurnitureData twice. + const master = (await readFurniData()) as Record< + string, + { furnitype: Array> } + >; const maps = await Promise.all( langsToBuild.map(async ({ lang, hotel }) => { const map = await fetchTranslationsForHotel(hotel); @@ -621,6 +588,7 @@ export async function patchLocalizedFurniDataEntries( const wanted = new Map(); for (const e of entries) wanted.set(e.classname, e); + const wantedSet = new Set(wanted.keys()); const results: LanguageFileResult[] = []; @@ -628,41 +596,43 @@ export async function patchLocalizedFurniDataEntries( try { const officialIsReal = officialIsRealTranslation(lang, hotel); const targets = await getLocalizedWritePaths(lang); - // Load the existing language file if present, otherwise clone master. + let patched = 0; + + // Load the existing language file if present, otherwise build it + // from master (official translations only — never LibreTranslate, + // to avoid 100k+ network calls on first creation). let data: Record> }>; try { const raw = await fs.readFile(targets[0], "utf-8"); data = JSON.parse(raw); } catch { - const master = (await readFurniData()) as Record< - string, - { furnitype: Array> } - >; - data = cloneFurniData(master as unknown as Record); - // Apply all translations for the fallback path (same as full build - // but only for the file that didn't exist — official only to avoid - // 100k+ LibreTranslate calls on first creation). - for (const section of ["roomitemtypes", "wallitemtypes"] as const) { - for (const entry of data[section]?.furnitype ?? []) { - const cn = - typeof entry.classname === "string" ? entry.classname : ""; - if (!cn) continue; - // Only real official translations count here — English names - // from a com-mapped hotel must not be written over the file - // as if they were localized. - if (!officialIsReal) continue; - const t = lookupTranslation(map, cn); - if (t?.name) entry.name = t.name; - if (t?.description) entry.description = t.description; + data = cloneFurniData(master); + if (officialIsReal) { + for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + for (const entry of data[section]?.furnitype ?? []) { + const cn = + typeof entry.classname === "string" ? entry.classname : ""; + if (!cn) continue; + const t = lookupTranslation(map, cn); + if (t?.name) entry.name = t.name; + if (t?.description) entry.description = t.description; + } } } } - let patched = 0; + // Pass 1 — touch ONLY the wanted classnames (most patches are a + // single import), collecting the texts that need LibreTranslate. + const refs: Array<{ + entry: Record; + field: "name" | "description"; + }> = []; + const pendingTexts = new Set(); for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const entry of data[section]?.furnitype ?? []) { const cn = typeof entry.classname === "string" ? entry.classname : ""; + if (!cn || !wantedSet.has(cn)) continue; const wantedEntry = wanted.get(cn); if (!wantedEntry) continue; const t = lookupTranslation(map, cn); @@ -677,28 +647,38 @@ export async function patchLocalizedFurniDataEntries( entry.name = t.name; didPatch = true; } else if (lang !== "en" && origName) { - const translated = await translateWithLibre(origName, lang); - if (translated && translated !== origName) { - entry.name = translated; - didPatch = true; - } + pendingTexts.add(origName); + refs.push({ entry, field: "name" }); } if (officialIsReal && t?.description) { entry.description = t.description; didPatch = true; } else if (lang !== "en" && origDesc) { - const translated = await translateWithLibre(origDesc, lang); - if (translated && translated !== origDesc) { - entry.description = translated; - didPatch = true; - } + pendingTexts.add(origDesc); + refs.push({ entry, field: "description" }); } if (didPatch) patched++; } } - // Also handle brand-new entries that weren't in the file yet (rare - // when a single import added a classname not previously in lang file - // because the file was stale). Append with translated names. + + // Batch-translate (dedup + concurrency-capped) instead of one + // HTTP call per text. + const translations = await translateManyWithLibre(pendingTexts, lang); + const counted = new Set>(); + for (const ref of refs) { + const original = ref.entry[ref.field]; + if (typeof original !== "string") continue; + const translated = translations.get(original.trim()); + if (translated === undefined || translated === original) continue; + ref.entry[ref.field] = translated; + if (!counted.has(ref.entry)) { + counted.add(ref.entry); + patched++; + } + } + + // New classnames not yet in the file: copy from master with + // translation applied (only for the wanted set). const existingClassnames = new Set(); for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const e of data[section]?.furnitype ?? []) { @@ -706,21 +686,25 @@ export async function patchLocalizedFurniDataEntries( if (cn) existingClassnames.add(cn); } } - // New classnames are already in the master; if our loaded data - // came from an old lang file before the import, the entry won't - // exist — copy it from master with translation applied. - const master = (await readFurniData()) as Record< - string, - { furnitype: Array> } - >; + const newItems: Array<{ + section: "roomitemtypes" | "wallitemtypes"; + found: Record; + name?: string; + description?: string; + translateName: boolean; + translateDesc: boolean; + }> = []; + const newTexts = new Set(); for (const { classname, name, description } of entries) { if (existingClassnames.has(classname)) continue; - // Find in master. let found: Record | undefined; let foundSection: "roomitemtypes" | "wallitemtypes" | undefined; for (const section of ["roomitemtypes", "wallitemtypes"] as const) { - const arr = master[section]?.furnitype; - const hit = arr?.find((e) => e.classname === classname) as + const hit = ( + master[section]?.furnitype as + | Array> + | undefined + )?.find((e) => e.classname === classname) as | Record | undefined; if (hit) { @@ -730,37 +714,62 @@ export async function patchLocalizedFurniDataEntries( } } if (!found || !foundSection) continue; - const clone = { ...found }; const t = lookupTranslation(map, classname); - if (officialIsReal && t?.name) clone.name = t.name; - else if (lang !== "en") { - const src = - name ?? (typeof found.name === "string" ? found.name : "") ?? ""; - if (src) { - const translated = await translateWithLibre(src, lang); - if (translated !== src) clone.name = translated; + const srcName = + name ?? (typeof found.name === "string" ? found.name : ""); + const srcDesc = + description ?? + (typeof found.description === "string" ? found.description : ""); + let finalName: string | undefined; + let finalDesc: string | undefined; + let translateName = false; + let translateDesc = false; + if (officialIsReal && t?.name) finalName = t.name; + else if (lang !== "en" && srcName) { + finalName = srcName; + translateName = true; + newTexts.add(srcName); + } else finalName = srcName; + if (officialIsReal && t?.description) finalDesc = t.description; + else if (lang !== "en" && srcDesc) { + finalDesc = srcDesc; + translateDesc = true; + newTexts.add(srcDesc); + } else finalDesc = srcDesc; + newItems.push({ + section: foundSection, + found, + name: finalName, + description: finalDesc, + translateName, + translateDesc, + }); + } + const newTranslations = await translateManyWithLibre(newTexts, lang); + for (const item of newItems) { + const clone = { ...item.found } as Record; + if (item.name !== undefined) { + if (item.translateName) { + const tr = newTranslations.get(item.name.trim()); + clone.name = tr && tr !== item.name ? tr : item.name; + } else { + clone.name = item.name; } } - if (officialIsReal && t?.description) - clone.description = t.description; - else if (lang !== "en") { - const src = - description ?? - (typeof found.description === "string" - ? found.description - : "") ?? - ""; - if (src) { - const translated = await translateWithLibre(src, lang); - if (translated !== src) clone.description = translated; + if (item.description !== undefined) { + if (item.translateDesc) { + const tr = newTranslations.get(item.description.trim()); + clone.description = + tr && tr !== item.description ? tr : item.description; + } else { + clone.description = item.description; } } - if (!data[foundSection]) data[foundSection] = { furnitype: [] }; - data[foundSection].furnitype.push(clone); + if (!data[item.section]) data[item.section] = { furnitype: [] }; + data[item.section].furnitype.push(clone); patched++; } - // Persist LibreTranslate cache once per language (batched). if (lang !== "en") await saveLibreCache(); const payload = JSON.stringify(data);