/** * Localized furnidata builder — generates FurnitureData_.json files * (nl/en/de/fr/es/tr/it/pt/fi/pl/ru/ar/ja) from the master FurnitureData.json * + official Habbo translations per hotel (where available) + LibreTranslate * fallback for customs. * * On each import we rebuild per-language files so new/updated furniture also * gets its translated names. Official translations are fetched from Habbo's * public gamedata (habbo.nl, habbo.com, habbo.de …) and cached in memory. */ import { existsSync, promises as fs } from "node:fs"; import path from "node:path"; import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel"; import { habboFurnidataUrl } from "@/lib/habbo-gamedata-hotel"; import { logger } from "@/lib/logger"; import { getGamedataRoot } from "@/lib/services/furni-asset-dirs"; import { getFurnitureDataWritePaths, readFurniData, withFurniDataLock, } from "@/lib/services/furni-data"; import { browserHeaders } from "@/lib/services/import/core/browser-headers"; import { siteSettings } from "@/lib/services/site-settings"; export const FURNIDATA_LANGUAGES = [ { lang: "nl", hotel: "nl" as HabboGamedataHotel }, { lang: "en", hotel: "com" as HabboGamedataHotel }, { lang: "de", hotel: "de" as HabboGamedataHotel }, { lang: "fr", hotel: "fr" as HabboGamedataHotel }, { lang: "es", hotel: "es" as HabboGamedataHotel }, { lang: "tr", hotel: "com.tr" as HabboGamedataHotel }, { lang: "it", hotel: "it" as HabboGamedataHotel }, { lang: "pt", hotel: "com.br" as HabboGamedataHotel }, { lang: "fi", hotel: "fi" as HabboGamedataHotel }, { lang: "pl", hotel: "com" as HabboGamedataHotel }, { lang: "ru", hotel: "com" as HabboGamedataHotel }, { lang: "ar", hotel: "com" as HabboGamedataHotel }, { lang: "ja", hotel: "com" as HabboGamedataHotel }, ] as const; export type FurnidataLang = (typeof FURNIDATA_LANGUAGES)[number]["lang"]; /** * Master switch for FurnitureData localization. Rebuilding the per-language * files deep-clones the 100k+ entry master 13× and hits LibreTranslate, which * can spike RAM/CPU on every import. Admins can disable it in settings to skip * the work entirely (the master FurnitureData.json is still written by the * import itself). The manual "build languages" action passes `force` to run * regardless. */ async function translationEnabled(): Promise { try { return await siteSettings.getBool("furnidata_translate_enabled", true); } catch { return true; } } export interface LanguageFileResult { lang: string; hotel: string; file: string; translatedRoom: number; translatedWall: number; total: number; ok: boolean; error?: string; } type Translation = { name: string; description: string }; const LANG_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours — translations rarely change interface LangCacheEntry { map: Map; timestamp: number; loading?: Promise>; } const langCache = new Map(); // ── LibreTranslate (gratis, self-hosted) ───────────────────────────── const LIBRE_DEFAULT_URL = "http://127.0.0.1:5000"; const LIBRE_CACHE_FILE = ".translations_libre_cache.json"; type LibreCache = Record; // key: `${target}|${text}` → translated let libreMemoryCache: Map | null = null; let libreCacheLoaded = false; async function getLibreTranslateUrl(): Promise { try { const v = (await siteSettings.get( "libretranslate_url", LIBRE_DEFAULT_URL, )) as string | null; const url = (v ?? LIBRE_DEFAULT_URL).trim() || LIBRE_DEFAULT_URL; return url.replace(/\/+$/, ""); } catch { return LIBRE_DEFAULT_URL; } } async function loadLibreCache(): Promise> { if (libreMemoryCache && libreCacheLoaded) return libreMemoryCache; try { const gamedataRoot = await getGamedataRoot(); const cachePath = gamedataRoot ? path.join(gamedataRoot, "config", LIBRE_CACHE_FILE) : path.join(process.cwd(), LIBRE_CACHE_FILE); if (existsSync(cachePath)) { const raw = await fs.readFile(cachePath, "utf-8"); const obj = JSON.parse(raw) as LibreCache; libreMemoryCache = new Map(Object.entries(obj)); } else { libreMemoryCache = new Map(); } } catch { libreMemoryCache = new Map(); } libreCacheLoaded = true; return libreMemoryCache ?? new Map(); } async function saveLibreCache(): Promise { if (!libreMemoryCache) return; try { const gamedataRoot = await getGamedataRoot(); const cachePath = gamedataRoot ? path.join(gamedataRoot, "config", LIBRE_CACHE_FILE) : path.join(process.cwd(), LIBRE_CACHE_FILE); const obj: LibreCache = Object.fromEntries(libreMemoryCache.entries()); await fs.mkdir(path.dirname(cachePath), { recursive: true }); await fs.writeFile(cachePath, JSON.stringify(obj), "utf-8"); } catch (err) { logger.warn("[furni-i18n] Failed to save Libre cache", { error: (err as Error).message, }); } } async function translateWithLibre( text: string, targetLang: string, ): Promise { const trimmed = text.trim(); if (!trimmed || targetLang === "en") return text; const cache = await loadLibreCache(); const key = `${targetLang}|${trimmed}`; const cached = cache.get(key); if (cached !== undefined) return cached; const url = await getLibreTranslateUrl(); try { const res = await fetch(`${url}/translate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ q: trimmed, source: "en", target: targetLang, format: "text", }), signal: AbortSignal.timeout(10_000), }); if (!res.ok) { logger.warn( `[furni-i18n] LibreTranslate ${targetLang} failed: ${res.status}`, ); return text; } const data = (await res.json()) as { translatedText?: string }; const out = typeof data.translatedText === "string" ? data.translatedText : text; cache.set(key, out); // Best-effort persist, don't await per-call to avoid hammering disk — save // in background throttled (every 50 new entries). if (cache.size % 50 === 0) void saveLibreCache(); return out; } catch (err) { logger.warn(`[furni-i18n] LibreTranslate error (${targetLang})`, { error: (err as Error).message, }); return text; } } const LIBRE_BATCH_SIZE = 48; const LIBRE_CONCURRENCY = 4; /** * Translate many texts in one go using LibreTranslate's array API * (q accepts string[] → translatedText: string[]). Deduplicates input, * serves everything already in the cache, then fetches the rest in * parallel batches. Returns a map of trimmed source text → translation. * Texts whose batch failed are simply absent from the result. */ async function translateManyWithLibre( texts: Iterable, targetLang: string, ): Promise> { const result = new Map(); if (targetLang === "en") return result; const cache = await loadLibreCache(); const pending = new Set(); for (const raw of texts) { const trimmed = raw.trim(); if (!trimmed) continue; const key = `${targetLang}|${trimmed}`; const cached = cache.get(key); if (cached !== undefined) { result.set(trimmed, cached); continue; } pending.add(trimmed); } if (pending.size === 0) return result; const chunks: string[][] = []; let current: string[] = []; for (const text of pending) { current.push(text); if (current.length >= LIBRE_BATCH_SIZE) { chunks.push(current); current = []; } } if (current.length > 0) chunks.push(current); const url = await getLibreTranslateUrl(); let failures = 0; let nextChunk = 0; const worker = async (): Promise => { while (true) { const i = nextChunk++; if (i >= chunks.length) return; const chunk = chunks[i]; try { const res = await fetch(`${url}/translate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ q: chunk, source: "en", target: targetLang, format: "text", }), signal: AbortSignal.timeout(30_000), }); if (!res.ok) { failures++; logger.warn( `[furni-i18n] LibreTranslate batch ${targetLang} failed: ${res.status}`, ); continue; } const data = (await res.json()) as { translatedText?: string[] | string; }; const out = Array.isArray(data.translatedText) ? data.translatedText : typeof data.translatedText === "string" ? [data.translatedText] : []; for (let j = 0; j < chunk.length; j++) { const translated = typeof out[j] === "string" && out[j] ? out[j] : chunk[j]; result.set(chunk[j], translated); cache.set(`${targetLang}|${chunk[j]}`, translated); } } catch (err) { failures++; logger.warn(`[furni-i18n] LibreTranslate batch error (${targetLang})`, { error: (err as Error).message, }); } } }; await Promise.all( Array.from({ length: Math.min(LIBRE_CONCURRENCY, chunks.length) }, () => worker(), ), ); if (failures > 0) { logger.warn( `[furni-i18n] LibreTranslate ${targetLang}: ${failures}/${chunks.length} batches failed`, ); } // Persist whatever succeeded so rebuilds don't re-call the API. await saveLibreCache(); return result; } function lookupTranslation( map: Map, classname: string, ): Translation | undefined { const exact = map.get(classname); if (exact) return exact; if (classname.includes("*")) { const base = classname.split("*")[0]; const v0 = map.get(`${base}*0`); if (v0) return v0; const baseExact = map.get(base); if (baseExact) return baseExact; } if (!classname.includes("*")) { const v0 = map.get(`${classname}*0`); if (v0) return v0; } return undefined; } /** * A language mapped to hotel "com" only carries English names, so it must * NOT be treated as a real translation — fall through to LibreTranslate for * those (pl/ru/ar/ja). Shared by the full build and the single-import patch * so both paths stay in sync. */ function officialIsRealTranslation( lang: string, hotel: HabboGamedataHotel, ): boolean { return lang === "en" || hotel !== "com"; } async function fetchTranslationsForHotel( hotel: HabboGamedataHotel, ): Promise> { const cached = langCache.get(hotel); const now = Date.now(); if (cached && !cached.loading && now - cached.timestamp < LANG_CACHE_TTL) { return cached.map; } if (cached?.loading) return cached.loading; const url = habboFurnidataUrl(hotel); const loading = (async () => { try { const res = await fetch(url, { signal: AbortSignal.timeout(20_000), headers: browserHeaders(), }); if (!res.ok) { logger.warn( `[furni-i18n] Fetch failed for habbo.${hotel}: ${res.status}`, ); return cached?.map ?? new Map(); } const data = await res.json(); const map = new Map(); for (const section of ["roomitemtypes", "wallitemtypes"] as const) { const arr = data?.[section]?.furnitype; if (!Array.isArray(arr)) continue; for (const e of arr) { const cn = typeof e?.classname === "string" ? e.classname : ""; if (!cn) continue; // Keep first occurrence — mirrors the official file order. if (map.has(cn)) continue; map.set(cn, { name: typeof e?.name === "string" ? e.name : "", description: typeof e?.description === "string" ? e.description : "", }); } } langCache.set(hotel, { map, timestamp: Date.now() }); logger.debug( `[furni-i18n] Cached ${map.size} translations from habbo.${hotel}`, ); return map; } catch (err) { logger.warn(`[furni-i18n] Fetch error for habbo.${hotel}`, { error: (err as Error).message, }); return cached?.map ?? new Map(); } })(); langCache.set(hotel, { map: cached?.map ?? new Map(), timestamp: cached?.timestamp ?? 0, loading, }); const result = await loading; // Clear loading flag but keep map+timestamp. const entry = langCache.get(hotel); if (entry) { entry.loading = undefined; entry.map = result; if (!entry.timestamp) entry.timestamp = Date.now(); } return result; } async function getLocalizedWritePaths(lang: string): Promise { const bases = await getFurnitureDataWritePaths(); const out: string[] = []; for (const base of bases) { const dir = path.dirname(base); const langFile = `FurnitureData_${lang}.json`; out.push(path.join(dir, langFile)); } // Deduplicate same normalized path. const seen = new Set(); return out.filter((p) => { const key = process.platform === "win32" ? p.toLowerCase() : p; if (seen.has(key)) return false; seen.add(key); return true; }); } function cloneFurniData( data: Record, ): Record> }> { // Deep clone via JSON — structure is pure JSON and 100k+ entries, this is // the simplest safe copy. Each language build gets its own clone. return JSON.parse(JSON.stringify(data)) as Record< string, { furnitype: Array> } >; } /** * Build all language files from the current master FurnitureData.json. * Each file is a full copy with name/description overridden where an official * translation exists for that classname. Runs under the furnidata lock so it * never interleaves with concurrent imports. */ export async function buildLocalizedFurniDataFiles( force = false, ): Promise { if (!force && !(await translationEnabled())) { logger.debug( "[furni-i18n] FurnitureData translation disabled by setting; skipping localized build.", ); return []; } return withFurniDataLock(async () => { const master = (await readFurniData()) as Record< string, { furnitype: Array> } >; // Fetch all language maps in parallel (uses per-hotel memory cache). const maps = await Promise.all( FURNIDATA_LANGUAGES.map(async ({ lang, hotel }) => { const map = await fetchTranslationsForHotel(hotel); return { lang, hotel, map }; }), ); const results: LanguageFileResult[] = []; for (const { lang, hotel, map } of maps) { try { const cloned = cloneFurniData( master as unknown as Record, ); const officialIsReal = officialIsRealTranslation(lang, hotel); // Pass 1 — apply official translations directly and collect the // texts that need LibreTranslate (customs, or languages whose // hotel only provides English) so they can be batch-translated. const pendingRefs: Array<{ entry: Record; field: "name" | "description"; section: "roomitemtypes" | "wallitemtypes"; }> = []; const pendingTexts = new Set(); for (const section of ["roomitemtypes", "wallitemtypes"] as const) { const arr = cloned[section]?.furnitype; if (!Array.isArray(arr)) continue; for (const entry of arr) { const cn = typeof entry.classname === "string" ? entry.classname : ""; if (!cn) continue; const t = lookupTranslation(map, cn); if (officialIsReal && t?.name) { entry.name = t.name; } else if ( lang !== "en" && typeof entry.name === "string" && entry.name.trim() ) { pendingTexts.add(entry.name.trim()); pendingRefs.push({ entry, field: "name", section }); } if (officialIsReal && t?.description) { entry.description = t.description; } else if ( lang !== "en" && typeof entry.description === "string" && entry.description.trim() ) { pendingTexts.add(entry.description.trim()); pendingRefs.push({ entry, field: "description", section }); } } } // Pass 2 — batch-translate everything pending in a handful of // parallel requests instead of one HTTP call per text. const translations = await translateManyWithLibre(pendingTexts, lang); let translatedRoom = 0; let translatedWall = 0; if (pendingRefs.length > 0) { const counted = new Set>(); for (const ref of pendingRefs) { const original = ref.entry[ref.field]; if (typeof original !== "string") continue; const translated = translations.get(original.trim()); if (translated === undefined || translated === original) continue; ref.entry[ref.field] = translated; if (!counted.has(ref.entry)) { counted.add(ref.entry); if (ref.section === "roomitemtypes") translatedRoom++; else translatedWall++; } } } const total = (cloned.roomitemtypes?.furnitype?.length ?? 0) + (cloned.wallitemtypes?.furnitype?.length ?? 0); const payload = JSON.stringify(cloned); const targets = await getLocalizedWritePaths(lang); for (const target of targets) { await fs.mkdir(path.dirname(target), { recursive: true }); const tmp = `${target}.tmp`; await fs.writeFile(tmp, payload, "utf-8"); await fs.rename(tmp, target); } // Persist the LibreTranslate cache once per language so the next // full build reuses translations instead of re-calling the API. if (lang !== "en") await saveLibreCache(); results.push({ lang, hotel, file: targets[0] ?? `FurnitureData_${lang}.json`, translatedRoom, translatedWall, total, ok: true, }); } catch (err) { results.push({ lang, hotel, file: `FurnitureData_${lang}.json`, translatedRoom: 0, translatedWall: 0, total: 0, ok: false, error: (err as Error).message, }); } } return results; }); } /** * Patch the language files for a small set of classnames (e.g. a single * import) without rebuilding every file from scratch. Used on single-import * paths where a full rebuild would be wasteful. */ export async function patchLocalizedFurniDataEntries( entries: Array<{ classname: string; name?: string; description?: string }>, force = false, ): Promise { if (entries.length === 0) return []; if (!force && !(await translationEnabled())) { logger.debug( "[furni-i18n] FurnitureData translation disabled by setting; skipping localized patch.", ); return []; } return withFurniDataLock(async () => { const maps = await Promise.all( FURNIDATA_LANGUAGES.map(async ({ lang, hotel }) => { const map = await fetchTranslationsForHotel(hotel); return { lang, hotel, map }; }), ); const wanted = new Map(); for (const e of entries) wanted.set(e.classname, e); const results: LanguageFileResult[] = []; for (const { lang, hotel, map } of maps) { try { const officialIsReal = officialIsRealTranslation(lang, hotel); const targets = await getLocalizedWritePaths(lang); // Load the existing language file if present, otherwise clone master. let data: Record> }>; try { const raw = await fs.readFile(targets[0], "utf-8"); data = JSON.parse(raw); } catch { const master = (await readFurniData()) as Record< string, { furnitype: Array> } >; data = cloneFurniData(master as unknown as Record); // Apply all translations for the fallback path (same as full build // but only for the file that didn't exist — official only to avoid // 100k+ LibreTranslate calls on first creation). for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const entry of data[section]?.furnitype ?? []) { const cn = typeof entry.classname === "string" ? entry.classname : ""; if (!cn) continue; // Only real official translations count here — English names // from a com-mapped hotel must not be written over the file // as if they were localized. if (!officialIsReal) continue; const t = lookupTranslation(map, cn); if (t?.name) entry.name = t.name; if (t?.description) entry.description = t.description; } } } let patched = 0; for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const entry of data[section]?.furnitype ?? []) { const cn = typeof entry.classname === "string" ? entry.classname : ""; const wantedEntry = wanted.get(cn); if (!wantedEntry) continue; const t = lookupTranslation(map, cn); let didPatch = false; const origName = wantedEntry.name ?? (typeof entry.name === "string" ? entry.name : ""); const origDesc = wantedEntry.description ?? (typeof entry.description === "string" ? entry.description : ""); if (officialIsReal && t?.name) { entry.name = t.name; didPatch = true; } else if (lang !== "en" && origName) { const translated = await translateWithLibre(origName, lang); if (translated && translated !== origName) { entry.name = translated; didPatch = true; } } if (officialIsReal && t?.description) { entry.description = t.description; didPatch = true; } else if (lang !== "en" && origDesc) { const translated = await translateWithLibre(origDesc, lang); if (translated && translated !== origDesc) { entry.description = translated; didPatch = true; } } if (didPatch) patched++; } } // Also handle brand-new entries that weren't in the file yet (rare // when a single import added a classname not previously in lang file // because the file was stale). Append with translated names. const existingClassnames = new Set(); for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const e of data[section]?.furnitype ?? []) { const cn = typeof e?.classname === "string" ? e.classname : ""; if (cn) existingClassnames.add(cn); } } // New classnames are already in the master; if our loaded data // came from an old lang file before the import, the entry won't // exist — copy it from master with translation applied. const master = (await readFurniData()) as Record< string, { furnitype: Array> } >; for (const { classname, name, description } of entries) { if (existingClassnames.has(classname)) continue; // Find in master. let found: Record | undefined; let foundSection: "roomitemtypes" | "wallitemtypes" | undefined; for (const section of ["roomitemtypes", "wallitemtypes"] as const) { const arr = master[section]?.furnitype; const hit = arr?.find((e) => e.classname === classname) as | Record | undefined; if (hit) { found = hit; foundSection = section; break; } } if (!found || !foundSection) continue; const clone = { ...found }; const t = lookupTranslation(map, classname); if (officialIsReal && t?.name) clone.name = t.name; else if (lang !== "en") { const src = name ?? (typeof found.name === "string" ? found.name : "") ?? ""; if (src) { const translated = await translateWithLibre(src, lang); if (translated !== src) clone.name = translated; } } if (officialIsReal && t?.description) clone.description = t.description; else if (lang !== "en") { const src = description ?? (typeof found.description === "string" ? found.description : "") ?? ""; if (src) { const translated = await translateWithLibre(src, lang); if (translated !== src) clone.description = translated; } } if (!data[foundSection]) data[foundSection] = { furnitype: [] }; data[foundSection].furnitype.push(clone); patched++; } // Persist LibreTranslate cache once per language (batched). if (lang !== "en") await saveLibreCache(); const payload = JSON.stringify(data); for (const target of targets) { await fs.mkdir(path.dirname(target), { recursive: true }); const tmp = `${target}.tmp`; await fs.writeFile(tmp, payload, "utf-8"); await fs.rename(tmp, target); } results.push({ lang, hotel, file: targets[0] ?? `FurnitureData_${lang}.json`, translatedRoom: patched, translatedWall: 0, total: patched, ok: true, }); } catch (err) { results.push({ lang, hotel, file: `FurnitureData_${lang}.json`, translatedRoom: 0, translatedWall: 0, total: 0, ok: false, error: (err as Error).message, }); } } return results; }); } /** Visible for tests — apply translations to a furnidata clone in place. */ export function applyTranslationsForTest( data: Record> }>, translations: Map, ): { translatedRoom: number; translatedWall: number } { let translatedRoom = 0; let translatedWall = 0; for (const section of ["roomitemtypes", "wallitemtypes"] as const) { for (const entry of data[section]?.furnitype ?? []) { const cn = typeof entry.classname === "string" ? entry.classname : ""; const t = lookupTranslation(translations, cn); if (!t) continue; let did = false; if (t.name) { entry.name = t.name; did = true; } if (t.description) { entry.description = t.description; did = true; } if (did) { if (section === "roomitemtypes") translatedRoom++; else translatedWall++; } } } return { translatedRoom, translatedWall }; }