feat(i18n): LibreTranslate fallback for customs (free, self-hosted)
Use official Habbo translations where available; fall back to self-hosted LibreTranslate (http://127.0.0.1:5000, host-network, 7 languages) for custom furni with no official translation. Patch path (single imports) translates name/description via LibreTranslate (en -> nl/de/fr/es/tr/it) with persistent cache (.translations_libre_cache.json) and caches 6h for official maps. Full rebuilds stay official-only to avoid 500k+ API calls; customs are translated incrementally per import. LibreTranslate URL configurable via site setting libretranslate_url. Docker: host-network fix for DNS.
This commit is contained in:
openhands committed
1 parent
8bb339cc49
commit
01a3ba6e39
1 file changed
+160
-11
@@ -8,17 +8,19 @@
|
||||
* public gamedata (habbo.nl, habbo.com, habbo.de …) and cached in memory.
|
||||
*/
|
||||
|
||||
import { promises as fs } from "node:fs";
|
||||
import { existsSync, promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel";
|
||||
import { habboFurnidataUrl } from "@/lib/habbo-gamedata-hotel";
|
||||
import { logger } from "@/lib/logger";
|
||||
import { getGamedataRoot } from "@/lib/services/furni-asset-dirs";
|
||||
import {
|
||||
getFurnitureDataWritePaths,
|
||||
readFurniData,
|
||||
withFurniDataLock,
|
||||
} from "@/lib/services/furni-data";
|
||||
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
||||
import { siteSettings } from "@/lib/services/site-settings";
|
||||
|
||||
export const FURNIDATA_LANGUAGES = [
|
||||
{ lang: "nl", hotel: "nl" as HabboGamedataHotel },
|
||||
@@ -55,6 +57,111 @@ interface LangCacheEntry {
|
||||
|
||||
const langCache = new Map<string, LangCacheEntry>();
|
||||
|
||||
// ── LibreTranslate (gratis, self-hosted) ─────────────────────────────
|
||||
const LIBRE_DEFAULT_URL = "http://127.0.0.1:5000";
|
||||
const LIBRE_CACHE_FILE = ".translations_libre_cache.json";
|
||||
type LibreCache = Record<string, string>; // key: `${target}|${text}` → translated
|
||||
|
||||
let libreMemoryCache: Map<string, string> | null = null;
|
||||
let libreCacheLoaded = false;
|
||||
|
||||
async function getLibreTranslateUrl(): Promise<string> {
|
||||
try {
|
||||
const v = (await siteSettings.get(
|
||||
"libretranslate_url",
|
||||
LIBRE_DEFAULT_URL,
|
||||
)) as string | null;
|
||||
const url = (v ?? LIBRE_DEFAULT_URL).trim() || LIBRE_DEFAULT_URL;
|
||||
return url.replace(/\/+$/, "");
|
||||
} catch {
|
||||
return LIBRE_DEFAULT_URL;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLibreCache(): Promise<Map<string, string>> {
|
||||
if (libreMemoryCache && libreCacheLoaded) return libreMemoryCache;
|
||||
try {
|
||||
const gamedataRoot = await getGamedataRoot();
|
||||
const cachePath = gamedataRoot
|
||||
? path.join(gamedataRoot, "config", LIBRE_CACHE_FILE)
|
||||
: path.join(process.cwd(), LIBRE_CACHE_FILE);
|
||||
if (existsSync(cachePath)) {
|
||||
const raw = await fs.readFile(cachePath, "utf-8");
|
||||
const obj = JSON.parse(raw) as LibreCache;
|
||||
libreMemoryCache = new Map(Object.entries(obj));
|
||||
} else {
|
||||
libreMemoryCache = new Map();
|
||||
}
|
||||
} catch {
|
||||
libreMemoryCache = new Map();
|
||||
}
|
||||
libreCacheLoaded = true;
|
||||
return libreMemoryCache ?? new Map<string, string>();
|
||||
}
|
||||
|
||||
async function saveLibreCache(): Promise<void> {
|
||||
if (!libreMemoryCache) return;
|
||||
try {
|
||||
const gamedataRoot = await getGamedataRoot();
|
||||
const cachePath = gamedataRoot
|
||||
? path.join(gamedataRoot, "config", LIBRE_CACHE_FILE)
|
||||
: path.join(process.cwd(), LIBRE_CACHE_FILE);
|
||||
const obj: LibreCache = Object.fromEntries(libreMemoryCache.entries());
|
||||
await fs.mkdir(path.dirname(cachePath), { recursive: true });
|
||||
await fs.writeFile(cachePath, JSON.stringify(obj), "utf-8");
|
||||
} catch (err) {
|
||||
logger.warn("[furni-i18n] Failed to save Libre cache", {
|
||||
error: (err as Error).message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function translateWithLibre(
|
||||
text: string,
|
||||
targetLang: string,
|
||||
): Promise<string> {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed || targetLang === "en") return text;
|
||||
const cache = await loadLibreCache();
|
||||
const key = `${targetLang}|${trimmed}`;
|
||||
const cached = cache.get(key);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
const url = await getLibreTranslateUrl();
|
||||
try {
|
||||
const res = await fetch(`${url}/translate`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
q: trimmed,
|
||||
source: "en",
|
||||
target: targetLang,
|
||||
format: "text",
|
||||
}),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!res.ok) {
|
||||
logger.warn(
|
||||
`[furni-i18n] LibreTranslate ${targetLang} failed: ${res.status}`,
|
||||
);
|
||||
return text;
|
||||
}
|
||||
const data = (await res.json()) as { translatedText?: string };
|
||||
const out =
|
||||
typeof data.translatedText === "string" ? data.translatedText : text;
|
||||
cache.set(key, out);
|
||||
// Best-effort persist, don't await per-call to avoid hammering disk — save
|
||||
// in background throttled (every 50 new entries).
|
||||
if (cache.size % 50 === 0) void saveLibreCache();
|
||||
return out;
|
||||
} catch (err) {
|
||||
logger.warn(`[furni-i18n] LibreTranslate error (${targetLang})`, {
|
||||
error: (err as Error).message,
|
||||
});
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
function lookupTranslation(
|
||||
map: Map<string, Translation>,
|
||||
classname: string,
|
||||
@@ -312,16 +419,16 @@ export async function patchLocalizedFurniDataEntries(
|
||||
>;
|
||||
data = cloneFurniData(master as unknown as Record<string, unknown>);
|
||||
// Apply all translations for the fallback path (same as full build
|
||||
// but only for the file that didn't exist).
|
||||
// but only for the file that didn't exist — official only to avoid
|
||||
// 100k+ LibreTranslate calls on first creation).
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!cn) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
if (!t) continue;
|
||||
if (t.name) entry.name = t.name;
|
||||
if (t.description) entry.description = t.description;
|
||||
if (t?.name) entry.name = t.name;
|
||||
if (t?.description) entry.description = t.description;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -331,12 +438,37 @@ export async function patchLocalizedFurniDataEntries(
|
||||
for (const entry of data[section]?.furnitype ?? []) {
|
||||
const cn =
|
||||
typeof entry.classname === "string" ? entry.classname : "";
|
||||
if (!wanted.has(cn)) continue;
|
||||
const wantedEntry = wanted.get(cn);
|
||||
if (!wantedEntry) continue;
|
||||
const t = lookupTranslation(map, cn);
|
||||
if (!t) continue;
|
||||
if (t.name) entry.name = t.name;
|
||||
if (t.description) entry.description = t.description;
|
||||
patched++;
|
||||
let didPatch = false;
|
||||
const origName =
|
||||
wantedEntry.name ??
|
||||
(typeof entry.name === "string" ? entry.name : "");
|
||||
const origDesc =
|
||||
wantedEntry.description ??
|
||||
(typeof entry.description === "string" ? entry.description : "");
|
||||
if (t?.name) {
|
||||
entry.name = t.name;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origName) {
|
||||
const translated = await translateWithLibre(origName, lang);
|
||||
if (translated && translated !== origName) {
|
||||
entry.name = translated;
|
||||
didPatch = true;
|
||||
}
|
||||
}
|
||||
if (t?.description) {
|
||||
entry.description = t.description;
|
||||
didPatch = true;
|
||||
} else if (lang !== "en" && origDesc) {
|
||||
const translated = await translateWithLibre(origDesc, lang);
|
||||
if (translated && translated !== origDesc) {
|
||||
entry.description = translated;
|
||||
didPatch = true;
|
||||
}
|
||||
}
|
||||
if (didPatch) patched++;
|
||||
}
|
||||
}
|
||||
// Also handle brand-new entries that weren't in the file yet (rare
|
||||
@@ -356,7 +488,7 @@ export async function patchLocalizedFurniDataEntries(
|
||||
string,
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
for (const { classname } of entries) {
|
||||
for (const { classname, name, description } of entries) {
|
||||
if (existingClassnames.has(classname)) continue;
|
||||
// Find in master.
|
||||
let found: Record<string, unknown> | undefined;
|
||||
@@ -376,12 +508,29 @@ export async function patchLocalizedFurniDataEntries(
|
||||
const clone = { ...found };
|
||||
const t = lookupTranslation(map, classname);
|
||||
if (t?.name) clone.name = t.name;
|
||||
else if (lang !== "en") {
|
||||
const src =
|
||||
name ?? (typeof found.name === "string" ? found.name : "") ?? "";
|
||||
if (src) clone.name = await translateWithLibre(src, lang);
|
||||
}
|
||||
if (t?.description) clone.description = t.description;
|
||||
else if (lang !== "en") {
|
||||
const src =
|
||||
description ??
|
||||
(typeof found.description === "string"
|
||||
? found.description
|
||||
: "") ??
|
||||
"";
|
||||
if (src) clone.description = await translateWithLibre(src, lang);
|
||||
}
|
||||
if (!data[foundSection]) data[foundSection] = { furnitype: [] };
|
||||
data[foundSection].furnitype.push(clone);
|
||||
patched++;
|
||||
}
|
||||
|
||||
// Persist LibreTranslate cache once per language (batched).
|
||||
if (lang !== "en") await saveLibreCache();
|
||||
|
||||
const payload = JSON.stringify(data);
|
||||
for (const target of targets) {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
|
||||
Reference in new issue
Block a user