feat(i18n): LibreTranslate fallback for customs (free, self-hosted)
Use official Habbo translations where available; fall back to self-hosted LibreTranslate (http://127.0.0.1:5000, host-network, 7 languages) for custom furni with no official translation. Patch path (single imports) translates name/description via LibreTranslate (en -> nl/de/fr/es/tr/it) with persistent cache (.translations_libre_cache.json) and caches 6h for official maps. Full rebuilds stay official-only to avoid 500k+ API calls; customs are translated incrementally per import. LibreTranslate URL configurable via site setting libretranslate_url. Docker: host-network fix for DNS.
This commit is contained in:
1 parent
8bb339cc49
commit
01a3ba6e39
1 file changed
+160
-11
@@ -8,17 +8,19 @@
|
|||||||
* public gamedata (habbo.nl, habbo.com, habbo.de …) and cached in memory.
|
* public gamedata (habbo.nl, habbo.com, habbo.de …) and cached in memory.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { promises as fs } from "node:fs";
|
import { existsSync, promises as fs } from "node:fs";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel";
|
import type { HabboGamedataHotel } from "@/lib/habbo-gamedata-hotel";
|
||||||
import { habboFurnidataUrl } from "@/lib/habbo-gamedata-hotel";
|
import { habboFurnidataUrl } from "@/lib/habbo-gamedata-hotel";
|
||||||
import { logger } from "@/lib/logger";
|
import { logger } from "@/lib/logger";
|
||||||
|
import { getGamedataRoot } from "@/lib/services/furni-asset-dirs";
|
||||||
import {
|
import {
|
||||||
getFurnitureDataWritePaths,
|
getFurnitureDataWritePaths,
|
||||||
readFurniData,
|
readFurniData,
|
||||||
withFurniDataLock,
|
withFurniDataLock,
|
||||||
} from "@/lib/services/furni-data";
|
} from "@/lib/services/furni-data";
|
||||||
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
||||||
|
import { siteSettings } from "@/lib/services/site-settings";
|
||||||
|
|
||||||
export const FURNIDATA_LANGUAGES = [
|
export const FURNIDATA_LANGUAGES = [
|
||||||
{ lang: "nl", hotel: "nl" as HabboGamedataHotel },
|
{ lang: "nl", hotel: "nl" as HabboGamedataHotel },
|
||||||
@@ -55,6 +57,111 @@ interface LangCacheEntry {
|
|||||||
|
|
||||||
const langCache = new Map<string, LangCacheEntry>();
|
const langCache = new Map<string, LangCacheEntry>();
|
||||||
|
|
||||||
|
// ── LibreTranslate (gratis, self-hosted) ─────────────────────────────
|
||||||
|
const LIBRE_DEFAULT_URL = "http://127.0.0.1:5000";
|
||||||
|
const LIBRE_CACHE_FILE = ".translations_libre_cache.json";
|
||||||
|
type LibreCache = Record<string, string>; // key: `${target}|${text}` → translated
|
||||||
|
|
||||||
|
let libreMemoryCache: Map<string, string> | null = null;
|
||||||
|
let libreCacheLoaded = false;
|
||||||
|
|
||||||
|
async function getLibreTranslateUrl(): Promise<string> {
|
||||||
|
try {
|
||||||
|
const v = (await siteSettings.get(
|
||||||
|
"libretranslate_url",
|
||||||
|
LIBRE_DEFAULT_URL,
|
||||||
|
)) as string | null;
|
||||||
|
const url = (v ?? LIBRE_DEFAULT_URL).trim() || LIBRE_DEFAULT_URL;
|
||||||
|
return url.replace(/\/+$/, "");
|
||||||
|
} catch {
|
||||||
|
return LIBRE_DEFAULT_URL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadLibreCache(): Promise<Map<string, string>> {
|
||||||
|
if (libreMemoryCache && libreCacheLoaded) return libreMemoryCache;
|
||||||
|
try {
|
||||||
|
const gamedataRoot = await getGamedataRoot();
|
||||||
|
const cachePath = gamedataRoot
|
||||||
|
? path.join(gamedataRoot, "config", LIBRE_CACHE_FILE)
|
||||||
|
: path.join(process.cwd(), LIBRE_CACHE_FILE);
|
||||||
|
if (existsSync(cachePath)) {
|
||||||
|
const raw = await fs.readFile(cachePath, "utf-8");
|
||||||
|
const obj = JSON.parse(raw) as LibreCache;
|
||||||
|
libreMemoryCache = new Map(Object.entries(obj));
|
||||||
|
} else {
|
||||||
|
libreMemoryCache = new Map();
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
libreMemoryCache = new Map();
|
||||||
|
}
|
||||||
|
libreCacheLoaded = true;
|
||||||
|
return libreMemoryCache ?? new Map<string, string>();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveLibreCache(): Promise<void> {
|
||||||
|
if (!libreMemoryCache) return;
|
||||||
|
try {
|
||||||
|
const gamedataRoot = await getGamedataRoot();
|
||||||
|
const cachePath = gamedataRoot
|
||||||
|
? path.join(gamedataRoot, "config", LIBRE_CACHE_FILE)
|
||||||
|
: path.join(process.cwd(), LIBRE_CACHE_FILE);
|
||||||
|
const obj: LibreCache = Object.fromEntries(libreMemoryCache.entries());
|
||||||
|
await fs.mkdir(path.dirname(cachePath), { recursive: true });
|
||||||
|
await fs.writeFile(cachePath, JSON.stringify(obj), "utf-8");
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn("[furni-i18n] Failed to save Libre cache", {
|
||||||
|
error: (err as Error).message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function translateWithLibre(
|
||||||
|
text: string,
|
||||||
|
targetLang: string,
|
||||||
|
): Promise<string> {
|
||||||
|
const trimmed = text.trim();
|
||||||
|
if (!trimmed || targetLang === "en") return text;
|
||||||
|
const cache = await loadLibreCache();
|
||||||
|
const key = `${targetLang}|${trimmed}`;
|
||||||
|
const cached = cache.get(key);
|
||||||
|
if (cached !== undefined) return cached;
|
||||||
|
|
||||||
|
const url = await getLibreTranslateUrl();
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${url}/translate`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
q: trimmed,
|
||||||
|
source: "en",
|
||||||
|
target: targetLang,
|
||||||
|
format: "text",
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(10_000),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
logger.warn(
|
||||||
|
`[furni-i18n] LibreTranslate ${targetLang} failed: ${res.status}`,
|
||||||
|
);
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as { translatedText?: string };
|
||||||
|
const out =
|
||||||
|
typeof data.translatedText === "string" ? data.translatedText : text;
|
||||||
|
cache.set(key, out);
|
||||||
|
// Best-effort persist, don't await per-call to avoid hammering disk — save
|
||||||
|
// in background throttled (every 50 new entries).
|
||||||
|
if (cache.size % 50 === 0) void saveLibreCache();
|
||||||
|
return out;
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(`[furni-i18n] LibreTranslate error (${targetLang})`, {
|
||||||
|
error: (err as Error).message,
|
||||||
|
});
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function lookupTranslation(
|
function lookupTranslation(
|
||||||
map: Map<string, Translation>,
|
map: Map<string, Translation>,
|
||||||
classname: string,
|
classname: string,
|
||||||
@@ -312,16 +419,16 @@ export async function patchLocalizedFurniDataEntries(
|
|||||||
>;
|
>;
|
||||||
data = cloneFurniData(master as unknown as Record<string, unknown>);
|
data = cloneFurniData(master as unknown as Record<string, unknown>);
|
||||||
// Apply all translations for the fallback path (same as full build
|
// Apply all translations for the fallback path (same as full build
|
||||||
// but only for the file that didn't exist).
|
// but only for the file that didn't exist — official only to avoid
|
||||||
|
// 100k+ LibreTranslate calls on first creation).
|
||||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||||
for (const entry of data[section]?.furnitype ?? []) {
|
for (const entry of data[section]?.furnitype ?? []) {
|
||||||
const cn =
|
const cn =
|
||||||
typeof entry.classname === "string" ? entry.classname : "";
|
typeof entry.classname === "string" ? entry.classname : "";
|
||||||
if (!cn) continue;
|
if (!cn) continue;
|
||||||
const t = lookupTranslation(map, cn);
|
const t = lookupTranslation(map, cn);
|
||||||
if (!t) continue;
|
if (t?.name) entry.name = t.name;
|
||||||
if (t.name) entry.name = t.name;
|
if (t?.description) entry.description = t.description;
|
||||||
if (t.description) entry.description = t.description;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -331,12 +438,37 @@ export async function patchLocalizedFurniDataEntries(
|
|||||||
for (const entry of data[section]?.furnitype ?? []) {
|
for (const entry of data[section]?.furnitype ?? []) {
|
||||||
const cn =
|
const cn =
|
||||||
typeof entry.classname === "string" ? entry.classname : "";
|
typeof entry.classname === "string" ? entry.classname : "";
|
||||||
if (!wanted.has(cn)) continue;
|
const wantedEntry = wanted.get(cn);
|
||||||
|
if (!wantedEntry) continue;
|
||||||
const t = lookupTranslation(map, cn);
|
const t = lookupTranslation(map, cn);
|
||||||
if (!t) continue;
|
let didPatch = false;
|
||||||
if (t.name) entry.name = t.name;
|
const origName =
|
||||||
if (t.description) entry.description = t.description;
|
wantedEntry.name ??
|
||||||
patched++;
|
(typeof entry.name === "string" ? entry.name : "");
|
||||||
|
const origDesc =
|
||||||
|
wantedEntry.description ??
|
||||||
|
(typeof entry.description === "string" ? entry.description : "");
|
||||||
|
if (t?.name) {
|
||||||
|
entry.name = t.name;
|
||||||
|
didPatch = true;
|
||||||
|
} else if (lang !== "en" && origName) {
|
||||||
|
const translated = await translateWithLibre(origName, lang);
|
||||||
|
if (translated && translated !== origName) {
|
||||||
|
entry.name = translated;
|
||||||
|
didPatch = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (t?.description) {
|
||||||
|
entry.description = t.description;
|
||||||
|
didPatch = true;
|
||||||
|
} else if (lang !== "en" && origDesc) {
|
||||||
|
const translated = await translateWithLibre(origDesc, lang);
|
||||||
|
if (translated && translated !== origDesc) {
|
||||||
|
entry.description = translated;
|
||||||
|
didPatch = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (didPatch) patched++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Also handle brand-new entries that weren't in the file yet (rare
|
// Also handle brand-new entries that weren't in the file yet (rare
|
||||||
@@ -356,7 +488,7 @@ export async function patchLocalizedFurniDataEntries(
|
|||||||
string,
|
string,
|
||||||
{ furnitype: Array<Record<string, unknown>> }
|
{ furnitype: Array<Record<string, unknown>> }
|
||||||
>;
|
>;
|
||||||
for (const { classname } of entries) {
|
for (const { classname, name, description } of entries) {
|
||||||
if (existingClassnames.has(classname)) continue;
|
if (existingClassnames.has(classname)) continue;
|
||||||
// Find in master.
|
// Find in master.
|
||||||
let found: Record<string, unknown> | undefined;
|
let found: Record<string, unknown> | undefined;
|
||||||
@@ -376,12 +508,29 @@ export async function patchLocalizedFurniDataEntries(
|
|||||||
const clone = { ...found };
|
const clone = { ...found };
|
||||||
const t = lookupTranslation(map, classname);
|
const t = lookupTranslation(map, classname);
|
||||||
if (t?.name) clone.name = t.name;
|
if (t?.name) clone.name = t.name;
|
||||||
|
else if (lang !== "en") {
|
||||||
|
const src =
|
||||||
|
name ?? (typeof found.name === "string" ? found.name : "") ?? "";
|
||||||
|
if (src) clone.name = await translateWithLibre(src, lang);
|
||||||
|
}
|
||||||
if (t?.description) clone.description = t.description;
|
if (t?.description) clone.description = t.description;
|
||||||
|
else if (lang !== "en") {
|
||||||
|
const src =
|
||||||
|
description ??
|
||||||
|
(typeof found.description === "string"
|
||||||
|
? found.description
|
||||||
|
: "") ??
|
||||||
|
"";
|
||||||
|
if (src) clone.description = await translateWithLibre(src, lang);
|
||||||
|
}
|
||||||
if (!data[foundSection]) data[foundSection] = { furnitype: [] };
|
if (!data[foundSection]) data[foundSection] = { furnitype: [] };
|
||||||
data[foundSection].furnitype.push(clone);
|
data[foundSection].furnitype.push(clone);
|
||||||
patched++;
|
patched++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Persist LibreTranslate cache once per language (batched).
|
||||||
|
if (lang !== "en") await saveLibreCache();
|
||||||
|
|
||||||
const payload = JSON.stringify(data);
|
const payload = JSON.stringify(data);
|
||||||
for (const target of targets) {
|
for (const target of targets) {
|
||||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||||
|
|||||||
Reference in new issue
Block a user