diff --git a/scripts/build-languages-full.py b/scripts/build-languages-full.py new file mode 100644 index 0000000000..df50c46719 --- /dev/null +++ b/scripts/build-languages-full.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +""" +Full furnidata translation script using LibreTranslate. +Translates all customs (missing official translations) for all 13 languages. +Run from /var/www/atom-nexst +""" + +import json +import pathlib +import os +import time +import concurrent.futures +import urllib.request +import sys + +# ── Config ───────────────────────────────────────────────────────────── +MASTER_PATH = "/var/www/Gamedata/config/FurnitureData.json" +OUT_DIR = "/var/www/Gamedata/config" +CACHE_PATH = "/var/www/Gamedata/config/.translations_libre_cache.json" +LIBRE_URL = "http://127.0.0.1:5000/translate" +CONCURRENCY = 6 +LANGUAGES = [ + ("nl", "nl"), + ("en", "com"), + ("de", "de"), + ("fr", "fr"), + ("es", "es"), + ("tr", "com.tr"), + ("it", "it"), + ("pt", "com.br"), + ("fi", "fi"), + ("pl", "com"), + ("ru", "com"), + ("ar", "com"), + ("ja", "com"), +] +OFFICIAL_HOTEL_LANGS = {"nl", "en", "de", "fr", "es", "tr", "it", "pt", "fi"} +# ──────────────────────────────────────────────────────────────────────── + +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + "Accept": "application/json,text/plain,*/*;q=0.9", + "Accept-Language": "en-US,en;q=0.9", + "Sec-Fetch-Site": "cross-site", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Dest": "empty", +} + +def load_json(path): + return json.loads(pathlib.Path(path).read_text()) + +def save_json(path, data): + tmp = path + ".tmp" + pathlib.Path(tmp).write_text(json.dumps(data, ensure_ascii=False)) + os.rename(tmp, path) + os.chmod(path, 0o640) + try: + import pwd, grp + os.chown(path, pwd.getpwnam("www-data").pw_uid, grp.getgrnam("www-data").gr_gid) + except: + pass + +def build_map(data): + m = {} + for sec in ("roomitemtypes", "wallitemtypes"): + for e in data[sec]["furnitype"]: + cn = e["classname"] + if cn not in m: + m[cn] = {"name": e.get("name", ""), "description": e.get("description", "")} + return m + +def is_meaningful(text, classname): + if not isinstance(text, str): + return False + t = text.strip() + if not t: + return False + if t == classname: + return False + if t == f"{classname} desc": + return False + if t == f"{classname} name": + return False + if len(t) < 2: + return False + return True + +def libre_translate(text, target, cache): + key = f"{target}|{text}" + if key in cache: + return cache[key] + data = json.dumps({"q": text, "source": "en", "target": target, "format": "text"}).encode() + req = urllib.request.Request(LIBRE_URL, data=data, headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=15) as r: + out = json.loads(r.read().decode()).get("translatedText", text) + except Exception as e: + print(f" LibreTranslate error for {target} '{text[:40]}': {e}") + out = text + cache[key] = out + return out + +def fetch_official(lang, hotel): + if lang not in OFFICIAL_HOTEL_LANGS: + return {} + url = f"https://www.habbo.{hotel}/gamedata/furnidata_json/1" + print(f" Fetching official {lang} from habbo.{hotel}...") + try: + req = urllib.request.Request(url, headers=HEADERS) + with urllib.request.urlopen(req, timeout=30) as r: + data = json.loads(r.read().decode()) + except Exception as e: + print(f" Failed to fetch official {lang}: {e}") + return {} + fmap = {} + for sec in ("roomitemtypes", "wallitemtypes"): + for e in data.get(sec, {}).get("furnitype", []): + cn = e.get("classname", "") + if not cn or cn in fmap: + continue + fmap[cn] = {"name": e.get("name", ""), "description": e.get("description", "")} + print(f" Official {lang}: {len(fmap)} translations") + return fmap + +def translate_batch(texts, target, cache, concurrency=CONCURRENCY): + if not texts: + return {} + print(f" Translating {len(texts)} unique texts to {target} with {concurrency} workers...") + start = time.time() + out = {} + missing = [t for t in texts if f"{target}|{t}" not in cache] + if not missing: + return {t: cache[f"{target}|{t}"] for t in texts} + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as ex: + futs = {ex.submit(libre_translate, t, target, cache): t for t in missing} + done = 0 + for f in concurrent.futures.as_completed(futs): + t = futs[f] + try: + out[t] = f.result() + except Exception as e: + print(f" Failed {t[:40]}: {e}") + out[t] = t + done += 1 + if done % 100 == 0: + elapsed = time.time() - start + print(f" {done}/{len(missing)} in {elapsed:.1f}s ({elapsed/max(done,1):.2f}s/req)") + elapsed = time.time() - start + print(f" Done {len(texts)} texts to {target} in {elapsed:.1f}s") + return out + +def main(): + print("Loading master FurnitureData.json...") + master = load_json(MASTER_PATH) + master_map = build_map(master) + print(f"Master: {len(master_map)} entries") + + # Load or create cache + cache = {} + if os.path.exists(CACHE_PATH): + try: + cache = load_json(CACHE_PATH) + print(f"Loaded cache: {len(cache)} entries") + except Exception: + print("Cache corrupt, starting fresh") + cache = {} + + # For each language + for lang, hotel in LANGUAGES: + if lang == "en": + print(f"\n=== {lang} (source) ===") + # en file is just copy of master + out_path = f"{OUT_DIR}/FurnitureData_en.json" + save_json(out_path, master) + print(f" Wrote {out_path}") + continue + + out_path = f"{OUT_DIR}/FurnitureData_{lang}.json" + print(f"\n=== {lang} (hotel: {hotel}) ===") + + # Fetch official translations if available + official = fetch_official(lang, hotel) if lang in OFFICIAL_HOTEL_LANGS else {} + + # Clone master + cloned = json.loads(json.dumps(master)) + translated = 0 + official_count = 0 + libre_count = 0 + + # Collect all texts needing translation (customs not covered by official) + missing_names = set() + missing_descs = set() + + for sec in ("roomitemtypes", "wallitemtypes"): + for e in cloned[sec]["furnitype"]: + cn = e["classname"] + t = official.get(cn) + if not t and "*" in cn: + base = cn.split("*")[0] + t = official.get(f"{base}*0") or official.get(base) + if not t and "*" not in cn: + t = official.get(f"{cn}*0") + + orig_name = e["name"] + orig_desc = e.get("description", "") + + # Apply official if available + if t: + did = False + if t.get("name"): + e["name"] = t["name"] + did = True + if t.get("description"): + e["description"] = t["description"] + did = True + if did: + translated += 1 + official_count += 1 + continue + + # Collect missing for LibreTranslate + if is_meaningful(orig_name, cn): + missing_names.add(orig_name.strip()) + if is_meaningful(orig_desc, cn): + missing_descs.add(orig_desc.strip()) + + print(f" Official applied: {official_count}, missing for Libre: {len(missing_names)} names, {len(missing_descs)} descs") + + # Translate missing via LibreTranslate + if missing_names: + tr_names = translate_batch(missing_names, lang, cache, CONCURRENCY) + for sec in ("roomitemtypes", "wallitemtypes"): + for e in cloned[sec]["furnitype"]: + cn = e["classname"] + orig = e["name"] + if is_meaningful(orig, cn): + tr = tr_names.get(orig.strip()) + if tr and tr != orig: + e["name"] = tr + translated += 1 + libre_count += 1 + + if missing_descs: + tr_descs = translate_batch(missing_descs, lang, cache, CONCURRENCY) + for sec in ("roomitemtypes", "wallitemtypes"): + for e in cloned[sec]["furnitype"]: + cn = e["classname"] + orig = e.get("description", "") + if is_meaningful(orig, cn): + tr = tr_descs.get(orig.strip()) + if tr and tr != orig: + e["description"] = tr + translated += 1 + libre_count += 1 + + # Save cache after each language + save_json(CACHE_PATH, cache) + print(f" LibreTranslate applied: {libre_count}, total translated: {translated}") + + # Write output + save_json(out_path, cloned) + print(f" Wrote {out_path} ({pathlib.Path(out_path).stat().st_size / 1024 / 1024:.1f} MB)") + + # Final cache save + save_json(CACHE_PATH, cache) + print(f"\n=== All done! Cache: {len(cache)} entries ===") + + # Summary + for lang, _ in LANGUAGES: + fmap = build_map(load_json(f"{OUT_DIR}/FurnitureData_{lang}.json")) + ident = sum(1 for k, v in master_map.items() if fmap.get(k) and fmap[k]["name"] == v["name"]) + pct = (len(master_map) - ident) / len(master_map) * 100 + print(f" {lang}: {len(master_map)-ident}/{len(master_map)} translated ({pct:.1f}%)") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/build-languages-full.ts b/scripts/build-languages-full.ts new file mode 100644 index 0000000000..9add253068 --- /dev/null +++ b/scripts/build-languages-full.ts @@ -0,0 +1,43 @@ +import "./load-env"; +import { buildLocalizedFurniDataFiles } from "../src/lib/services/furni-data-i18n"; + +async function main() { + const args = process.argv.slice(2); + const full = args.includes("--full"); + const maxPerLang = full + ? undefined + : args.includes("--limit") + ? parseInt(args[args.indexOf("--limit") + 1] || "2000", 10) + : 1500; + const concurrency = args.includes("--concurrency") + ? parseInt(args[args.indexOf("--concurrency") + 1] || "6", 10) + : 6; + + console.log( + `[build-languages] Starting ${full ? "FULL" : `limited ${maxPerLang} per lang`} build (concurrency ${concurrency})...`, + ); + const start = Date.now(); + try { + const results = await buildLocalizedFurniDataFiles({ + maxPerLang, + concurrency, + useLibre: true, + }); + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`[build-languages] Done in ${elapsed}s`); + for (const r of results) { + const totalTranslated = r.translatedRoom + r.translatedWall; + const pct = + r.total > 0 ? ((totalTranslated / r.total) * 100).toFixed(1) : "0.0"; + console.log( + ` ${r.lang} (${r.hotel}): ${totalTranslated}/${r.total} translated (${pct}%) -> ${r.file} ${r.ok ? "OK" : `FAIL ${r.error}`}`, + ); + } + process.exit(0); + } catch (e) { + console.error("[build-languages] Failed:", e); + process.exit(1); + } +} + +main(); diff --git a/scripts/translate-furnidata-full.ts b/scripts/translate-furnidata-full.ts new file mode 100644 index 0000000000..1638263a64 --- /dev/null +++ b/scripts/translate-furnidata-full.ts @@ -0,0 +1,32 @@ +import "./load-env"; +import { buildLocalizedFurniDataFiles } from "../src/lib/services/furni-data-i18n"; + +async function main() { + const args = process.argv.slice(2); + const maxPerLang = args.includes("--full") + ? undefined + : args.includes("--limit") + ? parseInt(args[args.indexOf("--limit") + 1] || "2000", 10) + : 1500; + const full = args.includes("--full"); + console.log( + `[translate-furnidata] Starting ${full ? "FULL" : `limited ${maxPerLang} per lang`} build...`, + ); + const start = Date.now(); + try { + const results = await buildLocalizedFurniDataFiles( + full ? {} : { maxPerLang }, + ); + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`[translate-furnidata] Done in ${elapsed}s`); + for (const r of results) { + console.log( + ` ${r.lang} (${r.hotel}): ${r.translatedRoom + r.translatedWall}/${r.total} translated -> ${r.file} ${r.ok ? "OK" : `FAIL ${r.error}`}`, + ); + } + } catch (e) { + console.error("Failed", e); + process.exit(1); + } +} +main();