scripts: add full build/translate scripts for furnidata i18n
This commit is contained in:
openhands committed
1 parent
2339485e9a
commit
e66b2c1a5a
3 files changed
+351
No files matched your search
@@ -0,0 +1,276 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Full furnidata translation script using LibreTranslate.
|
||||||
|
Translates all customs (missing official translations) for all 13 languages.
|
||||||
|
Run from /var/www/atom-nexst
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import concurrent.futures
|
||||||
|
import urllib.request
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# ── Config ─────────────────────────────────────────────────────────────
|
||||||
|
MASTER_PATH = "/var/www/Gamedata/config/FurnitureData.json"
|
||||||
|
OUT_DIR = "/var/www/Gamedata/config"
|
||||||
|
CACHE_PATH = "/var/www/Gamedata/config/.translations_libre_cache.json"
|
||||||
|
LIBRE_URL = "http://127.0.0.1:5000/translate"
|
||||||
|
CONCURRENCY = 6
|
||||||
|
LANGUAGES = [
|
||||||
|
("nl", "nl"),
|
||||||
|
("en", "com"),
|
||||||
|
("de", "de"),
|
||||||
|
("fr", "fr"),
|
||||||
|
("es", "es"),
|
||||||
|
("tr", "com.tr"),
|
||||||
|
("it", "it"),
|
||||||
|
("pt", "com.br"),
|
||||||
|
("fi", "fi"),
|
||||||
|
("pl", "com"),
|
||||||
|
("ru", "com"),
|
||||||
|
("ar", "com"),
|
||||||
|
("ja", "com"),
|
||||||
|
]
|
||||||
|
OFFICIAL_HOTEL_LANGS = {"nl", "en", "de", "fr", "es", "tr", "it", "pt", "fi"}
|
||||||
|
# ────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "application/json,text/plain,*/*;q=0.9",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
"Sec-Fetch-Site": "cross-site",
|
||||||
|
"Sec-Fetch-Mode": "cors",
|
||||||
|
"Sec-Fetch-Dest": "empty",
|
||||||
|
}
|
||||||
|
|
||||||
|
def load_json(path):
|
||||||
|
return json.loads(pathlib.Path(path).read_text())
|
||||||
|
|
||||||
|
def save_json(path, data):
|
||||||
|
tmp = path + ".tmp"
|
||||||
|
pathlib.Path(tmp).write_text(json.dumps(data, ensure_ascii=False))
|
||||||
|
os.rename(tmp, path)
|
||||||
|
os.chmod(path, 0o640)
|
||||||
|
try:
|
||||||
|
import pwd, grp
|
||||||
|
os.chown(path, pwd.getpwnam("www-data").pw_uid, grp.getgrnam("www-data").gr_gid)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def build_map(data):
|
||||||
|
m = {}
|
||||||
|
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||||
|
for e in data[sec]["furnitype"]:
|
||||||
|
cn = e["classname"]
|
||||||
|
if cn not in m:
|
||||||
|
m[cn] = {"name": e.get("name", ""), "description": e.get("description", "")}
|
||||||
|
return m
|
||||||
|
|
||||||
|
def is_meaningful(text, classname):
|
||||||
|
if not isinstance(text, str):
|
||||||
|
return False
|
||||||
|
t = text.strip()
|
||||||
|
if not t:
|
||||||
|
return False
|
||||||
|
if t == classname:
|
||||||
|
return False
|
||||||
|
if t == f"{classname} desc":
|
||||||
|
return False
|
||||||
|
if t == f"{classname} name":
|
||||||
|
return False
|
||||||
|
if len(t) < 2:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def libre_translate(text, target, cache):
|
||||||
|
key = f"{target}|{text}"
|
||||||
|
if key in cache:
|
||||||
|
return cache[key]
|
||||||
|
data = json.dumps({"q": text, "source": "en", "target": target, "format": "text"}).encode()
|
||||||
|
req = urllib.request.Request(LIBRE_URL, data=data, headers={"Content-Type": "application/json"})
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=15) as r:
|
||||||
|
out = json.loads(r.read().decode()).get("translatedText", text)
|
||||||
|
except Exception as e:
|
||||||
|
print(f" LibreTranslate error for {target} '{text[:40]}': {e}")
|
||||||
|
out = text
|
||||||
|
cache[key] = out
|
||||||
|
return out
|
||||||
|
|
||||||
|
def fetch_official(lang, hotel):
|
||||||
|
if lang not in OFFICIAL_HOTEL_LANGS:
|
||||||
|
return {}
|
||||||
|
url = f"https://www.habbo.{hotel}/gamedata/furnidata_json/1"
|
||||||
|
print(f" Fetching official {lang} from habbo.{hotel}...")
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(url, headers=HEADERS)
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
data = json.loads(r.read().decode())
|
||||||
|
except Exception as e:
|
||||||
|
print(f" Failed to fetch official {lang}: {e}")
|
||||||
|
return {}
|
||||||
|
fmap = {}
|
||||||
|
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||||
|
for e in data.get(sec, {}).get("furnitype", []):
|
||||||
|
cn = e.get("classname", "")
|
||||||
|
if not cn or cn in fmap:
|
||||||
|
continue
|
||||||
|
fmap[cn] = {"name": e.get("name", ""), "description": e.get("description", "")}
|
||||||
|
print(f" Official {lang}: {len(fmap)} translations")
|
||||||
|
return fmap
|
||||||
|
|
||||||
|
def translate_batch(texts, target, cache, concurrency=CONCURRENCY):
|
||||||
|
if not texts:
|
||||||
|
return {}
|
||||||
|
print(f" Translating {len(texts)} unique texts to {target} with {concurrency} workers...")
|
||||||
|
start = time.time()
|
||||||
|
out = {}
|
||||||
|
missing = [t for t in texts if f"{target}|{t}" not in cache]
|
||||||
|
if not missing:
|
||||||
|
return {t: cache[f"{target}|{t}"] for t in texts}
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as ex:
|
||||||
|
futs = {ex.submit(libre_translate, t, target, cache): t for t in missing}
|
||||||
|
done = 0
|
||||||
|
for f in concurrent.futures.as_completed(futs):
|
||||||
|
t = futs[f]
|
||||||
|
try:
|
||||||
|
out[t] = f.result()
|
||||||
|
except Exception as e:
|
||||||
|
print(f" Failed {t[:40]}: {e}")
|
||||||
|
out[t] = t
|
||||||
|
done += 1
|
||||||
|
if done % 100 == 0:
|
||||||
|
elapsed = time.time() - start
|
||||||
|
print(f" {done}/{len(missing)} in {elapsed:.1f}s ({elapsed/max(done,1):.2f}s/req)")
|
||||||
|
elapsed = time.time() - start
|
||||||
|
print(f" Done {len(texts)} texts to {target} in {elapsed:.1f}s")
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print("Loading master FurnitureData.json...")
|
||||||
|
master = load_json(MASTER_PATH)
|
||||||
|
master_map = build_map(master)
|
||||||
|
print(f"Master: {len(master_map)} entries")
|
||||||
|
|
||||||
|
# Load or create cache
|
||||||
|
cache = {}
|
||||||
|
if os.path.exists(CACHE_PATH):
|
||||||
|
try:
|
||||||
|
cache = load_json(CACHE_PATH)
|
||||||
|
print(f"Loaded cache: {len(cache)} entries")
|
||||||
|
except Exception:
|
||||||
|
print("Cache corrupt, starting fresh")
|
||||||
|
cache = {}
|
||||||
|
|
||||||
|
# For each language
|
||||||
|
for lang, hotel in LANGUAGES:
|
||||||
|
if lang == "en":
|
||||||
|
print(f"\n=== {lang} (source) ===")
|
||||||
|
# en file is just copy of master
|
||||||
|
out_path = f"{OUT_DIR}/FurnitureData_en.json"
|
||||||
|
save_json(out_path, master)
|
||||||
|
print(f" Wrote {out_path}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
out_path = f"{OUT_DIR}/FurnitureData_{lang}.json"
|
||||||
|
print(f"\n=== {lang} (hotel: {hotel}) ===")
|
||||||
|
|
||||||
|
# Fetch official translations if available
|
||||||
|
official = fetch_official(lang, hotel) if lang in OFFICIAL_HOTEL_LANGS else {}
|
||||||
|
|
||||||
|
# Clone master
|
||||||
|
cloned = json.loads(json.dumps(master))
|
||||||
|
translated = 0
|
||||||
|
official_count = 0
|
||||||
|
libre_count = 0
|
||||||
|
|
||||||
|
# Collect all texts needing translation (customs not covered by official)
|
||||||
|
missing_names = set()
|
||||||
|
missing_descs = set()
|
||||||
|
|
||||||
|
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||||
|
for e in cloned[sec]["furnitype"]:
|
||||||
|
cn = e["classname"]
|
||||||
|
t = official.get(cn)
|
||||||
|
if not t and "*" in cn:
|
||||||
|
base = cn.split("*")[0]
|
||||||
|
t = official.get(f"{base}*0") or official.get(base)
|
||||||
|
if not t and "*" not in cn:
|
||||||
|
t = official.get(f"{cn}*0")
|
||||||
|
|
||||||
|
orig_name = e["name"]
|
||||||
|
orig_desc = e.get("description", "")
|
||||||
|
|
||||||
|
# Apply official if available
|
||||||
|
if t:
|
||||||
|
did = False
|
||||||
|
if t.get("name"):
|
||||||
|
e["name"] = t["name"]
|
||||||
|
did = True
|
||||||
|
if t.get("description"):
|
||||||
|
e["description"] = t["description"]
|
||||||
|
did = True
|
||||||
|
if did:
|
||||||
|
translated += 1
|
||||||
|
official_count += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Collect missing for LibreTranslate
|
||||||
|
if is_meaningful(orig_name, cn):
|
||||||
|
missing_names.add(orig_name.strip())
|
||||||
|
if is_meaningful(orig_desc, cn):
|
||||||
|
missing_descs.add(orig_desc.strip())
|
||||||
|
|
||||||
|
print(f" Official applied: {official_count}, missing for Libre: {len(missing_names)} names, {len(missing_descs)} descs")
|
||||||
|
|
||||||
|
# Translate missing via LibreTranslate
|
||||||
|
if missing_names:
|
||||||
|
tr_names = translate_batch(missing_names, lang, cache, CONCURRENCY)
|
||||||
|
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||||
|
for e in cloned[sec]["furnitype"]:
|
||||||
|
cn = e["classname"]
|
||||||
|
orig = e["name"]
|
||||||
|
if is_meaningful(orig, cn):
|
||||||
|
tr = tr_names.get(orig.strip())
|
||||||
|
if tr and tr != orig:
|
||||||
|
e["name"] = tr
|
||||||
|
translated += 1
|
||||||
|
libre_count += 1
|
||||||
|
|
||||||
|
if missing_descs:
|
||||||
|
tr_descs = translate_batch(missing_descs, lang, cache, CONCURRENCY)
|
||||||
|
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||||
|
for e in cloned[sec]["furnitype"]:
|
||||||
|
cn = e["classname"]
|
||||||
|
orig = e.get("description", "")
|
||||||
|
if is_meaningful(orig, cn):
|
||||||
|
tr = tr_descs.get(orig.strip())
|
||||||
|
if tr and tr != orig:
|
||||||
|
e["description"] = tr
|
||||||
|
translated += 1
|
||||||
|
libre_count += 1
|
||||||
|
|
||||||
|
# Save cache after each language
|
||||||
|
save_json(CACHE_PATH, cache)
|
||||||
|
print(f" LibreTranslate applied: {libre_count}, total translated: {translated}")
|
||||||
|
|
||||||
|
# Write output
|
||||||
|
save_json(out_path, cloned)
|
||||||
|
print(f" Wrote {out_path} ({pathlib.Path(out_path).stat().st_size / 1024 / 1024:.1f} MB)")
|
||||||
|
|
||||||
|
# Final cache save
|
||||||
|
save_json(CACHE_PATH, cache)
|
||||||
|
print(f"\n=== All done! Cache: {len(cache)} entries ===")
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
for lang, _ in LANGUAGES:
|
||||||
|
fmap = build_map(load_json(f"{OUT_DIR}/FurnitureData_{lang}.json"))
|
||||||
|
ident = sum(1 for k, v in master_map.items() if fmap.get(k) and fmap[k]["name"] == v["name"])
|
||||||
|
pct = (len(master_map) - ident) / len(master_map) * 100
|
||||||
|
print(f" {lang}: {len(master_map)-ident}/{len(master_map)} translated ({pct:.1f}%)")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import "./load-env";
|
||||||
|
import { buildLocalizedFurniDataFiles } from "../src/lib/services/furni-data-i18n";
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const full = args.includes("--full");
|
||||||
|
const maxPerLang = full
|
||||||
|
? undefined
|
||||||
|
: args.includes("--limit")
|
||||||
|
? parseInt(args[args.indexOf("--limit") + 1] || "2000", 10)
|
||||||
|
: 1500;
|
||||||
|
const concurrency = args.includes("--concurrency")
|
||||||
|
? parseInt(args[args.indexOf("--concurrency") + 1] || "6", 10)
|
||||||
|
: 6;
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[build-languages] Starting ${full ? "FULL" : `limited ${maxPerLang} per lang`} build (concurrency ${concurrency})...`,
|
||||||
|
);
|
||||||
|
const start = Date.now();
|
||||||
|
try {
|
||||||
|
const results = await buildLocalizedFurniDataFiles({
|
||||||
|
maxPerLang,
|
||||||
|
concurrency,
|
||||||
|
useLibre: true,
|
||||||
|
});
|
||||||
|
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||||
|
console.log(`[build-languages] Done in ${elapsed}s`);
|
||||||
|
for (const r of results) {
|
||||||
|
const totalTranslated = r.translatedRoom + r.translatedWall;
|
||||||
|
const pct =
|
||||||
|
r.total > 0 ? ((totalTranslated / r.total) * 100).toFixed(1) : "0.0";
|
||||||
|
console.log(
|
||||||
|
` ${r.lang} (${r.hotel}): ${totalTranslated}/${r.total} translated (${pct}%) -> ${r.file} ${r.ok ? "OK" : `FAIL ${r.error}`}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
process.exit(0);
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[build-languages] Failed:", e);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import "./load-env";
|
||||||
|
import { buildLocalizedFurniDataFiles } from "../src/lib/services/furni-data-i18n";
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const maxPerLang = args.includes("--full")
|
||||||
|
? undefined
|
||||||
|
: args.includes("--limit")
|
||||||
|
? parseInt(args[args.indexOf("--limit") + 1] || "2000", 10)
|
||||||
|
: 1500;
|
||||||
|
const full = args.includes("--full");
|
||||||
|
console.log(
|
||||||
|
`[translate-furnidata] Starting ${full ? "FULL" : `limited ${maxPerLang} per lang`} build...`,
|
||||||
|
);
|
||||||
|
const start = Date.now();
|
||||||
|
try {
|
||||||
|
const results = await buildLocalizedFurniDataFiles(
|
||||||
|
full ? {} : { maxPerLang },
|
||||||
|
);
|
||||||
|
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||||
|
console.log(`[translate-furnidata] Done in ${elapsed}s`);
|
||||||
|
for (const r of results) {
|
||||||
|
console.log(
|
||||||
|
` ${r.lang} (${r.hotel}): ${r.translatedRoom + r.translatedWall}/${r.total} translated -> ${r.file} ${r.ok ? "OK" : `FAIL ${r.error}`}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error("Failed", e);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
main();
|
||||||
Reference in new issue
Block a user