scripts: add full build/translate scripts for furnidata i18n
This commit is contained in:
openhands committed
1 parent
2339485e9a
commit
e66b2c1a5a
3 files changed
+351
No files matched your search
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Full furnidata translation script using LibreTranslate.
|
||||
Translates all customs (missing official translations) for all 13 languages.
|
||||
Run from /var/www/atom-nexst
|
||||
"""
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import os
|
||||
import time
|
||||
import concurrent.futures
|
||||
import urllib.request
|
||||
import sys
|
||||
|
||||
# ── Config ─────────────────────────────────────────────────────────────
|
||||
MASTER_PATH = "/var/www/Gamedata/config/FurnitureData.json"
|
||||
OUT_DIR = "/var/www/Gamedata/config"
|
||||
CACHE_PATH = "/var/www/Gamedata/config/.translations_libre_cache.json"
|
||||
LIBRE_URL = "http://127.0.0.1:5000/translate"
|
||||
CONCURRENCY = 6
|
||||
LANGUAGES = [
|
||||
("nl", "nl"),
|
||||
("en", "com"),
|
||||
("de", "de"),
|
||||
("fr", "fr"),
|
||||
("es", "es"),
|
||||
("tr", "com.tr"),
|
||||
("it", "it"),
|
||||
("pt", "com.br"),
|
||||
("fi", "fi"),
|
||||
("pl", "com"),
|
||||
("ru", "com"),
|
||||
("ar", "com"),
|
||||
("ja", "com"),
|
||||
]
|
||||
OFFICIAL_HOTEL_LANGS = {"nl", "en", "de", "fr", "es", "tr", "it", "pt", "fi"}
|
||||
# ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json,text/plain,*/*;q=0.9",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Sec-Fetch-Site": "cross-site",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
}
|
||||
|
||||
def load_json(path):
|
||||
return json.loads(pathlib.Path(path).read_text())
|
||||
|
||||
def save_json(path, data):
|
||||
tmp = path + ".tmp"
|
||||
pathlib.Path(tmp).write_text(json.dumps(data, ensure_ascii=False))
|
||||
os.rename(tmp, path)
|
||||
os.chmod(path, 0o640)
|
||||
try:
|
||||
import pwd, grp
|
||||
os.chown(path, pwd.getpwnam("www-data").pw_uid, grp.getgrnam("www-data").gr_gid)
|
||||
except:
|
||||
pass
|
||||
|
||||
def build_map(data):
|
||||
m = {}
|
||||
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||
for e in data[sec]["furnitype"]:
|
||||
cn = e["classname"]
|
||||
if cn not in m:
|
||||
m[cn] = {"name": e.get("name", ""), "description": e.get("description", "")}
|
||||
return m
|
||||
|
||||
def is_meaningful(text, classname):
|
||||
if not isinstance(text, str):
|
||||
return False
|
||||
t = text.strip()
|
||||
if not t:
|
||||
return False
|
||||
if t == classname:
|
||||
return False
|
||||
if t == f"{classname} desc":
|
||||
return False
|
||||
if t == f"{classname} name":
|
||||
return False
|
||||
if len(t) < 2:
|
||||
return False
|
||||
return True
|
||||
|
||||
def libre_translate(text, target, cache):
|
||||
key = f"{target}|{text}"
|
||||
if key in cache:
|
||||
return cache[key]
|
||||
data = json.dumps({"q": text, "source": "en", "target": target, "format": "text"}).encode()
|
||||
req = urllib.request.Request(LIBRE_URL, data=data, headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
out = json.loads(r.read().decode()).get("translatedText", text)
|
||||
except Exception as e:
|
||||
print(f" LibreTranslate error for {target} '{text[:40]}': {e}")
|
||||
out = text
|
||||
cache[key] = out
|
||||
return out
|
||||
|
||||
def fetch_official(lang, hotel):
|
||||
if lang not in OFFICIAL_HOTEL_LANGS:
|
||||
return {}
|
||||
url = f"https://www.habbo.{hotel}/gamedata/furnidata_json/1"
|
||||
print(f" Fetching official {lang} from habbo.{hotel}...")
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
data = json.loads(r.read().decode())
|
||||
except Exception as e:
|
||||
print(f" Failed to fetch official {lang}: {e}")
|
||||
return {}
|
||||
fmap = {}
|
||||
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||
for e in data.get(sec, {}).get("furnitype", []):
|
||||
cn = e.get("classname", "")
|
||||
if not cn or cn in fmap:
|
||||
continue
|
||||
fmap[cn] = {"name": e.get("name", ""), "description": e.get("description", "")}
|
||||
print(f" Official {lang}: {len(fmap)} translations")
|
||||
return fmap
|
||||
|
||||
def translate_batch(texts, target, cache, concurrency=CONCURRENCY):
|
||||
if not texts:
|
||||
return {}
|
||||
print(f" Translating {len(texts)} unique texts to {target} with {concurrency} workers...")
|
||||
start = time.time()
|
||||
out = {}
|
||||
missing = [t for t in texts if f"{target}|{t}" not in cache]
|
||||
if not missing:
|
||||
return {t: cache[f"{target}|{t}"] for t in texts}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as ex:
|
||||
futs = {ex.submit(libre_translate, t, target, cache): t for t in missing}
|
||||
done = 0
|
||||
for f in concurrent.futures.as_completed(futs):
|
||||
t = futs[f]
|
||||
try:
|
||||
out[t] = f.result()
|
||||
except Exception as e:
|
||||
print(f" Failed {t[:40]}: {e}")
|
||||
out[t] = t
|
||||
done += 1
|
||||
if done % 100 == 0:
|
||||
elapsed = time.time() - start
|
||||
print(f" {done}/{len(missing)} in {elapsed:.1f}s ({elapsed/max(done,1):.2f}s/req)")
|
||||
elapsed = time.time() - start
|
||||
print(f" Done {len(texts)} texts to {target} in {elapsed:.1f}s")
|
||||
return out
|
||||
|
||||
def main():
|
||||
print("Loading master FurnitureData.json...")
|
||||
master = load_json(MASTER_PATH)
|
||||
master_map = build_map(master)
|
||||
print(f"Master: {len(master_map)} entries")
|
||||
|
||||
# Load or create cache
|
||||
cache = {}
|
||||
if os.path.exists(CACHE_PATH):
|
||||
try:
|
||||
cache = load_json(CACHE_PATH)
|
||||
print(f"Loaded cache: {len(cache)} entries")
|
||||
except Exception:
|
||||
print("Cache corrupt, starting fresh")
|
||||
cache = {}
|
||||
|
||||
# For each language
|
||||
for lang, hotel in LANGUAGES:
|
||||
if lang == "en":
|
||||
print(f"\n=== {lang} (source) ===")
|
||||
# en file is just copy of master
|
||||
out_path = f"{OUT_DIR}/FurnitureData_en.json"
|
||||
save_json(out_path, master)
|
||||
print(f" Wrote {out_path}")
|
||||
continue
|
||||
|
||||
out_path = f"{OUT_DIR}/FurnitureData_{lang}.json"
|
||||
print(f"\n=== {lang} (hotel: {hotel}) ===")
|
||||
|
||||
# Fetch official translations if available
|
||||
official = fetch_official(lang, hotel) if lang in OFFICIAL_HOTEL_LANGS else {}
|
||||
|
||||
# Clone master
|
||||
cloned = json.loads(json.dumps(master))
|
||||
translated = 0
|
||||
official_count = 0
|
||||
libre_count = 0
|
||||
|
||||
# Collect all texts needing translation (customs not covered by official)
|
||||
missing_names = set()
|
||||
missing_descs = set()
|
||||
|
||||
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||
for e in cloned[sec]["furnitype"]:
|
||||
cn = e["classname"]
|
||||
t = official.get(cn)
|
||||
if not t and "*" in cn:
|
||||
base = cn.split("*")[0]
|
||||
t = official.get(f"{base}*0") or official.get(base)
|
||||
if not t and "*" not in cn:
|
||||
t = official.get(f"{cn}*0")
|
||||
|
||||
orig_name = e["name"]
|
||||
orig_desc = e.get("description", "")
|
||||
|
||||
# Apply official if available
|
||||
if t:
|
||||
did = False
|
||||
if t.get("name"):
|
||||
e["name"] = t["name"]
|
||||
did = True
|
||||
if t.get("description"):
|
||||
e["description"] = t["description"]
|
||||
did = True
|
||||
if did:
|
||||
translated += 1
|
||||
official_count += 1
|
||||
continue
|
||||
|
||||
# Collect missing for LibreTranslate
|
||||
if is_meaningful(orig_name, cn):
|
||||
missing_names.add(orig_name.strip())
|
||||
if is_meaningful(orig_desc, cn):
|
||||
missing_descs.add(orig_desc.strip())
|
||||
|
||||
print(f" Official applied: {official_count}, missing for Libre: {len(missing_names)} names, {len(missing_descs)} descs")
|
||||
|
||||
# Translate missing via LibreTranslate
|
||||
if missing_names:
|
||||
tr_names = translate_batch(missing_names, lang, cache, CONCURRENCY)
|
||||
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||
for e in cloned[sec]["furnitype"]:
|
||||
cn = e["classname"]
|
||||
orig = e["name"]
|
||||
if is_meaningful(orig, cn):
|
||||
tr = tr_names.get(orig.strip())
|
||||
if tr and tr != orig:
|
||||
e["name"] = tr
|
||||
translated += 1
|
||||
libre_count += 1
|
||||
|
||||
if missing_descs:
|
||||
tr_descs = translate_batch(missing_descs, lang, cache, CONCURRENCY)
|
||||
for sec in ("roomitemtypes", "wallitemtypes"):
|
||||
for e in cloned[sec]["furnitype"]:
|
||||
cn = e["classname"]
|
||||
orig = e.get("description", "")
|
||||
if is_meaningful(orig, cn):
|
||||
tr = tr_descs.get(orig.strip())
|
||||
if tr and tr != orig:
|
||||
e["description"] = tr
|
||||
translated += 1
|
||||
libre_count += 1
|
||||
|
||||
# Save cache after each language
|
||||
save_json(CACHE_PATH, cache)
|
||||
print(f" LibreTranslate applied: {libre_count}, total translated: {translated}")
|
||||
|
||||
# Write output
|
||||
save_json(out_path, cloned)
|
||||
print(f" Wrote {out_path} ({pathlib.Path(out_path).stat().st_size / 1024 / 1024:.1f} MB)")
|
||||
|
||||
# Final cache save
|
||||
save_json(CACHE_PATH, cache)
|
||||
print(f"\n=== All done! Cache: {len(cache)} entries ===")
|
||||
|
||||
# Summary
|
||||
for lang, _ in LANGUAGES:
|
||||
fmap = build_map(load_json(f"{OUT_DIR}/FurnitureData_{lang}.json"))
|
||||
ident = sum(1 for k, v in master_map.items() if fmap.get(k) and fmap[k]["name"] == v["name"])
|
||||
pct = (len(master_map) - ident) / len(master_map) * 100
|
||||
print(f" {lang}: {len(master_map)-ident}/{len(master_map)} translated ({pct:.1f}%)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
import "./load-env";
|
||||
import { buildLocalizedFurniDataFiles } from "../src/lib/services/furni-data-i18n";
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const full = args.includes("--full");
|
||||
const maxPerLang = full
|
||||
? undefined
|
||||
: args.includes("--limit")
|
||||
? parseInt(args[args.indexOf("--limit") + 1] || "2000", 10)
|
||||
: 1500;
|
||||
const concurrency = args.includes("--concurrency")
|
||||
? parseInt(args[args.indexOf("--concurrency") + 1] || "6", 10)
|
||||
: 6;
|
||||
|
||||
console.log(
|
||||
`[build-languages] Starting ${full ? "FULL" : `limited ${maxPerLang} per lang`} build (concurrency ${concurrency})...`,
|
||||
);
|
||||
const start = Date.now();
|
||||
try {
|
||||
const results = await buildLocalizedFurniDataFiles({
|
||||
maxPerLang,
|
||||
concurrency,
|
||||
useLibre: true,
|
||||
});
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`[build-languages] Done in ${elapsed}s`);
|
||||
for (const r of results) {
|
||||
const totalTranslated = r.translatedRoom + r.translatedWall;
|
||||
const pct =
|
||||
r.total > 0 ? ((totalTranslated / r.total) * 100).toFixed(1) : "0.0";
|
||||
console.log(
|
||||
` ${r.lang} (${r.hotel}): ${totalTranslated}/${r.total} translated (${pct}%) -> ${r.file} ${r.ok ? "OK" : `FAIL ${r.error}`}`,
|
||||
);
|
||||
}
|
||||
process.exit(0);
|
||||
} catch (e) {
|
||||
console.error("[build-languages] Failed:", e);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,32 @@
|
||||
import "./load-env";
|
||||
import { buildLocalizedFurniDataFiles } from "../src/lib/services/furni-data-i18n";
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const maxPerLang = args.includes("--full")
|
||||
? undefined
|
||||
: args.includes("--limit")
|
||||
? parseInt(args[args.indexOf("--limit") + 1] || "2000", 10)
|
||||
: 1500;
|
||||
const full = args.includes("--full");
|
||||
console.log(
|
||||
`[translate-furnidata] Starting ${full ? "FULL" : `limited ${maxPerLang} per lang`} build...`,
|
||||
);
|
||||
const start = Date.now();
|
||||
try {
|
||||
const results = await buildLocalizedFurniDataFiles(
|
||||
full ? {} : { maxPerLang },
|
||||
);
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`[translate-furnidata] Done in ${elapsed}s`);
|
||||
for (const r of results) {
|
||||
console.log(
|
||||
` ${r.lang} (${r.hotel}): ${r.translatedRoom + r.translatedWall}/${r.total} translated -> ${r.file} ${r.ok ? "OK" : `FAIL ${r.error}`}`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Failed", e);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
main();
|
||||
Reference in new issue
Block a user