feat(db): add self-tuned mariadb-turbo container and bulk JSON importer

- docker-compose: opt-in mariadb-turbo service (profile 'db') with inline
  mysqld tuning (max_allowed_packet=512M, innodb_flush_log_at_trx_commit=2,
  2G buffer pool, net_read/write_timeout=600) for >50 MB JSON bulk loads;
  named volume for the datadir + node_modules host-sync guidance
- scripts/bulk-import-json.ts: chunked (2000-row) idempotent importer with
  ON DUPLICATE KEY UPDATE, resumable, habbo furnidata or flat-array input
- src/db/schema-gamedata.ts: promote+index JSON storage schema (furnidata,
  docs, texts) incl. VIRTUAL generated columns for MariaDB
- package.json: add db:bulk, db:up, db:down, db:introspect, db:schema:generate
This commit is contained in:
openhands committed 2026-09-07 16:54:44 +02:00
1 parent a640e40a5d
commit 52459c0b74
5 files changed
+540 -2

No files matched your search

+97 -1
View File
@@ -36,6 +36,16 @@ services:
# same directory, so mount it into the container at the same absolute path.
# Must be RW so furniture/badge imports can write mirrors to it.
- /var/www/Gamedata:/var/www/Gamedata
#
# ── node_modules corruption (§ host-sync) ──
# pnpm node_modules must NEVER be a host bind-mount: shared-filesystem
# sync (Docker Desktop gRPC-FUSE/VirtioFS, Unison, Syncthing) corrupts
# pnpm's hardlinked store and .bin shims. The production image already
# bakes node_modules in at build time and is NOT bind-mounted here.
# For local dev use a *named volume* instead of a host bind:
# - node_modules:/app/node_modules
# and never share `node_modules` / `pnpm store` via the Docker bind.
# On macOS add `:cached`/`:delegated` consistency labels per mount.
healthcheck:
test: ["CMD", "node", "-e", "fetch('http://localhost:3002/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
@@ -78,4 +88,90 @@ services:
# - "3030:3030"
# restart: unless-stopped
# volumes:
# - /var/www/Gamedata:/var/www/Gamedata
# - /var/www/Gamedata:/var/www/Gamedata
# ── MariaDB "Turbo" (heavy JSON / bulk-loads) ──
# Dedicated MariaDB tuned for >50 MB JSON imports. All tuning lives inline
# in the service `command:` (bulk_insert_buffer_size,
# innodb_flush_log_at_trx_commit=2, max_allowed_packet=512M, ...) so the
# container configures itself — no external .cnf to mount or keep in sync.
# Opt-in via profile so `docker compose up` keeps running the standalone
# stack as today.
#
# Start: docker compose --profile db up -d (= pnpm db:up)
# Note: host networking binds 127.0.0.1:3306 → STOP the host MariaDB
# first (the app's .env DATABASE_URL points at localhost:3306).
# Fixes the "Pulling schema from database..." hang: raise
# net_read/net_write_timeout (done above) + stop imports while
# running `pnpm db:generate` / drizzle-kit introspection.
mariadb-turbo:
image: mariadb:11
container_name: mariadb-turbo
profiles: ["db"]
network_mode: host
restart: unless-stopped
environment:
# mariadb:11 uses MARIADB_*; MYSQL_* also works but is deprecated there.
- MARIADB_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD:-root}
- MARIADB_DATABASE=${MARIADB_DATABASE:-habbo}
- MARIADB_USER=${MARIADB_USER:-cms}
- MARIADB_PASSWORD=${MARIADB_PASSWORD:-cms}
- MARIADB_AUTO_UPGRADE=1
# MariaDB tunes itself — the Official image appends these flags to mysqld,
# no external .cnf file needed.
command: [
# Charset
"--character-set-server=utf8mb4",
"--collation-server=utf8mb4_unicode_ci",
# 50 MB JSON batches: raise the 16 MB default packet ceiling.
"--max-allowed-packet=512M",
"--net-buffer-length=1M",
# Timeouts — fixes the "Pulling schema from database..." hang
# (drizzle-kit introspection no longer starves behind big imports).
"--connect-timeout=30",
"--wait-timeout=3600",
"--interactive-timeout=3600",
"--net-read-timeout=600",
"--net-write-timeout=600",
# InnoDB: durability/performance trade-off for bulk writes.
"--innodb-flush-log-at-trx-commit=2",
"--innodb-buffer-pool-size=2G",
"--innodb-buffer-pool-instances=4",
"--innodb-log-file-size=1G",
"--innodb-log-buffer-size=64M",
"--innodb-flush-method=O_DIRECT",
"--innodb-autoextend-increment=512",
"--innodb-max-dirty-pages-pct=90",
"--bulk-insert-buffer-size=512M",
# Raise so the engine nearly never double-writes during a 50 MB load.
"--innodb-doublewrite=0",
# libaio/native_aio misbehaves in some containers; io_uring path is fine.
"--innodb-use-native-aio=0",
# Temp tables used when MariaDB scans JSON blobs cannot be indexed.
"--tmp-table-size=256M",
"--max-heap-table-size=256M",
"--read-buffer-size=4M",
"--read-rnd-buffer-size=16M",
# Save ~1-2 GB RAM; bulk loads don't need the instrumentation.
"--performance-schema=OFF",
"--skip-name-resolve",
]
volumes:
# Named volume, NOT a host bind — shared-filesystem sync trashes InnoDB
# files the same way it trashes pnpm's node_modules. Named volumes live
# inside the container filesystem, so 50 MB of JSON writes never cross a
# host-sync boundary.
- mariadb-turbo-data:/var/lib/mysql
healthcheck:
# healthcheck.sh ships in the official mariadb image.
test: ["CMD-SHELL", "healthcheck.sh --connect --innodb_initialized || mariadb-admin ping --silent"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
mem_limit: 4g
# Named volumes declared once; used by the MariaDB service above.
volumes:
mariadb-turbo-data:
driver: local