feat(db): add self-tuned mariadb-turbo container and bulk JSON importer
- docker-compose: opt-in mariadb-turbo service (profile 'db') with inline mysqld tuning (max_allowed_packet=512M, innodb_flush_log_at_trx_commit=2, 2G buffer pool, net_read/write_timeout=600) for >50 MB JSON bulk loads; named volume for the datadir + node_modules host-sync guidance - scripts/bulk-import-json.ts: chunked (2000-row) idempotent importer with ON DUPLICATE KEY UPDATE, resumable, habbo furnidata or flat-array input - src/db/schema-gamedata.ts: promote+index JSON storage schema (furnidata, docs, texts) incl. VIRTUAL generated columns for MariaDB - package.json: add db:bulk, db:up, db:down, db:introspect, db:schema:generate
This commit is contained in:
1 parent
a640e40a5d
commit
52459c0b74
5 files changed
+540
-2
No files matched your search
+97
-1
@@ -36,6 +36,16 @@ services:
|
||||
# same directory, so mount it into the container at the same absolute path.
|
||||
# Must be RW so furniture/badge imports can write mirrors to it.
|
||||
- /var/www/Gamedata:/var/www/Gamedata
|
||||
#
|
||||
# ── node_modules corruption (§ host-sync) ──
|
||||
# pnpm node_modules must NEVER be a host bind-mount: shared-filesystem
|
||||
# sync (Docker Desktop gRPC-FUSE/VirtioFS, Unison, Syncthing) corrupts
|
||||
# pnpm's hardlinked store and .bin shims. The production image already
|
||||
# bakes node_modules in at build time and is NOT bind-mounted here.
|
||||
# For local dev use a *named volume* instead of a host bind:
|
||||
# - node_modules:/app/node_modules
|
||||
# and never share `node_modules` / `pnpm store` via the Docker bind.
|
||||
# On macOS add `:cached`/`:delegated` consistency labels per mount.
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD", "node", "-e", "fetch('http://localhost:3002/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
|
||||
@@ -78,4 +88,90 @@ services:
|
||||
# - "3030:3030"
|
||||
# restart: unless-stopped
|
||||
# volumes:
|
||||
# - /var/www/Gamedata:/var/www/Gamedata
|
||||
# - /var/www/Gamedata:/var/www/Gamedata
|
||||
|
||||
# ── MariaDB "Turbo" (heavy JSON / bulk-loads) ──
|
||||
# Dedicated MariaDB tuned for >50 MB JSON imports. All tuning lives inline
|
||||
# in the service `command:` (bulk_insert_buffer_size,
|
||||
# innodb_flush_log_at_trx_commit=2, max_allowed_packet=512M, ...) so the
|
||||
# container configures itself — no external .cnf to mount or keep in sync.
|
||||
# Opt-in via profile so `docker compose up` keeps running the standalone
|
||||
# stack as today.
|
||||
#
|
||||
# Start: docker compose --profile db up -d (= pnpm db:up)
|
||||
# Note: host networking binds 127.0.0.1:3306 → STOP the host MariaDB
|
||||
# first (the app's .env DATABASE_URL points at localhost:3306).
|
||||
# Fixes the "Pulling schema from database..." hang: raise
|
||||
# net_read/net_write_timeout (done above) + stop imports while
|
||||
# running `pnpm db:generate` / drizzle-kit introspection.
|
||||
mariadb-turbo:
|
||||
image: mariadb:11
|
||||
container_name: mariadb-turbo
|
||||
profiles: ["db"]
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# mariadb:11 uses MARIADB_*; MYSQL_* also works but is deprecated there.
|
||||
- MARIADB_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD:-root}
|
||||
- MARIADB_DATABASE=${MARIADB_DATABASE:-habbo}
|
||||
- MARIADB_USER=${MARIADB_USER:-cms}
|
||||
- MARIADB_PASSWORD=${MARIADB_PASSWORD:-cms}
|
||||
- MARIADB_AUTO_UPGRADE=1
|
||||
# MariaDB tunes itself — the Official image appends these flags to mysqld,
|
||||
# no external .cnf file needed.
|
||||
command: [
|
||||
# Charset
|
||||
"--character-set-server=utf8mb4",
|
||||
"--collation-server=utf8mb4_unicode_ci",
|
||||
# 50 MB JSON batches: raise the 16 MB default packet ceiling.
|
||||
"--max-allowed-packet=512M",
|
||||
"--net-buffer-length=1M",
|
||||
# Timeouts — fixes the "Pulling schema from database..." hang
|
||||
# (drizzle-kit introspection no longer starves behind big imports).
|
||||
"--connect-timeout=30",
|
||||
"--wait-timeout=3600",
|
||||
"--interactive-timeout=3600",
|
||||
"--net-read-timeout=600",
|
||||
"--net-write-timeout=600",
|
||||
# InnoDB: durability/performance trade-off for bulk writes.
|
||||
"--innodb-flush-log-at-trx-commit=2",
|
||||
"--innodb-buffer-pool-size=2G",
|
||||
"--innodb-buffer-pool-instances=4",
|
||||
"--innodb-log-file-size=1G",
|
||||
"--innodb-log-buffer-size=64M",
|
||||
"--innodb-flush-method=O_DIRECT",
|
||||
"--innodb-autoextend-increment=512",
|
||||
"--innodb-max-dirty-pages-pct=90",
|
||||
"--bulk-insert-buffer-size=512M",
|
||||
# Raise so the engine nearly never double-writes during a 50 MB load.
|
||||
"--innodb-doublewrite=0",
|
||||
# libaio/native_aio misbehaves in some containers; io_uring path is fine.
|
||||
"--innodb-use-native-aio=0",
|
||||
# Temp tables used when MariaDB scans JSON blobs cannot be indexed.
|
||||
"--tmp-table-size=256M",
|
||||
"--max-heap-table-size=256M",
|
||||
"--read-buffer-size=4M",
|
||||
"--read-rnd-buffer-size=16M",
|
||||
# Save ~1-2 GB RAM; bulk loads don't need the instrumentation.
|
||||
"--performance-schema=OFF",
|
||||
"--skip-name-resolve",
|
||||
]
|
||||
volumes:
|
||||
# Named volume, NOT a host bind — shared-filesystem sync trashes InnoDB
|
||||
# files the same way it trashes pnpm's node_modules. Named volumes live
|
||||
# inside the container filesystem, so 50 MB of JSON writes never cross a
|
||||
# host-sync boundary.
|
||||
- mariadb-turbo-data:/var/lib/mysql
|
||||
healthcheck:
|
||||
# healthcheck.sh ships in the official mariadb image.
|
||||
test: ["CMD-SHELL", "healthcheck.sh --connect --innodb_initialized || mariadb-admin ping --silent"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
mem_limit: 4g
|
||||
|
||||
# Named volumes declared once; used by the MariaDB service above.
|
||||
volumes:
|
||||
mariadb-turbo-data:
|
||||
driver: local
|
||||
Reference in new issue
Block a user