feat(db): add self-tuned mariadb-turbo container and bulk JSON importer
- docker-compose: opt-in mariadb-turbo service (profile 'db') with inline mysqld tuning (max_allowed_packet=512M, innodb_flush_log_at_trx_commit=2, 2G buffer pool, net_read/write_timeout=600) for >50 MB JSON bulk loads; named volume for the datadir + node_modules host-sync guidance - scripts/bulk-import-json.ts: chunked (2000-row) idempotent importer with ON DUPLICATE KEY UPDATE, resumable, habbo furnidata or flat-array input - src/db/schema-gamedata.ts: promote+index JSON storage schema (furnidata, docs, texts) incl. VIRTUAL generated columns for MariaDB - package.json: add db:bulk, db:up, db:down, db:introspect, db:schema:generate
This commit is contained in:
1 parent
a640e40a5d
commit
52459c0b74
5 files changed
+540
-2
No files matched your search
@@ -0,0 +1,106 @@
|
||||
// Gamedata / large-JSON storage schema for MariaDB.
|
||||
//
|
||||
// MariaDB's JSON type is an alias for LONGTEXT (+ a `CHECK (json_valid(...))`
|
||||
// constraint), so it can NEVER be indexed directly. The "smart" pattern:
|
||||
// 1. keep the raw JSON document in longtext / mediumtext — full fidelity,
|
||||
// no validation overhead, no 4 GB fetch if only a key is needed;
|
||||
// 2. promote the fields you actually WHERE/ORDER BY into real columns;
|
||||
// 3. index those columns (optionally as VIRTUAL generated columns using
|
||||
// JSON_EXTRACT so you don't duplicate data in the seed script).
|
||||
//
|
||||
// Not part of drizzle's auto-generated src/db/schema.ts (which is regenerated
|
||||
// from the live emulator DB) — this file is a standalone schema module you
|
||||
// pass to `drizzle(conn, { schema })` in bulk scripts or import directly.
|
||||
|
||||
import { sql } from "drizzle-orm";
|
||||
import {
|
||||
index,
|
||||
int,
|
||||
longtext,
|
||||
mediumtext,
|
||||
mysqlEnum,
|
||||
mysqlTable,
|
||||
uniqueIndex,
|
||||
varchar,
|
||||
} from "drizzle-orm/mysql-core";
|
||||
|
||||
// ── Furnidata: one row per furniture/clothing item ───────────────────────────
|
||||
// Compatible with the habbo furnidata_json shape:
|
||||
// { "roomitemtypes": { "furnitype": [ { id, classname, public_name, ... } ] },
|
||||
// "wallitemtypes": { "furnitype": [ ... ] },
|
||||
// "effecttypes": { "effecttype": [ ... ] } }
|
||||
export const GamedataFurnidata = mysqlTable(
|
||||
"gamedata_furnidata",
|
||||
{
|
||||
id: int("id", { unsigned: true }).autoincrement().primaryKey().notNull(),
|
||||
source: varchar("source", { length: 64 }).notNull().default("habbo"),
|
||||
spriteId: int("sprite_id", { unsigned: true }).notNull(),
|
||||
// "s" = floor/room item, "i" = wall item, "e" = clothing/effect.
|
||||
kind: mysqlEnum("kind", ["s", "i", "e"]).notNull().default("s"),
|
||||
className: varchar("class_name", { length: 128 }).notNull(),
|
||||
// Promoted from JSON so hot lookups & LIST/WHERE never touch the blob.
|
||||
publicName: varchar("public_name", { length: 128 }).notNull().default(""),
|
||||
// Raw JSON document, kept intact for exact round-trips (catalog export).
|
||||
payload: longtext("payload").notNull(),
|
||||
},
|
||||
(t) => [
|
||||
// Idempotent seeds upsert on this key (source, sprite_id).
|
||||
uniqueIndex("gamedata_furnidata_source_sprite").on(t.source, t.spriteId),
|
||||
index("gamedata_furnidata_class").on(t.className),
|
||||
index("gamedata_furnidata_kind").on(t.kind),
|
||||
],
|
||||
);
|
||||
|
||||
// ── Generic JSON document store ───────────────────────────────────────────────
|
||||
// Use for any >50 MB JSON dump split into per-key documents (external_texts,
|
||||
// product data, achievements, etc.). `title` is a VIRTUAL generated column
|
||||
// extracted from the payload via JSON_EXTRACT and indexed — MariaDB supports
|
||||
// secondary indexes on virtual columns (10.2+), so filtering by a payload
|
||||
// field stays fast without storing a duplicate copy.
|
||||
export const GamedataDocs = mysqlTable(
|
||||
"gamedata_docs",
|
||||
{
|
||||
id: int("id", { unsigned: true }).autoincrement().primaryKey().notNull(),
|
||||
category: varchar("category", { length: 64 }).notNull(),
|
||||
docKey: varchar("doc_key", { length: 128 }).notNull(),
|
||||
payload: longtext("payload").notNull(),
|
||||
// SHA-1 of the raw payload → cheap change detection on re-runs.
|
||||
payloadSha1: varchar("payload_sha1", { length: 40 }).notNull().default(""),
|
||||
// VIRTUAL (default) column; the JSON_EXTRACT runs at query/index time.
|
||||
// NOT NULL forces the extract to resolve — use .notNull() only when
|
||||
// every document guarantees the field.
|
||||
title: varchar("title", { length: 255 })
|
||||
.generatedAlwaysAs(
|
||||
sql`CAST(JSON_UNQUOTE(JSON_EXTRACT(payload, '$.title')) AS CHAR(255))`,
|
||||
{ mode: "virtual" },
|
||||
)
|
||||
.notNull(),
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex("gamedata_docs_cat_key").on(t.category, t.docKey),
|
||||
index("gamedata_docs_title").on(t.title),
|
||||
],
|
||||
);
|
||||
|
||||
// ── Key/value texts (external_texts style, split by category) ───────────────
|
||||
// One row per string; bulk-loads stay resumable via (category, text_key).
|
||||
export const GamedataTexts = mysqlTable(
|
||||
"gamedata_texts",
|
||||
{
|
||||
id: int("id", { unsigned: true }).autoincrement().primaryKey().notNull(),
|
||||
category: varchar("category", { length: 64 }).notNull(),
|
||||
textKey: varchar("text_key", { length: 255 }).notNull(),
|
||||
value: mediumtext("value").notNull(),
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex("gamedata_texts_cat_key").on(t.category, t.textKey),
|
||||
index("gamedata_texts_key").on(t.textKey),
|
||||
],
|
||||
);
|
||||
|
||||
export type GamedataFurnidataRow = typeof GamedataFurnidata.$inferSelect;
|
||||
export type GamedataFurnidataInsert = typeof GamedataFurnidata.$inferInsert;
|
||||
export type GamedataDocsRow = typeof GamedataDocs.$inferSelect;
|
||||
export type GamedataDocsInsert = typeof GamedataDocs.$inferInsert;
|
||||
export type GamedataTextsRow = typeof GamedataTexts.$inferSelect;
|
||||
export type GamedataTextsInsert = typeof GamedataTexts.$inferInsert;
|
||||
Reference in new issue
Block a user