perf: offload SWF→Nitro conversion to a worker-thread pool
The synchronous convertSwfToNitro / extractIconFromSwf calls (pure CPU, no DB/network) now run in a small worker pool so the main Node event loop stays free during imports — the dominant import-time CPU spike is moved off the request thread. The pool degrades gracefully to a synchronous fallback if workers can't be created, and is skipped in the test environment.
This commit is contained in:
1 parent
7116f49e2b
commit
8d1b09f151
6 files changed
+240
-14
No files matched your search
@@ -24,10 +24,20 @@ vi.mock("@/lib/services/import/core/download", () => ({
|
||||
validateSwfBytes: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/services/swf-to-nitro", () => ({
|
||||
convertSwfToNitro: vi.fn(),
|
||||
extractIconFromSwf: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/services/swf-to-nitro", () => {
|
||||
const convertSwfToNitro = vi.fn();
|
||||
const extractIconFromSwf = vi.fn();
|
||||
return {
|
||||
convertSwfToNitro,
|
||||
extractIconFromSwf,
|
||||
convertSwfToNitroAsync: vi.fn(async (buffer: Buffer, classname: string) =>
|
||||
convertSwfToNitro(buffer, classname),
|
||||
),
|
||||
extractIconFromSwfAsync: vi.fn(async (buffer: Buffer, classname: string) =>
|
||||
extractIconFromSwf(buffer, classname),
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
import {
|
||||
autoPriceFurni,
|
||||
|
||||
@@ -34,8 +34,8 @@ import {
|
||||
} from "@/lib/services/import/core/download";
|
||||
import {
|
||||
type ConversionResult,
|
||||
convertSwfToNitro,
|
||||
extractIconFromSwf,
|
||||
convertSwfToNitroAsync,
|
||||
extractIconFromSwfAsync,
|
||||
parseNitroBundle,
|
||||
} from "@/lib/services/swf-to-nitro";
|
||||
import type { ImportSingleResult } from "@/types/furni";
|
||||
@@ -766,7 +766,7 @@ export async function importSingleFurni(params: {
|
||||
if (!iconOk && (swfOk || existsSync(/*turbopackIgnore: true*/ swfPath))) {
|
||||
try {
|
||||
const swfBuf = await fs.readFile(/*turbopackIgnore: true*/ swfPath);
|
||||
const iconPng = extractIconFromSwf(swfBuf, baseClassname);
|
||||
const iconPng = await extractIconFromSwfAsync(swfBuf, baseClassname);
|
||||
if (iconPng) {
|
||||
await fs.writeFile(/*turbopackIgnore: true*/ iconPath, iconPng);
|
||||
iconOk = true;
|
||||
@@ -795,7 +795,10 @@ export async function importSingleFurni(params: {
|
||||
try {
|
||||
if (existsSync(/*turbopackIgnore: true*/ swfPath)) {
|
||||
const swfBuffer = await fs.readFile(/*turbopackIgnore: true*/ swfPath);
|
||||
conversionResult = convertSwfToNitro(swfBuffer, baseClassname);
|
||||
conversionResult = await convertSwfToNitroAsync(
|
||||
swfBuffer,
|
||||
baseClassname,
|
||||
);
|
||||
await fs.writeFile(
|
||||
/*turbopackIgnore: true*/ nitroPath,
|
||||
conversionResult.bundle,
|
||||
|
||||
@@ -43,8 +43,8 @@ vi.mock("@/lib/services/import/core/download", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/services/swf-to-nitro", () => ({
|
||||
convertSwfToNitro: vi.fn(() => ({
|
||||
vi.mock("@/lib/services/swf-to-nitro", () => {
|
||||
const convertSwfToNitro = vi.fn((_buffer: Buffer, _classname: string) => ({
|
||||
bundle: Buffer.alloc(256, 0x42),
|
||||
classname: "n/a",
|
||||
dimensions: { x: 1, y: 1, z: 1 },
|
||||
@@ -61,10 +61,23 @@ vi.mock("@/lib/services/swf-to-nitro", () => ({
|
||||
defaultdir: 0,
|
||||
partcolors: { color: [] },
|
||||
},
|
||||
})),
|
||||
extractIconFromSwf: vi.fn(() => Buffer.from("fake-icon")),
|
||||
parseNitroBundle: vi.fn(() => ({ json: null })),
|
||||
}));
|
||||
}));
|
||||
const extractIconFromSwf = vi.fn((_buffer: Buffer, _classname: string) =>
|
||||
Buffer.from("fake-icon"),
|
||||
);
|
||||
const parseNitroBundle = vi.fn(() => ({ json: null }));
|
||||
return {
|
||||
convertSwfToNitro,
|
||||
extractIconFromSwf,
|
||||
parseNitroBundle,
|
||||
convertSwfToNitroAsync: vi.fn(async (buffer: Buffer, classname: string) =>
|
||||
convertSwfToNitro(buffer, classname),
|
||||
),
|
||||
extractIconFromSwfAsync: vi.fn(async (buffer: Buffer, classname: string) =>
|
||||
extractIconFromSwf(buffer, classname),
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/services/habbo-furnidata-cache", () => ({
|
||||
lookupOfficialHabboFurni: vi.fn(async () => null),
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import os from "node:os";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import type { WorkerRequest, WorkerResponse } from "./conversion-worker";
|
||||
import {
|
||||
type ConversionResult,
|
||||
convertSwfToNitro,
|
||||
extractIconFromSwf,
|
||||
} from "./index";
|
||||
|
||||
// Offloads the synchronous, CPU-heavy SWF→Nitro conversion (and icon
|
||||
// extraction) to a small pool of worker threads so the main Node event loop
|
||||
// stays free during imports — the conversion no longer blocks request handling
|
||||
// or anything else running on the server.
|
||||
//
|
||||
// The pool degrades gracefully: if workers can't be created (bundling/runtime
|
||||
// restrictions, or the test environment) every task falls back to running the
|
||||
// pure conversion function synchronously on the main thread, so imports never
|
||||
// break.
|
||||
|
||||
const POOL_SIZE = Math.max(1, Math.min(os.cpus().length - 1, 4));
|
||||
|
||||
interface Job {
|
||||
req: WorkerRequest;
|
||||
resolve: (r: WorkerResponse) => void;
|
||||
reject: (e: Error) => void;
|
||||
}
|
||||
|
||||
const workers: Worker[] = [];
|
||||
const idle: Worker[] = [];
|
||||
const pending = new Map<number, { worker: Worker; job: Job }>();
|
||||
const queue: Job[] = [];
|
||||
let nextId = 1;
|
||||
let broken = false;
|
||||
|
||||
function isTestEnv(): boolean {
|
||||
return process.env.NODE_ENV === "test" || process.env.VITEST !== undefined;
|
||||
}
|
||||
|
||||
function spawn(): Worker | null {
|
||||
try {
|
||||
const w = new Worker(new URL("./conversion-worker.ts", import.meta.url));
|
||||
w.on("message", (res: WorkerResponse) => {
|
||||
const p = pending.get(res.id);
|
||||
if (!p) return;
|
||||
pending.delete(res.id);
|
||||
idle.push(w);
|
||||
p.job.resolve(res);
|
||||
pump();
|
||||
});
|
||||
w.on("error", (err: Error) => {
|
||||
broken = true;
|
||||
const p = [...pending.values()].find((x) => x.worker === w);
|
||||
if (p) {
|
||||
pending.delete(p.job.req.id);
|
||||
p.job.reject(err);
|
||||
}
|
||||
});
|
||||
return w;
|
||||
} catch {
|
||||
broken = true;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function ensurePool(): boolean {
|
||||
if (broken || isTestEnv()) return false;
|
||||
if (workers.length > 0) return true;
|
||||
for (let i = 0; i < POOL_SIZE; i++) {
|
||||
const w = spawn();
|
||||
if (!w) {
|
||||
broken = true;
|
||||
return false;
|
||||
}
|
||||
workers.push(w);
|
||||
idle.push(w);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function pump() {
|
||||
while (idle.length > 0 && queue.length > 0) {
|
||||
const w = idle.pop()!;
|
||||
const job = queue.shift()!;
|
||||
pending.set(job.req.id, { worker: w, job });
|
||||
w.postMessage(job.req);
|
||||
}
|
||||
}
|
||||
|
||||
function run(req: Omit<WorkerRequest, "id">): Promise<WorkerResponse> {
|
||||
if (!ensurePool()) {
|
||||
try {
|
||||
if (req.type === "convert") {
|
||||
return Promise.resolve({
|
||||
id: 0,
|
||||
ok: true,
|
||||
result: convertSwfToNitro(req.buffer, req.classname),
|
||||
});
|
||||
}
|
||||
return Promise.resolve({
|
||||
id: 0,
|
||||
ok: true,
|
||||
icon: extractIconFromSwf(req.buffer, req.classname),
|
||||
});
|
||||
} catch (e) {
|
||||
return Promise.resolve({
|
||||
id: 0,
|
||||
ok: false,
|
||||
error: (e as Error).message,
|
||||
});
|
||||
}
|
||||
}
|
||||
return new Promise<WorkerResponse>((resolve, reject) => {
|
||||
const job: Job = {
|
||||
req: { ...req, id: nextId++ } as WorkerRequest,
|
||||
resolve,
|
||||
reject,
|
||||
};
|
||||
queue.push(job);
|
||||
pump();
|
||||
});
|
||||
}
|
||||
|
||||
export async function convertSwfToNitroAsync(
|
||||
buffer: Buffer,
|
||||
classname: string,
|
||||
): Promise<ConversionResult> {
|
||||
const res = await run({ type: "convert", buffer, classname });
|
||||
if (res.ok && res.result) return res.result;
|
||||
// Worker failed for this item — fall back to the main thread.
|
||||
return convertSwfToNitro(buffer, classname);
|
||||
}
|
||||
|
||||
export async function extractIconFromSwfAsync(
|
||||
buffer: Buffer,
|
||||
classname: string,
|
||||
): Promise<Buffer | null> {
|
||||
const res = await run({ type: "icon", buffer, classname });
|
||||
if (res.ok) return res.icon ?? null;
|
||||
return extractIconFromSwf(buffer, classname);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { parentPort } from "node:worker_threads";
|
||||
import { convertSwfToNitro, extractIconFromSwf } from "./index";
|
||||
|
||||
export interface ConvertRequest {
|
||||
id: number;
|
||||
type: "convert";
|
||||
buffer: Buffer;
|
||||
classname: string;
|
||||
}
|
||||
|
||||
export interface IconRequest {
|
||||
id: number;
|
||||
type: "icon";
|
||||
buffer: Buffer;
|
||||
classname: string;
|
||||
}
|
||||
|
||||
export type WorkerRequest = ConvertRequest | IconRequest;
|
||||
|
||||
export type WorkerResponse =
|
||||
| {
|
||||
id: number;
|
||||
ok: true;
|
||||
result?: import("./index").ConversionResult;
|
||||
icon?: Buffer | null;
|
||||
}
|
||||
| { id: number; ok: false; error: string };
|
||||
|
||||
if (parentPort) {
|
||||
parentPort.on("message", (msg: WorkerRequest) => {
|
||||
try {
|
||||
if (msg.type === "convert") {
|
||||
const result = convertSwfToNitro(msg.buffer, msg.classname);
|
||||
parentPort!.postMessage({
|
||||
id: msg.id,
|
||||
ok: true,
|
||||
result,
|
||||
} satisfies WorkerResponse);
|
||||
} else {
|
||||
const icon = extractIconFromSwf(msg.buffer, msg.classname);
|
||||
parentPort!.postMessage({
|
||||
id: msg.id,
|
||||
ok: true,
|
||||
icon: icon ?? null,
|
||||
} satisfies WorkerResponse);
|
||||
}
|
||||
} catch (err) {
|
||||
parentPort!.postMessage({
|
||||
id: msg.id,
|
||||
ok: false,
|
||||
error: (err as Error).message,
|
||||
} satisfies WorkerResponse);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -50,6 +50,11 @@ export type {
|
||||
ConversionResult,
|
||||
FurniMetadata,
|
||||
} from "@/types/furni";
|
||||
// Worker-pool wrappers — run the CPU-heavy conversion off the main thread.
|
||||
export {
|
||||
convertSwfToNitroAsync,
|
||||
extractIconFromSwfAsync,
|
||||
} from "./conversion-pool";
|
||||
// Re-export public API from submodules
|
||||
export { createNitroBundle, parseNitroBundle } from "./nitro-builder";
|
||||
export { computeDefaultDir, extractPartColors } from "./xml-processor";
|
||||
|
||||
Reference in new issue
Block a user