feat(import): run catalog synchronizations as durable jobs
This commit is contained in:
1 parent
a2954e4408
commit
76f0420d64
14 files changed
+559
-801
No files matched your search
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,97 +1,8 @@
|
||||
import { withAdmin } from "@/lib/api-handler";
|
||||
import { db, ItemsBase } from "@/lib/db";
|
||||
import { PERMS } from "@/lib/permissions";
|
||||
import { importSingleFurni } from "@/lib/services/furni-import";
|
||||
import { getOfficialHabboFurnidata } from "@/lib/services/habbo-furnidata-cache";
|
||||
import { runSseBatch } from "@/lib/services/import/core/sse-batch";
|
||||
|
||||
import { enqueueFurnitureSync } from "@/lib/services/furni-sync-queue";
|
||||
export const POST = withAdmin(
|
||||
{ permission: PERMS.ASSETS_IMPORT },
|
||||
async (request) => {
|
||||
// Haal alle officiële furnidata
|
||||
const furnidata = await getOfficialHabboFurnidata();
|
||||
|
||||
// Haal alle bestaande classnames uit de database
|
||||
const existingClassnames = new Set(
|
||||
await db
|
||||
.select({ itemName: ItemsBase.itemName })
|
||||
.from(ItemsBase)
|
||||
.then((rows) => rows.map((r) => r.itemName)),
|
||||
);
|
||||
|
||||
// Filter naar alleen items die niet in de database staan
|
||||
const missingItems = Array.from(furnidata.entries())
|
||||
.filter(([, entry]) => !existingClassnames.has(entry.classname))
|
||||
.map(([, entry]) => ({
|
||||
classname: entry.classname,
|
||||
name: entry.name,
|
||||
description: entry.description ?? "",
|
||||
type: entry.category === "wallitem" ? "wallitem" : "flooritem",
|
||||
revision: entry.revision ?? 0,
|
||||
category: entry.category ?? "unknown",
|
||||
}));
|
||||
|
||||
if (missingItems.length === 0) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: "All items are already in the database",
|
||||
imported: 0,
|
||||
}),
|
||||
{
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (missingItems.length > 500) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
message: `Too many missing items (${missingItems.length}). Consider importing in smaller batches.`,
|
||||
}),
|
||||
{
|
||||
headers: { "Content-Type": "application/json" },
|
||||
status: 400,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Importeren via SSE batch
|
||||
return runSseBatch({
|
||||
items: missingItems,
|
||||
concurrency: 3,
|
||||
signal: request.signal,
|
||||
labelOf: (it) => it.classname,
|
||||
worker: async (it, _index, report) => {
|
||||
const result = await importSingleFurni({
|
||||
id: 0,
|
||||
classname: it.classname,
|
||||
name: it.name,
|
||||
description: it.description,
|
||||
type: it.type,
|
||||
revision: it.revision,
|
||||
category: it.category,
|
||||
onProgress: report,
|
||||
});
|
||||
|
||||
if (result.ok) {
|
||||
return {
|
||||
ok: true,
|
||||
warnings: result.warnings,
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
warnings: result.warnings,
|
||||
error: result.error,
|
||||
};
|
||||
},
|
||||
flush: async () => {
|
||||
// FurnitureData.json wordt automatisch geschreven door importSingleFurni
|
||||
},
|
||||
});
|
||||
},
|
||||
(request, ctx) =>
|
||||
enqueueFurnitureSync(request, ctx.session.user.id, "official"),
|
||||
);
|
||||
@@ -2,6 +2,7 @@ import { after } from "next/server";
|
||||
import { z } from "zod";
|
||||
import { apiError, apiOk } from "@/lib/api";
|
||||
import { withAdmin } from "@/lib/api-handler";
|
||||
import { getRequestId } from "@/lib/foundation/request-context";
|
||||
import { validateClassnames } from "@/lib/furni/studio-inspection";
|
||||
import { PERMS } from "@/lib/permission-slugs";
|
||||
import { redis } from "@/lib/redis";
|
||||
@@ -92,6 +93,7 @@ export const POST = withAdmin(
|
||||
const job = await store.create({
|
||||
...body,
|
||||
userId: ctx.session.user.id,
|
||||
operationId: getRequestId(),
|
||||
createdAt,
|
||||
updatedAt: createdAt,
|
||||
state: "queued",
|
||||
|
||||
Reference in new issue
Block a user