Support Gitea configuration and safe Catalog Studio furniture completion
CI / check (push) Successful in 1m31s
CI / deploy (push) Successful in 1m18s
CI / e2e (push) Successful in 21s

This commit is contained in:
Simo committed 2026-09-05 13:33:40 +02:00
1 parent 26f071117b
commit 9f791ba284
22 files changed
+1120 -166

No files matched your search

+3
View File
@@ -71,6 +71,9 @@ RUN if [ "$PACKAGE_MANAGER" = "yarn" ] || { [ -z "$PACKAGE_MANAGER" ] && [ -f ya
# --- Runtime stage ---
FROM node:26.8.1-bookworm-slim AS runner
# Catalog Studio publishes to self-hosted Git repositories over HTTPS.
RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/*
# The CMS writes to bind-mounted host directories (/var/www/Gamedata is owned by
# the host's www-data user, UID/GID 33). The node base image already ships a
# www-data user with UID/GID 33, which matches that ownership — so we run as
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,63 @@
[data-nitro-editor] {
color: var(--admin-text);
background: var(--admin-surface);
border-color: var(--admin-border);
}
[data-nitro-editor] section,
[data-nitro-editor] .border,
[data-nitro-editor] .border-b,
[data-nitro-editor] .border-t {
border-color: var(--admin-border);
}
[data-nitro-editor] section {
background: var(--admin-canvas);
}
[data-nitro-editor] h3 {
font-size: 14px;
margin: 0;
}
[data-nitro-editor] [data-slot="switch"] {
width: 40px;
height: 24px;
background: var(--admin-border);
border: 1px solid var(--admin-text-muted);
}
[data-nitro-editor] [data-slot="switch"][data-checked] {
background: var(--admin-accent);
border-color: var(--admin-accent);
}
[data-nitro-editor] [data-slot="switch-thumb"] {
width: 18px;
height: 18px;
background: var(--admin-text);
color: var(--admin-canvas);
left: 2px;
}
[data-nitro-editor] [data-slot="switch-thumb"][data-checked] {
left: calc(100% - 20px);
}
[data-nitro-editor] .nitro-direction {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 3px;
min-height: 52px;
border: 1px solid var(--admin-border);
border-radius: 8px;
background: var(--admin-surface);
color: var(--admin-text-muted);
font-size: 12px;
}
[data-nitro-editor] .nitro-direction[aria-pressed="true"] {
border-color: var(--admin-accent);
background: var(--admin-accent);
color: var(--admin-accent-foreground);
}
[data-nitro-editor] .nitro-direction:focus-visible {
outline: 2px solid var(--admin-accent);
outline-offset: 3px;
}
[data-nitro-editor] [data-slot="dialog-footer"] {
flex-shrink: 0;
}
@@ -3,7 +3,11 @@
import { useCallback, useEffect, useState } from "react";
import { adminFetch } from "@/lib/admin-fetch";
import { CatalogGitConfiguration } from "./catalog-git-configuration";
type ExportStatus = {
remote?: string;
branch?: string;
enabled: boolean;
pending: number;
active: number;
@@ -13,7 +17,11 @@ type ExportStatus = {
error?: string;
};
export function CatalogExportClient() {
export function CatalogExportClient({
canConfigure = false,
}: {
canConfigure?: boolean;
}) {
const [status, setStatus] = useState<ExportStatus | null>(null);
const [error, setError] = useState("");
const [busy, setBusy] = useState(false);
@@ -55,26 +63,34 @@ export function CatalogExportClient() {
<h2 className="text-lg font-semibold">Catalog repository export</h2>
<p className="text-sm text-muted-foreground">
Automatically publishes bundles, icons, gamedata, translations and
catalog SQL to Epicnabbo-Catalogus-Updated-Daily (Beta-3).
catalog SQL to your configured Gitea repository.
</p>
{canConfigure && (
<CatalogGitConfiguration onSaved={() => void refresh()} />
)}
{status?.remote && (
<p className="text-xs break-all">
{status.remote} · {status.branch}
</p>
)}
<div role="status" className="text-sm" aria-live="polite">
{!status
? "Loading…"
: !status.enabled
? "Not configured on this server."
? "Export disabled. Configure the Gitea connection below."
: status.active
? `Waiting for ${status.active} operation(s) to finish.`
: status.running
? "Export in progress…"
: status.pending
? `${status.pending} update(s) queued for the worker.`
? `${status.pending} update(s) queued for export.`
: "No pending updates."}
{status?.commit && (
<p>
Last published commit:{" "}
<a
className="underline"
href={`https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily/commit/${status.commit}`}
href={`${status.remote?.replace(/\.git$/, "")}/commit/${status.commit}`}
target="_blank"
rel="noreferrer"
>
File diff suppressed because it is too large. Load diff
+7 -1
View File
@@ -14,7 +14,13 @@ export default async function StudioSyncPage(_props: {
}
return (
<div className="space-y-8">
<CatalogExportClient />
<CatalogExportClient
canConfigure={canAccess(
permissions,
PERMS.SETTINGS_EDIT,
session.user.rank,
)}
/>
<h1 className="text-2xl font-bold tracking-tight">Sync All Sources</h1>
<p className="text-muted-foreground mt-2">
Scan configured sources and automatically import any furniture that is
@@ -0,0 +1,89 @@
// @vitest-environment node
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { NextRequest } from "next/server";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({ connection: vi.fn(), guard: vi.fn() }));
vi.mock("@/lib/api-handler", () => ({
withAdmin: (options: unknown, handler: unknown) => {
mocks.guard(options);
return handler;
},
}));
vi.mock("@/lib/services/catalog-git-managed", () => ({
testCatalogConnection: mocks.connection,
}));
import { PERMS } from "@/lib/permission-slugs";
import { GET, POST } from "./route";
let root: string;
const input = {
action: "save",
enabled: true,
remote: "https://gitea.example/user/catalog",
branch: "main",
username: "user",
token: "test-private-token",
};
const request = (data = input) =>
new NextRequest("http://localhost/api/admin/catalog-export/config", {
method: "POST",
body: JSON.stringify(data),
});
describe("Gitea configuration endpoint", () => {
beforeEach(async () => {
root = await fs.mkdtemp(path.join(os.tmpdir(), "catalog-config-test-"));
vi.stubEnv("CATALOG_GIT_STATE_DIR", root);
vi.stubEnv("AUTH_SECRET", "test-secret");
mocks.connection.mockReset().mockResolvedValue(undefined);
});
afterEach(async () => {
vi.unstubAllEnvs();
await fs.rm(root, { recursive: true, force: true });
});
it("requires settings permission and never returns the token", async () => {
expect(mocks.guard).toHaveBeenCalledWith({
permission: PERMS.SETTINGS_EDIT,
});
const response = await POST(request());
expect(response.status).toBe(200);
const data = await response.json();
expect(data.hasToken).toBe(true);
expect(data.token).toBeUndefined();
expect(data.encryptedToken).toBeUndefined();
expect(
await fs.readFile(path.join(root, "config.json"), "utf8"),
).not.toContain(input.token);
expect(
await (
await GET(
new NextRequest("http://localhost/api/admin/catalog-export/config"),
)
).text(),
).not.toContain(input.token);
});
it("does not save unverified enabled configurations", async () => {
mocks.connection.mockRejectedValue(new Error("Cannot connect"));
expect((await POST(request())).status).toBe(400);
expect(await fs.readdir(root)).toEqual([]);
});
it("rejects configuration changes during an export", async () => {
await fs.writeFile(path.join(root, "worker.lock"), "active");
expect((await POST(request())).status).toBe(409);
expect(mocks.connection).not.toHaveBeenCalled();
});
it("prevents moving queued changes to a different repository", async () => {
await POST(request());
await fs.writeFile(path.join(root, "change.pending"), "pending");
expect(
(
await POST(
request({ ...input, remote: "https://other.example/user/repo" }),
)
).status,
).toBe(409);
});
});
@@ -0,0 +1,91 @@
import { promises as fs } from "node:fs";
import { hostname } from "node:os";
import path from "node:path";
import { withAdmin } from "@/lib/api-handler";
import { apiError, apiOk } from "@/lib/api-response";
import { PERMS } from "@/lib/permission-slugs";
import {
catalogStateRoot,
publicCatalogConfig,
readManagedCatalogConfig,
saveCatalogConfig,
validateCatalogConfig,
} from "@/lib/services/catalog-git-config";
import { testCatalogConnection } from "@/lib/services/catalog-git-managed";
export const GET = withAdmin({ permission: PERMS.SETTINGS_EDIT }, async () =>
apiOk(publicCatalogConfig()),
);
export const POST = withAdmin(
{ permission: PERMS.SETTINGS_EDIT },
async (request) => {
const body = await request.json().catch(() => null);
if (!body || !["save", "test"].includes(body.action))
return apiError("Choose save or test", 400);
let config: ReturnType<typeof validateCatalogConfig>;
try {
config = validateCatalogConfig(body, readManagedCatalogConfig());
} catch (error) {
return apiError((error as Error).message, 400);
}
if (body.action === "test") {
try {
await testCatalogConnection(config);
return apiOk({
message:
"Gitea repository and branch are readable. Write access is checked when exporting.",
});
} catch (error) {
return apiError((error as Error).message, 400);
}
}
const root = catalogStateRoot();
await fs.mkdir(root, { recursive: true });
const lock = await fs
.open(path.join(root, "worker.lock"), "wx")
.catch((error: NodeJS.ErrnoException) => {
if (error.code === "EEXIST") return null;
throw error;
});
if (!lock)
return apiError("An export is running. Retry after it finishes.", 409);
try {
await lock.writeFile(
JSON.stringify({ pid: process.pid, host: hostname() }),
);
const entries = await fs.readdir(root);
if (entries.some((name) => name.endsWith(".active")))
return apiError(
"Wait for active catalog operations to finish before changing Git configuration.",
409,
);
const previous = readManagedCatalogConfig();
if (
previous &&
(previous.remote !== config.remote ||
previous.branch !== config.branch) &&
entries.some((name) => name.endsWith(".pending"))
)
return apiError(
"Export pending updates before changing repository or branch.",
409,
);
if (config.enabled) {
try {
await testCatalogConnection(config);
} catch (error) {
return apiError((error as Error).message, 400);
}
}
await saveCatalogConfig(config);
if (
previous &&
(previous.remote !== config.remote || previous.branch !== config.branch)
)
await fs.rm(path.join(root, "status.json"), { force: true });
return apiOk(publicCatalogConfig(config));
} finally {
await lock.close();
await fs.rm(path.join(root, "worker.lock"), { force: true });
}
},
);
+16 -3
View File
@@ -1,18 +1,31 @@
import { withAdmin } from "@/lib/api-handler";
import { apiError, apiOk } from "@/lib/api-response";
import { PERMS } from "@/lib/permissions";
import { publicCatalogConfig } from "@/lib/services/catalog-git-config";
import {
catalogExportEnabled,
catalogExportQueue,
catalogExportStatus,
scheduleCatalogExport,
} from "@/lib/services/catalog-git-queue";
export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () =>
apiOk(await catalogExportStatus()),
);
export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => {
const status = await catalogExportStatus();
if (
status.enabled &&
status.pending > 0 &&
status.active === 0 &&
!("running" in status && status.running) &&
!("error" in status && status.error)
)
await scheduleCatalogExport();
const config = publicCatalogConfig();
return apiOk({ ...status, remote: config.remote, branch: config.branch });
});
export const POST = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => {
if (!catalogExportEnabled())
return apiError("Catalog repository export is not configured", 400);
await catalogExportQueue().request();
await scheduleCatalogExport();
return apiOk(await catalogExportStatus());
});
@@ -125,6 +125,7 @@ export const POST = withAdmin(
revision: item.revision ?? 0,
category: item.category ?? "unknown",
skipFurniDataWrite: false,
repairExisting: body.repairExisting === true,
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
nitroBaseUrl: source?.nitroBaseUrl,
iconBaseUrl: source?.iconBaseUrl,
+2
View File
@@ -507,6 +507,7 @@ export const POST = withAdmin(
nitroBaseUrl: source?.nitroBaseUrl || undefined,
iconBaseUrl: source?.iconBaseUrl || undefined,
skipFurniDataWrite: false,
repairExisting: body.repairExisting === true,
});
if (result.ok) {
@@ -679,6 +680,7 @@ export const POST = withAdmin(
revision: revision ?? 0,
category: category ?? "unknown",
updateExisting: updateExisting === true,
repairExisting: body.repairExisting === true,
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
nitroBaseUrl: source?.nitroBaseUrl || undefined,
iconBaseUrl: source?.iconBaseUrl || undefined,
@@ -42,11 +42,7 @@ export function ImportReview({
const complete = items.every((item) => localByName.has(item.classname));
const ready = items.filter((item) => {
const local = localByName.get(item.classname);
return (
local &&
local.sql.length === 0 &&
compareFurniture(item, local).state !== "conflict"
);
return local && compareFurniture(item, local).state !== "conflict";
});
const blocked = items.filter((item) => {
const local = localByName.get(item.classname);
@@ -119,7 +115,7 @@ export function ImportReview({
<>
<p role="status" className="text-sm">
{ready.length} ready to import · {existing} already in SQL
(skipped) · {blocked} conflicts (skipped)
(complete missing data) · {blocked} conflicts (skipped)
</p>
{inspection.items.some(
(item) =>
@@ -184,7 +180,10 @@ export function ImportReview({
<>
<p className="text-sm">
Only the {ready.length} eligible items below will be
submitted. Existing SQL items and conflicts are excluded.
submitted. Existing furniture keeps its local IDs, SQL
settings and catalog offers. Missing assets and catalog
entries are completed; furnidata is refreshed. Conflicts are
excluded.
</p>
<p className="text-xs text-[var(--admin-text-muted)]">
Automatic categories and prices below are estimates. Source
@@ -692,6 +692,7 @@ export function StudioClient({
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
repairExisting: true,
id: item.id,
classname: item.classname,
name: item.name,
@@ -779,6 +780,7 @@ export function StudioClient({
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
sourceId: activeSource || undefined,
repairExisting: true,
items: toImport.map((i) => ({
id: i.id,
classname: i.classname,
@@ -2323,6 +2325,16 @@ export function StudioClient({
</Button>
) : (
<div className="space-y-2">
<Button
variant="outline"
className="w-full gap-1.5"
disabled={importingId !== null}
onClick={() =>
setReview({ items: [detail], single: true })
}
>
Complete missing data
</Button>
{!detail.nitroExists && (
<>
<p className="rounded-md border border-[var(--admin-warning)]/30 bg-[var(--admin-warning)]/10 p-2 text-[0.65rem] text-[var(--admin-warning)]">
+2 -2
View File
@@ -5,8 +5,8 @@ import { canAccess, getApiAdminContext } from "@/lib/permissions";
import { logServerError } from "@/lib/server-log";
import {
beginCatalogExport,
catalogExportEnabled,
catalogExportQueue,
isCatalogMutation,
} from "@/lib/services/catalog-git-queue";
@@ -92,7 +92,7 @@ export function withAdmin(
catalogExportEnabled() &&
isCatalogMutation(request.method, request.nextUrl.pathname)
) {
finishExport = await catalogExportQueue().begin();
finishExport = await beginCatalogExport();
}
const response = await handler(request, context, routeContext);
if (
@@ -0,0 +1,65 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
catalogGitEnvironment,
decryptCatalogToken,
publicCatalogConfig,
validateCatalogConfig,
} from "./catalog-git-config";
const input = {
enabled: true,
remote: "https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily",
branch: "Beta-3",
username: "remco",
token: "private-token",
};
describe("Gitea catalog configuration", () => {
beforeEach(() => vi.stubEnv("AUTH_SECRET", "test-auth-secret"));
afterEach(() => vi.unstubAllEnvs());
it("supports self-hosted Gitea and encrypts the token", () => {
const config = validateCatalogConfig(input, null);
expect(config.remote).toBe(`${input.remote}.git`);
expect(config.encryptedToken).not.toContain(input.token);
expect(decryptCatalogToken(config.encryptedToken)).toBe(input.token);
expect(JSON.stringify(publicCatalogConfig(config))).not.toContain(
input.token,
);
expect(publicCatalogConfig(config).provider).toBe("gitea");
});
it("preserves saved token only for the same repository and user", () => {
const previous = validateCatalogConfig(input, null);
expect(
validateCatalogConfig({ ...input, token: "" }, previous).encryptedToken,
).toBe(previous.encryptedToken);
expect(() =>
validateCatalogConfig(
{ ...input, token: "", remote: "https://other.example/user/repo" },
previous,
),
).toThrow();
});
it("rejects embedded credentials and unsafe branches", () => {
expect(() =>
validateCatalogConfig(
{ ...input, remote: "https://user:secret@host/user/repo" },
null,
),
).toThrow();
for (const branch of [
"--help",
"main..evil",
"../branch",
"a.lock",
"a\\b",
])
expect(() => validateCatalogConfig({ ...input, branch }, null)).toThrow();
});
it("supplies credentials through environment and disables redirects", () => {
const config = validateCatalogConfig(input, null);
const env = catalogGitEnvironment(config);
expect(env.GIT_CONFIG_KEY_0).toBe(
"http.https://gitlab.epicnabbo.nl/.extraHeader",
);
expect(env.GIT_CONFIG_VALUE_1).toBe("false");
});
});
File diff suppressed because it is too large. Load diff
+2 -1
View File
@@ -107,6 +107,7 @@ export async function publishCatalogFiles(options: {
remote: string;
branch: string;
files: CatalogFile[];
env?: NodeJS.ProcessEnv;
}) {
const { checkout, remote, branch, files } = options;
const git = async (...args: string[]) => {
@@ -114,7 +115,7 @@ export async function publishCatalogFiles(options: {
cwd: checkout,
timeout: 120_000,
maxBuffer: 16 * 1024 * 1024,
env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
env: { ...process.env, ...options.env, GIT_TERMINAL_PROMPT: "0" },
});
return stdout.trim();
};
+15 -4
View File
@@ -1,18 +1,22 @@
import { promises as fs } from "node:fs";
import { hostname } from "node:os";
import path from "node:path";
import {
catalogGitEnvironment,
readManagedCatalogConfig,
} from "./catalog-git-config";
import {
CATALOG_BRANCH,
CATALOG_REMOTE,
publishCatalogFiles,
recoverCatalogQueue,
} from "./catalog-git-core";
import { ensureManagedCatalogCheckout } from "./catalog-git-managed";
import { catalogExportEnabled, catalogExportQueue } from "./catalog-git-queue";
import { createCatalogSnapshot } from "./catalog-git-snapshot";
export async function runCatalogExport() {
const checkout = process.env.CATALOG_GIT_CHECKOUT?.trim();
if (!catalogExportEnabled() || !checkout) return;
if (!catalogExportEnabled()) return;
const queue = catalogExportQueue();
await queue.entries();
await recoverCatalogQueue(queue.root);
@@ -51,10 +55,17 @@ export async function runCatalogExport() {
// Any operation started/completed during capture invalidates the snapshot.
const current = await queue.batch();
if (!current || JSON.stringify(current) !== JSON.stringify(batch)) return;
const config = readManagedCatalogConfig();
if (config && !config.enabled) return;
const checkout = config
? await ensureManagedCatalogCheckout(config)
: process.env.CATALOG_GIT_CHECKOUT?.trim();
if (!checkout) return;
const commit = await publishCatalogFiles({
checkout,
remote: CATALOG_REMOTE,
branch: CATALOG_BRANCH,
remote: config?.remote ?? CATALOG_REMOTE,
branch: config?.branch ?? CATALOG_BRANCH,
env: config ? catalogGitEnvironment(config) : undefined,
files,
});
await queue.complete(batch);
+72
View File
@@ -0,0 +1,72 @@
import { execFile } from "node:child_process";
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import path from "node:path";
import { promisify } from "node:util";
import {
catalogGitEnvironment,
type ManagedCatalogConfig,
managedCatalogCheckout,
} from "./catalog-git-config";
const exec = promisify(execFile);
export async function testCatalogConnection(config: ManagedCatalogConfig) {
try {
const result = await exec(
"git",
[
"ls-remote",
"--exit-code",
"--heads",
config.remote,
`refs/heads/${config.branch}`,
],
{
env: catalogGitEnvironment(config),
timeout: 30000,
maxBuffer: 1024 * 1024,
},
);
if (!result.stdout.trim()) throw new Error("Missing branch");
} catch {
throw new Error(
"Cannot read this Gitea repository and branch. Check the URL, username, token and server Git installation.",
);
}
}
export async function ensureManagedCatalogCheckout(
config: ManagedCatalogConfig,
) {
const checkout = managedCatalogCheckout(config);
try {
await fs.access(path.join(checkout, ".git"));
return checkout;
} catch {
/* first export */
}
await fs.mkdir(path.dirname(checkout), { recursive: true });
const temporary = `${checkout}.clone-${randomUUID()}`;
try {
await exec(
"git",
[
"clone",
"--single-branch",
"--branch",
config.branch,
"--",
config.remote,
temporary,
],
{
env: catalogGitEnvironment(config),
timeout: 600000,
maxBuffer: 4 * 1024 * 1024,
},
);
await fs.rename(temporary, checkout);
return checkout;
} finally {
await fs.rm(temporary, { recursive: true, force: true });
}
}
+30 -6
View File
@@ -1,15 +1,19 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import {
catalogStateRoot,
readManagedCatalogConfig,
} from "./catalog-git-config";
import { CatalogExportQueue } from "./catalog-git-core";
export function catalogExportEnabled() {
return Boolean(process.env.CATALOG_GIT_CHECKOUT?.trim());
const config = readManagedCatalogConfig();
return config
? config.enabled
: Boolean(process.env.CATALOG_GIT_CHECKOUT?.trim());
}
export function catalogExportQueue() {
return new CatalogExportQueue(
process.env.CATALOG_GIT_STATE_DIR ||
path.join(process.cwd(), "storage", "catalog-git"),
);
return new CatalogExportQueue(catalogStateRoot());
}
export function isCatalogMutation(method: string, pathname: string) {
return (
@@ -45,10 +49,30 @@ export async function withCatalogExport<T>(
operation: () => Promise<T>,
): Promise<T> {
if (!catalogExportEnabled()) return operation();
const finish = await catalogExportQueue().begin();
const finish = await beginCatalogExport();
try {
return await operation();
} finally {
await finish();
}
}
export async function scheduleCatalogExport() {
if (!readManagedCatalogConfig()?.enabled) return;
try {
const { after } = await import("next/server");
after(async () => {
const { runCatalogExport } = await import("./catalog-git-export");
await runCatalogExport();
});
} catch {
/* Standalone worker contexts use their scheduled retry. */
}
}
export async function beginCatalogExport() {
const finish = await catalogExportQueue().begin();
return async () => {
await finish();
await scheduleCatalogExport();
};
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff