This commit is contained in:
Simo committed 2026-09-09 18:29:06 +02:00
commit b3a6531d7b
35 files changed
+398 -1048

No files matched your search

-3
View File
@@ -34,8 +34,5 @@ public/cache
public/tmp public/tmp
db_backup_*.sql db_backup_*.sql
*.log *.log
playwright-report
test-results
blob-report
.codex .codex
.agents .agents
+2 -2
View File
@@ -76,8 +76,8 @@ PAYPAL_API=https://api-m.sandbox.paypal.com
# --- LOGGING --- # --- LOGGING ---
LOG_LEVEL=error LOG_LEVEL=error
# --- FLARESOLVERR (Cloudflare bypass for clone sources) --- # --- BYPARR (Cloudflare bypass for clone sources) ---
FLARESOLVERR_URL=http://localhost:8191 BYPARR_URL=http://localhost:8191
# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials. # Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials.
CATALOG_GIT_CHECKOUT= CATALOG_GIT_CHECKOUT=
+2 -5
View File
@@ -53,14 +53,11 @@ jobs:
BCRYPT_ROUNDS: 4 BCRYPT_ROUNDS: 4
run: | run: |
if [ -x /usr/bin/time ]; then if [ -x /usr/bin/time ]; then
/usr/bin/time -f 'Tests: %e seconds; peak process RSS: %M KiB' pnpm test --maxWorkers=2 /usr/bin/time -f 'Tests: %e seconds; peak process RSS: %M KiB' pnpm test:coverage --maxWorkers=4
else else
time pnpm test --maxWorkers=2 time pnpm test:coverage --maxWorkers=4
fi fi
- name: Knip (unused files/exports)
run: pnpm knip
# ───────────────────────────────────────────── # ─────────────────────────────────────────────
# Docker build & deploy # Docker build & deploy
# Draait op de host (self-hosted) zodat Docker # Draait op de host (self-hosted) zodat Docker
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
pnpm install --frozen-lockfile pnpm install --frozen-lockfile
pnpm typecheck pnpm typecheck
pnpm biome:lint pnpm biome:lint
pnpm test pnpm test:coverage
env: env:
SKIP_ENV_VALIDATION: 1 SKIP_ENV_VALIDATION: 1
NODE_ENV: test NODE_ENV: test
-5
View File
@@ -33,11 +33,6 @@ public/tmp/
# Test coverage reports # Test coverage reports
coverage/ coverage/
# Playwright test artifacts
test-results/
playwright-report/
blob-report/
.aider* .aider*
/public/vendor/tinymce/ /public/vendor/tinymce/
-1
View File
@@ -1 +0,0 @@
pnpm exec lint-staged
-2
View File
@@ -1,2 +0,0 @@
pnpm typecheck
pnpm test
+22 -7
View File
@@ -4,22 +4,37 @@ FROM node:26.8.1-alpine AS migrations
WORKDIR /app WORKDIR /app
ENV NEXT_TELEMETRY_DISABLED=1 ENV NEXT_TELEMETRY_DISABLED=1
# Keep the bootstrap aligned with package.json packageManager. # Keep the bootstrap aligned with package.json packageManager.
RUN apk add --no-cache git \ # The apk cache is persisted in a BuildKit cache mount so git is not
# re-downloaded on every build.
RUN --mount=type=cache,target=/var/cache/apk \
apk add --no-cache git \
&& npm install -g [email protected] && npm install -g [email protected]
# The pnpm store is kept in a BuildKit cache mount that persists across builds
# on the builder. This is what stops disk usage from growing unbounded: the
# downloaded dependency store is shared and reused instead of being copied into
# a fresh image layer on every build. Unlike an image layer it is also prunable
# independently, so a hard cap (see ci-deploy.sh) keeps it bounded.
ENV PNPM_HOME=/pnpm PNPM_STORE=/pnpm/store
# pnpm-workspace.yaml + .npmrc must be present too: the lockfile records the # pnpm-workspace.yaml + .npmrc must be present too: the lockfile records the
# overrides from pnpm-workspace.yaml, and --frozen-lockfile rejects a build # overrides from pnpm-workspace.yaml, and --frozen-lockfile rejects a build
# where the workspace config is absent (ERR_PNPM_LOCKFILE_CONFIG_MISMATCH). # where the workspace config is absent (ERR_PNPM_LOCKFILE_CONFIG_MISMATCH).
COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml* .npmrc* ./ COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml* .npmrc* ./
# pnpm fetch: download all deps into $PNPM_STORE first, so only the lockfile # pnpm fetch: download all deps into the shared cache-mounted store.
# change (not source changes) invalidates the network-heavy download layer. RUN --mount=type=cache,target=/pnpm \
RUN pnpm fetch --ignore-scripts pnpm fetch --ignore-scripts
RUN pnpm install --frozen-lockfile --ignore-scripts --offline # Install offline from the cache-mounted store; the store itself stays in the
# build cache between builds.
RUN --mount=type=cache,target=/pnpm \
pnpm install --frozen-lockfile --ignore-scripts --offline
COPY . . COPY . .
FROM migrations AS builder FROM migrations AS builder
ARG NEXT_DEPLOYMENT_ID="unknown" ARG NEXT_DEPLOYMENT_ID="unknown"
ENV NEXT_DEPLOYMENT_ID="$NEXT_DEPLOYMENT_ID" ENV NEXT_DEPLOYMENT_ID="$NEXT_DEPLOYMENT_ID"
# Fixture values exist only for this build command; production secrets are runtime-only. # Fixture values exist only for this build command; production secrets are runtime-only.
RUN DATABASE_URL="mysql://build:[email protected]:9/build" \ # Cache Next.js build output and webpack caches so rebuilds only redo the
# changed parts.
RUN --mount=type=cache,target=/app/.next/cache \
DATABASE_URL="mysql://build:[email protected]:9/build" \
HOTEL_NAME="Build fixture" APP_URL="http://localhost:3002" \ HOTEL_NAME="Build fixture" APP_URL="http://localhost:3002" \
AUTH_SECRET="build-fixture-not-for-runtime-use-000000000000" \ AUTH_SECRET="build-fixture-not-for-runtime-use-000000000000" \
pnpm run build pnpm run build
@@ -32,7 +47,7 @@ ENV NODE_ENV=production \
NEXT_TELEMETRY_DISABLED=1 \ NEXT_TELEMETRY_DISABLED=1 \
PORT=3002 \ PORT=3002 \
HOSTNAME=0.0.0.0 HOSTNAME=0.0.0.0
RUN apk add --no-cache tini \ RUN apk add --no-cache tini curl \
&& addgroup -g 33 -S nextjs && adduser -u 33 -S -G nextjs nextjs \ && addgroup -g 33 -S nextjs && adduser -u 33 -S -G nextjs nextjs \
&& mkdir -p /app/storage /app/public/nitro-assets /app/public/swf /var/www/Gamedata \ && mkdir -p /app/storage /app/public/nitro-assets /app/public/swf /var/www/Gamedata \
&& chown -R 33:33 /app/storage /app/public /var/www/Gamedata && chown -R 33:33 /app/storage /app/public /var/www/Gamedata
+4 -4
View File
@@ -812,16 +812,16 @@ pnpm jobs:worker # or: npm run jobs:worker / yarn jobs:worker
Optional OpenAI-powered moderation for comments and guestbook posts. Set `OPENAI_API_KEY`. Optional OpenAI-powered moderation for comments and guestbook posts. Set `OPENAI_API_KEY`.
### FlareSolverr (Cloudflare Bypass) ### Byparr (Cloudflare Bypass)
Some clone sources are protected by Cloudflare. FlareSolverr solves these challenges automatically. Some clone sources are protected by Cloudflare. Byparr (a FlareSolverr successor) solves these challenges automatically.
```bash ```bash
docker compose up -d flaresolverr docker compose up -d byparr
``` ```
```dotenv ```dotenv
FLARESOLVERR_URL=http://localhost:8191 BYPARR_URL=http://localhost:8191
``` ```
### Furniture Import with Auto-Translation ### Furniture Import with Auto-Translation
+1 -1
View File
@@ -7,7 +7,7 @@
}, },
"files": { "files": {
"ignoreUnknown": false, "ignoreUnknown": false,
"includes": ["**", "!setup", "!*.cjs", "!coverage"] "includes": ["**", "!setup", "!*.cjs", "!coverage", "!drizzle/drafts/meta"]
}, },
"formatter": { "formatter": {
"enabled": true, "enabled": true,
+8 -9
View File
@@ -58,19 +58,18 @@ services:
start_period: 40s start_period: 40s
mem_limit: 2g mem_limit: 2g
# ── FlareSolverr (Cloudflare bypass for clone sources) ── # ── Byparr (Cloudflare bypass for clone sources) ──
# Solves Cloudflare/TLS-fingerprint blocks via a headless Chrome browser. # Solves Cloudflare/TLS-fingerprint blocks via a headless Chrome browser.
# The CMS calls this on http://localhost:8191 for sources that block Node's # Drop-in successor to FlareSolverr. The CMS calls this on
# TLS fingerprint (e.g. Leet.city). Used by src/lib/services/flare-solver.ts. # http://localhost:8191 for sources that block Node's TLS fingerprint
flaresolverr: # (e.g. Leet.city). Used by src/lib/services/byparr.ts.
image: ghcr.io/flaresolverr/flaresolverr:latest byparr:
container_name: flaresolverr image: ghcr.io/thephaseless/byparr:latest
container_name: byparr
network_mode: host network_mode: host
restart: unless-stopped restart: unless-stopped
environment: environment:
- LOG_LEVEL=info - LOG_LEVEL=INFO
- BROWSER_TIMEOUT=60000
- BROWSER_WORKERS=1
mem_limit: 2g mem_limit: 2g
healthcheck: healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:8191/health"] test: ["CMD", "curl", "-sf", "http://localhost:8191/health"]
+1 -1
View File
@@ -44,7 +44,7 @@ stack is diagnostic evidence, not an automatic root-cause determination.
## Dependencies ## Dependencies
`pnpm install --frozen-lockfile`, `pnpm deps:audit`, `pnpm typecheck`, `pnpm test`, `pnpm install --frozen-lockfile`, `pnpm deps:audit`, `pnpm typecheck`, `pnpm test`,
`pnpm knip`, `pnpm biome:lint`, and `pnpm build` are the verification sequence. `pnpm biome:lint`, and `pnpm build` are the verification sequence.
`pnpm analyze --output` writes a Next.js bundle analysis (not an application build). `pnpm analyze --output` writes a Next.js bundle analysis (not an application build).
TinyMCE 8.9 is pinned in pnpm. `pnpm assets:editor` copies its runtime files and TinyMCE 8.9 is pinned in pnpm. `pnpm assets:editor` copies its runtime files and
-13
View File
@@ -1,13 +0,0 @@
import { expect, test } from "@playwright/test";
test("health endpoint is reachable", async ({ request }) => {
const res = await request.get("/api/health");
expect(res.ok()).toBeTruthy();
const body = await res.json();
expect(body).toHaveProperty("status");
});
test("homepage renders", async ({ page }) => {
await page.goto("/");
await expect(page).toHaveTitle(/.+/);
});
-17
View File
@@ -1,17 +0,0 @@
{
"$schema": "https://unpkg.com/knip@6/schema.json",
"entry": [
"src/app/**/page.{ts,tsx}",
"src/app/**/layout.{ts,tsx}",
"src/app/**/route.{ts,tsx}",
"src/app/**/{error,not-found,loading,template,default,global-error}.{ts,tsx}",
"scripts/migrate-aes-cbc-to-gcm.ts",
"scripts/build-languages-full.ts",
"scripts/translate-furnidata-full.ts",
"scripts/align-db-ids-with-furnidata.ts",
"scripts/furni-diagnose-now.ts"
],
"project": ["src/**/*.{ts,tsx,css}", "scripts/**/*.{ts,js}"],
"ignoreDependencies": ["pino-pretty"],
"ignoreBinaries": ["sendmail"]
}
+2 -12
View File
@@ -14,11 +14,10 @@
"lint": "biome check .", "lint": "biome check .",
"biome:lint": "biome check .", "biome:lint": "biome check .",
"format": "biome format --write .", "format": "biome format --write .",
"knip": "knip --include files,dependencies,devDependencies,unlisted,binaries",
"diag:permissions": "tsx scripts/diagnose-permission-page.ts", "diag:permissions": "tsx scripts/diagnose-permission-page.ts",
"jobs:worker": "node --conditions=react-server --import tsx scripts/jobs-worker.ts", "jobs:worker": "node --conditions=react-server --import tsx scripts/jobs-worker.ts",
"test": "vitest run", "test": "vitest run --coverage.enabled=false",
"test:e2e": "playwright test", "test:coverage": "vitest run",
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",
"db:generate": "drizzle-kit generate", "db:generate": "drizzle-kit generate",
"db:introspect": "drizzle-kit introspect", "db:introspect": "drizzle-kit introspect",
@@ -32,17 +31,12 @@
"gamedata:compress": "node scripts/compress-gamedata.mjs", "gamedata:compress": "node scripts/compress-gamedata.mjs",
"hk:matrix:check": "tsx scripts/verify-housekeeping-matrix.ts", "hk:matrix:check": "tsx scripts/verify-housekeeping-matrix.ts",
"test:housekeeping": "vitest run --coverage.enabled=false src/features/housekeeping src/lib/admin-theme-source-audit.test.ts src/lib/admin/authorization-contract.test.ts", "test:housekeeping": "vitest run --coverage.enabled=false src/features/housekeeping src/lib/admin-theme-source-audit.test.ts src/lib/admin/authorization-contract.test.ts",
"prepare": "node scripts/prepare-hooks.mjs",
"assets:editor": "node scripts/copy-editor-assets.mjs", "assets:editor": "node scripts/copy-editor-assets.mjs",
"deps:audit": "pnpm audit --audit-level=high", "deps:audit": "pnpm audit --audit-level=high",
"analyze": "next experimental-analyze", "analyze": "next experimental-analyze",
"i18n:check": "node scripts/audit-cms-translations.mjs --check", "i18n:check": "node scripts/audit-cms-translations.mjs --check",
"i18n:audit": "node scripts/audit-cms-translations.mjs" "i18n:audit": "node scripts/audit-cms-translations.mjs"
}, },
"lint-staged": {
"*.{js,ts,jsx,tsx,json}": "biome check --write --no-errors-on-unmatched",
"*.{ts,tsx}": "node scripts/check-admin-colors.mjs"
},
"dependencies": { "dependencies": {
"@base-ui/react": "1.8.0", "@base-ui/react": "1.8.0",
"@dnd-kit/core": "6.3.1", "@dnd-kit/core": "6.3.1",
@@ -86,7 +80,6 @@
"devDependencies": { "devDependencies": {
"@babel/parser": "7.29.8", "@babel/parser": "7.29.8",
"@biomejs/biome": "2.5.12", "@biomejs/biome": "2.5.12",
"@playwright/test": "^1.62.1",
"@tailwindcss/forms": "0.5.11", "@tailwindcss/forms": "0.5.11",
"@tailwindcss/postcss": "4.3.3", "@tailwindcss/postcss": "4.3.3",
"@tailwindcss/typography": "0.5.20", "@tailwindcss/typography": "0.5.20",
@@ -95,9 +88,6 @@
"@types/react-dom": "19.2.7", "@types/react-dom": "19.2.7",
"@vitest/coverage-v8": "5.0.0", "@vitest/coverage-v8": "5.0.0",
"drizzle-kit": "0.31.10", "drizzle-kit": "0.31.10",
"husky": "9.1.7",
"knip": "6.34.0",
"lint-staged": "17.4.1",
"pino-pretty": "13.1.3", "pino-pretty": "13.1.3",
"postcss": "8.5.28", "postcss": "8.5.28",
"tailwindcss": "4.3.3", "tailwindcss": "4.3.3",
-20
View File
@@ -1,20 +0,0 @@
import { defineConfig, devices } from "@playwright/test";
const baseURL = process.env.PLAYWRIGHT_BASE_URL ?? "http://127.0.0.1:3000";
export default defineConfig({
testDir: "./e2e",
fullyParallel: true,
retries: process.env.CI ? 2 : 0,
reporter: process.env.CI ? "github" : "list",
use: { baseURL, trace: "on-first-retry" },
webServer: process.env.PLAYWRIGHT_BASE_URL
? undefined
: {
command: "pnpm dev --port 3000",
url: "http://127.0.0.1:3000/api/health",
reuseExistingServer: !process.env.CI,
timeout: 120_000,
},
projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }],
});
+193 -800
View File
File diff suppressed because it is too large. Load diff
+2 -5
View File
@@ -93,8 +93,6 @@ for managed_name in epicnext-cms epicnext-cms-app; do
done done
cp "$deploy_dir/.env" .env cp "$deploy_dir/.env" .env
pnpm install --frozen-lockfile pnpm install --frozen-lockfile
# Prepare browser before cutover so installation failures cannot interrupt the site.
pnpm exec playwright install chromium
echo "Building $image" echo "Building $image"
DOCKER_BUILDKIT=1 docker build --network=host --progress=plain --cache-from epicnext-cms:latest \ DOCKER_BUILDKIT=1 docker build --network=host --progress=plain --cache-from epicnext-cms:latest \
@@ -161,8 +159,7 @@ candidate_attempted=1
) )
healthy healthy
node scripts/verify-deployed-release.mjs http://127.0.0.1:3002/api/health "$sha" node scripts/verify-deployed-release.mjs http://127.0.0.1:3002/api/health "$sha"
PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e # Publish the latest alias only after health and release checks pass.
# Publish the latest alias only after health and browser checks pass.
docker tag "$image" epicnext-cms:latest docker tag "$image" epicnext-cms:latest
cutover_started=0 cutover_started=0
if [ "$backup_created" -eq 1 ]; then docker rm "$backup_name" || true; fi if [ "$backup_created" -eq 1 ]; then docker rm "$backup_name" || true; fi
@@ -176,5 +173,5 @@ while IFS= read -r tag; do
if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi
fi fi
done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms) done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms)
docker builder prune -af --filter "until=72h" --keep-storage=2g || true docker builder prune -af --filter "until=72h" --max-used-space=4g || true
docker image prune -f --filter "until=168h" || true docker image prune -f --filter "until=168h" || true
+7 -7
View File
@@ -1,20 +1,20 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
URL="${FLARESOLVERR_URL:-http://localhost:8191}" URL="${BYPARR_URL:-${FLARESOLVERR_URL:-http://localhost:8191}}"
MAX_RETRIES="${FLARESOLVERR_MAX_RETRIES:-30}" MAX_RETRIES="${BYPARR_MAX_RETRIES:-30}"
RETRY_INTERVAL="${FLARESOLVERR_RETRY_INTERVAL:-2}" RETRY_INTERVAL="${BYPARR_RETRY_INTERVAL:-2}"
echo "Waiting for FlareSolverr at ${URL}..." echo "Waiting for Byparr at ${URL}..."
for i in $(seq 1 "$MAX_RETRIES"); do for i in $(seq 1 "$MAX_RETRIES"); do
if curl -sf "${URL}/health" >/dev/null 2>&1; then if curl -sf "${URL}/health" >/dev/null 2>&1; then
echo "FlareSolverr is healthy." echo "Byparr is healthy."
exit 0 exit 0
fi fi
echo "Attempt ${i}/${MAX_RETRIES} - FlareSolverr not ready, retrying in ${RETRY_INTERVAL}s..." echo "Attempt ${i}/${MAX_RETRIES} - Byparr not ready, retrying in ${RETRY_INTERVAL}s..."
sleep "$RETRY_INTERVAL" sleep "$RETRY_INTERVAL"
done done
echo "ERROR: FlareSolverr did not become healthy after ${MAX_RETRIES} attempts." echo "ERROR: Byparr did not become healthy after ${MAX_RETRIES} attempts."
exit 1 exit 1
-5
View File
@@ -1,5 +0,0 @@
// Production builds and CI do not need Git hooks or dev-only executables.
if (process.env.NODE_ENV !== "production" && !process.env.CI) {
const { default: husky } = await import("husky");
husky();
}
+1 -1
View File
@@ -14,7 +14,7 @@ export function ArticleForm({
action, action,
defaultValues, defaultValues,
}: { }: {
action: (formData: FormData) => Promise<ArticleSaveResult | void>; action: (formData: FormData) => Promise<ArticleSaveResult | undefined>;
edit?: boolean; edit?: boolean;
defaultValues?: { defaultValues?: {
title?: string; title?: string;
+7 -5
View File
@@ -5,11 +5,13 @@ const state = vi.hoisted(() => ({
queried: [] as unknown[], queried: [] as unknown[],
})); }));
vi.mock("@/lib/api-handler", () => ({ vi.mock("@/lib/api-handler", () => ({
withAdmin: (_: unknown, handler: Function) => (request: unknown) => withAdmin:
handler(request, { (_: unknown, handler: (...args: unknown[]) => unknown) =>
session: { user: { id: 1, rank: 7 } }, (request: unknown) =>
permissions: { isSuperAdmin: false }, handler(request, {
}), session: { user: { id: 1, rank: 7 } },
permissions: { isSuperAdmin: false },
}),
})); }));
vi.mock("@/lib/rate-limit", () => ({ rateLimit: async () => ({ ok: true }) })); vi.mock("@/lib/rate-limit", () => ({ rateLimit: async () => ({ ok: true }) }));
vi.mock("@/lib/permissions", async () => ({ vi.mock("@/lib/permissions", async () => ({
+1 -1
View File
@@ -77,7 +77,7 @@ describe("deployment transaction", () => {
expect(result.calls.indexOf("pnpm db:migrate")).toBeLessThan( expect(result.calls.indexOf("pnpm db:migrate")).toBeLessThan(
result.calls.indexOf("docker stop"), result.calls.indexOf("docker stop"),
); );
expect(result.calls.indexOf("pnpm test:e2e")).toBeLessThan( expect(result.calls.indexOf("verify-deployed-release.mjs")).toBeLessThan(
result.calls.indexOf( result.calls.indexOf(
`docker tag epicnext-cms:${sha} epicnext-cms:latest`, `docker tag epicnext-cms:${sha} epicnext-cms:latest`,
), ),
+4 -3
View File
@@ -25,7 +25,7 @@ describe("CI workflow", () => {
it("check runs lint, typecheck, and test", () => { it("check runs lint, typecheck, and test", () => {
expect(workflow).toContain("pnpm biome:lint"); expect(workflow).toContain("pnpm biome:lint");
expect(workflow).toContain("pnpm typecheck"); expect(workflow).toContain("pnpm typecheck");
expect(workflow).toContain("pnpm test"); expect(workflow).toContain("pnpm test:coverage");
}); });
it("check job runs on self-hosted runner", () => { it("check job runs on self-hosted runner", () => {
@@ -61,7 +61,8 @@ describe("CI workflow", () => {
it("smoke-tests the deployed app within the deployment transaction", () => { it("smoke-tests the deployed app within the deployment transaction", () => {
expect(workflow).toContain("bash scripts/ci-deploy.sh"); expect(workflow).toContain("bash scripts/ci-deploy.sh");
const deploy = readFileSync("scripts/ci-deploy.sh", "utf8"); const deploy = readFileSync("scripts/ci-deploy.sh", "utf8");
expect(deploy).toContain("pnpm test:e2e"); expect(deploy).toContain(
expect(deploy).toContain("PLAYWRIGHT_BASE_URL"); "node scripts/verify-deployed-release.mjs http://127.0.0.1:3002/api/health",
);
}); });
}); });
+3 -3
View File
@@ -4,11 +4,11 @@ import { expect, it } from "vitest";
const workflow = readFileSync(".gitea/workflows/ci.yaml", "utf8"); const workflow = readFileSync(".gitea/workflows/ci.yaml", "utf8");
const deploy = readFileSync("scripts/ci-deploy.sh", "utf8"); const deploy = readFileSync("scripts/ci-deploy.sh", "utf8");
it("uses two test workers and runs smoke checks in the deployment transaction", () => { it("uses two test workers and runs smoke checks in the deployment transaction", () => {
expect(workflow).toContain("pnpm test --maxWorkers=2"); expect(workflow).toContain("pnpm test:coverage --maxWorkers=4");
expect(workflow).not.toContain("\n e2e:"); expect(workflow).not.toContain("\n e2e:");
expect(workflow).toContain("bash scripts/ci-deploy.sh"); expect(workflow).toContain("bash scripts/ci-deploy.sh");
expect(deploy).toContain( expect(deploy).toContain(
"PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e", "node scripts/verify-deployed-release.mjs http://127.0.0.1:3002/api/health",
); );
}); });
it("locks CI and scheduled deployments using the same production lock", () => { it("locks CI and scheduled deployments using the same production lock", () => {
@@ -27,7 +27,7 @@ it("preserves production runtime configuration and recent cache", () => {
expect(deploy).toContain("/app/storage"); expect(deploy).toContain("/app/storage");
expect(deploy).not.toContain("--env-file"); expect(deploy).not.toContain("--env-file");
expect(deploy).toContain( expect(deploy).toContain(
'docker builder prune -af --filter "until=72h" --keep-storage=2g', 'docker builder prune -af --filter "until=72h" --max-used-space=4g',
); );
expect(deploy).not.toContain("docker volume prune"); expect(deploy).not.toContain("docker volume prune");
}); });
+2 -2
View File
@@ -115,12 +115,12 @@ export class ErrorStore {
name.slice(0, 10) < cutoff name.slice(0, 10) < cutoff
) )
await unlink(path.join(this.directory, name)).catch(() => {}); await unlink(path.join(this.directory, name)).catch(() => {});
const file = path.join(this.directory, record.at.slice(0, 10) + ".jsonl"); const file = path.join(this.directory, `${record.at.slice(0, 10)}.jsonl`);
const size = await stat(file) const size = await stat(file)
.then((s) => s.size) .then((s) => s.size)
.catch(() => 0); .catch(() => 0);
if (size >= DAY_LIMIT) throw new Error("Daily error storage limit reached"); if (size >= DAY_LIMIT) throw new Error("Daily error storage limit reached");
await appendFile(file, JSON.stringify(record) + "\n", { mode: 0o600 }); await appendFile(file, `${JSON.stringify(record)}\n`, { mode: 0o600 });
} }
async resolve(fingerprint: string) { async resolve(fingerprint: string) {
if (!/^[a-f0-9]{16}$/.test(fingerprint)) if (!/^[a-f0-9]{16}$/.test(fingerprint))
+90
View File
@@ -0,0 +1,90 @@
const BYPARR_URL =
process.env.BYPARR_URL ??
process.env.FLARESOLVERR_URL ?? // legacy env fallback
"http://localhost:8191";
const BYPARR_API = `${BYPARR_URL}/v1`;
type ByparrCookie = {
name: string;
value: string;
domain?: string;
path?: string;
};
type ByparrSolution = {
response: string;
status: number;
cookies: ByparrCookie[];
};
type ByparrResult = {
status?: "ok" | "error";
message?: string;
solution?: ByparrSolution;
error?: string;
};
async function byparrRequest(
url: string,
timeout = 30000,
): Promise<ByparrSolution> {
const res = await fetch(BYPARR_API, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
cmd: "request.get",
url,
maxTimeout: timeout,
}),
signal: AbortSignal.timeout(timeout + 5000),
});
if (!res.ok) {
throw new Error(`Byparr request failed: ${res.status} ${res.statusText}`);
}
const data = (await res.json()) as ByparrResult;
if (data.error) {
throw new Error(`Byparr error: ${data.error}`);
}
// Byparr returns HTTP 200 with `status: "error"` for invalid requests
// (FlareSolverr used a top-level `error` field instead).
if (data.status && data.status !== "ok") {
throw new Error(`Byparr error: ${data.message ?? "request failed"}`);
}
if (!data.solution) {
throw new Error("Byparr: no solution in response");
}
return data.solution;
}
export async function fetchWithByparr(
url: string,
timeout = 30000,
): Promise<string> {
const solution = await byparrRequest(url, timeout);
return solution.response;
}
/**
* Solve a Cloudflare challenge for the given URL and return the clearance
* cookies as a `-b "name=value; ..."` argument string suitable for curl.
* Returns null if Byparr is unavailable or no cookies were set.
*/
export async function getByparrCookies(
url: string,
timeout = 30000,
): Promise<string | null> {
try {
const solution = await byparrRequest(url, timeout);
if (!solution.cookies?.length) return null;
return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
} catch {
return null;
}
}
+13 -13
View File
@@ -12,7 +12,7 @@ const {
selectLimit, selectLimit,
fsUnlink, fsUnlink,
fsReadFile, fsReadFile,
fetchWithFlareSolver, fetchWithByparr,
curlFetchText, curlFetchText,
curlDownload, curlDownload,
} = vi.hoisted(() => ({ } = vi.hoisted(() => ({
@@ -23,7 +23,7 @@ const {
selectLimit: vi.fn<AnyFn>(async () => []), selectLimit: vi.fn<AnyFn>(async () => []),
fsUnlink: vi.fn<AnyFn>(async () => {}), fsUnlink: vi.fn<AnyFn>(async () => {}),
fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")), fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")),
fetchWithFlareSolver: vi.fn<AnyFn>(async () => ""), fetchWithByparr: vi.fn<AnyFn>(async () => ""),
curlFetchText: vi.fn<AnyFn>(async () => ""), curlFetchText: vi.fn<AnyFn>(async () => ""),
curlDownload: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })), curlDownload: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
})); }));
@@ -33,7 +33,7 @@ vi.mock("@/lib/services/import/core/curl-fetch", () => ({
curlDownload, curlDownload,
curlFetchText, curlFetchText,
})); }));
vi.mock("@/lib/services/flare-solver", () => ({ fetchWithFlareSolver })); vi.mock("@/lib/services/byparr", () => ({ fetchWithByparr }));
vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry })); vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry }));
vi.mock("@/lib/services/furni-import", () => ({ vi.mock("@/lib/services/furni-import", () => ({
ensureDirectories: vi.fn<AnyFn>(async () => {}), ensureDirectories: vi.fn<AnyFn>(async () => {}),
@@ -217,8 +217,8 @@ describe("fetchSourceFurnidata", () => {
const HTML_WRAPPED = `<html><head><meta charset="utf-8"></head><body><pre>${JSON_BODY}</pre></body></html>`; const HTML_WRAPPED = `<html><head><meta charset="utf-8"></head><body><pre>${JSON_BODY}</pre></body></html>`;
beforeEach(() => { beforeEach(() => {
fetchWithFlareSolver.mockReset(); fetchWithByparr.mockReset();
fetchWithFlareSolver.mockResolvedValue(""); fetchWithByparr.mockResolvedValue("");
vi.unstubAllGlobals(); vi.unstubAllGlobals();
}); });
@@ -235,10 +235,10 @@ describe("fetchSourceFurnidata", () => {
); );
const list = await fetchSourceFurnidata("https://a.test/fd.json", 1); const list = await fetchSourceFurnidata("https://a.test/fd.json", 1);
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]); expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
expect(fetchWithFlareSolver).not.toHaveBeenCalled(); expect(fetchWithByparr).not.toHaveBeenCalled();
}); });
it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via FlareSolverr)", async () => { it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via Byparr)", async () => {
vi.stubGlobal( vi.stubGlobal(
"fetch", "fetch",
vi.fn( vi.fn(
@@ -249,13 +249,13 @@ describe("fetchSourceFurnidata", () => {
}), }),
), ),
); );
fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED); fetchWithByparr.mockResolvedValue(HTML_WRAPPED);
const list = await fetchSourceFurnidata("https://b.test/fd.json", 1); const list = await fetchSourceFurnidata("https://b.test/fd.json", 1);
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]); expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
expect(fetchWithFlareSolver).toHaveBeenCalledWith("https://b.test/fd.json"); expect(fetchWithByparr).toHaveBeenCalledWith("https://b.test/fd.json");
}); });
it("falls back to FlareSolverr on HTML body even when status is 200", async () => { it("falls back to Byparr on HTML body even when status is 200", async () => {
vi.stubGlobal( vi.stubGlobal(
"fetch", "fetch",
vi.fn( vi.fn(
@@ -266,7 +266,7 @@ describe("fetchSourceFurnidata", () => {
}), }),
), ),
); );
fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED); fetchWithByparr.mockResolvedValue(HTML_WRAPPED);
const list = await fetchSourceFurnidata("https://c.test/fd.json", 1); const list = await fetchSourceFurnidata("https://c.test/fd.json", 1);
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]); expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
}); });
@@ -284,7 +284,7 @@ describe("fetchSourceFurnidata", () => {
); );
const list = await fetchSourceFurnidata("https://g.test/fd.json", 1); const list = await fetchSourceFurnidata("https://g.test/fd.json", 1);
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]); expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
expect(fetchWithFlareSolver).not.toHaveBeenCalled(); expect(fetchWithByparr).not.toHaveBeenCalled();
}); });
it("throws when HTML has no embedded JSON payload", async () => { it("throws when HTML has no embedded JSON payload", async () => {
@@ -298,7 +298,7 @@ describe("fetchSourceFurnidata", () => {
}), }),
), ),
); );
fetchWithFlareSolver.mockResolvedValue("<html>still a challenge</html>"); fetchWithByparr.mockResolvedValue("<html>still a challenge</html>");
await expect( await expect(
fetchSourceFurnidata("https://d.test/fd.json", 1), fetchSourceFurnidata("https://d.test/fd.json", 1),
).rejects.toThrow(/Cloudflare challenge/); ).rejects.toThrow(/Cloudflare challenge/);
+9 -9
View File
@@ -5,9 +5,9 @@ import { autoDetectInteraction } from "@/lib/furni/auto-interaction";
import { normalizeClassname } from "@/lib/furni/classname"; import { normalizeClassname } from "@/lib/furni/classname";
import { HABBO_GAMEDATA_HOTEL_SETTING_KEY } from "@/lib/habbo-gamedata-hotel"; import { HABBO_GAMEDATA_HOTEL_SETTING_KEY } from "@/lib/habbo-gamedata-hotel";
import { logger } from "@/lib/logger"; import { logger } from "@/lib/logger";
import { fetchWithByparr } from "@/lib/services/byparr";
import { extractFurniIconPng } from "@/lib/services/clone-icon"; import { extractFurniIconPng } from "@/lib/services/clone-icon";
import type { CloneSource } from "@/lib/services/clone-sources"; import type { CloneSource } from "@/lib/services/clone-sources";
import { fetchWithFlareSolver } from "@/lib/services/flare-solver";
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs"; import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
import { appendFurniEntry } from "@/lib/services/furni-data"; import { appendFurniEntry } from "@/lib/services/furni-data";
import { import {
@@ -101,7 +101,7 @@ const TTL = 5 * 60 * 1000;
/** /**
* Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's * Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
* JSON is embedded in a `<pre>` block inside the page, often surfaced by the * JSON is embedded in a `<pre>` block inside the page, often surfaced by the
* FlareSolverr fallback). Extract the JSON payload before giving up. * Byparr fallback). Extract the JSON payload before giving up.
*/ */
function extractJsonFromHtml(body: string): string | null { function extractJsonFromHtml(body: string): string | null {
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/); const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
@@ -117,20 +117,20 @@ function extractJsonFromHtml(body: string): string | null {
} }
/** /**
* Fetch a source's furnidata via FlareSolverr, falling back to a curl * Fetch a source's furnidata via Byparr, falling back to a curl
* subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while * subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while
* the same request succeeds from curl — so curl is a useful second fallback * the same request succeeds from curl — so curl is a useful second fallback
* when FlareSolverr is unavailable. * when Byparr is unavailable.
*/ */
async function fetchWithFallback(url: string): Promise<string> { async function fetchWithFallback(url: string): Promise<string> {
try { try {
return await fetchWithFlareSolver(url); return await fetchWithByparr(url);
} catch { } catch {
try { try {
return await curlFetchText(url); return await curlFetchText(url);
} catch (err) { } catch (err) {
throw new Error( throw new Error(
`FlareSolverr + curl fallback failed for ${url}: ${(err as Error).message}`, `Byparr + curl fallback failed for ${url}: ${(err as Error).message}`,
); );
} }
} }
@@ -150,8 +150,8 @@ export async function fetchSourceFurnidata(
}); });
if (res.ok) { if (res.ok) {
// Some sources (e.g. GitHub raw) serve JSON with a text/plain // Some sources (e.g. GitHub raw) serve JSON with a text/plain
// content-type — only route to FlareSolverr when the body is // content-type — only route to Byparr when the body is actually
// actually HTML, otherwise parse the JSON directly. // HTML, otherwise parse the JSON directly.
const text = await res.text(); const text = await res.text();
if ( if (
text.trimStart().startsWith("{") || text.trimStart().startsWith("{") ||
@@ -171,7 +171,7 @@ export async function fetchSourceFurnidata(
const extracted = extractJsonFromHtml(body); const extracted = extractJsonFromHtml(body);
if (!extracted) { if (!extracted) {
throw new Error( throw new Error(
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`, `Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start Byparr and check its logs.`,
); );
} }
body = extracted; body = extracted;
+9 -1
View File
@@ -142,12 +142,20 @@ export const DEFAULT_SOURCES: CloneSource[] = [
{ {
id: "default-virtualcity", id: "default-virtualcity",
name: "VirtualCity", name: "VirtualCity",
furnidataUrl: "https://virtualc.nl/gamedata/furnidata_json/1", furnidataUrl: "https://virtualc.nl/gamedata/FurnitureData.json",
nitroBaseUrl: "https://virtualc.nl/nitro/bundled/furniture", nitroBaseUrl: "https://virtualc.nl/nitro/bundled/furniture",
iconBaseUrl: "https://virtualc.nl/nitro/icons", iconBaseUrl: "https://virtualc.nl/nitro/icons",
hotel: "com", hotel: "com",
sourceSwfBaseUrl: "https://virtualc.nl/dcr", sourceSwfBaseUrl: "https://virtualc.nl/dcr",
}, },
{
id: "default-sodastudios",
name: "SodaStudios",
furnidataUrl: "https://assets.sodaho.tel/gamedata/FurnitureData.json",
nitroBaseUrl: "https://assets.sodaho.tel/bundled/furniture",
iconBaseUrl: "https://assets.sodaho.tel/dcr/hof_furni/icons",
hotel: "com",
},
{ {
id: "default-habboon", id: "default-habboon",
name: "Habboon", name: "Habboon",
-82
View File
@@ -1,82 +0,0 @@
const FLARESOLVERR_URL =
process.env.FLARESOLVERR_URL ?? "http://localhost:8191";
const FLARESOLVERR_API = `${FLARESOLVERR_URL}/v1`;
type FlareCookie = {
name: string;
value: string;
domain: string;
path: string;
};
type FlareSolution = {
response: string;
status: number;
cookies: FlareCookie[];
};
type FlareResult = {
solution?: FlareSolution;
error?: string;
};
async function flareRequest(
url: string,
timeout = 30000,
): Promise<FlareSolution> {
const res = await fetch(FLARESOLVERR_API, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
cmd: "request.get",
url,
maxTimeout: timeout,
}),
signal: AbortSignal.timeout(timeout + 5000),
});
if (!res.ok) {
throw new Error(
`FlareSolverr request failed: ${res.status} ${res.statusText}`,
);
}
const data = (await res.json()) as FlareResult;
if (data.error) {
throw new Error(`FlareSolverr error: ${data.error}`);
}
if (!data.solution) {
throw new Error("FlareSolverr: no solution in response");
}
return data.solution;
}
export async function fetchWithFlareSolver(
url: string,
timeout = 30000,
): Promise<string> {
const solution = await flareRequest(url, timeout);
return solution.response;
}
/**
* Solve a Cloudflare challenge for the given URL and return the clearance
* cookies as a `-b "name=value; ..."` argument string suitable for curl.
* Returns null if FlareSolverr is unavailable or no cookies were set.
*/
export async function getFlareSolverrCookies(
url: string,
timeout = 30000,
): Promise<string | null> {
try {
const solution = await flareRequest(url, timeout);
if (!solution.cookies?.length) return null;
return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
} catch {
return null;
}
}
+4 -4
View File
@@ -1,5 +1,5 @@
import { promises as fs } from "node:fs"; import { promises as fs } from "node:fs";
import { getFlareSolverrCookies } from "@/lib/services/flare-solver"; import { getByparrCookies } from "@/lib/services/byparr";
import { parseNitroBundle } from "../../swf/nitro-builder"; import { parseNitroBundle } from "../../swf/nitro-builder";
import { browserHeaders } from "./browser-headers"; import { browserHeaders } from "./browser-headers";
import { curlDownload } from "./curl-fetch"; import { curlDownload } from "./curl-fetch";
@@ -62,7 +62,7 @@ export async function downloadFile(
const curlFallback = async (): Promise<boolean> => { const curlFallback = async (): Promise<boolean> => {
let curlRes = await curlDownload(url, destPath); let curlRes = await curlDownload(url, destPath);
if (!curlRes.ok) { if (!curlRes.ok) {
const cookieHeader = await getFlareSolverrCookies(url); const cookieHeader = await getByparrCookies(url);
curlRes = await curlDownload( curlRes = await curlDownload(
url, url,
destPath, destPath,
@@ -108,7 +108,7 @@ export async function downloadFile(
res.status === 404 || res.status === 410 || res.status === 403; res.status === 404 || res.status === 410 || res.status === 403;
if (deterministic || attempt === maxRetries) { if (deterministic || attempt === maxRetries) {
// HTTP 403 is often a TLS-fingerprint / Cloudflare challenge // HTTP 403 is often a TLS-fingerprint / Cloudflare challenge
// block (e.g. Leet.city) — retry via curl, with FlareSolverr // block (e.g. Leet.city) — retry via curl, with Byparr
// clearance cookies if the plain curl is also challenged. // clearance cookies if the plain curl is also challenged.
if (res.status === 403 && (await curlFallback())) { if (res.status === 403 && (await curlFallback())) {
const stat = await fs.stat(destPath); const stat = await fs.stat(destPath);
@@ -148,7 +148,7 @@ export async function downloadFile(
return { ok: true, size: buffer.length }; return { ok: true, size: buffer.length };
} catch (err) { } catch (err) {
if (attempt === maxRetries || stage === "write") { if (attempt === maxRetries || stage === "write") {
// TLS fingerprint aborts land here too — try curl/FlareSolverr // TLS fingerprint aborts land here too — try curl/Byparr
// before reporting the network failure. // before reporting the network failure.
if (stage === "download" && (await curlFallback())) { if (stage === "download" && (await curlFallback())) {
const stat = await fs.stat(destPath); const stat = await fs.stat(destPath);
+8 -2
View File
@@ -13,7 +13,6 @@ flock() { echo "lock" >> "$TEST_DIR/calls"; [ "$SCENARIO" != lock-failure ]; }
pnpm() { pnpm() {
echo "pnpm $*" >> "$TEST_DIR/calls" echo "pnpm $*" >> "$TEST_DIR/calls"
if [ "$1" = db:migrate ] && [ "$SCENARIO" = migration-failure ]; then return 1; fi if [ "$1" = db:migrate ] && [ "$SCENARIO" = migration-failure ]; then return 1; fi
if [ "$1" = test:e2e ] && [ "$SCENARIO" = smoke-failure ]; then return 1; fi
return 0 return 0
} }
curl() { curl() {
@@ -43,5 +42,12 @@ docker() {
} }
export -f git flock pnpm curl sleep docker export -f git flock pnpm curl sleep docker
node() { echo "node $*" >> "$TEST_DIR/calls"; [ "$SCENARIO" != release-failure ]; } node() {
echo "node $*" >> "$TEST_DIR/calls"
case "$SCENARIO" in
release-failure) return 1 ;;
smoke-failure) case "$*" in *verify-deployed-release.mjs*) return 1 ;; esac ;;
esac
return 0
}
export -f node export -f node
+1 -1
View File
@@ -34,5 +34,5 @@
".next-staging/types/**/*.ts", ".next-staging/types/**/*.ts",
".next-staging/dev/types/**/*.ts" ".next-staging/dev/types/**/*.ts"
], ],
"exclude": ["node_modules", "e2e", "playwright.config.ts", "src/generated"] "exclude": ["node_modules", "src/generated"]
} }
+1 -1
View File
@@ -1282,7 +1282,7 @@ with open(out, "w", encoding="utf-8") as f:
# Sanity-check a .env file for the failure modes that actually break a deploy: # Sanity-check a .env file for the failure modes that actually break a deploy:
# duplicate keys, and a value concatenated onto another key (missing newline), # duplicate keys, and a value concatenated onto another key (missing newline),
# e.g. FLARESOLVERR_URL=http://localhost:8191AUTH_TRUST_HOST="..." # e.g. BYPARR_URL=http://localhost:8191AUTH_TRUST_HOST="..."
validate_env_file() { validate_env_file() {
local base_dir="${1:-$SCRIPT_DIR}" local base_dir="${1:-$SCRIPT_DIR}"
local en="$base_dir/.env" local en="$base_dir/.env"