This commit is contained in:
openhands committed
1 parent
8efd032cc6
commit
df38dccbf1
735 files changed
+128321
-120870
No files matched your search
@@ -16,8 +16,8 @@ import { prisma } from "@/lib/prisma";
|
||||
// explicitly-flagged AI result). Pure server module; uses global fetch only.
|
||||
|
||||
export interface ModerationResult {
|
||||
ok: boolean;
|
||||
reason?: string;
|
||||
ok: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const OPENAI_MODERATIONS_URL = "https://api.openai.com/v1/moderations";
|
||||
@@ -33,26 +33,31 @@ let wordFilterLoadedAt = 0;
|
||||
const WORD_FILTER_TTL_MS = 60_000;
|
||||
|
||||
async function loadWordFilter(): Promise<string[]> {
|
||||
const now = Date.now();
|
||||
if (wordFilterCache === null || now - wordFilterLoadedAt > WORD_FILTER_TTL_MS) {
|
||||
try {
|
||||
const rows = await prisma.websiteWordfilter.findMany({
|
||||
select: { word: true },
|
||||
});
|
||||
wordFilterCache = rows.map((r) => r.word.trim().toLowerCase()).filter((w) => w.length > 0);
|
||||
wordFilterLoadedAt = now;
|
||||
} catch {
|
||||
// DB unavailable — return an empty filter WITHOUT caching, so the next
|
||||
// call retries. Fail-open: a missing blocklist must not block content.
|
||||
return [];
|
||||
}
|
||||
}
|
||||
return wordFilterCache;
|
||||
const now = Date.now();
|
||||
if (
|
||||
wordFilterCache === null ||
|
||||
now - wordFilterLoadedAt > WORD_FILTER_TTL_MS
|
||||
) {
|
||||
try {
|
||||
const rows = await prisma.websiteWordfilter.findMany({
|
||||
select: { word: true },
|
||||
});
|
||||
wordFilterCache = rows
|
||||
.map((r) => r.word.trim().toLowerCase())
|
||||
.filter((w) => w.length > 0);
|
||||
wordFilterLoadedAt = now;
|
||||
} catch {
|
||||
// DB unavailable — return an empty filter WITHOUT caching, so the next
|
||||
// call retries. Fail-open: a missing blocklist must not block content.
|
||||
return [];
|
||||
}
|
||||
}
|
||||
return wordFilterCache;
|
||||
}
|
||||
|
||||
/** Invalidate the cached word filter after the blocklist is edited. */
|
||||
export function reloadWordFilter(): void {
|
||||
wordFilterCache = null;
|
||||
wordFilterCache = null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -61,13 +66,13 @@ export function reloadWordFilter(): void {
|
||||
* AtomCMS's behaviour (filtered words are blocked even inside other words).
|
||||
*/
|
||||
async function wordFilterHit(text: string): Promise<string | null> {
|
||||
const words = await loadWordFilter();
|
||||
if (words.length === 0) return null;
|
||||
const haystack = text.toLowerCase();
|
||||
for (const word of words) {
|
||||
if (haystack.includes(word)) return word;
|
||||
}
|
||||
return null;
|
||||
const words = await loadWordFilter();
|
||||
if (words.length === 0) return null;
|
||||
const haystack = text.toLowerCase();
|
||||
for (const word of words) {
|
||||
if (haystack.includes(word)) return word;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -76,34 +81,35 @@ async function wordFilterHit(text: string): Promise<string | null> {
|
||||
* malformed body, timeout) returns false — fail-open.
|
||||
*/
|
||||
async function openAiFlagged(text: string): Promise<boolean> {
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) return false;
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) return false;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
|
||||
try {
|
||||
const res = await fetch(OPENAI_MODERATIONS_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({ input: text }),
|
||||
signal: controller.signal,
|
||||
cache: "no-store",
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
|
||||
try {
|
||||
const res = await fetch(OPENAI_MODERATIONS_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({ input: text }),
|
||||
signal: controller.signal,
|
||||
cache: "no-store",
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
|
||||
const data: unknown = await res.json();
|
||||
const results = (data as { results?: Array<{ flagged?: boolean }> })?.results;
|
||||
if (!Array.isArray(results)) return false;
|
||||
return results.some((r) => r?.flagged === true);
|
||||
} catch {
|
||||
// Network error / abort / parse failure — fail-open.
|
||||
return false;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
const data: unknown = await res.json();
|
||||
const results = (data as { results?: Array<{ flagged?: boolean }> })
|
||||
?.results;
|
||||
if (!Array.isArray(results)) return false;
|
||||
return results.some((r) => r?.flagged === true);
|
||||
} catch {
|
||||
// Network error / abort / parse failure — fail-open.
|
||||
return false;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -112,22 +118,22 @@ async function openAiFlagged(text: string): Promise<boolean> {
|
||||
* or non-string input is treated as allowed (nothing to moderate).
|
||||
*/
|
||||
export async function isAllowed(text: string): Promise<ModerationResult> {
|
||||
if (typeof text !== "string" || text.trim().length === 0) {
|
||||
return { ok: true };
|
||||
}
|
||||
if (typeof text !== "string" || text.trim().length === 0) {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// Layer 1: local blocklist (cheap, cached).
|
||||
const hit = await wordFilterHit(text);
|
||||
if (hit) {
|
||||
return { ok: false, reason: `Blocked by word filter: "${hit}"` };
|
||||
}
|
||||
// Layer 1: local blocklist (cheap, cached).
|
||||
const hit = await wordFilterHit(text);
|
||||
if (hit) {
|
||||
return { ok: false, reason: `Blocked by word filter: "${hit}"` };
|
||||
}
|
||||
|
||||
// Layer 2: OpenAI moderation (only when configured).
|
||||
if (await openAiFlagged(text)) {
|
||||
return { ok: false, reason: "Blocked by automated content moderation" };
|
||||
}
|
||||
// Layer 2: OpenAI moderation (only when configured).
|
||||
if (await openAiFlagged(text)) {
|
||||
return { ok: false, reason: "Blocked by automated content moderation" };
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -136,8 +142,8 @@ export async function isAllowed(text: string): Promise<ModerationResult> {
|
||||
* early. Resolves silently when the content is allowed.
|
||||
*/
|
||||
export async function moderateOrThrow(text: string): Promise<void> {
|
||||
const result = await isAllowed(text);
|
||||
if (!result.ok) {
|
||||
throw new Error(result.reason ?? "Content not allowed");
|
||||
}
|
||||
const result = await isAllowed(text);
|
||||
if (!result.ok) {
|
||||
throw new Error(result.reason ?? "Content not allowed");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user