You've already forked Atomcms-edit
Initial commit
This commit is contained in:
root
Executable
+274
@@ -0,0 +1,274 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Models\Miscellaneous\WebsiteSetting;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
readonly class ContentModerationService
|
||||
{
|
||||
private const int MODERATION_CACHE_DURATION_MINUTES = 5;
|
||||
|
||||
private const string SETTINGS_CACHE_KEY = 'content_moderation_settings';
|
||||
|
||||
private const int SETTINGS_CACHE_DURATION = 300;
|
||||
|
||||
private const string BLOCKED_WORDS_CACHE_KEY = 'blocked_words';
|
||||
|
||||
private string $openAiApiKey;
|
||||
|
||||
private bool $useOpenAI;
|
||||
|
||||
private float $sensitivity;
|
||||
|
||||
/** @var array<mixed> */
|
||||
private array $blockedWords;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->openAiApiKey = config('services.openai.api_key', '');
|
||||
$this->useOpenAI = $this->openAiApiKey !== '';
|
||||
$this->sensitivity = (float) ($this->getSetting('ai_filter_sensitivity') ?? 0.8);
|
||||
$this->blockedWords = $this->loadBlockedWords();
|
||||
}
|
||||
|
||||
public function moderate(string $content, string $type = 'general'): array
|
||||
{
|
||||
$trimmedContent = trim($content);
|
||||
if ($trimmedContent === '') {
|
||||
return [
|
||||
'approved' => true,
|
||||
'score' => 0,
|
||||
'reason' => null,
|
||||
'detected_words' => [],
|
||||
];
|
||||
}
|
||||
|
||||
$localResult = $this->localModerate($trimmedContent);
|
||||
|
||||
if ($localResult['blocked']) {
|
||||
return [
|
||||
'approved' => false,
|
||||
'score' => 1.0,
|
||||
'reason' => 'Blocked content detected',
|
||||
'detected_words' => $localResult['words'],
|
||||
'method' => 'local',
|
||||
];
|
||||
}
|
||||
|
||||
if ($this->useOpenAI) {
|
||||
return $this->openAIModerate($trimmedContent, $type);
|
||||
}
|
||||
|
||||
return [
|
||||
'approved' => true,
|
||||
'score' => $localResult['score'],
|
||||
'reason' => null,
|
||||
'detected_words' => [],
|
||||
'method' => 'local',
|
||||
];
|
||||
}
|
||||
|
||||
private function localModerate(string $content): array
|
||||
{
|
||||
$text = strtolower(strip_tags($content));
|
||||
$detectedWords = array_filter(
|
||||
$this->blockedWords,
|
||||
fn (string $word): bool => stripos($text, $word) !== false,
|
||||
);
|
||||
|
||||
if ($detectedWords !== []) {
|
||||
return [
|
||||
'blocked' => true,
|
||||
'words' => array_values($detectedWords),
|
||||
'score' => 1.0,
|
||||
];
|
||||
}
|
||||
|
||||
$suspiciousPatterns = [
|
||||
'/(.)\1{4,}/i',
|
||||
'/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/',
|
||||
'/\b(\w*?[\x27\`]+\w*?)+\b/i',
|
||||
];
|
||||
|
||||
$patternScore = array_reduce(
|
||||
$suspiciousPatterns,
|
||||
fn (float $score, string $pattern): float => preg_match($pattern, $text) ? $score + 0.3 : $score,
|
||||
0.0,
|
||||
);
|
||||
|
||||
return [
|
||||
'blocked' => $patternScore >= $this->sensitivity,
|
||||
'words' => [],
|
||||
'score' => min($patternScore, 1.0),
|
||||
];
|
||||
}
|
||||
|
||||
private function openAIModerate(string $content, string $type): array
|
||||
{
|
||||
$cacheKey = 'moderation_' . md5($content . $type);
|
||||
|
||||
return Cache::remember($cacheKey, now()->addMinutes(self::MODERATION_CACHE_DURATION_MINUTES), function () use ($content): array {
|
||||
try {
|
||||
$response = Http::withHeaders([
|
||||
'Authorization' => 'Bearer ' . $this->openAiApiKey,
|
||||
'Content-Type' => 'application/json',
|
||||
])->post('https://api.openai.com/v1/moderations', [
|
||||
'input' => $content,
|
||||
]);
|
||||
|
||||
if ($response->successful()) {
|
||||
$result = $response->json();
|
||||
$moderationResult = $result['results'][0];
|
||||
|
||||
$categories = $moderationResult['categories'];
|
||||
$categoryScores = $moderationResult['category_scores'];
|
||||
|
||||
$maxScore = 0.0;
|
||||
$triggeredCategories = [];
|
||||
|
||||
foreach ($categories as $category => $flagged) {
|
||||
$categoryScore = $categoryScores[$category] ?? 0;
|
||||
if ($flagged || $categoryScore > $this->sensitivity) {
|
||||
$triggeredCategories[] = $category;
|
||||
$maxScore = max($maxScore, $categoryScore);
|
||||
}
|
||||
}
|
||||
|
||||
$approved = $triggeredCategories === [] || $maxScore < $this->sensitivity;
|
||||
|
||||
return [
|
||||
'approved' => $approved,
|
||||
'score' => $maxScore,
|
||||
'reason' => $approved ? null : implode(', ', $triggeredCategories),
|
||||
'detected_words' => $moderationResult['flagged_tokens'] ?? [],
|
||||
'method' => 'openai',
|
||||
'categories' => $triggeredCategories,
|
||||
];
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
report($e);
|
||||
}
|
||||
|
||||
return [
|
||||
'approved' => true,
|
||||
'score' => 0,
|
||||
'reason' => null,
|
||||
'detected_words' => [],
|
||||
'method' => 'fallback',
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
public function addBlockedWord(string $word): bool
|
||||
{
|
||||
$cleanWord = strtolower(trim($word));
|
||||
$key = 'filter_word_' . $cleanWord;
|
||||
|
||||
WebsiteSetting::updateOrCreate(
|
||||
['key' => $key],
|
||||
['value' => '1', 'comment' => 'Blocked word: ' . $word],
|
||||
);
|
||||
|
||||
Cache::forget(self::BLOCKED_WORDS_CACHE_KEY);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function removeBlockedWord(string $word): bool
|
||||
{
|
||||
$cleanWord = strtolower(trim($word));
|
||||
$key = 'filter_word_' . $cleanWord;
|
||||
|
||||
WebsiteSetting::where('key', $key)->delete();
|
||||
|
||||
Cache::forget(self::BLOCKED_WORDS_CACHE_KEY);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<string>
|
||||
*/
|
||||
public function getBlockedWords(): array
|
||||
{
|
||||
return $this->blockedWords;
|
||||
}
|
||||
|
||||
public function setSensitivity(float $sensitivity): void
|
||||
{
|
||||
$newSensitivity = max(0, min(1, $sensitivity));
|
||||
|
||||
WebsiteSetting::updateOrCreate(
|
||||
['key' => 'ai_filter_sensitivity'],
|
||||
['value' => (string) $newSensitivity, 'comment' => 'AI filter sensitivity (0-1)'],
|
||||
);
|
||||
|
||||
Cache::forget(self::SETTINGS_CACHE_KEY);
|
||||
CacheService::clearContentModeration();
|
||||
}
|
||||
|
||||
public function getSensitivity(): float
|
||||
{
|
||||
return $this->sensitivity;
|
||||
}
|
||||
|
||||
public function isUsingOpenAI(): bool
|
||||
{
|
||||
return $this->useOpenAI;
|
||||
}
|
||||
|
||||
public function isEnabled(): bool
|
||||
{
|
||||
return (bool) $this->getSetting('ai_filter_enabled');
|
||||
}
|
||||
|
||||
public function shouldAutoReject(): bool
|
||||
{
|
||||
return (bool) $this->getSetting('ai_filter_auto_reject');
|
||||
}
|
||||
|
||||
public function getStats(): array
|
||||
{
|
||||
return [
|
||||
'enabled' => $this->isEnabled(),
|
||||
'auto_reject' => $this->shouldAutoReject(),
|
||||
'method' => $this->useOpenAI ? 'openai' : 'local',
|
||||
'sensitivity' => $this->sensitivity,
|
||||
'blocked_words_count' => count($this->blockedWords),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<string>
|
||||
*/
|
||||
private function loadBlockedWords(): array
|
||||
{
|
||||
return Cache::remember(self::BLOCKED_WORDS_CACHE_KEY, self::SETTINGS_CACHE_DURATION, fn (): array => WebsiteSetting::where('key', 'like', 'filter_word_%')
|
||||
->where('value', '1')
|
||||
->pluck('key')
|
||||
->map(fn (string $key): string => str_replace('filter_word_', '', $key))
|
||||
->values()
|
||||
->toArray());
|
||||
}
|
||||
|
||||
private function getSetting(string $key): ?string
|
||||
{
|
||||
return $this->getSettings()[$key] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string|null>
|
||||
*/
|
||||
private function getSettings(): array
|
||||
{
|
||||
return Cache::remember(self::SETTINGS_CACHE_KEY, self::SETTINGS_CACHE_DURATION, fn (): array => WebsiteSetting::whereIn('key', [
|
||||
'ai_filter_sensitivity',
|
||||
'ai_filter_enabled',
|
||||
'ai_filter_auto_reject',
|
||||
])->pluck('value', 'key')->toArray());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user