refactor: remove unused text analysis module and integrate Qdrant enhancements
Build & Deploy (Nix) / build-and-deploy (backend) (push) Successful in 2m30s
Build & Deploy (Nix) / build-and-deploy (discord-gateway) (push) Successful in 3m7s
Build & Deploy (Nix) / build-and-deploy (proxy) (push) Successful in 3m20s
Build & Deploy (Nix) / build-and-deploy (backend) (push) Successful in 2m30s
Build & Deploy (Nix) / build-and-deploy (discord-gateway) (push) Successful in 3m7s
Build & Deploy (Nix) / build-and-deploy (proxy) (push) Successful in 3m20s
- Deleted the text analysis prompt constants and helpers as they are no longer needed. - Added batch search functionality for Qdrant to optimize vector searches. - Implemented methods for deleting expired Qdrant points and invalidating cache based on content hash. - Updated text batch processor to use new timeout configurations and modified content building for moderation prompts. - Enhanced text cache store to support new Qdrant integration and improved cache invalidation logic. - Introduced a new user reputation model with a more nuanced trust scoring system, including penalties and rewards for user behavior. - Added unit tests for the new trust model to ensure correctness of penalty and trust gain calculations. - Updated configuration schema to reflect new timeout settings and removed deprecated OpenAI moderation keys.
This commit is contained in:
+1
-5
@@ -97,6 +97,7 @@ AI_LLM_MAX_CONCURRENT=5 # Max concurrent LLM API calls (default:
|
|||||||
AI_LLM_IMAGE_MAX_DIMENSION=1024 # Max image dimension in pixels before resize (default: 1024)
|
AI_LLM_IMAGE_MAX_DIMENSION=1024 # Max image dimension in pixels before resize (default: 1024)
|
||||||
AI_LLM_TEXT_BATCH_SIZE=20 # Max messages per text-only moderation batch (default: 20)
|
AI_LLM_TEXT_BATCH_SIZE=20 # Max messages per text-only moderation batch (default: 20)
|
||||||
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000 # Timeout in ms for media analysis calls (default: 60000)
|
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000 # Timeout in ms for media analysis calls (default: 60000)
|
||||||
|
AI_LLM_TEXT_ANALYSIS_TIMEOUT_MS=30000 # Timeout in ms for text-only analysis calls (default: 30000)
|
||||||
|
|
||||||
# === AI Analysis Tuning ===
|
# === AI Analysis Tuning ===
|
||||||
AI_ANALYSIS_DEBOUNCE_MS=500 # Debounce window for batching messages in ms (default: 500)
|
AI_ANALYSIS_DEBOUNCE_MS=500 # Debounce window for batching messages in ms (default: 500)
|
||||||
@@ -110,11 +111,6 @@ AI_ANALYSIS_PROCESSING_TIMEOUT_MS=120000 # Conversation lock timeout in ms (defa
|
|||||||
AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50 # Max concurrent individual-fallback jobs (default: 50)
|
AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50 # Max concurrent individual-fallback jobs (default: 50)
|
||||||
AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50 # Consecutive errors before circuit breaker trips (default: 50)
|
AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50 # Consecutive errors before circuit breaker trips (default: 50)
|
||||||
|
|
||||||
# === OpenAI Moderation (optional separate provider) ===
|
|
||||||
# OPENAI_MODERATION_API_KEY= # OpenAI API key for moderation endpoint
|
|
||||||
# OPENAI_MODERATION_BASE_URL=https://api.openai.com/v1 # OpenAI moderation base URL (default)
|
|
||||||
# OPENAI_MODERATION_MODEL=omni-moderation-latest # OpenAI moderation model (default)
|
|
||||||
|
|
||||||
# === Auto-Delete ===
|
# === Auto-Delete ===
|
||||||
AUTO_DELETE_FLAGGED_ENABLED=true # Enable auto-deletion of flagged messages (default: true)
|
AUTO_DELETE_FLAGGED_ENABLED=true # Enable auto-deletion of flagged messages (default: true)
|
||||||
AUTO_DELETE_FLAGGED_DRY_RUN=true # Dry-run mode: log but do not delete (default: false)
|
AUTO_DELETE_FLAGGED_DRY_RUN=true # Dry-run mode: log but do not delete (default: false)
|
||||||
|
|||||||
@@ -5,11 +5,15 @@
|
|||||||
*
|
*
|
||||||
* ## Pipeline
|
* ## Pipeline
|
||||||
*
|
*
|
||||||
* Message → LLM evaluator (with conversation context + media evidence)
|
* Message batch → runModerationAnalysis (orchestrator) → LLM evaluator
|
||||||
|
* (with conversation context + media evidence) → per-message verdicts
|
||||||
*
|
*
|
||||||
* Every message is judged by the LLM — there is no regex/heuristic
|
* The orchestrator splits text-only vs media internally and runs both
|
||||||
* pre-classification. A failed LLM call yields an explicit "error" status
|
* paths in parallel with ONE LLM call per sub-batch — a 20-message text
|
||||||
* (never a heuristic verdict), and the recovery worker retries it later.
|
* batch costs 1 LLM call, not 20. Every message is judged by the LLM —
|
||||||
|
* there is no regex/heuristic pre-classification. A failed LLM call yields
|
||||||
|
* an explicit "error" status (never a heuristic verdict), and the recovery
|
||||||
|
* worker retries it later.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { createChildLogger } from "@/shared/logger/index";
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
@@ -115,7 +119,10 @@ export default async function workerRouter(
|
|||||||
if (!config.AI_LLM_API_KEY) {
|
if (!config.AI_LLM_API_KEY) {
|
||||||
const errorMsg =
|
const errorMsg =
|
||||||
"AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials.";
|
"AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials.";
|
||||||
logger.error({ error: errorMsg }, "AI_LLM_API_KEY is missing from environment");
|
logger.error(
|
||||||
|
{ error: errorMsg },
|
||||||
|
"AI_LLM_API_KEY is missing from environment",
|
||||||
|
);
|
||||||
|
|
||||||
if (job.type === "batch") {
|
if (job.type === "batch") {
|
||||||
return {
|
return {
|
||||||
@@ -172,60 +179,9 @@ export default async function workerRouter(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Single-pass LLM pipeline
|
// Result normalization
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
|
||||||
* Runs the LLM moderation analysis on a single message.
|
|
||||||
*
|
|
||||||
* The LLM verdict IS the result — confidence, severity, flags and
|
|
||||||
* analysis all come from the model. On failure the message is marked
|
|
||||||
* "error" (explicit, retryable) instead of receiving a heuristic verdict.
|
|
||||||
*/
|
|
||||||
async function runLLMAnalysis(
|
|
||||||
message: MessageRecord,
|
|
||||||
contextText: string,
|
|
||||||
attachments: Awaited<ReturnType<typeof messageStore.getAttachmentsForMessages>>,
|
|
||||||
): Promise<AnalysisResult> {
|
|
||||||
try {
|
|
||||||
const moderationResult = await runModerationAnalysis({
|
|
||||||
targets: [message],
|
|
||||||
contextText,
|
|
||||||
attachments,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (moderationResult.results.length === 0) {
|
|
||||||
return buildFallbackResult(message.id, "No LLM result returned");
|
|
||||||
}
|
|
||||||
|
|
||||||
const llmResult = moderationResult.results[0] as unknown as AnalysisResult;
|
|
||||||
if (llmResult.status === "error") {
|
|
||||||
return llmResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
messageId: llmResult.messageId,
|
|
||||||
status: llmResult.status ?? "clean",
|
|
||||||
flags: llmResult.flags ?? [],
|
|
||||||
categories: llmResult.categories ?? [],
|
|
||||||
severity: llmResult.severity ?? "none",
|
|
||||||
confidence: normalizeConfidence(llmResult.confidence),
|
|
||||||
recommendedAction: llmResult.recommendedAction ?? "none",
|
|
||||||
score: llmResult.score ?? 0,
|
|
||||||
analysis:
|
|
||||||
llmResult.analysis?.trim() ||
|
|
||||||
buildFallbackAnalysis(message, llmResult.status ?? "clean"),
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
||||||
logger.warn(
|
|
||||||
{ messageId: message.id, error: errorMsg },
|
|
||||||
"LLM analysis failed for message",
|
|
||||||
);
|
|
||||||
return buildFallbackResult(message.id, errorMsg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Clamp LLM-provided confidence to [0, 1]; default by status when missing. */
|
/** Clamp LLM-provided confidence to [0, 1]; default by status when missing. */
|
||||||
function normalizeConfidence(raw: number | undefined | null): number {
|
function normalizeConfidence(raw: number | undefined | null): number {
|
||||||
if (typeof raw === "number" && Number.isFinite(raw)) {
|
if (typeof raw === "number" && Number.isFinite(raw)) {
|
||||||
@@ -239,10 +195,7 @@ function normalizeConfidence(raw: number | undefined | null): number {
|
|||||||
* Quotes the actual message content so the log still explains WHAT was said
|
* Quotes the actual message content so the log still explains WHAT was said
|
||||||
* instead of a bare template like "Tidak ada indikasi pelanggaran."
|
* instead of a bare template like "Tidak ada indikasi pelanggaran."
|
||||||
*/
|
*/
|
||||||
function buildFallbackAnalysis(
|
function buildFallbackAnalysis(message: MessageRecord, status: string): string {
|
||||||
message: MessageRecord,
|
|
||||||
status: string,
|
|
||||||
): string {
|
|
||||||
const raw = (message.edited_content ?? message.content ?? "").trim();
|
const raw = (message.edited_content ?? message.content ?? "").trim();
|
||||||
const snippet = raw.length > 120 ? `${raw.slice(0, 120).trimEnd()}…` : raw;
|
const snippet = raw.length > 120 ? `${raw.slice(0, 120).trimEnd()}…` : raw;
|
||||||
|
|
||||||
@@ -273,8 +226,31 @@ function buildFallbackResult(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Normalize one orchestrator result into the worker's AnalysisResult shape. */
|
||||||
|
function normalizeResult(
|
||||||
|
result: AnalysisResult,
|
||||||
|
message: MessageRecord | undefined,
|
||||||
|
): AnalysisResult {
|
||||||
|
const status = result.status ?? "clean";
|
||||||
|
return {
|
||||||
|
messageId: result.messageId,
|
||||||
|
status,
|
||||||
|
flags: result.flags ?? [],
|
||||||
|
categories: result.categories ?? [],
|
||||||
|
severity: result.severity ?? "none",
|
||||||
|
confidence: normalizeConfidence(result.confidence),
|
||||||
|
recommendedAction: result.recommendedAction ?? "none",
|
||||||
|
score: result.score ?? 0,
|
||||||
|
analysis:
|
||||||
|
result.analysis?.trim() ||
|
||||||
|
(message
|
||||||
|
? buildFallbackAnalysis(message, status)
|
||||||
|
: buildFallbackResult(result.messageId, "Missing message").analysis),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Batch handler
|
// Batch handler — ONE orchestrator call for the whole batch
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function processBatch(job: {
|
async function processBatch(job: {
|
||||||
@@ -286,7 +262,7 @@ async function processBatch(job: {
|
|||||||
const firstMessage = messages[0];
|
const firstMessage = messages[0];
|
||||||
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
||||||
|
|
||||||
// Fetch context
|
// Fetch context + attachments ONCE for the whole batch.
|
||||||
const contextBefore = await messageStore.getConversationContextBefore({
|
const contextBefore = await messageStore.getConversationContextBefore({
|
||||||
channelId: firstMessage.channel_id,
|
channelId: firstMessage.channel_id,
|
||||||
threadId: firstMessage.thread_id,
|
threadId: firstMessage.thread_id,
|
||||||
@@ -301,31 +277,31 @@ async function processBatch(job: {
|
|||||||
});
|
});
|
||||||
const contextText = contextLines.join("\n");
|
const contextText = contextLines.join("\n");
|
||||||
|
|
||||||
// Fetch attachments
|
|
||||||
const targetIds = messages.map((m) => m.id);
|
const targetIds = messages.map((m) => m.id);
|
||||||
const contextIds = contextBefore.map((m) => m.id);
|
const contextIds = contextBefore.map((m) => m.id);
|
||||||
const allMessageIds = [...targetIds, ...contextIds];
|
const attachments = await messageStore.getAttachmentsForMessages([
|
||||||
const attachments =
|
...targetIds,
|
||||||
await messageStore.getAttachmentsForMessages(allMessageIds);
|
...contextIds,
|
||||||
|
]);
|
||||||
|
|
||||||
// Run LLM analysis for each message
|
// The orchestrator handles text/media split + caching + parallel paths
|
||||||
const analysisResults = await Promise.all(
|
// internally, so a 20-message batch = 1 text LLM call (+1 media call
|
||||||
messages.map(async (msg) => {
|
// when media is present), not N per-message calls.
|
||||||
try {
|
const moderationResult = await runModerationAnalysis({
|
||||||
return await runLLMAnalysis(msg, contextText, attachments);
|
targets: messages,
|
||||||
} catch (error) {
|
contextText,
|
||||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
attachments,
|
||||||
logger.error(
|
});
|
||||||
{ messageId: msg.id, error: errorMsg },
|
|
||||||
"LLM analysis failed for message",
|
const results = moderationResult.results.map((r) =>
|
||||||
);
|
normalizeResult(
|
||||||
return buildFallbackResult(msg.id, errorMsg);
|
r as unknown as AnalysisResult,
|
||||||
}
|
messages.find((m) => m.id === r.messageId),
|
||||||
}),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Save results to DB
|
// Save results to DB
|
||||||
const updates = analysisResults.map((result) => ({
|
const updates = results.map((result) => ({
|
||||||
messageId: result.messageId,
|
messageId: result.messageId,
|
||||||
result: {
|
result: {
|
||||||
status: result.status,
|
status: result.status,
|
||||||
@@ -393,8 +369,28 @@ async function processIndividual(job: {
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await runLLMAnalysis(message, contextText, attachments);
|
const moderationResult = await runModerationAnalysis({
|
||||||
return { ok: true, results: [result] };
|
targets: [message],
|
||||||
|
contextText,
|
||||||
|
attachments,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (moderationResult.results.length === 0) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
results: [buildFallbackResult(message.id, "No LLM result returned")],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const llmResult = moderationResult.results[0] as unknown as AnalysisResult;
|
||||||
|
if (llmResult.status === "error") {
|
||||||
|
return { ok: true, results: [llmResult] };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
results: [normalizeResult(llmResult, message)],
|
||||||
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { createChildLogger } from "@/shared/logger/index";
|
|
||||||
import type { Client } from "discord.js-selfbot-v13";
|
import type { Client } from "discord.js-selfbot-v13";
|
||||||
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
import type { EventBroadcaster } from "../event-broadcaster/index.js";
|
import type { EventBroadcaster } from "../event-broadcaster/index.js";
|
||||||
import { messageStore } from "../message-capture/messageStore.js";
|
import { messageStore } from "../message-capture/messageStore.js";
|
||||||
@@ -33,9 +33,17 @@ import {
|
|||||||
setModerationClient,
|
setModerationClient,
|
||||||
setSharedEventBroadcaster,
|
setSharedEventBroadcaster,
|
||||||
} from "./moderationState.js";
|
} from "./moderationState.js";
|
||||||
|
import { deleteExpiredQdrantPoints } from "./qdrantClient.js";
|
||||||
|
import { pruneExpiredTexts } from "./textCacheStore.js";
|
||||||
|
|
||||||
const logger = createChildLogger("ai-analyzer");
|
const logger = createChildLogger("ai-analyzer");
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Cache hygiene (expired verdict sweep)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
const CACHE_PRUNE_INTERVAL_MS = 6 * 60 * 60 * 1000; // every 6 hours
|
||||||
|
let lastCachePruneAt = 0;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Re-exports from sub-modules (preserving original public API)
|
// Re-exports from sub-modules (preserving original public API)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -133,6 +141,26 @@ export function startPendingAIAnalysisWorker(
|
|||||||
.catch(console.error);
|
.catch(console.error);
|
||||||
|
|
||||||
setInterval(() => {
|
setInterval(() => {
|
||||||
|
// [D] Periodic cache hygiene: purge expired moderation verdicts from
|
||||||
|
// Postgres and Qdrant. Expired entries are never reused (filters check
|
||||||
|
// expires_at) but accumulate forever without this sweep.
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastCachePruneAt >= CACHE_PRUNE_INTERVAL_MS) {
|
||||||
|
lastCachePruneAt = now;
|
||||||
|
Promise.all([pruneExpiredTexts(), deleteExpiredQdrantPoints()])
|
||||||
|
.then(([pgDeleted, qdDeleted]) => {
|
||||||
|
if (pgDeleted > 0 || qdDeleted > 0) {
|
||||||
|
logger.info(
|
||||||
|
{ pgDeleted, qdDeleted },
|
||||||
|
"Expired moderation cache pruned",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
logger.warn({ error: String(err) }, "Moderation cache prune failed");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
messageStore.revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
messageStore.revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
||||||
logger.error(
|
logger.error(
|
||||||
{ error: String(err) },
|
{ error: String(err) },
|
||||||
|
|||||||
@@ -1,327 +0,0 @@
|
|||||||
import { createChildLogger } from "@/shared/logger/index";
|
|
||||||
import { config } from "../../shared/config/config.js";
|
|
||||||
import { initializeDatabase } from "../../shared/database/drizzle.js";
|
|
||||||
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
|
|
||||||
import { messageStore } from "../message-capture/messageStore.js";
|
|
||||||
import type {
|
|
||||||
AnalysisResult,
|
|
||||||
MessageRecord,
|
|
||||||
} from "../message-capture/types.js";
|
|
||||||
import { buildConversationContext } from "./conversationContext.js";
|
|
||||||
import { runModerationAnalysis } from "./moderationOrchestrator.js";
|
|
||||||
import { runSimpleTextFallback } from "./simpleFallback.js";
|
|
||||||
|
|
||||||
const logger = createChildLogger("aiAnalysisWorker");
|
|
||||||
|
|
||||||
let dbInitialized = false;
|
|
||||||
let dbInitPromise: Promise<any> | null = null;
|
|
||||||
|
|
||||||
async function ensureDb() {
|
|
||||||
if (dbInitialized) return;
|
|
||||||
if (!dbInitPromise) {
|
|
||||||
dbInitPromise = initializeDatabase().then(() => {
|
|
||||||
dbInitialized = true;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
await dbInitPromise;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Job types — the default export routes on `type`
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
type WorkerJob =
|
|
||||||
| { type: "batch"; conversationKey: string; messages: MessageRecord[] }
|
|
||||||
| { type: "individual"; message: MessageRecord; skipNormalAnalysis: boolean };
|
|
||||||
|
|
||||||
type BatchOkResponse = {
|
|
||||||
ok: true;
|
|
||||||
conversationKey: string;
|
|
||||||
rows: MessageRecord[];
|
|
||||||
};
|
|
||||||
type BatchErrorResponse = {
|
|
||||||
ok: false;
|
|
||||||
conversationKey: string;
|
|
||||||
rows: MessageRecord[];
|
|
||||||
error: string;
|
|
||||||
};
|
|
||||||
type IndividualOkResponse = { ok: true; results: AnalysisResult[] };
|
|
||||||
type IndividualErrorResponse = {
|
|
||||||
ok: false;
|
|
||||||
results: AnalysisResult[];
|
|
||||||
error: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type WorkerResponse =
|
|
||||||
| BatchOkResponse
|
|
||||||
| BatchErrorResponse
|
|
||||||
| IndividualOkResponse
|
|
||||||
| IndividualErrorResponse;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default export — Piscina worker entry point.
|
|
||||||
* Routes to the correct handler based on `type` field.
|
|
||||||
*/
|
|
||||||
export default async function workerRouter(
|
|
||||||
job: WorkerJob,
|
|
||||||
): Promise<WorkerResponse> {
|
|
||||||
if (!config.AI_LLM_API_KEY) {
|
|
||||||
const errorMsg =
|
|
||||||
"AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials.";
|
|
||||||
logger.error(
|
|
||||||
{ error: errorMsg },
|
|
||||||
"AI_LLM_API_KEY is missing from environment",
|
|
||||||
);
|
|
||||||
|
|
||||||
if (job.type === "batch") {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
conversationKey: job.conversationKey,
|
|
||||||
rows: [],
|
|
||||||
error: errorMsg,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { ok: false, results: [], error: errorMsg };
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await ensureDb();
|
|
||||||
} catch (dbError) {
|
|
||||||
const msg = dbError instanceof Error ? dbError.message : String(dbError);
|
|
||||||
if (job.type === "batch") {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
conversationKey: job.conversationKey,
|
|
||||||
rows: [],
|
|
||||||
error: `Database init failed: ${msg}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { ok: false, results: [], error: `Database init failed: ${msg}` };
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (job.type === "batch") {
|
|
||||||
return await processBatch(job);
|
|
||||||
}
|
|
||||||
return await processIndividual(job);
|
|
||||||
} catch (error) {
|
|
||||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
||||||
const errorStack = error instanceof Error ? error.stack : undefined;
|
|
||||||
logger.error(
|
|
||||||
{ type: job.type, error: errorMessage, stack: errorStack },
|
|
||||||
"Worker job failed",
|
|
||||||
);
|
|
||||||
if (job.type === "batch") {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
conversationKey: job.conversationKey,
|
|
||||||
rows: [],
|
|
||||||
error: errorMessage,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { ok: false, results: [], error: errorMessage };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Batch handler
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async function processBatch(job: {
|
|
||||||
type: "batch";
|
|
||||||
conversationKey: string;
|
|
||||||
messages: MessageRecord[];
|
|
||||||
}): Promise<BatchOkResponse | BatchErrorResponse> {
|
|
||||||
const { conversationKey, messages } = job;
|
|
||||||
const firstMessage = messages[0];
|
|
||||||
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
|
||||||
|
|
||||||
const contextBefore = await messageStore.getConversationContextBefore({
|
|
||||||
channelId: firstMessage.channel_id,
|
|
||||||
threadId: firstMessage.thread_id,
|
|
||||||
beforeCreatedAt: firstMessage.created_at,
|
|
||||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
|
||||||
});
|
|
||||||
|
|
||||||
const contextLines = buildConversationContext({
|
|
||||||
contextBefore,
|
|
||||||
targets: messages,
|
|
||||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
|
||||||
});
|
|
||||||
|
|
||||||
const targetIds = messages.map((m) => m.id);
|
|
||||||
const contextIds = contextBefore.map((m) => m.id);
|
|
||||||
const allMessageIds = [...targetIds, ...contextIds];
|
|
||||||
const attachments =
|
|
||||||
await messageStore.getAttachmentsForMessages(allMessageIds);
|
|
||||||
|
|
||||||
// ── Split: text-only vs media ──────────────────────────────────────
|
|
||||||
// Text-only analysis runs fast (single LLM call, no vision).
|
|
||||||
// Media analysis is slow (download + vision → LLM).
|
|
||||||
// Messages with BOTH text and media go into both arrays:
|
|
||||||
// - text batch → analyzes the text content immediately
|
|
||||||
// - media batch → analyzes images/video when ready
|
|
||||||
// By splitting here, text results are saved to DB immediately
|
|
||||||
// instead of waiting for media downloads to finish.
|
|
||||||
// ────────────────────────────────────────────────────────────────────
|
|
||||||
const textOnly: MessageRecord[] = [];
|
|
||||||
const media: MessageRecord[] = [];
|
|
||||||
|
|
||||||
for (const msg of messages) {
|
|
||||||
const meta = msg.metadata
|
|
||||||
? extractMessageMediaEvidence(msg.metadata)
|
|
||||||
: null;
|
|
||||||
if (
|
|
||||||
meta &&
|
|
||||||
(meta.attachments.length > 0 ||
|
|
||||||
meta.stickers.length > 0 ||
|
|
||||||
meta.embeds.length > 0)
|
|
||||||
) {
|
|
||||||
media.push(msg);
|
|
||||||
// If the message also has text content, analyze it in the text batch too
|
|
||||||
const rawContent = msg.edited_content ?? msg.content;
|
|
||||||
if (rawContent.trim().length > 0) {
|
|
||||||
textOnly.push(msg);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
textOnly.push(msg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const allRows: MessageRecord[] = [];
|
|
||||||
|
|
||||||
// ── Parallel: text-only + media analysis run concurrently ──────────
|
|
||||||
// Text-only → fast LLM call. Media → download + vision + LLM.
|
|
||||||
// Running both in parallel means media downloads overlap with text LLM call.
|
|
||||||
// Each path saves to DB as soon as its own results are ready.
|
|
||||||
// ────────────────────────────────────────────────────────────────────
|
|
||||||
const textPromise =
|
|
||||||
textOnly.length > 0
|
|
||||||
? runModerationAnalysis({
|
|
||||||
targets: textOnly,
|
|
||||||
contextText: contextLines.join("\n"),
|
|
||||||
attachments,
|
|
||||||
}).then((result) => {
|
|
||||||
const updates = result.results.map((analysisResult) => ({
|
|
||||||
messageId: analysisResult.messageId,
|
|
||||||
result: {
|
|
||||||
status: analysisResult.status,
|
|
||||||
flags: JSON.stringify(analysisResult.flags),
|
|
||||||
score: analysisResult.score,
|
|
||||||
analysis: analysisResult.analysis,
|
|
||||||
categories: analysisResult.categories,
|
|
||||||
severity: analysisResult.severity,
|
|
||||||
confidence: analysisResult.confidence,
|
|
||||||
recommendedAction: analysisResult.recommendedAction,
|
|
||||||
analyzedAt: Date.now(),
|
|
||||||
error: null,
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
if (updates.length > 0) {
|
|
||||||
return messageStore
|
|
||||||
.updateMessagesAIAnalysisBulk(updates)
|
|
||||||
.then((rows) => {
|
|
||||||
allRows.push(...rows);
|
|
||||||
logger.info(
|
|
||||||
{ count: updates.length, conversationKey },
|
|
||||||
"Text-only batch saved — media analysis still in progress",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
})
|
|
||||||
: Promise.resolve();
|
|
||||||
|
|
||||||
const mediaPromise =
|
|
||||||
media.length > 0
|
|
||||||
? runModerationAnalysis({
|
|
||||||
targets: media,
|
|
||||||
contextText: contextLines.join("\n"),
|
|
||||||
attachments,
|
|
||||||
}).then((result) => {
|
|
||||||
const updates = result.results.map((analysisResult) => ({
|
|
||||||
messageId: analysisResult.messageId,
|
|
||||||
result: {
|
|
||||||
status: analysisResult.status,
|
|
||||||
flags: JSON.stringify(analysisResult.flags),
|
|
||||||
score: analysisResult.score,
|
|
||||||
analysis: analysisResult.analysis,
|
|
||||||
categories: analysisResult.categories,
|
|
||||||
severity: analysisResult.severity,
|
|
||||||
confidence: analysisResult.confidence,
|
|
||||||
recommendedAction: analysisResult.recommendedAction,
|
|
||||||
analyzedAt: Date.now(),
|
|
||||||
error: null,
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
if (updates.length > 0) {
|
|
||||||
return messageStore
|
|
||||||
.updateMessagesAIAnalysisBulk(updates)
|
|
||||||
.then((rows) => {
|
|
||||||
allRows.push(...rows);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
})
|
|
||||||
: Promise.resolve();
|
|
||||||
|
|
||||||
// Wait for both to complete
|
|
||||||
await Promise.all([textPromise, mediaPromise]);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
total: messages.length,
|
|
||||||
textOnly: textOnly.length,
|
|
||||||
media: media.length,
|
|
||||||
saved: allRows.length,
|
|
||||||
},
|
|
||||||
"Batch analysis complete",
|
|
||||||
);
|
|
||||||
|
|
||||||
return { ok: true, conversationKey, rows: allRows };
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Individual fallback handler (offloaded from main thread)
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async function processIndividual(job: {
|
|
||||||
type: "individual";
|
|
||||||
message: MessageRecord;
|
|
||||||
skipNormalAnalysis: boolean;
|
|
||||||
}): Promise<IndividualOkResponse | IndividualErrorResponse> {
|
|
||||||
const { message, skipNormalAnalysis } = job;
|
|
||||||
|
|
||||||
const contextBefore = await messageStore.getConversationContextBefore({
|
|
||||||
channelId: message.channel_id,
|
|
||||||
threadId: message.thread_id,
|
|
||||||
beforeCreatedAt: message.created_at,
|
|
||||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
|
||||||
});
|
|
||||||
|
|
||||||
const contextLines = buildConversationContext({
|
|
||||||
contextBefore,
|
|
||||||
targets: [message],
|
|
||||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
|
||||||
});
|
|
||||||
|
|
||||||
const contextIds = contextBefore.map((m) => m.id);
|
|
||||||
const attachments = await messageStore.getAttachmentsForMessages([
|
|
||||||
message.id,
|
|
||||||
...contextIds,
|
|
||||||
]);
|
|
||||||
|
|
||||||
let results: AnalysisResult[];
|
|
||||||
|
|
||||||
if (skipNormalAnalysis) {
|
|
||||||
const simpleResult = await runSimpleTextFallback(message);
|
|
||||||
results = [simpleResult];
|
|
||||||
} else {
|
|
||||||
const moderationResult = await runModerationAnalysis({
|
|
||||||
targets: [message],
|
|
||||||
contextText: contextLines.join("\n"),
|
|
||||||
attachments,
|
|
||||||
});
|
|
||||||
results = moderationResult.results;
|
|
||||||
}
|
|
||||||
|
|
||||||
return { ok: true, results };
|
|
||||||
}
|
|
||||||
@@ -9,9 +9,10 @@
|
|||||||
*
|
*
|
||||||
* Both sides now import from this module instead.
|
* Both sides now import from this module instead.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type { ChatCompletion } from "openai/resources/chat/completions";
|
||||||
import { createChildLogger } from "@/shared/logger/index";
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
import { delay, retryWithBackoff } from "@/shared/utils/index";
|
import { delay, retryWithBackoff } from "@/shared/utils/index";
|
||||||
import type { ChatCompletion } from "openai/resources/chat/completions";
|
|
||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
import type { AnalysisResult } from "../message-capture/types.js";
|
import type { AnalysisResult } from "../message-capture/types.js";
|
||||||
import { llmChat } from "./llmClient.js";
|
import { llmChat } from "./llmClient.js";
|
||||||
@@ -27,11 +28,24 @@ export interface RetryState {
|
|||||||
lastInvalidContent: string | null;
|
lastInvalidContent: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Content builder contract: produces the moderation prompt split into
|
||||||
|
* SYSTEM (rules / output schema / context — stable) and USER (the actual
|
||||||
|
* `<messages_to_analyze>` payload). Kept as two roles so routers and
|
||||||
|
* providers that treat system messages differently get the correct framing.
|
||||||
|
*/
|
||||||
|
export interface ModerationPromptContent {
|
||||||
|
system: string;
|
||||||
|
user: string;
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Shared LLM call + parse + fallback helper
|
// Shared LLM call + parse + fallback helper
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
export async function callModerationLLM(
|
export async function callModerationLLM(
|
||||||
buildContent: (state: RetryState) => Promise<string>,
|
buildContent: (
|
||||||
|
state: RetryState,
|
||||||
|
) => Promise<string | ModerationPromptContent>,
|
||||||
targetIds: string[],
|
targetIds: string[],
|
||||||
label: string,
|
label: string,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
@@ -52,8 +66,15 @@ export async function callModerationLLM(
|
|||||||
async () => {
|
async () => {
|
||||||
try {
|
try {
|
||||||
const content = await buildContent(state);
|
const content = await buildContent(state);
|
||||||
|
const messages =
|
||||||
|
typeof content === "string"
|
||||||
|
? [{ role: "user" as const, content }]
|
||||||
|
: [
|
||||||
|
{ role: "system" as const, content: content.system },
|
||||||
|
{ role: "user" as const, content: content.user },
|
||||||
|
];
|
||||||
const completion = await llmChat({
|
const completion = await llmChat({
|
||||||
messages: [{ role: "user", content }],
|
messages,
|
||||||
max_tokens: 16384,
|
max_tokens: 16384,
|
||||||
jsonResponse: { type: "json_object" },
|
jsonResponse: { type: "json_object" },
|
||||||
retries: 0,
|
retries: 0,
|
||||||
@@ -133,6 +154,23 @@ export async function callModerationLLM(
|
|||||||
);
|
);
|
||||||
parsed = analysis.parsed;
|
parsed = analysis.parsed;
|
||||||
result = analysis.result;
|
result = analysis.result;
|
||||||
|
|
||||||
|
// [I] Token usage accounting — surface provider-reported usage per batch
|
||||||
|
// so cost per channel/guild can be tracked (routers bill per token).
|
||||||
|
const usage = result?.usage;
|
||||||
|
if (usage && (usage.prompt_tokens || usage.completion_tokens)) {
|
||||||
|
log.info(
|
||||||
|
{
|
||||||
|
label,
|
||||||
|
targetIds,
|
||||||
|
model: config.AI_LLM_MODEL,
|
||||||
|
prompt_tokens: usage.prompt_tokens,
|
||||||
|
completion_tokens: usage.completion_tokens,
|
||||||
|
total_tokens: usage.total_tokens,
|
||||||
|
},
|
||||||
|
`LLM usage (${label})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof Error && err.name === "AbortError") throw err;
|
if (err instanceof Error && err.name === "AbortError") throw err;
|
||||||
|
|
||||||
|
|||||||
@@ -6,10 +6,10 @@
|
|||||||
* defaults are maintained in one place.
|
* defaults are maintained in one place.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { createChildLogger } from "@/shared/logger/index";
|
|
||||||
import { retryWithBackoff } from "@/shared/utils/index";
|
|
||||||
import OpenAI from "openai";
|
import OpenAI from "openai";
|
||||||
import pLimit from "p-limit";
|
import pLimit from "p-limit";
|
||||||
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
|
import { retryWithBackoff } from "@/shared/utils/index";
|
||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
|
|
||||||
const log = createChildLogger("llm-client");
|
const log = createChildLogger("llm-client");
|
||||||
@@ -245,39 +245,13 @@ export async function llmChat(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Convenience for the legacy text-only badword detection call in
|
|
||||||
* `indonesianTextNormalizer`. Returns parsed flags or [].
|
|
||||||
*/
|
|
||||||
export async function llmDetectBadwords(text: string): Promise<string[]> {
|
|
||||||
const completion = await llmChat({
|
|
||||||
messages: [
|
|
||||||
{
|
|
||||||
role: "user",
|
|
||||||
content:
|
|
||||||
"Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " +
|
|
||||||
'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' +
|
|
||||||
Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") +
|
|
||||||
". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " +
|
|
||||||
text,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
max_tokens: 200,
|
|
||||||
temperature: 0.1,
|
|
||||||
top_p: 0.9,
|
|
||||||
jsonResponse: { type: "json_object" },
|
|
||||||
retries: 2,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!completion) return [];
|
|
||||||
const content = completion.choices[0]?.message?.content?.trim();
|
|
||||||
if (!content) return [];
|
|
||||||
return extractFlagsFromContent(content);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Convenience for vision (image/sticker/emoji) analysis.
|
* Convenience for vision (image/sticker/emoji) analysis.
|
||||||
* Returns the raw completion content (trimmed) or null.
|
* Returns the raw completion content (trimmed) or null.
|
||||||
|
*
|
||||||
|
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
|
||||||
|
* wraps this call in its own 3-attempt loop with exponential backoff.
|
||||||
|
* A second retry layer would multiply worst-case API calls (3×3=9/image).
|
||||||
*/
|
*/
|
||||||
export async function llmVision(
|
export async function llmVision(
|
||||||
promptText: string,
|
promptText: string,
|
||||||
@@ -297,91 +271,9 @@ export async function llmVision(
|
|||||||
max_tokens: 500,
|
max_tokens: 500,
|
||||||
temperature: 0.1,
|
temperature: 0.1,
|
||||||
top_p: 0.9,
|
top_p: 0.9,
|
||||||
retries: 2,
|
retries: 0,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!completion) return null;
|
if (!completion) return null;
|
||||||
return completion.choices[0]?.message?.content?.trim() ?? null;
|
return completion.choices[0]?.message?.content?.trim() ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Flag extraction (reused from indonesianTextNormalizer)
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const VALID_PRIMARY_AI_FLAGS = new Set([
|
|
||||||
"spam",
|
|
||||||
"hate_speech",
|
|
||||||
"sara",
|
|
||||||
"hoaks",
|
|
||||||
"harassment",
|
|
||||||
"vulgar_language",
|
|
||||||
"sexual_content",
|
|
||||||
"sexual_deviation",
|
|
||||||
"violence",
|
|
||||||
"self_harm",
|
|
||||||
"doxxing",
|
|
||||||
"scam",
|
|
||||||
"misinformation",
|
|
||||||
"nsfw_image",
|
|
||||||
"gore_image",
|
|
||||||
"illegal_content",
|
|
||||||
"gambling",
|
|
||||||
"drugs",
|
|
||||||
"child_safety",
|
|
||||||
"financial_scam",
|
|
||||||
"religious_insult",
|
|
||||||
"self_promo",
|
|
||||||
"conflict_instigation",
|
|
||||||
"offensive_username",
|
|
||||||
"potential_evasion",
|
|
||||||
"unclear_context",
|
|
||||||
]);
|
|
||||||
|
|
||||||
function normalizeFlag(value: string): string | null {
|
|
||||||
const lower = value
|
|
||||||
.trim()
|
|
||||||
.toLowerCase()
|
|
||||||
.replace(/[\s-]+/g, "_");
|
|
||||||
if (!lower) return null;
|
|
||||||
if (VALID_PRIMARY_AI_FLAGS.has(lower)) return lower;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function extractFlagsFromContent(content: string): string[] {
|
|
||||||
const flags = new Set<string>();
|
|
||||||
let parsed: unknown;
|
|
||||||
try {
|
|
||||||
parsed = JSON.parse(content);
|
|
||||||
} catch {
|
|
||||||
parsed = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const addValue = (v: unknown) => {
|
|
||||||
if (typeof v !== "string") return;
|
|
||||||
const n = normalizeFlag(v);
|
|
||||||
if (n) flags.add(n);
|
|
||||||
};
|
|
||||||
|
|
||||||
if (Array.isArray(parsed)) {
|
|
||||||
for (const item of parsed) addValue(item);
|
|
||||||
} else if (parsed && typeof parsed === "object") {
|
|
||||||
const obj = parsed as Record<string, unknown>;
|
|
||||||
for (const key of ["flags", "categories", "badwords"]) {
|
|
||||||
const val = obj[key];
|
|
||||||
if (Array.isArray(val)) {
|
|
||||||
for (const item of val) addValue(item);
|
|
||||||
} else {
|
|
||||||
addValue(val);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (flags.size > 0) return Array.from(flags);
|
|
||||||
|
|
||||||
const lower = content.toLowerCase();
|
|
||||||
for (const flag of VALID_PRIMARY_AI_FLAGS) {
|
|
||||||
if (lower.includes(flag)) flags.add(flag);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Array.from(flags);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import type {
|
|||||||
MessageRecord,
|
MessageRecord,
|
||||||
} from "../message-capture/types.js";
|
} from "../message-capture/types.js";
|
||||||
import { getChannelCulture } from "./channelCultureStore.js";
|
import { getChannelCulture } from "./channelCultureStore.js";
|
||||||
import { prepareMediaMessage } from "./mediaAnalysisClient.js";
|
|
||||||
import type { RetryState } from "./llmCaller.js";
|
import type { RetryState } from "./llmCaller.js";
|
||||||
import { callModerationLLM } from "./llmCaller.js";
|
import { callModerationLLM } from "./llmCaller.js";
|
||||||
|
import { prepareMediaMessage } from "./mediaAnalysisClient.js";
|
||||||
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
|
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
|
||||||
import { buildCorrectedFewShotExamples } from "./textBatchProcessor.js";
|
import { buildCorrectedFewShotExamples } from "./textBatchProcessor.js";
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ export async function runMediaBatch(
|
|||||||
});
|
});
|
||||||
|
|
||||||
const messagesBlock = prepared.map((p) => p.messageBlock).join("\n");
|
const messagesBlock = prepared.map((p) => p.messageBlock).join("\n");
|
||||||
const userContent = `${systemText}\n\n<messages_to_analyze>\n${messagesBlock}\n</messages_to_analyze>`;
|
const userContent = `<messages_to_analyze>\n${messagesBlock}\n</messages_to_analyze>`;
|
||||||
|
|
||||||
const perMsgTimeout = config.AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS ?? 60000;
|
const perMsgTimeout = config.AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS ?? 60000;
|
||||||
const batchTimeout = Math.min(
|
const batchTimeout = Math.min(
|
||||||
@@ -79,7 +79,7 @@ export async function runMediaBatch(
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await callModerationLLM(
|
const result = await callModerationLLM(
|
||||||
async (_state: RetryState) => userContent,
|
async (_state: RetryState) => ({ system: systemText, user: userContent }),
|
||||||
targetIds,
|
targetIds,
|
||||||
`media-batch:${targetIds.length}msgs`,
|
`media-batch:${targetIds.length}msgs`,
|
||||||
abortController.signal,
|
abortController.signal,
|
||||||
|
|||||||
@@ -12,15 +12,19 @@ import type {
|
|||||||
AttachmentRecord,
|
AttachmentRecord,
|
||||||
MessageRecord,
|
MessageRecord,
|
||||||
} from "../message-capture/types.js";
|
} from "../message-capture/types.js";
|
||||||
|
import { embedTexts, isEmbeddingEnabled } from "./embeddingClient.js";
|
||||||
import { hasMediaContent } from "./mediaAnalysisClient.js";
|
import { hasMediaContent } from "./mediaAnalysisClient.js";
|
||||||
import { runMediaBatch } from "./mediaBatchProcessor.js";
|
import { runMediaBatch } from "./mediaBatchProcessor.js";
|
||||||
|
import { isQdrantConfigured, searchQdrantBatch } from "./qdrantClient.js";
|
||||||
|
import { logCacheEvent } from "./responseLogger.js";
|
||||||
import { initSearxngCache } from "./searxngSearch.js";
|
import { initSearxngCache } from "./searxngSearch.js";
|
||||||
import { runTextOnlyBatch } from "./textBatchProcessor.js";
|
import { runTextOnlyBatch } from "./textBatchProcessor.js";
|
||||||
import { embedText, isEmbeddingEnabled } from "./embeddingClient.js";
|
|
||||||
import {
|
import {
|
||||||
findSimilarTextModeration,
|
findSimilarTextModeration,
|
||||||
getCachedTextModeration,
|
getCachedTextModeration,
|
||||||
|
makeModerationContextKey,
|
||||||
makeTextModerationCacheKey,
|
makeTextModerationCacheKey,
|
||||||
|
parseQdrantVerdict,
|
||||||
setCachedTextModeration,
|
setCachedTextModeration,
|
||||||
} from "./textCacheStore.js";
|
} from "./textCacheStore.js";
|
||||||
|
|
||||||
@@ -47,6 +51,13 @@ export interface ModerationOutput {
|
|||||||
/**
|
/**
|
||||||
* Runs LLM-based moderation analysis on messages.
|
* Runs LLM-based moderation analysis on messages.
|
||||||
* Splits text-only vs media, runs both paths in parallel, applies caching.
|
* Splits text-only vs media, runs both paths in parallel, applies caching.
|
||||||
|
*
|
||||||
|
* Cache strategy (two-phase, batched):
|
||||||
|
* 1. Exact-hash lookups (no API) — key is content + conversation context
|
||||||
|
* (channel/thread) because LLM verdicts depend on context.
|
||||||
|
* 2. Semantic near-duplicate lookup — ONE embeddings call for all uncached
|
||||||
|
* text targets, then ONE Qdrant batch search (index-aligned), instead of
|
||||||
|
* N sequential embed→search round-trips.
|
||||||
*/
|
*/
|
||||||
export async function runModerationAnalysis(
|
export async function runModerationAnalysis(
|
||||||
input: ModerationInput,
|
input: ModerationInput,
|
||||||
@@ -56,10 +67,11 @@ export async function runModerationAnalysis(
|
|||||||
initSearxngCache(config.REDIS_URL);
|
initSearxngCache(config.REDIS_URL);
|
||||||
if (!targets.length) throw new Error("No targets provided for analysis");
|
if (!targets.length) throw new Error("No targets provided for analysis");
|
||||||
|
|
||||||
// Per-user moderation cache check (text-only)
|
// ── Phase 1: exact-hash cache (per conversation context) ────────────────
|
||||||
const cacheHits: AnalysisResult[] = [];
|
const cacheHits: AnalysisResult[] = [];
|
||||||
const uncachedTargets: MessageRecord[] = [];
|
const uncachedTargets: MessageRecord[] = [];
|
||||||
const seenCacheKeys = new Set<string>();
|
// cacheKey → result for identical-content dedupe within one batch
|
||||||
|
const hitByKey = new Map<string, AnalysisResult>();
|
||||||
// Embedding per exact cache key — computed once during lookup, reused
|
// Embedding per exact cache key — computed once during lookup, reused
|
||||||
// when the fresh LLM verdict is written back to the semantic cache.
|
// when the fresh LLM verdict is written back to the semantic cache.
|
||||||
const embeddingsByKey = new Map<string, number[]>();
|
const embeddingsByKey = new Map<string, number[]>();
|
||||||
@@ -77,17 +89,16 @@ export async function runModerationAnalysis(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const cacheKey = makeTextModerationCacheKey(rawContent);
|
const cacheKey = makeTextModerationCacheKey(
|
||||||
if (seenCacheKeys.has(cacheKey)) {
|
rawContent,
|
||||||
const previousHit = cacheHits.find((h) => h.messageId !== target.id);
|
makeModerationContextKey(target),
|
||||||
if (previousHit) {
|
);
|
||||||
cacheHits.push({ ...previousHit, messageId: target.id });
|
const seen = hitByKey.get(cacheKey);
|
||||||
} else {
|
if (seen) {
|
||||||
uncachedTargets.push(target);
|
// Same content already resolved this batch — reuse the verdict.
|
||||||
}
|
cacheHits.push({ ...seen, messageId: target.id });
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
seenCacheKeys.add(cacheKey);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const cached = await getCachedTextModeration(cacheKey);
|
const cached = await getCachedTextModeration(cacheKey);
|
||||||
@@ -122,7 +133,7 @@ export async function runModerationAnalysis(
|
|||||||
"Cache entry contains error artifact — treating as miss",
|
"Cache entry contains error artifact — treating as miss",
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
cacheHits.push({
|
const hit: AnalysisResult = {
|
||||||
messageId: target.id,
|
messageId: target.id,
|
||||||
status: cached.status,
|
status: cached.status,
|
||||||
flags: cached.flags,
|
flags: cached.flags,
|
||||||
@@ -135,7 +146,10 @@ export async function runModerationAnalysis(
|
|||||||
cached.recommendedAction as AnalysisResult["recommendedAction"],
|
cached.recommendedAction as AnalysisResult["recommendedAction"],
|
||||||
policyVersion: "cached-user-moderation-2026-06",
|
policyVersion: "cached-user-moderation-2026-06",
|
||||||
evidence: [],
|
evidence: [],
|
||||||
} as AnalysisResult);
|
};
|
||||||
|
cacheHits.push(hit);
|
||||||
|
hitByKey.set(cacheKey, hit);
|
||||||
|
logCacheEvent("hit", cacheKey, "text");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,47 +157,130 @@ export async function runModerationAnalysis(
|
|||||||
/* proceed */
|
/* proceed */
|
||||||
}
|
}
|
||||||
|
|
||||||
// Semantic cache: reuse verdicts for near-duplicate text (requires the
|
uncachedTargets.push(target);
|
||||||
// configured embedding model). Only non-trivial text-only messages
|
}
|
||||||
// qualify — media and empty text never take this path.
|
|
||||||
if (isEmbeddingEnabled() && rawContent.trim().length >= 5) {
|
// ── Phase 2: semantic cache — batched (one embed call + one Qdrant
|
||||||
const embedding = await embedText(rawContent);
|
// batch search for ALL uncached text targets) ─────────────────────────
|
||||||
if (embedding) {
|
if (isEmbeddingEnabled()) {
|
||||||
embeddingsByKey.set(cacheKey, embedding);
|
const semanticCandidates = uncachedTargets
|
||||||
const semantic = await findSimilarTextModeration(
|
.map((t) => ({
|
||||||
embedding,
|
target: t,
|
||||||
config.AI_LLM_EMBEDDING_MIN_SIMILARITY,
|
cacheKey: makeTextModerationCacheKey(
|
||||||
config.AI_LLM_EMBEDDING_MAX_CANDIDATES,
|
t.edited_content ?? t.content,
|
||||||
|
makeModerationContextKey(t),
|
||||||
|
),
|
||||||
|
}))
|
||||||
|
.filter(({ target }) => {
|
||||||
|
const raw = (target.edited_content ?? target.content).trim();
|
||||||
|
if (raw.length < 5) return false;
|
||||||
|
if (hasMediaContent(target, attachments)) return false;
|
||||||
|
return !hitByKey.has(
|
||||||
|
makeTextModerationCacheKey(raw, makeModerationContextKey(target)),
|
||||||
);
|
);
|
||||||
if (semantic) {
|
});
|
||||||
log.debug(
|
|
||||||
{
|
if (semanticCandidates.length > 0) {
|
||||||
messageId: target.id,
|
const texts = semanticCandidates.map(
|
||||||
similarity: Number(semantic.similarity.toFixed(4)),
|
({ target }) => target.edited_content ?? target.content,
|
||||||
status: semantic.status,
|
);
|
||||||
},
|
const embeddings = await embedTexts(texts);
|
||||||
"Semantic moderation cache hit — reusing stored verdict",
|
if (embeddings && embeddings.length === texts.length) {
|
||||||
|
// index-aligned with semanticCandidates
|
||||||
|
for (let i = 0; i < semanticCandidates.length; i++) {
|
||||||
|
const { target, cacheKey } = semanticCandidates[i];
|
||||||
|
embeddingsByKey.set(cacheKey, embeddings[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isQdrantConfigured()) {
|
||||||
|
const batchHits = await searchQdrantBatch(
|
||||||
|
embeddings,
|
||||||
|
config.AI_LLM_EMBEDDING_MAX_CANDIDATES,
|
||||||
|
config.AI_LLM_EMBEDDING_MIN_SIMILARITY,
|
||||||
);
|
);
|
||||||
cacheHits.push({
|
for (let i = 0; i < semanticCandidates.length; i++) {
|
||||||
messageId: target.id,
|
const { target, cacheKey } = semanticCandidates[i];
|
||||||
status: semantic.status,
|
const hits = batchHits[i] ?? [];
|
||||||
flags: semantic.flags,
|
if (hits.length === 0) continue;
|
||||||
score: semantic.score,
|
const verdict = parseQdrantVerdict(hits[0].payload, hits[0].score);
|
||||||
analysis: semantic.analysis,
|
if (!verdict) continue;
|
||||||
categories: semantic.categories,
|
log.debug(
|
||||||
severity: semantic.severity as AnalysisResult["severity"],
|
{
|
||||||
confidence: semantic.confidence,
|
messageId: target.id,
|
||||||
recommendedAction:
|
similarity: Number(verdict.similarity.toFixed(4)),
|
||||||
semantic.recommendedAction as AnalysisResult["recommendedAction"],
|
status: verdict.status,
|
||||||
policyVersion: "semantic-cache-2026-07",
|
},
|
||||||
evidence: [],
|
"Semantic moderation cache hit — reusing stored verdict",
|
||||||
} as AnalysisResult);
|
);
|
||||||
continue;
|
const hit: AnalysisResult = {
|
||||||
|
messageId: target.id,
|
||||||
|
status: verdict.status,
|
||||||
|
flags: verdict.flags,
|
||||||
|
score: verdict.score,
|
||||||
|
analysis: verdict.analysis,
|
||||||
|
categories: verdict.categories,
|
||||||
|
severity: verdict.severity as AnalysisResult["severity"],
|
||||||
|
confidence: verdict.confidence,
|
||||||
|
recommendedAction:
|
||||||
|
verdict.recommendedAction as AnalysisResult["recommendedAction"],
|
||||||
|
policyVersion: "semantic-cache-2026-07",
|
||||||
|
evidence: [],
|
||||||
|
};
|
||||||
|
cacheHits.push(hit);
|
||||||
|
hitByKey.set(cacheKey, hit);
|
||||||
|
logCacheEvent("hit", cacheKey, "text");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Legacy Postgres fallback path (no Qdrant): per-candidate scan.
|
||||||
|
for (let i = 0; i < semanticCandidates.length; i++) {
|
||||||
|
const { target, cacheKey } = semanticCandidates[i];
|
||||||
|
const semantic = await findSimilarTextModeration(
|
||||||
|
embeddings[i],
|
||||||
|
config.AI_LLM_EMBEDDING_MIN_SIMILARITY,
|
||||||
|
config.AI_LLM_EMBEDDING_MAX_CANDIDATES,
|
||||||
|
);
|
||||||
|
if (!semantic) continue;
|
||||||
|
log.debug(
|
||||||
|
{
|
||||||
|
messageId: target.id,
|
||||||
|
similarity: Number(semantic.similarity.toFixed(4)),
|
||||||
|
status: semantic.status,
|
||||||
|
},
|
||||||
|
"Semantic moderation cache hit (PG fallback) — reusing stored verdict",
|
||||||
|
);
|
||||||
|
const hit: AnalysisResult = {
|
||||||
|
messageId: target.id,
|
||||||
|
status: semantic.status,
|
||||||
|
flags: semantic.flags,
|
||||||
|
score: semantic.score,
|
||||||
|
analysis: semantic.analysis,
|
||||||
|
categories: semantic.categories,
|
||||||
|
severity: semantic.severity as AnalysisResult["severity"],
|
||||||
|
confidence: semantic.confidence,
|
||||||
|
recommendedAction:
|
||||||
|
semantic.recommendedAction as AnalysisResult["recommendedAction"],
|
||||||
|
policyVersion: "semantic-cache-2026-07",
|
||||||
|
evidence: [],
|
||||||
|
};
|
||||||
|
cacheHits.push(hit);
|
||||||
|
hitByKey.set(cacheKey, hit);
|
||||||
|
logCacheEvent("hit", cacheKey, "text");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop semantic hits from the LLM work queue.
|
||||||
|
for (let i = uncachedTargets.length - 1; i >= 0; i--) {
|
||||||
|
const t = uncachedTargets[i];
|
||||||
|
const key = makeTextModerationCacheKey(
|
||||||
|
t.edited_content ?? t.content,
|
||||||
|
makeModerationContextKey(t),
|
||||||
|
);
|
||||||
|
if (hitByKey.has(key)) {
|
||||||
|
uncachedTargets.splice(i, 1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
uncachedTargets.push(target);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cacheHits.length > 0) {
|
if (cacheHits.length > 0) {
|
||||||
@@ -248,7 +345,10 @@ export async function runModerationAnalysis(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const cacheKey = makeTextModerationCacheKey(rawContent);
|
const cacheKey = makeTextModerationCacheKey(
|
||||||
|
rawContent,
|
||||||
|
makeModerationContextKey(target),
|
||||||
|
);
|
||||||
setCachedTextModeration(
|
setCachedTextModeration(
|
||||||
cacheKey,
|
cacheKey,
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
/**
|
|
||||||
* Text analysis prompt constants and helpers for LLM moderation.
|
|
||||||
*
|
|
||||||
* Contains shared types and utilities for text-based analysis scenarios.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export type {
|
|
||||||
BuildSystemPromptOptions,
|
|
||||||
PromptMode,
|
|
||||||
} from "./system.js";
|
|
||||||
export { buildSystemPrompt, sanitizeAiContent } from "./system.js";
|
|
||||||
@@ -22,6 +22,9 @@ export interface QdrantVerdictPayload {
|
|||||||
flags: string; // JSON string of the full moderation result
|
flags: string; // JSON string of the full moderation result
|
||||||
analyzed_at: number;
|
analyzed_at: number;
|
||||||
expires_at: number;
|
expires_at: number;
|
||||||
|
/** Bare content hash (16 hex chars) — enables content-based invalidation
|
||||||
|
* regardless of the (context-scoped) point id. */
|
||||||
|
content_hash?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function baseUrl(): string {
|
function baseUrl(): string {
|
||||||
@@ -215,6 +218,150 @@ export async function searchQdrant(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Batch search: one HTTP round-trip for N vectors (Qdrant
|
||||||
|
* `/points/search/batch`). Result is index-aligned with `vectors` — each
|
||||||
|
* entry is the top hits for that vector (or [] on per-vector failure).
|
||||||
|
* Used by the orchestrator to avoid N sequential embed→search round-trips.
|
||||||
|
*/
|
||||||
|
export async function searchQdrantBatch(
|
||||||
|
vectors: number[][],
|
||||||
|
limit: number,
|
||||||
|
scoreThreshold: number,
|
||||||
|
): Promise<QdrantSearchHit[][]> {
|
||||||
|
if (vectors.length === 0) return [];
|
||||||
|
try {
|
||||||
|
const json = (await request(
|
||||||
|
"POST",
|
||||||
|
`/collections/${collectionName()}/points/search/batch`,
|
||||||
|
{
|
||||||
|
searches: vectors.map((vector) => ({
|
||||||
|
vector,
|
||||||
|
limit,
|
||||||
|
score_threshold: scoreThreshold,
|
||||||
|
with_payload: true,
|
||||||
|
filter: {
|
||||||
|
must: [
|
||||||
|
{
|
||||||
|
key: "expires_at",
|
||||||
|
range: { gte: Date.now() },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
)) as {
|
||||||
|
result?: Array<{
|
||||||
|
result?: Array<{
|
||||||
|
id?: number;
|
||||||
|
score?: number;
|
||||||
|
payload?: QdrantVerdictPayload;
|
||||||
|
}>;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
return (json.result ?? []).map((entry) =>
|
||||||
|
(entry.result ?? [])
|
||||||
|
.filter((hit) => hit.payload?.flags)
|
||||||
|
.map((hit) => ({
|
||||||
|
cacheKey: `qdrant:${hit.id ?? "?"}`,
|
||||||
|
score: hit.score ?? 0,
|
||||||
|
payload: hit.payload as QdrantVerdictPayload,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
log.warn(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Qdrant batch search failed — semantic cache skipped",
|
||||||
|
);
|
||||||
|
return vectors.map(() => []);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete expired verdict points from the collection. Best-effort: 404
|
||||||
|
* (collection missing) and failures are swallowed — the periodic pruner
|
||||||
|
* just retries next sweep.
|
||||||
|
*/
|
||||||
|
export async function deleteExpiredQdrantPoints(): Promise<number> {
|
||||||
|
try {
|
||||||
|
const json = (await request(
|
||||||
|
"POST",
|
||||||
|
`/collections/${collectionName()}/points/delete`,
|
||||||
|
{
|
||||||
|
filter: {
|
||||||
|
must: [
|
||||||
|
{
|
||||||
|
key: "expires_at",
|
||||||
|
range: { lt: Date.now() },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)) as { result?: { deleted?: number } | null };
|
||||||
|
|
||||||
|
return json.result?.deleted ?? 0;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Error && error.message.includes("-> 404")) {
|
||||||
|
log.debug({}, "Qdrant collection absent — nothing to prune");
|
||||||
|
} else {
|
||||||
|
log.warn(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Qdrant expired-point prune failed",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete the verdict point for an exact cache key (used by cache
|
||||||
|
* invalidation when a moderator corrects a verdict).
|
||||||
|
*/
|
||||||
|
export async function deleteQdrantPoint(cacheKey: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await request("POST", `/collections/${collectionName()}/points/delete`, {
|
||||||
|
points: [qdrantPointId(cacheKey)],
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
log.warn(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Qdrant point delete failed",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete all verdict points whose payload carries a given bare content hash.
|
||||||
|
* Used by cache invalidation for corrected verdicts — matches context-scoped
|
||||||
|
* points that share the same content regardless of their point ids.
|
||||||
|
*/
|
||||||
|
export async function deleteQdrantPointsByContentHash(
|
||||||
|
bareHash: string,
|
||||||
|
): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await request("POST", `/collections/${collectionName()}/points/delete`, {
|
||||||
|
filter: {
|
||||||
|
must: [
|
||||||
|
{
|
||||||
|
key: "content_hash",
|
||||||
|
match: { value: bareHash },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
log.warn(
|
||||||
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
|
"Qdrant content-hash point delete failed",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** True when Qdrant is configured (non-empty URL). */
|
/** True when Qdrant is configured (non-empty URL). */
|
||||||
export function isQdrantConfigured(): boolean {
|
export function isQdrantConfigured(): boolean {
|
||||||
return Boolean(config.QDRANT_URL);
|
return Boolean(config.QDRANT_URL);
|
||||||
|
|||||||
@@ -12,13 +12,13 @@ import type {
|
|||||||
MessageRecord,
|
MessageRecord,
|
||||||
} from "../message-capture/types.js";
|
} from "../message-capture/types.js";
|
||||||
import { getChannelCulture } from "./channelCultureStore.js";
|
import { getChannelCulture } from "./channelCultureStore.js";
|
||||||
|
import type { ModerationPromptContent, RetryState } from "./llmCaller.js";
|
||||||
|
import { callModerationLLM } from "./llmCaller.js";
|
||||||
import {
|
import {
|
||||||
buildReferenceXml,
|
buildReferenceXml,
|
||||||
escapeXml,
|
escapeXml,
|
||||||
getAnalysisContent,
|
getAnalysisContent,
|
||||||
} from "./moderationBuilders.js";
|
} from "./moderationBuilders.js";
|
||||||
import type { RetryState } from "./llmCaller.js";
|
|
||||||
import { callModerationLLM } from "./llmCaller.js";
|
|
||||||
import {
|
import {
|
||||||
buildSystemPrompt as buildSystemPromptModular,
|
buildSystemPrompt as buildSystemPromptModular,
|
||||||
sanitizeAiContent,
|
sanitizeAiContent,
|
||||||
@@ -74,7 +74,7 @@ export async function runTextOnlyBatch(
|
|||||||
if (!targets.length) return { results: [], raw: null };
|
if (!targets.length) return { results: [], raw: null };
|
||||||
|
|
||||||
const maxBatchSize = config.AI_LLM_TEXT_BATCH_SIZE ?? 20;
|
const maxBatchSize = config.AI_LLM_TEXT_BATCH_SIZE ?? 20;
|
||||||
const timeoutMs = config.AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS ?? 60000;
|
const timeoutMs = config.AI_LLM_TEXT_ANALYSIS_TIMEOUT_MS ?? 30000;
|
||||||
|
|
||||||
// Parallel: URL fetch + SearXNG
|
// Parallel: URL fetch + SearXNG
|
||||||
const urlFetchPromise = (async () => {
|
const urlFetchPromise = (async () => {
|
||||||
@@ -193,7 +193,9 @@ export async function runTextOnlyBatch(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const buildContent = async (state: RetryState): Promise<string> => {
|
const buildContent = async (
|
||||||
|
state: RetryState,
|
||||||
|
): Promise<ModerationPromptContent> => {
|
||||||
const correction = state.lastParseError
|
const correction = state.lastParseError
|
||||||
? {
|
? {
|
||||||
error: state.lastParseError,
|
error: state.lastParseError,
|
||||||
@@ -241,7 +243,10 @@ export async function runTextOnlyBatch(
|
|||||||
)
|
)
|
||||||
.join("\n")}\n</web_searches>`
|
.join("\n")}\n</web_searches>`
|
||||||
: "";
|
: "";
|
||||||
return `${systemText}${searxngBlock}\n\n<messages_to_analyze>\n${messagesBlock}\n</messages_to_analyze>`;
|
return {
|
||||||
|
system: systemText,
|
||||||
|
user: `${searxngBlock}\n\n<messages_to_analyze>\n${messagesBlock}\n</messages_to_analyze>`,
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const abortController = new AbortController();
|
const abortController = new AbortController();
|
||||||
@@ -286,7 +291,15 @@ export async function runTextOnlyBatch(
|
|||||||
config.AI_LLM_MODEL,
|
config.AI_LLM_MODEL,
|
||||||
batchResult.results,
|
batchResult.results,
|
||||||
0,
|
0,
|
||||||
undefined,
|
(
|
||||||
|
batchResult.raw as {
|
||||||
|
usage?: {
|
||||||
|
prompt_tokens: number;
|
||||||
|
completion_tokens: number;
|
||||||
|
total_tokens: number;
|
||||||
|
};
|
||||||
|
} | null
|
||||||
|
)?.usage ?? undefined,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,11 @@ import { createChildLogger } from "@/shared/logger/index";
|
|||||||
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
|
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
|
||||||
import { findBestEmbeddingMatch } from "./embeddingClient.js";
|
import { findBestEmbeddingMatch } from "./embeddingClient.js";
|
||||||
import {
|
import {
|
||||||
|
deleteExpiredQdrantPoints,
|
||||||
|
deleteQdrantPoint,
|
||||||
|
deleteQdrantPointsByContentHash,
|
||||||
isQdrantConfigured,
|
isQdrantConfigured,
|
||||||
|
type QdrantVerdictPayload,
|
||||||
searchQdrant,
|
searchQdrant,
|
||||||
upsertQdrantPoint,
|
upsertQdrantPoint,
|
||||||
} from "./qdrantClient.js";
|
} from "./qdrantClient.js";
|
||||||
@@ -19,84 +23,6 @@ export interface TextCacheEntry {
|
|||||||
hit_count: number;
|
hit_count: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Lookup cached analysis result for a normalized text string.
|
|
||||||
* Returns null if not found or expired.
|
|
||||||
*/
|
|
||||||
export async function getCachedText(
|
|
||||||
text: string,
|
|
||||||
): Promise<TextCacheEntry | null> {
|
|
||||||
try {
|
|
||||||
const row = await executeGet(
|
|
||||||
`SELECT text, flags, source, analyzed_at, expires_at, hit_count
|
|
||||||
FROM text_analysis_cache
|
|
||||||
WHERE text = $1 AND expires_at > $2`,
|
|
||||||
[text, Date.now()],
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!row) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
text: row.text,
|
|
||||||
flags: JSON.parse(row.flags),
|
|
||||||
source: row.source,
|
|
||||||
analyzed_at: row.analyzed_at,
|
|
||||||
expires_at: row.expires_at,
|
|
||||||
hit_count: row.hit_count,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
logger.error(
|
|
||||||
{ error: error instanceof Error ? error.message : String(error) },
|
|
||||||
"Failed to get cached text",
|
|
||||||
);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Insert or update a text analysis cache entry.
|
|
||||||
*/
|
|
||||||
export async function upsertCachedText(
|
|
||||||
text: string,
|
|
||||||
flags: string[],
|
|
||||||
source: "local" | "primary_ai" | "vision_llm",
|
|
||||||
expiresAt: number,
|
|
||||||
): Promise<void> {
|
|
||||||
const now = Date.now();
|
|
||||||
|
|
||||||
try {
|
|
||||||
await executeAll(
|
|
||||||
`INSERT INTO text_analysis_cache (text, flags, source, analyzed_at, expires_at, hit_count)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, 0)
|
|
||||||
ON CONFLICT (text) DO UPDATE SET
|
|
||||||
flags = EXCLUDED.flags,
|
|
||||||
source = EXCLUDED.source,
|
|
||||||
analyzed_at = EXCLUDED.analyzed_at,
|
|
||||||
expires_at = EXCLUDED.expires_at`,
|
|
||||||
[text, JSON.stringify(flags), source, now, expiresAt],
|
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
logger.error(
|
|
||||||
{ error: error instanceof Error ? error.message : String(error) },
|
|
||||||
"Failed to upsert cached text",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Increment hit count for a cached text entry (called on cache hit).
|
|
||||||
*/
|
|
||||||
export async function incrementTextCacheHit(text: string): Promise<void> {
|
|
||||||
try {
|
|
||||||
await executeAll(
|
|
||||||
`UPDATE text_analysis_cache SET hit_count = hit_count + 1 WHERE text = $1`,
|
|
||||||
[text],
|
|
||||||
);
|
|
||||||
} catch (_error) {
|
|
||||||
// Silent fail — this is just a counter, not critical
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete expired cache entries. Run periodically to keep the table clean.
|
* Delete expired cache entries. Run periodically to keep the table clean.
|
||||||
*/
|
*/
|
||||||
@@ -116,47 +42,6 @@ export async function pruneExpiredTexts(): Promise<number> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cache statistics for observability.
|
|
||||||
*/
|
|
||||||
export async function getTextCacheStats(): Promise<{
|
|
||||||
total: number;
|
|
||||||
expired: number;
|
|
||||||
bySource: Record<string, number>;
|
|
||||||
}> {
|
|
||||||
try {
|
|
||||||
const now = Date.now();
|
|
||||||
|
|
||||||
const [totalRow, expiredRow, sourceRows] = await Promise.all([
|
|
||||||
executeAll(`SELECT count(*) as cnt FROM text_analysis_cache`),
|
|
||||||
executeAll(
|
|
||||||
`SELECT count(*) as cnt FROM text_analysis_cache WHERE expires_at < $1`,
|
|
||||||
[now],
|
|
||||||
),
|
|
||||||
executeAll(
|
|
||||||
`SELECT source, count(*) as cnt FROM text_analysis_cache GROUP BY source`,
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const bySource: Record<string, number> = {};
|
|
||||||
for (const row of sourceRows) {
|
|
||||||
bySource[row.source] = row.cnt;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
total: totalRow[0]?.cnt ?? 0,
|
|
||||||
expired: expiredRow[0]?.cnt ?? 0,
|
|
||||||
bySource,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
logger.error(
|
|
||||||
{ error: error instanceof Error ? error.message : String(error) },
|
|
||||||
"Failed to get text cache stats",
|
|
||||||
);
|
|
||||||
return { total: 0, expired: 0, bySource: {} };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Media / vision analysis cache helpers (reuses text_analysis_cache table)
|
// Media / vision analysis cache helpers (reuses text_analysis_cache table)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -295,15 +180,67 @@ export async function deleteCachedMediaAnalysis(
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate a deterministic cache key for a per-user moderation result.
|
* Generate a deterministic cache key for a per-conversation moderation result.
|
||||||
*
|
*
|
||||||
* Format: user_mod:<userId>:<sha256(content).slice(0,16)>
|
* Format: text_mod:<context>:<sha256(content).slice(0,16)>
|
||||||
* Two users sending the same text get separate cache entries so that
|
*
|
||||||
* per-user action history (e.g. repeated spam) can be tracked later.
|
* `context` is "channel:thread" — the LLM verdict depends on conversation
|
||||||
|
* context, so the same text in a different channel/thread is analyzed
|
||||||
|
* separately instead of silently reusing a context-free verdict.
|
||||||
|
* (Previously the key was content-only, which made the "per-user" comment
|
||||||
|
* misleading — the user ID never participates in the key.)
|
||||||
*/
|
*/
|
||||||
export function makeTextModerationCacheKey(content: string): string {
|
export function makeTextModerationCacheKey(
|
||||||
|
content: string,
|
||||||
|
context?: string,
|
||||||
|
): string {
|
||||||
const hash = createHash("sha256").update(content).digest("hex").slice(0, 16);
|
const hash = createHash("sha256").update(content).digest("hex").slice(0, 16);
|
||||||
return `text_mod:${hash}`;
|
const ctx = context ? `${context}:` : "";
|
||||||
|
return `text_mod:${ctx}${hash}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Conversation context signature used in moderation cache keys.
|
||||||
|
* Thread-scoped messages use the thread id (discussions inside a thread
|
||||||
|
* share context), everything else uses the channel id.
|
||||||
|
*/
|
||||||
|
export function makeModerationContextKey(message: {
|
||||||
|
channel_id: string;
|
||||||
|
thread_id?: string | null;
|
||||||
|
}): string {
|
||||||
|
return message.thread_id ?? message.channel_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Invalidate cached moderation verdicts for a piece of content: removes
|
||||||
|
* matching Postgres rows AND Qdrant points. Called when a moderator
|
||||||
|
* corrects a verdict so a stale/wrong cached decision cannot resurface.
|
||||||
|
*
|
||||||
|
* Handles both key formats:
|
||||||
|
* - legacy `text_mod:<hash>` (content-only, pre-context keys)
|
||||||
|
* - current `text_mod:<context>:<hash>` (channel/thread-scoped)
|
||||||
|
*/
|
||||||
|
export async function invalidateTextModerationCache(
|
||||||
|
content: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const bareHash = createHash("sha256")
|
||||||
|
.update(content)
|
||||||
|
.digest("hex")
|
||||||
|
.slice(0, 16);
|
||||||
|
const legacyKey = `text_mod:${bareHash}`;
|
||||||
|
|
||||||
|
const queries: Promise<unknown>[] = [
|
||||||
|
executeAll(`DELETE FROM text_analysis_cache WHERE text LIKE $1`, [
|
||||||
|
`text_mod:%${bareHash}`,
|
||||||
|
]).catch(() => {}),
|
||||||
|
];
|
||||||
|
if (isQdrantConfigured()) {
|
||||||
|
queries.push(
|
||||||
|
deleteQdrantPoint(legacyKey).catch(() => {}),
|
||||||
|
deleteQdrantPointsByContentHash(bareHash).catch(() => {}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await Promise.all(queries).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -360,6 +297,54 @@ export async function getCachedTextModeration(cacheKey: string): Promise<{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse a Qdrant verdict payload into the result shape shared by the
|
||||||
|
* semantic cache lookups. Returns null on malformed payloads (callers then
|
||||||
|
* fall through to the LLM).
|
||||||
|
*/
|
||||||
|
export function parseQdrantVerdict(
|
||||||
|
payload: QdrantVerdictPayload,
|
||||||
|
similarity: number,
|
||||||
|
): {
|
||||||
|
text: string;
|
||||||
|
similarity: number;
|
||||||
|
status: "clean" | "warn" | "flagged";
|
||||||
|
flags: string[];
|
||||||
|
score: number;
|
||||||
|
analysis: string;
|
||||||
|
categories: string[];
|
||||||
|
severity: string;
|
||||||
|
confidence: number;
|
||||||
|
recommendedAction: string;
|
||||||
|
} | null {
|
||||||
|
let parsed: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(payload.flags) as Record<string, unknown>;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!parsed || typeof parsed !== "object") return null;
|
||||||
|
|
||||||
|
const storedStatus = (parsed.status as string) ?? "clean";
|
||||||
|
const status: "clean" | "warn" | "flagged" =
|
||||||
|
storedStatus === "warn" || storedStatus === "flagged"
|
||||||
|
? storedStatus
|
||||||
|
: "clean";
|
||||||
|
|
||||||
|
return {
|
||||||
|
text: payload.text,
|
||||||
|
similarity,
|
||||||
|
status,
|
||||||
|
flags: (parsed.flags as string[]) ?? [],
|
||||||
|
score: (parsed.score as number) ?? 0,
|
||||||
|
analysis: (parsed.analysis as string) ?? "",
|
||||||
|
categories: (parsed.categories as string[]) ?? [],
|
||||||
|
severity: (parsed.severity as string) ?? "none",
|
||||||
|
confidence: (parsed.confidence as number) ?? 0,
|
||||||
|
recommendedAction: (parsed.recommendedAction as string) ?? "none",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Semantic moderation cache lookup.
|
* Semantic moderation cache lookup.
|
||||||
*
|
*
|
||||||
@@ -389,29 +374,7 @@ export async function findSimilarTextModeration(
|
|||||||
const hits = await searchQdrant(embedding, limit, minSimilarity);
|
const hits = await searchQdrant(embedding, limit, minSimilarity);
|
||||||
if (hits.length > 0) {
|
if (hits.length > 0) {
|
||||||
const hit = hits[0];
|
const hit = hits[0];
|
||||||
let parsed: Record<string, unknown>;
|
return parseQdrantVerdict(hit.payload, hit.score);
|
||||||
try {
|
|
||||||
parsed = JSON.parse(hit.payload.flags) as Record<string, unknown>;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const storedStatus = (parsed.status as string) ?? "clean";
|
|
||||||
const status: "clean" | "warn" | "flagged" =
|
|
||||||
storedStatus === "warn" || storedStatus === "flagged"
|
|
||||||
? storedStatus
|
|
||||||
: "clean";
|
|
||||||
return {
|
|
||||||
text: hit.payload.text,
|
|
||||||
similarity: hit.score,
|
|
||||||
status,
|
|
||||||
flags: (parsed.flags as string[]) ?? [],
|
|
||||||
score: (parsed.score as number) ?? 0,
|
|
||||||
analysis: (parsed.analysis as string) ?? "",
|
|
||||||
categories: (parsed.categories as string[]) ?? [],
|
|
||||||
severity: (parsed.severity as string) ?? "none",
|
|
||||||
confidence: (parsed.confidence as number) ?? 0,
|
|
||||||
recommendedAction: (parsed.recommendedAction as string) ?? "none",
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
// No Qdrant hit — fall through to Postgres legacy rows.
|
// No Qdrant hit — fall through to Postgres legacy rows.
|
||||||
}
|
}
|
||||||
@@ -521,6 +484,7 @@ export async function setCachedTextModeration(
|
|||||||
flags: JSON.stringify(result),
|
flags: JSON.stringify(result),
|
||||||
analyzed_at: now,
|
analyzed_at: now,
|
||||||
expires_at: now + USER_MOD_CACHE_TTL_MS,
|
expires_at: now + USER_MOD_CACHE_TTL_MS,
|
||||||
|
content_hash: cacheKey.split(":").pop() ?? "",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -666,6 +630,12 @@ export async function getRecentCorrectedModerations(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Store a corrected moderation entry for future few-shot injection.
|
* Store a corrected moderation entry for future few-shot injection.
|
||||||
|
*
|
||||||
|
* Also invalidates any cached verdicts for the corrected content (both
|
||||||
|
* Postgres rows and Qdrant points) so the corrected decision propagates
|
||||||
|
* immediately instead of being shadowed by a stale cache entry. Full
|
||||||
|
* content is looked up by message_id when available — more precise than
|
||||||
|
* the (possibly truncated) snippet.
|
||||||
*/
|
*/
|
||||||
export async function insertCorrectedModeration(entry: {
|
export async function insertCorrectedModeration(entry: {
|
||||||
messageId: string;
|
messageId: string;
|
||||||
@@ -696,5 +666,20 @@ export async function insertCorrectedModeration(entry: {
|
|||||||
{ error: error instanceof Error ? error.message : String(error) },
|
{ error: error instanceof Error ? error.message : String(error) },
|
||||||
"Failed to insert corrected moderation",
|
"Failed to insert corrected moderation",
|
||||||
);
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Best-effort invalidation: prefer full content from the messages table.
|
||||||
|
try {
|
||||||
|
const row = await executeGet(
|
||||||
|
`SELECT content, edited_content FROM messages WHERE id = $1`,
|
||||||
|
[entry.messageId],
|
||||||
|
);
|
||||||
|
const fullContent = (row?.edited_content ?? row?.content ?? "").trim();
|
||||||
|
await invalidateTextModerationCache(
|
||||||
|
fullContent || entry.contentSnippet,
|
||||||
|
).catch(() => {});
|
||||||
|
} catch {
|
||||||
|
await invalidateTextModerationCache(entry.contentSnippet).catch(() => {});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { createChildLogger } from "@/shared/logger/index";
|
|
||||||
import { and, desc, eq } from "drizzle-orm";
|
import { and, desc, eq } from "drizzle-orm";
|
||||||
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
import { getDatabase } from "../../shared/database/drizzle.js";
|
import { getDatabase } from "../../shared/database/drizzle.js";
|
||||||
import {
|
import {
|
||||||
messagesTable,
|
messagesTable,
|
||||||
@@ -9,6 +9,131 @@ import {
|
|||||||
|
|
||||||
const logger = createChildLogger("userReputationStore");
|
const logger = createChildLogger("userReputationStore");
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Trust model v2 — fair, recoverable, escalation-aware
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// Problems with v1 that this fixes:
|
||||||
|
// 1. Trust practically could NOT rise: +2 per 100 clean messages meant a
|
||||||
|
// single -15 "high" penalty required 750 clean messages to repay.
|
||||||
|
// 2. Flat penalties regardless of history: first-timers and repeat
|
||||||
|
// offenders were punished identically.
|
||||||
|
// 3. Minor infractions could zero out a user (low=-2 at score 2 → 0),
|
||||||
|
// which is disproportionate.
|
||||||
|
//
|
||||||
|
// v2 model:
|
||||||
|
// - GAIN: +1 trust per 15 consecutive clean messages (cap 100). Recovery
|
||||||
|
// is real but earned — consistent good behavior rebuilds trust.
|
||||||
|
// - PENALTY: severity table low=3 / medium=6 / high=12 / critical=25.
|
||||||
|
// - FIRST OFFENSE: penalty halved (leniency for a single slip).
|
||||||
|
// - REPEAT OFFENDER: infraction within the last 7 days → ×1.5 (escalation).
|
||||||
|
// - FLOOR: low/medium infractions cannot push trust below 10/5 — minor
|
||||||
|
// offenses never permanently cripple a user; high/critical can still
|
||||||
|
// zero out (severe behavior has severe consequences).
|
||||||
|
// - Streak resets on infraction; time-based recovery still happens through
|
||||||
|
// the clean-message gain (no arbitrary idle-decay).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export const TRUST_DEFAULTS = {
|
||||||
|
DEFAULT_TRUST: 50,
|
||||||
|
MAX_TRUST: 100,
|
||||||
|
MIN_TRUST: 0,
|
||||||
|
CLEAN_MESSAGES_PER_POINT: 15,
|
||||||
|
REPEAT_OFFENSE_WINDOW_MS: 7 * 24 * 60 * 60 * 1000, // 7 days
|
||||||
|
REPEAT_OFFENSE_MULTIPLIER: 1.5,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export const INFRACTION_PENALTIES: Record<
|
||||||
|
"low" | "medium" | "high" | "critical",
|
||||||
|
number
|
||||||
|
> = {
|
||||||
|
low: 3,
|
||||||
|
medium: 6,
|
||||||
|
high: 12,
|
||||||
|
critical: 25,
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Trust floors per severity — minor offenses can't tank a user to zero. */
|
||||||
|
export const INFRACTION_FLOORS: Record<
|
||||||
|
"low" | "medium" | "high" | "critical",
|
||||||
|
number
|
||||||
|
> = {
|
||||||
|
low: 10,
|
||||||
|
medium: 5,
|
||||||
|
high: 0,
|
||||||
|
critical: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
function clampTrust(score: number): number {
|
||||||
|
return Math.min(
|
||||||
|
TRUST_DEFAULTS.MAX_TRUST,
|
||||||
|
Math.max(TRUST_DEFAULTS.MIN_TRUST, Math.round(score)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InfractionContext {
|
||||||
|
totalInfractions: number;
|
||||||
|
lastInfractionAt: number | null;
|
||||||
|
severity: "low" | "medium" | "high" | "critical";
|
||||||
|
now?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InfractionOutcome {
|
||||||
|
penalty: number;
|
||||||
|
appliedRules: {
|
||||||
|
firstOffense: boolean;
|
||||||
|
repeatEscalation: boolean;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pure penalty computation for the trust model (unit-testable, no DB).
|
||||||
|
* - First offense ever → halved (leniency for a single slip).
|
||||||
|
* - Repeat offense within the 7-day window → ×1.5 (escalation).
|
||||||
|
*/
|
||||||
|
export function computeInfractionPenalty(
|
||||||
|
ctx: InfractionContext,
|
||||||
|
): InfractionOutcome {
|
||||||
|
const basePenalty = INFRACTION_PENALTIES[ctx.severity];
|
||||||
|
let penalty = basePenalty;
|
||||||
|
const isFirstOffense = ctx.totalInfractions === 0;
|
||||||
|
|
||||||
|
if (isFirstOffense) {
|
||||||
|
penalty = Math.ceil(basePenalty / 2);
|
||||||
|
} else if (
|
||||||
|
ctx.lastInfractionAt &&
|
||||||
|
(ctx.now ?? Date.now()) - ctx.lastInfractionAt <=
|
||||||
|
TRUST_DEFAULTS.REPEAT_OFFENSE_WINDOW_MS
|
||||||
|
) {
|
||||||
|
penalty = Math.ceil(basePenalty * TRUST_DEFAULTS.REPEAT_OFFENSE_MULTIPLIER);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
penalty,
|
||||||
|
appliedRules: {
|
||||||
|
firstOffense: isFirstOffense,
|
||||||
|
repeatEscalation: !isFirstOffense && penalty > basePenalty,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CleanGainOutcome {
|
||||||
|
newStreak: number;
|
||||||
|
trustGain: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pure clean-message gain computation (unit-testable, no DB).
|
||||||
|
* +1 trust every CLEAN_MESSAGES_PER_POINT consecutive clean messages;
|
||||||
|
* the streak keeps counting past the threshold (gains compound).
|
||||||
|
*/
|
||||||
|
export function computeCleanTrustGain(currentStreak: number): CleanGainOutcome {
|
||||||
|
const newStreak = currentStreak + 1;
|
||||||
|
const trustGain =
|
||||||
|
newStreak % TRUST_DEFAULTS.CLEAN_MESSAGES_PER_POINT === 0 ? 1 : 0;
|
||||||
|
return { newStreak, trustGain };
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensures a user reputation record exists.
|
* Ensures a user reputation record exists.
|
||||||
*/
|
*/
|
||||||
@@ -33,7 +158,7 @@ export async function initializeUserReputation(
|
|||||||
.values({
|
.values({
|
||||||
user_id: userId,
|
user_id: userId,
|
||||||
guild_id: guildId,
|
guild_id: guildId,
|
||||||
trust_score: 50,
|
trust_score: TRUST_DEFAULTS.DEFAULT_TRUST,
|
||||||
clean_message_streak: 0,
|
clean_message_streak: 0,
|
||||||
total_infractions: 0,
|
total_infractions: 0,
|
||||||
created_at: Date.now(),
|
created_at: Date.now(),
|
||||||
@@ -85,7 +210,11 @@ export async function getUserReputation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment the clean message streak and update trust score if threshold is met.
|
* Increment the clean message streak and grow trust — +1 per
|
||||||
|
* CLEAN_MESSAGES_PER_POINT consecutive clean messages (cap 100). The streak
|
||||||
|
* keeps counting past the threshold so gains compound with continued good
|
||||||
|
* behavior (no more wasted progress at 100, and recovery is genuinely
|
||||||
|
* reachable after an infraction).
|
||||||
*/
|
*/
|
||||||
export async function recordCleanMessage(
|
export async function recordCleanMessage(
|
||||||
userId: string,
|
userId: string,
|
||||||
@@ -93,14 +222,11 @@ export async function recordCleanMessage(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const rep = await initializeUserReputation(userId, guildId);
|
const rep = await initializeUserReputation(userId, guildId);
|
||||||
const db = getDatabase();
|
const db = getDatabase();
|
||||||
let newStreak = rep.clean_message_streak + 1;
|
const { newStreak, trustGain } = computeCleanTrustGain(
|
||||||
let newScore = rep.trust_score;
|
rep.clean_message_streak,
|
||||||
|
);
|
||||||
// Every 100 clean messages, give +2 trust score up to 100
|
const newScore =
|
||||||
if (newStreak >= 100) {
|
trustGain > 0 ? clampTrust(rep.trust_score + trustGain) : rep.trust_score;
|
||||||
newScore = Math.min(100, newScore + 2);
|
|
||||||
newStreak = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.update(userReputationsTable)
|
.update(userReputationsTable)
|
||||||
@@ -119,6 +245,12 @@ export async function recordCleanMessage(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Apply an infraction penalty to a user.
|
* Apply an infraction penalty to a user.
|
||||||
|
*
|
||||||
|
* Fairness rules:
|
||||||
|
* - First offense ever → penalty halved (leniency, rounded up).
|
||||||
|
* - Repeat offense within the 7-day window → ×1.5 (escalation).
|
||||||
|
* - Severity floor prevents minor infractions from zeroing a user.
|
||||||
|
* - Streak resets — trust must be re-earned through clean behavior.
|
||||||
*/
|
*/
|
||||||
export async function recordInfraction(
|
export async function recordInfraction(
|
||||||
userId: string,
|
userId: string,
|
||||||
@@ -127,23 +259,16 @@ export async function recordInfraction(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const rep = await initializeUserReputation(userId, guildId);
|
const rep = await initializeUserReputation(userId, guildId);
|
||||||
const db = getDatabase();
|
const db = getDatabase();
|
||||||
let penalty = 0;
|
|
||||||
switch (severity) {
|
|
||||||
case "low":
|
|
||||||
penalty = 2;
|
|
||||||
break;
|
|
||||||
case "medium":
|
|
||||||
penalty = 5;
|
|
||||||
break;
|
|
||||||
case "high":
|
|
||||||
penalty = 15;
|
|
||||||
break;
|
|
||||||
case "critical":
|
|
||||||
penalty = 30;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
const newScore = Math.max(0, rep.trust_score - penalty);
|
const outcome = computeInfractionPenalty({
|
||||||
|
totalInfractions: rep.total_infractions,
|
||||||
|
lastInfractionAt: rep.last_infraction_at,
|
||||||
|
severity,
|
||||||
|
});
|
||||||
|
const { penalty } = outcome;
|
||||||
|
|
||||||
|
const floor = INFRACTION_FLOORS[severity];
|
||||||
|
const newScore = Math.max(floor, clampTrust(rep.trust_score - penalty));
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.update(userReputationsTable)
|
.update(userReputationsTable)
|
||||||
@@ -160,8 +285,12 @@ export async function recordInfraction(
|
|||||||
{
|
{
|
||||||
userId,
|
userId,
|
||||||
severity,
|
severity,
|
||||||
|
basePenalty: INFRACTION_PENALTIES[severity],
|
||||||
penalty,
|
penalty,
|
||||||
|
appliedRules: outcome.appliedRules,
|
||||||
|
previousScore: rep.trust_score,
|
||||||
newScore,
|
newScore,
|
||||||
|
floor,
|
||||||
totalInfractions: rep.total_infractions + 1,
|
totalInfractions: rep.total_infractions + 1,
|
||||||
},
|
},
|
||||||
"Infraction recorded",
|
"Infraction recorded",
|
||||||
|
|||||||
@@ -163,21 +163,14 @@ export const configSchema = z
|
|||||||
.int()
|
.int()
|
||||||
.positive()
|
.positive()
|
||||||
.default(60000),
|
.default(60000),
|
||||||
// ── AI Model (new unified keys) ───────────────────────────────────
|
// Text-only moderation batches are cheaper than media (no downloads /
|
||||||
AI_MODEL_FAST_CLASSIFIER_ENABLED: z
|
// vision pre-pass), so they get their own (shorter) timeout instead of
|
||||||
.string()
|
// being tied to the media budget.
|
||||||
.optional()
|
AI_LLM_TEXT_ANALYSIS_TIMEOUT_MS: z.coerce
|
||||||
.transform((v) => v === "true")
|
|
||||||
.default(true)
|
|
||||||
.describe("Enable Layer 1 fast heuristic classifier"),
|
|
||||||
AI_MODEL_LLM_TIMEOUT_MS: z.coerce
|
|
||||||
.number()
|
.number()
|
||||||
.int()
|
.int()
|
||||||
.positive()
|
.positive()
|
||||||
.default(30000)
|
.default(30000),
|
||||||
.describe("Timeout for individual LLM moderation calls"),
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// ── AI Analysis Timing ──────────────────────────────────────────────
|
// ── AI Analysis Timing ──────────────────────────────────────────────
|
||||||
AI_ANALYSIS_DEBOUNCE_MS: z.coerce.number().positive().default(500),
|
AI_ANALYSIS_DEBOUNCE_MS: z.coerce.number().positive().default(500),
|
||||||
@@ -210,7 +203,11 @@ export const configSchema = z
|
|||||||
.int()
|
.int()
|
||||||
.positive()
|
.positive()
|
||||||
.default(50),
|
.default(50),
|
||||||
PISCINA_MAX_THREADS: z.coerce.number().int().positive().optional(),
|
// Worker pool size. Default 4 (not availableParallelism) because each
|
||||||
|
// Piscina thread owns its own pLimit(5) semaphore — on big VPSes
|
||||||
|
// availableParallelism × 5 concurrent LLM calls would overwhelm the
|
||||||
|
// router. Keep threads modest; concurrency is capped per-thread anyway.
|
||||||
|
PISCINA_MAX_THREADS: z.coerce.number().int().positive().default(4),
|
||||||
|
|
||||||
// ── Voice Transcription ────────────────────────────────────────────────
|
// ── Voice Transcription ────────────────────────────────────────────────
|
||||||
AI_VOICE_TRANSCRIPTION_ENABLED: z
|
AI_VOICE_TRANSCRIPTION_ENABLED: z
|
||||||
@@ -219,14 +216,6 @@ export const configSchema = z
|
|||||||
.transform((v) => v === "true")
|
.transform((v) => v === "true")
|
||||||
.default(false),
|
.default(false),
|
||||||
|
|
||||||
// ── OpenAI Moderation ───────────────────────────────────────────────
|
|
||||||
OPENAI_MODERATION_API_KEY: z.string().optional(),
|
|
||||||
OPENAI_MODERATION_BASE_URL: z
|
|
||||||
.string()
|
|
||||||
.url()
|
|
||||||
.default("https://api.openai.com/v1"),
|
|
||||||
OPENAI_MODERATION_MODEL: z.string().default("omni-moderation-latest"),
|
|
||||||
|
|
||||||
// ── Auto Delete ─────────────────────────────────────────────────────
|
// ── Auto Delete ─────────────────────────────────────────────────────
|
||||||
AUTO_DELETE_FLAGGED_ENABLED: z
|
AUTO_DELETE_FLAGGED_ENABLED: z
|
||||||
.string()
|
.string()
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
// Trust model v2 — pure math tests (no DB required)
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
computeCleanTrustGain,
|
||||||
|
computeInfractionPenalty,
|
||||||
|
INFRACTION_FLOORS,
|
||||||
|
INFRACTION_PENALTIES,
|
||||||
|
TRUST_DEFAULTS,
|
||||||
|
} from "../src/modules/ai-moderation/userReputationStore.js";
|
||||||
|
|
||||||
|
describe("computeCleanTrustGain — trust CAN rise", () => {
|
||||||
|
it("grants +1 every CLEAN_MESSAGES_PER_POINT clean messages", () => {
|
||||||
|
const before = computeCleanTrustGain(14);
|
||||||
|
expect(before.newStreak).toBe(15);
|
||||||
|
expect(before.trustGain).toBe(1);
|
||||||
|
|
||||||
|
const after = computeCleanTrustGain(15);
|
||||||
|
expect(after.newStreak).toBe(16);
|
||||||
|
expect(after.trustGain).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps compounding past the threshold (no wasted progress)", () => {
|
||||||
|
expect(computeCleanTrustGain(29).trustGain).toBe(1);
|
||||||
|
expect(computeCleanTrustGain(44).trustGain).toBe(1);
|
||||||
|
// 45 clean messages from a fresh start → 3 points of recovery
|
||||||
|
let gain = 0;
|
||||||
|
let streak = 0;
|
||||||
|
for (let i = 0; i < 45; i++) {
|
||||||
|
const r = computeCleanTrustGain(streak);
|
||||||
|
streak = r.newStreak;
|
||||||
|
gain += r.trustGain;
|
||||||
|
}
|
||||||
|
expect(gain).toBe(3);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("computeInfractionPenalty — fair and escalating", () => {
|
||||||
|
const NOW = Date.now();
|
||||||
|
|
||||||
|
it("applies base penalty for a repeat offender outside the window", () => {
|
||||||
|
const r = computeInfractionPenalty({
|
||||||
|
totalInfractions: 3,
|
||||||
|
lastInfractionAt: NOW - TRUST_DEFAULTS.REPEAT_OFFENSE_WINDOW_MS - 1000,
|
||||||
|
severity: "medium",
|
||||||
|
now: NOW,
|
||||||
|
});
|
||||||
|
expect(r.penalty).toBe(INFRACTION_PENALTIES.medium); // 6
|
||||||
|
expect(r.appliedRules.firstOffense).toBe(false);
|
||||||
|
expect(r.appliedRules.repeatEscalation).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("halves the penalty for a first offense (leniency)", () => {
|
||||||
|
const r = computeInfractionPenalty({
|
||||||
|
totalInfractions: 0,
|
||||||
|
lastInfractionAt: null,
|
||||||
|
severity: "high",
|
||||||
|
now: NOW,
|
||||||
|
});
|
||||||
|
expect(r.penalty).toBe(Math.ceil(INFRACTION_PENALTIES.high / 2)); // 6
|
||||||
|
expect(r.appliedRules.firstOffense).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("escalates ×1.5 for a repeat offense within 7 days", () => {
|
||||||
|
const r = computeInfractionPenalty({
|
||||||
|
totalInfractions: 2,
|
||||||
|
lastInfractionAt: NOW - 60 * 60 * 1000, // 1h ago
|
||||||
|
severity: "medium",
|
||||||
|
now: NOW,
|
||||||
|
});
|
||||||
|
expect(r.penalty).toBe(Math.ceil(INFRACTION_PENALTIES.medium * 1.5)); // 9
|
||||||
|
expect(r.appliedRules.repeatEscalation).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("critical first offense still hurts but is halved", () => {
|
||||||
|
const r = computeInfractionPenalty({
|
||||||
|
totalInfractions: 0,
|
||||||
|
lastInfractionAt: null,
|
||||||
|
severity: "critical",
|
||||||
|
now: NOW,
|
||||||
|
});
|
||||||
|
expect(r.penalty).toBe(Math.ceil(INFRACTION_PENALTIES.critical / 2)); // 13
|
||||||
|
});
|
||||||
|
|
||||||
|
it("severity floors prevent minor offenses from zeroing a user", () => {
|
||||||
|
expect(INFRACTION_FLOORS.low).toBeGreaterThan(0);
|
||||||
|
expect(INFRACTION_FLOORS.medium).toBeGreaterThan(0);
|
||||||
|
// high/critical can still reach zero — severe behavior has consequences
|
||||||
|
expect(INFRACTION_FLOORS.high).toBe(0);
|
||||||
|
expect(INFRACTION_FLOORS.critical).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { defineConfig } from "vitest/config";
|
||||||
|
|
||||||
|
// Resolves the "@/*" tsconfig path alias so vitest can import src modules
|
||||||
|
// (the pre-existing test suite was broken without this).
|
||||||
|
export default defineConfig({
|
||||||
|
resolve: {
|
||||||
|
alias: {
|
||||||
|
"@": fileURLToPath(new URL("./src", import.meta.url)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
test: {
|
||||||
|
include: ["tests/**/*.test.ts"],
|
||||||
|
// Loaded before module imports — satisfies the config singleton
|
||||||
|
// (DISCORD_TOKEN required) and DB-agnostic pure-function tests.
|
||||||
|
env: {
|
||||||
|
DISCORD_TOKEN: "test-discord-token",
|
||||||
|
DATABASE_URL: "postgres://localhost:5432/test",
|
||||||
|
AI_ANALYSIS_ENABLED: "true",
|
||||||
|
AI_LLM_API_KEY: "sk-test",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user