feat(ai-moderation): introduce user reputation and channel culture context
Implements a context-aware moderation system by tracking user behavior and channel-specific norms to improve AI decision-making accuracy. - Adds `user_reputations` table to track trust scores, clean streaks, and infraction history. - Adds `channel_cultures` table to store AI-generated summaries of channel-specific norms and slang. - Implements `userReputationStore` to autonomously update user scores based on moderation outcomes (clean vs. flagged). - Implements `cultureLearner` and `channelCultureStore` to manage evolving channel contexts. - Enhances LLM prompts to inject user reputation (trust scores, history) and channel culture summaries, enabling "wisdom-based" moderation (e.g., giving benefit of the doubt to high-trust users). - Integrates reputation and culture updates into the existing `aiAnalyzer` pipeline.
This commit is contained in:
@@ -488,6 +488,17 @@ async function processIndividualFallback(
|
|||||||
broadcastAnalysisCompleted(row);
|
broadcastAnalysisCompleted(row);
|
||||||
invalidateAnalyticsCache(row.guild_id);
|
invalidateAnalyticsCache(row.guild_id);
|
||||||
scheduleAutoDelete(row);
|
scheduleAutoDelete(row);
|
||||||
|
|
||||||
|
// Update reputation autonomously (Belajar & Kebijaksanaan)
|
||||||
|
if (row.ai_status === "clean") {
|
||||||
|
import("./userReputationStore.js")
|
||||||
|
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
||||||
|
.catch((e) => logger.error({ error: e }, "Failed to record clean message streak in fallback"));
|
||||||
|
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
||||||
|
import("./userReputationStore.js")
|
||||||
|
.then((store) => store.recordInfraction(row.user_id, row.guild_id, row.ai_severity as "low"|"medium"|"high"|"critical"))
|
||||||
|
.catch((e) => logger.error({ error: e }, "Failed to record infraction penalty in fallback"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const resultSummary = analysisResult.results[0];
|
const resultSummary = analysisResult.results[0];
|
||||||
@@ -701,6 +712,17 @@ async function processBatch(
|
|||||||
if (!isApiFailure) {
|
if (!isApiFailure) {
|
||||||
broadcastAnalysisCompleted(row);
|
broadcastAnalysisCompleted(row);
|
||||||
scheduleAutoDelete(row);
|
scheduleAutoDelete(row);
|
||||||
|
|
||||||
|
// Update reputation autonomously (Belajar & Kebijaksanaan)
|
||||||
|
if (row.ai_status === "clean") {
|
||||||
|
import("./userReputationStore.js")
|
||||||
|
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
||||||
|
.catch((e) => logger.error({ error: e }, "Failed to record clean message streak"));
|
||||||
|
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
||||||
|
import("./userReputationStore.js")
|
||||||
|
.then((store) => store.recordInfraction(row.user_id, row.guild_id, row.ai_severity as "low"|"medium"|"high"|"critical"))
|
||||||
|
.catch((e) => logger.error({ error: e }, "Failed to record infraction penalty"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1097,6 +1119,8 @@ export function startPendingAIAnalysisWorker(
|
|||||||
_redisEventBroadcaster = eventBroadcaster;
|
_redisEventBroadcaster = eventBroadcaster;
|
||||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||||
|
|
||||||
|
import("./cultureLearner.js").then(m => m.startCultureLearnerWorker()).catch(console.error);
|
||||||
|
|
||||||
setInterval(() => {
|
setInterval(() => {
|
||||||
revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import { eq } from "drizzle-orm";
|
||||||
|
import { getDatabase } from "../../shared/database/drizzle.js";
|
||||||
|
import {
|
||||||
|
channelCulturesTable,
|
||||||
|
ChannelCulture,
|
||||||
|
} from "../../shared/database/schema.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch the AI-generated culture summary for a channel.
|
||||||
|
*/
|
||||||
|
export async function getChannelCulture(channelId: string): Promise<ChannelCulture | null> {
|
||||||
|
const db = getDatabase();
|
||||||
|
const existing = await db
|
||||||
|
.select()
|
||||||
|
.from(channelCulturesTable)
|
||||||
|
.where(eq(channelCulturesTable.channel_id, channelId))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
return existing[0] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update the AI-generated culture summary for a channel.
|
||||||
|
*/
|
||||||
|
export async function updateChannelCulture(
|
||||||
|
channelId: string,
|
||||||
|
guildId: string,
|
||||||
|
cultureSummary: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const db = getDatabase();
|
||||||
|
|
||||||
|
await db
|
||||||
|
.insert(channelCulturesTable)
|
||||||
|
.values({
|
||||||
|
channel_id: channelId,
|
||||||
|
guild_id: guildId,
|
||||||
|
culture_summary: cultureSummary,
|
||||||
|
last_analyzed_at: Date.now(),
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: channelCulturesTable.channel_id,
|
||||||
|
set: {
|
||||||
|
culture_summary: cultureSummary,
|
||||||
|
last_analyzed_at: Date.now(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
import { eq, desc, sql, and } from "drizzle-orm";
|
||||||
|
import { getDatabase } from "../../shared/database/drizzle.js";
|
||||||
|
import {
|
||||||
|
messagesTable,
|
||||||
|
channelCulturesTable,
|
||||||
|
} from "../../shared/database/schema.js";
|
||||||
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { createChildLogger } from "@bete/shared/logger";
|
||||||
|
import { llmChat } from "./llmClient.js";
|
||||||
|
import { updateChannelCulture } from "./channelCultureStore.js";
|
||||||
|
|
||||||
|
const CULTURE_LEARNING_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours
|
||||||
|
const log = createChildLogger("cultureLearner");
|
||||||
|
|
||||||
|
async function learnChannelCulture(channelId: string, guildId: string): Promise<void> {
|
||||||
|
const db = getDatabase();
|
||||||
|
|
||||||
|
// Get recent clean messages for this channel
|
||||||
|
const recentMessages = await db
|
||||||
|
.select({
|
||||||
|
content: messagesTable.content,
|
||||||
|
username: messagesTable.username,
|
||||||
|
})
|
||||||
|
.from(messagesTable)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(messagesTable.channel_id, channelId),
|
||||||
|
eq(messagesTable.ai_status, "clean")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.orderBy(desc(messagesTable.created_at))
|
||||||
|
.limit(100);
|
||||||
|
|
||||||
|
if (recentMessages.length < 10) {
|
||||||
|
log.debug({ channelId }, "Not enough messages to learn culture");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const messagesText = recentMessages
|
||||||
|
.reverse()
|
||||||
|
.map(m => `${m.username}: ${m.content}`)
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
const prompt = `Anda adalah AI ahli perilaku sosiologis dan budaya online.
|
||||||
|
Tugas Anda adalah merangkum budaya (culture) dari sebuah channel chat berdasarkan riwayat pesan-pesan yang dianggap bersih (clean/tidak melanggar).
|
||||||
|
|
||||||
|
Pesan-pesan terakhir:
|
||||||
|
<messages>
|
||||||
|
${messagesText}
|
||||||
|
</messages>
|
||||||
|
|
||||||
|
Berdasarkan pesan-pesan di atas, buatlah ringkasan singkat (maksimal 3 paragraf) mengenai gaya bahasa, topik obrolan, dan norma sosial di channel ini. Ringkasan ini akan digunakan oleh sistem AI moderasi untuk memahami konteks dan "inside jokes" yang wajar di channel ini.
|
||||||
|
Jangan menambahkan teks basa-basi, langsung berikan ringkasannya.`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const completion = await llmChat({
|
||||||
|
messages: [{ role: "user", content: prompt }],
|
||||||
|
max_tokens: 500,
|
||||||
|
temperature: 0.7, // Higher temp for summarization
|
||||||
|
retries: 2,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!completion) throw new Error("Empty response from LLM");
|
||||||
|
const text = completion.choices[0]?.message?.content?.trim();
|
||||||
|
if (!text) throw new Error("Empty response from LLM");
|
||||||
|
|
||||||
|
await updateChannelCulture(channelId, guildId, text);
|
||||||
|
log.info({ channelId, guildId }, "Successfully learned and updated channel culture");
|
||||||
|
} catch (error) {
|
||||||
|
log.error({ channelId, error }, "Failed to learn channel culture");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runCultureLearningCycle(): Promise<void> {
|
||||||
|
const db = getDatabase();
|
||||||
|
log.info("Starting culture learning cycle");
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Find channels that haven't been analyzed recently
|
||||||
|
// We do a simple distinct channel_id query with a left join to see if it's stale
|
||||||
|
const staleChannels = await db.execute(sql`
|
||||||
|
SELECT m.channel_id, m.guild_id
|
||||||
|
FROM (SELECT DISTINCT channel_id, guild_id FROM messages) m
|
||||||
|
LEFT JOIN channel_cultures c ON m.channel_id = c.channel_id
|
||||||
|
WHERE c.last_analyzed_at IS NULL
|
||||||
|
OR c.last_analyzed_at < ${Date.now() - CULTURE_LEARNING_INTERVAL}
|
||||||
|
LIMIT 50
|
||||||
|
`);
|
||||||
|
|
||||||
|
for (const row of staleChannels.rows || staleChannels) {
|
||||||
|
// Cast the row because execute() returns untyped Record<string, unknown>[]
|
||||||
|
const channelId = String(row.channel_id);
|
||||||
|
const guildId = String(row.guild_id);
|
||||||
|
await learnChannelCulture(channelId, guildId);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
log.error({ error }, "Error in culture learning cycle");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let cultureInterval: NodeJS.Timeout | null = null;
|
||||||
|
|
||||||
|
export function startCultureLearnerWorker(): void {
|
||||||
|
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||||
|
if (cultureInterval) return;
|
||||||
|
|
||||||
|
// Run once on startup after 1 minute, then every 1 hour
|
||||||
|
setTimeout(() => {
|
||||||
|
runCultureLearningCycle().catch(e => log.error(e));
|
||||||
|
}, 60000);
|
||||||
|
|
||||||
|
cultureInterval = setInterval(() => {
|
||||||
|
runCultureLearningCycle().catch(e => log.error(e));
|
||||||
|
}, 1000 * 60 * 60); // Check every hour for channels that reached 12h expiry
|
||||||
|
|
||||||
|
log.info("Started background culture learner worker");
|
||||||
|
}
|
||||||
@@ -15,6 +15,11 @@ import type {
|
|||||||
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
||||||
import { llmChat, llmVision } from "./llmClient.js";
|
import { llmChat, llmVision } from "./llmClient.js";
|
||||||
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
|
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
|
||||||
|
import {
|
||||||
|
initializeUserReputation,
|
||||||
|
getUserRecentInfractions,
|
||||||
|
} from "./userReputationStore.js";
|
||||||
|
import { getChannelCulture } from "./channelCultureStore.js";
|
||||||
import { logModerationAnalysis, logModerationError } from "./responseLogger.js";
|
import { logModerationAnalysis, logModerationError } from "./responseLogger.js";
|
||||||
import {
|
import {
|
||||||
getStickerFromCache,
|
getStickerFromCache,
|
||||||
@@ -1077,11 +1082,33 @@ async function runTextOnlyBatch(
|
|||||||
const allResults: AnalysisResult[] = [];
|
const allResults: AnalysisResult[] = [];
|
||||||
let lastRaw: unknown = null;
|
let lastRaw: unknown = null;
|
||||||
|
|
||||||
|
const channelId = targets.length > 0 ? targets[0].channel_id : "";
|
||||||
|
const guildId = targets.length > 0 ? targets[0].guild_id : "";
|
||||||
|
const channelCultureObj = channelId ? await getChannelCulture(channelId) : null;
|
||||||
|
const channelCulture = channelCultureObj ? channelCultureObj.culture_summary : undefined;
|
||||||
|
|
||||||
// Run sub-batches sequentially to avoid rate limits
|
// Run sub-batches sequentially to avoid rate limits
|
||||||
for (let i = 0; i < subBatches.length; i++) {
|
for (let i = 0; i < subBatches.length; i++) {
|
||||||
const batch = subBatches[i];
|
const batch = subBatches[i];
|
||||||
const targetIds = batch.map((t) => t.id);
|
const targetIds = batch.map((t) => t.id);
|
||||||
|
|
||||||
|
// Fetch user context for this batch
|
||||||
|
const userContexts = new Map<string, string>();
|
||||||
|
for (const msg of batch) {
|
||||||
|
if (!userContexts.has(msg.user_id)) {
|
||||||
|
const rep = await initializeUserReputation(msg.user_id, msg.guild_id);
|
||||||
|
const history = await getUserRecentInfractions(msg.user_id);
|
||||||
|
|
||||||
|
let historyStr = "";
|
||||||
|
if (history.length > 0) {
|
||||||
|
historyStr = `\n <user_history>\n${history.map(h => ` - Flagged for ${h.flags} (Severity: ${h.severity}) pada pesan: "${h.content}"`).join("\n")}\n </user_history>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contextStr = `<user_reputation trust_score="${rep.trust_score}" clean_streak="${rep.clean_message_streak}" total_infractions="${rep.total_infractions}" />${historyStr}`;
|
||||||
|
userContexts.set(msg.user_id, contextStr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const buildContent = async (state: RetryState): Promise<string> => {
|
const buildContent = async (state: RetryState): Promise<string> => {
|
||||||
const correction = state.lastParseError
|
const correction = state.lastParseError
|
||||||
? {
|
? {
|
||||||
@@ -1097,6 +1124,7 @@ async function runTextOnlyBatch(
|
|||||||
mode: "text",
|
mode: "text",
|
||||||
correction,
|
correction,
|
||||||
correctedExamples,
|
correctedExamples,
|
||||||
|
channelCulture,
|
||||||
});
|
});
|
||||||
|
|
||||||
const messagesBlock = batch
|
const messagesBlock = batch
|
||||||
@@ -1116,9 +1144,10 @@ async function runTextOnlyBatch(
|
|||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join("\n");
|
.join("\n");
|
||||||
const webContext = urlContexts ? `\n${urlContexts}` : "";
|
const webContext = urlContexts ? `\n${urlContexts}` : "";
|
||||||
|
const userCtx = userContexts.get(msg.user_id) ?? "";
|
||||||
|
|
||||||
// XML delimiters wrap each message for prompt safety (R1)
|
// XML delimiters wrap each message for prompt safety (R1)
|
||||||
return `<message id="${msg.id}" user="${msg.username}">${content}${textContext}${webContext}</message>`;
|
return `<message id="${msg.id}" user="${msg.username}">\n ${userCtx}\n <content>${content}</content>${textContext}${webContext}\n</message>`;
|
||||||
})
|
})
|
||||||
.join("\n");
|
.join("\n");
|
||||||
|
|
||||||
@@ -1616,8 +1645,20 @@ async function _runSingleMediaAnalysis(
|
|||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join(" ");
|
.join(" ");
|
||||||
|
|
||||||
|
const channelId = target.channel_id;
|
||||||
|
const channelCultureObj = channelId ? await getChannelCulture(channelId) : null;
|
||||||
|
const channelCulture = channelCultureObj ? channelCultureObj.culture_summary : undefined;
|
||||||
|
|
||||||
|
const rep = await initializeUserReputation(target.user_id, target.guild_id);
|
||||||
|
const history = await getUserRecentInfractions(target.user_id);
|
||||||
|
let historyStr = "";
|
||||||
|
if (history.length > 0) {
|
||||||
|
historyStr = `\n <user_history>\n${history.map(h => ` - Flagged for ${h.flags} (Severity: ${h.severity}) pada pesan: "${h.content}"`).join("\n")}\n </user_history>`;
|
||||||
|
}
|
||||||
|
const userCtx = `<user_reputation trust_score="${rep.trust_score}" clean_streak="${rep.clean_message_streak}" total_infractions="${rep.total_infractions}" />${historyStr}`;
|
||||||
|
|
||||||
// XML delimiters wrap the message content (R1)
|
// XML delimiters wrap the message content (R1)
|
||||||
const messageBlock = `<message id="${target.id}" user="${target.username}">${content}${mediaContext ? ` ${mediaContext}` : ""}${textContext}${webContext}${mediaAnalysisContext}</message>`;
|
const messageBlock = `<message id="${target.id}" user="${target.username}">\n ${userCtx}\n <content>${content}</content>${mediaContext ? ` ${mediaContext}` : ""}${textContext}${webContext}${mediaAnalysisContext}\n</message>`;
|
||||||
|
|
||||||
// Modular system prompt with XML delimiters (R1, R7, R8)
|
// Modular system prompt with XML delimiters (R1, R7, R8)
|
||||||
const correctedExamples = await buildCorrectedFewShotExamples();
|
const correctedExamples = await buildCorrectedFewShotExamples();
|
||||||
@@ -1625,6 +1666,7 @@ async function _runSingleMediaAnalysis(
|
|||||||
contextText,
|
contextText,
|
||||||
mode: "mixed",
|
mode: "mixed",
|
||||||
correctedExamples,
|
correctedExamples,
|
||||||
|
channelCulture,
|
||||||
});
|
});
|
||||||
|
|
||||||
const userContent = `${systemText}\n\n<messages_to_analyze>\n${messageBlock}\n</messages_to_analyze>`;
|
const userContent = `${systemText}\n\n<messages_to_analyze>\n${messageBlock}\n</messages_to_analyze>`;
|
||||||
|
|||||||
@@ -395,6 +395,10 @@ export interface BuildSystemPromptOptions {
|
|||||||
* examples. Injected between static examples and output instructions.
|
* examples. Injected between static examples and output instructions.
|
||||||
*/
|
*/
|
||||||
correctedExamples?: string;
|
correctedExamples?: string;
|
||||||
|
/**
|
||||||
|
* Formatted XML block containing the AI-generated channel culture summary.
|
||||||
|
*/
|
||||||
|
channelCulture?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
|
export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
|
||||||
@@ -404,6 +408,7 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
|
|||||||
includeMediaInstructions,
|
includeMediaInstructions,
|
||||||
correction,
|
correction,
|
||||||
correctedExamples,
|
correctedExamples,
|
||||||
|
channelCulture,
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
// Backward compatibility: if mode is not set but includeMediaInstructions is,
|
// Backward compatibility: if mode is not set but includeMediaInstructions is,
|
||||||
@@ -433,6 +438,13 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
|
|||||||
parts.push(correctedExamples);
|
parts.push(correctedExamples);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Channel Culture Injection (Learning)
|
||||||
|
if (channelCulture) {
|
||||||
|
parts.push(`## Kultur Channel (Pembelajaran AI)\n${channelCulture}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
parts.push(`## Konteks Pengguna (Ingatan & Kebijaksanaan)\nSetiap pesan mungkin memiliki tag <user_reputation> dan <user_history>. *Gunakan Kebijaksanaan: Jika trust_score tinggi, beri benefit of the doubt pada ambiguitas. Jika trust_score rendah dan memiliki riwayat pelanggaran serupa, jadilah lebih tegas.*`);
|
||||||
|
|
||||||
parts.push(OUTPUT_INSTRUCTIONS);
|
parts.push(OUTPUT_INSTRUCTIONS);
|
||||||
|
|
||||||
// XML-delimited context — prevents prompt injection
|
// XML-delimited context — prevents prompt injection
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
import { eq, and, desc } from "drizzle-orm";
|
||||||
|
import { getDatabase } from "../../shared/database/drizzle.js";
|
||||||
|
import {
|
||||||
|
userReputationsTable,
|
||||||
|
messagesTable,
|
||||||
|
UserReputation,
|
||||||
|
} from "../../shared/database/schema.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensures a user reputation record exists.
|
||||||
|
*/
|
||||||
|
export async function initializeUserReputation(
|
||||||
|
userId: string,
|
||||||
|
guildId: string,
|
||||||
|
): Promise<UserReputation> {
|
||||||
|
const db = getDatabase();
|
||||||
|
const existing = await db
|
||||||
|
.select()
|
||||||
|
.from(userReputationsTable)
|
||||||
|
.where(eq(userReputationsTable.user_id, userId))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (existing.length > 0) {
|
||||||
|
return existing[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
const [inserted] = await db
|
||||||
|
.insert(userReputationsTable)
|
||||||
|
.values({
|
||||||
|
user_id: userId,
|
||||||
|
guild_id: guildId,
|
||||||
|
trust_score: 50,
|
||||||
|
clean_message_streak: 0,
|
||||||
|
total_infractions: 0,
|
||||||
|
created_at: Date.now(),
|
||||||
|
updated_at: Date.now(),
|
||||||
|
})
|
||||||
|
.onConflictDoNothing()
|
||||||
|
.returning();
|
||||||
|
|
||||||
|
if (!inserted) {
|
||||||
|
// If concurrent insert happened
|
||||||
|
const retry = await db
|
||||||
|
.select()
|
||||||
|
.from(userReputationsTable)
|
||||||
|
.where(eq(userReputationsTable.user_id, userId))
|
||||||
|
.limit(1);
|
||||||
|
return retry[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
return inserted;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch a user's reputation score. Returns default 50 if none exists.
|
||||||
|
*/
|
||||||
|
export async function getUserReputation(userId: string): Promise<UserReputation | null> {
|
||||||
|
const db = getDatabase();
|
||||||
|
const existing = await db
|
||||||
|
.select()
|
||||||
|
.from(userReputationsTable)
|
||||||
|
.where(eq(userReputationsTable.user_id, userId))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
return existing[0] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increment the clean message streak and update trust score if threshold is met.
|
||||||
|
*/
|
||||||
|
export async function recordCleanMessage(userId: string, guildId: string): Promise<void> {
|
||||||
|
const rep = await initializeUserReputation(userId, guildId);
|
||||||
|
const db = getDatabase();
|
||||||
|
let newStreak = rep.clean_message_streak + 1;
|
||||||
|
let newScore = rep.trust_score;
|
||||||
|
|
||||||
|
// Every 100 clean messages, give +2 trust score up to 100
|
||||||
|
if (newStreak >= 100) {
|
||||||
|
newScore = Math.min(100, newScore + 2);
|
||||||
|
newStreak = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(userReputationsTable)
|
||||||
|
.set({
|
||||||
|
clean_message_streak: newStreak,
|
||||||
|
trust_score: newScore,
|
||||||
|
updated_at: Date.now(),
|
||||||
|
})
|
||||||
|
.where(eq(userReputationsTable.user_id, userId));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply an infraction penalty to a user.
|
||||||
|
*/
|
||||||
|
export async function recordInfraction(
|
||||||
|
userId: string,
|
||||||
|
guildId: string,
|
||||||
|
severity: "low" | "medium" | "high" | "critical",
|
||||||
|
): Promise<void> {
|
||||||
|
const rep = await initializeUserReputation(userId, guildId);
|
||||||
|
const db = getDatabase();
|
||||||
|
let penalty = 0;
|
||||||
|
switch (severity) {
|
||||||
|
case "low":
|
||||||
|
penalty = 2;
|
||||||
|
break;
|
||||||
|
case "medium":
|
||||||
|
penalty = 5;
|
||||||
|
break;
|
||||||
|
case "high":
|
||||||
|
penalty = 15;
|
||||||
|
break;
|
||||||
|
case "critical":
|
||||||
|
penalty = 30;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const newScore = Math.max(0, rep.trust_score - penalty);
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(userReputationsTable)
|
||||||
|
.set({
|
||||||
|
trust_score: newScore,
|
||||||
|
clean_message_streak: 0, // Reset streak on infraction
|
||||||
|
total_infractions: rep.total_infractions + 1,
|
||||||
|
last_infraction_at: Date.now(),
|
||||||
|
updated_at: Date.now(),
|
||||||
|
})
|
||||||
|
.where(eq(userReputationsTable.user_id, userId));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch a user's past N flagged messages for context injection.
|
||||||
|
*/
|
||||||
|
export async function getUserRecentInfractions(userId: string, limit: number = 3) {
|
||||||
|
const db = getDatabase();
|
||||||
|
return await db
|
||||||
|
.select({
|
||||||
|
content: messagesTable.content,
|
||||||
|
flags: messagesTable.ai_moderation_flags,
|
||||||
|
severity: messagesTable.ai_severity,
|
||||||
|
created_at: messagesTable.created_at,
|
||||||
|
})
|
||||||
|
.from(messagesTable)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(messagesTable.user_id, userId),
|
||||||
|
eq(messagesTable.ai_status, "flagged")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.orderBy(desc(messagesTable.created_at))
|
||||||
|
.limit(limit);
|
||||||
|
}
|
||||||
@@ -248,6 +248,45 @@ export const pgVoiceRecordingsTable = pgTable(
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* User Reputations Table (PostgreSQL)
|
||||||
|
* Tracks user trust score and infractions to provide context to AI.
|
||||||
|
*/
|
||||||
|
export const pgUserReputationsTable = pgTable(
|
||||||
|
"user_reputations",
|
||||||
|
{
|
||||||
|
user_id: pgText("user_id").primaryKey(),
|
||||||
|
guild_id: pgText("guild_id").notNull(),
|
||||||
|
trust_score: pgInteger("trust_score").notNull().default(50),
|
||||||
|
clean_message_streak: pgInteger("clean_message_streak").notNull().default(0),
|
||||||
|
total_infractions: pgInteger("total_infractions").notNull().default(0),
|
||||||
|
last_infraction_at: pgBigint("last_infraction_at", { mode: "number" }),
|
||||||
|
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
|
||||||
|
updated_at: pgBigint("updated_at", { mode: "number" }).notNull(),
|
||||||
|
},
|
||||||
|
(table) => ({
|
||||||
|
guildIdx: pgIndex("idx_user_reputations_guild_id").on(table.guild_id),
|
||||||
|
scoreIdx: pgIndex("idx_user_reputations_trust_score").on(table.trust_score),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Channel Cultures Table (PostgreSQL)
|
||||||
|
* Stores AI-generated summaries of channel norms and slang to inject as context.
|
||||||
|
*/
|
||||||
|
export const pgChannelCulturesTable = pgTable(
|
||||||
|
"channel_cultures",
|
||||||
|
{
|
||||||
|
channel_id: pgText("channel_id").primaryKey(),
|
||||||
|
guild_id: pgText("guild_id").notNull(),
|
||||||
|
culture_summary: pgText("culture_summary").notNull(),
|
||||||
|
last_analyzed_at: pgBigint("last_analyzed_at", { mode: "number" }).notNull(),
|
||||||
|
},
|
||||||
|
(table) => ({
|
||||||
|
guildIdx: pgIndex("idx_channel_cultures_guild_id").on(table.guild_id),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Message Reviews Table (PostgreSQL)
|
* Message Reviews Table (PostgreSQL)
|
||||||
* Tracks manual reviews of messages flagged by AI moderation
|
* Tracks manual reviews of messages flagged by AI moderation
|
||||||
@@ -464,6 +503,8 @@ export const retentionPoliciesTable = pgRetentionPoliciesTable;
|
|||||||
export const textAnalysisCacheTable = pgTextAnalysisCacheTable;
|
export const textAnalysisCacheTable = pgTextAnalysisCacheTable;
|
||||||
export const stickerCacheTable = pgStickerCacheTable;
|
export const stickerCacheTable = pgStickerCacheTable;
|
||||||
export const correctedModerationsTable = pgCorrectedModerationsTable;
|
export const correctedModerationsTable = pgCorrectedModerationsTable;
|
||||||
|
export const userReputationsTable = pgUserReputationsTable;
|
||||||
|
export const channelCulturesTable = pgChannelCulturesTable;
|
||||||
|
|
||||||
// Export table types for use in queries
|
// Export table types for use in queries
|
||||||
export type MuxerJob = typeof muxerJobsTable.$inferSelect;
|
export type MuxerJob = typeof muxerJobsTable.$inferSelect;
|
||||||
@@ -499,3 +540,9 @@ export type StickerCacheInsert = typeof stickerCacheTable.$inferInsert;
|
|||||||
export type CorrectedModeration = typeof correctedModerationsTable.$inferSelect;
|
export type CorrectedModeration = typeof correctedModerationsTable.$inferSelect;
|
||||||
export type CorrectedModerationInsert =
|
export type CorrectedModerationInsert =
|
||||||
typeof correctedModerationsTable.$inferInsert;
|
typeof correctedModerationsTable.$inferInsert;
|
||||||
|
|
||||||
|
export type UserReputation = typeof userReputationsTable.$inferSelect;
|
||||||
|
export type UserReputationInsert = typeof userReputationsTable.$inferInsert;
|
||||||
|
|
||||||
|
export type ChannelCulture = typeof channelCulturesTable.$inferSelect;
|
||||||
|
export type ChannelCultureInsert = typeof channelCulturesTable.$inferInsert;
|
||||||
|
|||||||
Reference in New Issue
Block a user