refactor(gateway): remove user reputation feature entirely
Drop trust-score/infraction system: delete userReputationStore, remove call sites in fallback/batch processors, drop formatReputationAttrs, drop user_reputations table (migration 0016), delete trust-model test, update docs.
This commit is contained in:
@@ -71,7 +71,7 @@ handles a whole batch (text + media split internally, parallel paths).
|
|||||||
- `embeddingClient.ts` + `qdrantClient.ts` — semantic cache (one embed call +
|
- `embeddingClient.ts` + `qdrantClient.ts` — semantic cache (one embed call +
|
||||||
one batched Qdrant search for all uncached targets).
|
one batched Qdrant search for all uncached targets).
|
||||||
- `textCacheStore.ts` / `channelCultureStore.ts` / `userProfileStore.ts` /
|
- `textCacheStore.ts` / `channelCultureStore.ts` / `userProfileStore.ts` /
|
||||||
`userReputationStore.ts` — caches & learned per-channel/user state.
|
`userProfileStore.ts` — caches learned user profile summaries (optional).
|
||||||
|
|
||||||
### Concurrency model
|
### Concurrency model
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ Orchestration/caching: `moderationOrchestrator.ts` (exact hash → batched
|
|||||||
semantic Qdrant → LLM), `textBatchProcessor.ts` / `mediaBatchProcessor.ts`
|
semantic Qdrant → LLM), `textBatchProcessor.ts` / `mediaBatchProcessor.ts`
|
||||||
(one LLM call per sub-batch), `llmClient.ts` (central streaming client),
|
(one LLM call per sub-batch), `llmClient.ts` (central streaming client),
|
||||||
`embeddingClient.ts` + `qdrantClient.ts` (semantic cache), plus
|
`embeddingClient.ts` + `qdrantClient.ts` (semantic cache), plus
|
||||||
`channelCultureStore.ts` / `userProfileStore.ts` / `userReputationStore.ts`.
|
`channelCultureStore.ts` / `userProfileStore.ts`.
|
||||||
|
|
||||||
### voice-recording
|
### voice-recording
|
||||||
`voiceController.ts` (connect/disconnect/list) + `recorder.ts` (orchestration)
|
`voiceController.ts` (connect/disconnect/list) + `recorder.ts` (orchestration)
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
-- Remove the user reputation feature entirely (trust scores, infractions).
|
||||||
|
-- The feature was removed from the codebase; this drops the orphaned table.
|
||||||
|
DROP TABLE IF EXISTS "user_reputations";
|
||||||
@@ -113,6 +113,13 @@
|
|||||||
"when": 1787184000000,
|
"when": 1787184000000,
|
||||||
"tag": "0015_add_moderation_explainability",
|
"tag": "0015_add_moderation_explainability",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 16,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1787185000000,
|
||||||
|
"tag": "0016_drop_user_reputations",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -128,30 +128,6 @@ export async function skipAgeRestrictedMessages(
|
|||||||
// Batch pipeline
|
// Batch pipeline
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function postBatchReputationUpdate(rows: MessageRecord[]): Promise<void> {
|
|
||||||
for (const row of rows) {
|
|
||||||
if (row.ai_status === "clean") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error({ error: e }, "Failed to record clean message streak"),
|
|
||||||
);
|
|
||||||
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) =>
|
|
||||||
store.recordInfraction(
|
|
||||||
row.user_id,
|
|
||||||
row.guild_id,
|
|
||||||
row.ai_severity as "low" | "medium" | "high" | "critical",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error({ error: e }, "Failed to record infraction penalty"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function processBatch(
|
export async function processBatch(
|
||||||
conversationKey: string,
|
conversationKey: string,
|
||||||
messages: MessageRecord[],
|
messages: MessageRecord[],
|
||||||
@@ -196,21 +172,6 @@ export async function processBatch(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Post-batch reputation updates (fire-and-forget)
|
|
||||||
postBatchReputationUpdate(
|
|
||||||
result.rows.filter((r) => {
|
|
||||||
if (r.ai_status === "error") {
|
|
||||||
try {
|
|
||||||
const flags = JSON.parse(r.ai_moderation_flags ?? "[]") as string[];
|
|
||||||
return !flags.includes("analysis_api_failed");
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!result.ok) {
|
if (!result.ok) {
|
||||||
recordConversationBatchFailure(conversationKey);
|
recordConversationBatchFailure(conversationKey);
|
||||||
|
|
||||||
|
|||||||
@@ -123,33 +123,6 @@ async function processIndividualFallback(
|
|||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
broadcastAnalysisCompleted(row);
|
broadcastAnalysisCompleted(row);
|
||||||
scheduleAutoDelete(row);
|
scheduleAutoDelete(row);
|
||||||
|
|
||||||
// Update reputation autonomously
|
|
||||||
if (row.ai_status === "clean") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
{ error: e },
|
|
||||||
"Failed to record clean message streak in fallback",
|
|
||||||
),
|
|
||||||
);
|
|
||||||
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) =>
|
|
||||||
store.recordInfraction(
|
|
||||||
row.user_id,
|
|
||||||
row.guild_id,
|
|
||||||
row.ai_severity as "low" | "medium" | "high" | "critical",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
{ error: e },
|
|
||||||
"Failed to record infraction penalty in fallback",
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const resultSummary = analysisResult.results[0];
|
const resultSummary = analysisResult.results[0];
|
||||||
|
|||||||
@@ -127,56 +127,10 @@ export function buildUserProfileRef(userId: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// User reputation — richer than a bare trust score.
|
// Per-user history context (last flagged messages only — no trust model).
|
||||||
//
|
// context to AI moderation.
|
||||||
// The trust model tracks total_infractions, a clean-message streak and the
|
|
||||||
// last infraction timestamp. Feeding all of it to the LLM lets it tell a
|
|
||||||
// first-timer (same score, 1 infraction) from a repeat offender (score 50,
|
|
||||||
// 3 infractions, last one yesterday) — the same score means very different
|
|
||||||
// things in those two contexts.
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export interface ReputationAttrsSource {
|
|
||||||
trust_score: number;
|
|
||||||
total_infractions: number;
|
|
||||||
clean_message_streak: number;
|
|
||||||
last_infraction_at: number | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
const REPEAT_OFFENSE_WINDOW_MS = 7 * DAY_MS;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Formats reputation fields into XML attributes for `<user_reputation .../>`.
|
|
||||||
* Derived signals: last_offense_days_ago (0 = today) and repeat_offender
|
|
||||||
* (infraction within the last 7 days) are computed here so both the text and
|
|
||||||
* media paths emit the exact same shape.
|
|
||||||
*/
|
|
||||||
export function formatReputationAttrs(
|
|
||||||
rep: ReputationAttrsSource,
|
|
||||||
now: number = Date.now(),
|
|
||||||
): string {
|
|
||||||
const attrs = [
|
|
||||||
`trust_score="${rep.trust_score}"`,
|
|
||||||
`total_infractions="${rep.total_infractions}"`,
|
|
||||||
`clean_streak="${rep.clean_message_streak}"`,
|
|
||||||
];
|
|
||||||
if (
|
|
||||||
typeof rep.last_infraction_at === "number" &&
|
|
||||||
rep.last_infraction_at > 0
|
|
||||||
) {
|
|
||||||
const daysAgo = Math.max(
|
|
||||||
0,
|
|
||||||
Math.floor((now - rep.last_infraction_at) / DAY_MS),
|
|
||||||
);
|
|
||||||
attrs.push(`last_offense_days_ago="${daysAgo}"`);
|
|
||||||
const isRepeat =
|
|
||||||
rep.total_infractions > 0 &&
|
|
||||||
now - rep.last_infraction_at <= REPEAT_OFFENSE_WINDOW_MS;
|
|
||||||
if (isRepeat) attrs.push(`repeat_offender="true"`);
|
|
||||||
}
|
|
||||||
return attrs.join(" ");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Builds an optional `<user_history>` block (last flagged messages) from
|
* Builds an optional `<user_history>` block (last flagged messages) from
|
||||||
|
|||||||
@@ -1,324 +0,0 @@
|
|||||||
import { and, desc, eq } from "drizzle-orm";
|
|
||||||
import { createChildLogger } from "@/shared/logger/index";
|
|
||||||
import { getDatabase } from "../../shared/database/drizzle.js";
|
|
||||||
import {
|
|
||||||
messagesTable,
|
|
||||||
type UserReputation,
|
|
||||||
userReputationsTable,
|
|
||||||
} from "../../shared/database/schema.js";
|
|
||||||
|
|
||||||
const logger = createChildLogger("userReputationStore");
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Trust model v2 — fair, recoverable, escalation-aware
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
//
|
|
||||||
// Problems with v1 that this fixes:
|
|
||||||
// 1. Trust practically could NOT rise: +2 per 100 clean messages meant a
|
|
||||||
// single -15 "high" penalty required 750 clean messages to repay.
|
|
||||||
// 2. Flat penalties regardless of history: first-timers and repeat
|
|
||||||
// offenders were punished identically.
|
|
||||||
// 3. Minor infractions could zero out a user (low=-2 at score 2 → 0),
|
|
||||||
// which is disproportionate.
|
|
||||||
//
|
|
||||||
// v2 model:
|
|
||||||
// - GAIN: +1 trust per 15 consecutive clean messages (cap 100). Recovery
|
|
||||||
// is real but earned — consistent good behavior rebuilds trust.
|
|
||||||
// - PENALTY: severity table low=3 / medium=6 / high=12 / critical=25.
|
|
||||||
// - FIRST OFFENSE: penalty halved (leniency for a single slip).
|
|
||||||
// - REPEAT OFFENDER: infraction within the last 7 days → ×1.5 (escalation).
|
|
||||||
// - FLOOR: low/medium infractions cannot push trust below 10/5 — minor
|
|
||||||
// offenses never permanently cripple a user; high/critical can still
|
|
||||||
// zero out (severe behavior has severe consequences).
|
|
||||||
// - Streak resets on infraction; time-based recovery still happens through
|
|
||||||
// the clean-message gain (no arbitrary idle-decay).
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
export const TRUST_DEFAULTS = {
|
|
||||||
DEFAULT_TRUST: 50,
|
|
||||||
MAX_TRUST: 100,
|
|
||||||
MIN_TRUST: 0,
|
|
||||||
CLEAN_MESSAGES_PER_POINT: 15,
|
|
||||||
REPEAT_OFFENSE_WINDOW_MS: 7 * 24 * 60 * 60 * 1000, // 7 days
|
|
||||||
REPEAT_OFFENSE_MULTIPLIER: 1.5,
|
|
||||||
} as const;
|
|
||||||
|
|
||||||
export const INFRACTION_PENALTIES: Record<
|
|
||||||
"low" | "medium" | "high" | "critical",
|
|
||||||
number
|
|
||||||
> = {
|
|
||||||
low: 3,
|
|
||||||
medium: 6,
|
|
||||||
high: 12,
|
|
||||||
critical: 25,
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Trust floors per severity — minor offenses can't tank a user to zero. */
|
|
||||||
export const INFRACTION_FLOORS: Record<
|
|
||||||
"low" | "medium" | "high" | "critical",
|
|
||||||
number
|
|
||||||
> = {
|
|
||||||
low: 10,
|
|
||||||
medium: 5,
|
|
||||||
high: 0,
|
|
||||||
critical: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
function clampTrust(score: number): number {
|
|
||||||
return Math.min(
|
|
||||||
TRUST_DEFAULTS.MAX_TRUST,
|
|
||||||
Math.max(TRUST_DEFAULTS.MIN_TRUST, Math.round(score)),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface InfractionContext {
|
|
||||||
totalInfractions: number;
|
|
||||||
lastInfractionAt: number | null;
|
|
||||||
severity: "low" | "medium" | "high" | "critical";
|
|
||||||
now?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface InfractionOutcome {
|
|
||||||
penalty: number;
|
|
||||||
appliedRules: {
|
|
||||||
firstOffense: boolean;
|
|
||||||
repeatEscalation: boolean;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Pure penalty computation for the trust model (unit-testable, no DB).
|
|
||||||
* - First offense ever → halved (leniency for a single slip).
|
|
||||||
* - Repeat offense within the 7-day window → ×1.5 (escalation).
|
|
||||||
*/
|
|
||||||
export function computeInfractionPenalty(
|
|
||||||
ctx: InfractionContext,
|
|
||||||
): InfractionOutcome {
|
|
||||||
const basePenalty = INFRACTION_PENALTIES[ctx.severity];
|
|
||||||
let penalty = basePenalty;
|
|
||||||
const isFirstOffense = ctx.totalInfractions === 0;
|
|
||||||
|
|
||||||
if (isFirstOffense) {
|
|
||||||
penalty = Math.ceil(basePenalty / 2);
|
|
||||||
} else if (
|
|
||||||
ctx.lastInfractionAt &&
|
|
||||||
(ctx.now ?? Date.now()) - ctx.lastInfractionAt <=
|
|
||||||
TRUST_DEFAULTS.REPEAT_OFFENSE_WINDOW_MS
|
|
||||||
) {
|
|
||||||
penalty = Math.ceil(basePenalty * TRUST_DEFAULTS.REPEAT_OFFENSE_MULTIPLIER);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
penalty,
|
|
||||||
appliedRules: {
|
|
||||||
firstOffense: isFirstOffense,
|
|
||||||
repeatEscalation: !isFirstOffense && penalty > basePenalty,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CleanGainOutcome {
|
|
||||||
newStreak: number;
|
|
||||||
trustGain: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Pure clean-message gain computation (unit-testable, no DB).
|
|
||||||
* +1 trust every CLEAN_MESSAGES_PER_POINT consecutive clean messages;
|
|
||||||
* the streak keeps counting past the threshold (gains compound).
|
|
||||||
*/
|
|
||||||
export function computeCleanTrustGain(currentStreak: number): CleanGainOutcome {
|
|
||||||
const newStreak = currentStreak + 1;
|
|
||||||
const trustGain =
|
|
||||||
newStreak % TRUST_DEFAULTS.CLEAN_MESSAGES_PER_POINT === 0 ? 1 : 0;
|
|
||||||
return { newStreak, trustGain };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Ensures a user reputation record exists.
|
|
||||||
*/
|
|
||||||
export async function initializeUserReputation(
|
|
||||||
userId: string,
|
|
||||||
guildId: string,
|
|
||||||
): Promise<UserReputation> {
|
|
||||||
const db = getDatabase();
|
|
||||||
const existing = await db
|
|
||||||
.select()
|
|
||||||
.from(userReputationsTable)
|
|
||||||
.where(eq(userReputationsTable.user_id, userId))
|
|
||||||
.limit(1);
|
|
||||||
|
|
||||||
if (existing.length > 0) {
|
|
||||||
logger.debug({ userId }, "Reputation record already exists");
|
|
||||||
return existing[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
const [inserted] = await db
|
|
||||||
.insert(userReputationsTable)
|
|
||||||
.values({
|
|
||||||
user_id: userId,
|
|
||||||
guild_id: guildId,
|
|
||||||
trust_score: TRUST_DEFAULTS.DEFAULT_TRUST,
|
|
||||||
clean_message_streak: 0,
|
|
||||||
total_infractions: 0,
|
|
||||||
created_at: Date.now(),
|
|
||||||
updated_at: Date.now(),
|
|
||||||
})
|
|
||||||
.onConflictDoNothing()
|
|
||||||
.returning();
|
|
||||||
|
|
||||||
if (!inserted) {
|
|
||||||
// If concurrent insert happened
|
|
||||||
logger.debug({ userId }, "Concurrent reputation insert detected, retrying");
|
|
||||||
const retry = await db
|
|
||||||
.select()
|
|
||||||
.from(userReputationsTable)
|
|
||||||
.where(eq(userReputationsTable.user_id, userId))
|
|
||||||
.limit(1);
|
|
||||||
return retry[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
{ userId, trustScore: inserted.trust_score },
|
|
||||||
"Initialized user reputation",
|
|
||||||
);
|
|
||||||
return inserted;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetch a user's reputation score. Returns default 50 if none exists.
|
|
||||||
*/
|
|
||||||
export async function getUserReputation(
|
|
||||||
userId: string,
|
|
||||||
): Promise<UserReputation | null> {
|
|
||||||
const db = getDatabase();
|
|
||||||
const existing = await db
|
|
||||||
.select()
|
|
||||||
.from(userReputationsTable)
|
|
||||||
.where(eq(userReputationsTable.user_id, userId))
|
|
||||||
.limit(1);
|
|
||||||
|
|
||||||
if (existing[0]) {
|
|
||||||
logger.debug(
|
|
||||||
{ userId, trustScore: existing[0].trust_score },
|
|
||||||
"Fetched user reputation",
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
logger.debug({ userId }, "No reputation record found, returning null");
|
|
||||||
}
|
|
||||||
return existing[0] || null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Increment the clean message streak and grow trust — +1 per
|
|
||||||
* CLEAN_MESSAGES_PER_POINT consecutive clean messages (cap 100). The streak
|
|
||||||
* keeps counting past the threshold so gains compound with continued good
|
|
||||||
* behavior (no more wasted progress at 100, and recovery is genuinely
|
|
||||||
* reachable after an infraction).
|
|
||||||
*/
|
|
||||||
export async function recordCleanMessage(
|
|
||||||
userId: string,
|
|
||||||
guildId: string,
|
|
||||||
): Promise<void> {
|
|
||||||
const rep = await initializeUserReputation(userId, guildId);
|
|
||||||
const db = getDatabase();
|
|
||||||
const { newStreak, trustGain } = computeCleanTrustGain(
|
|
||||||
rep.clean_message_streak,
|
|
||||||
);
|
|
||||||
const newScore =
|
|
||||||
trustGain > 0 ? clampTrust(rep.trust_score + trustGain) : rep.trust_score;
|
|
||||||
|
|
||||||
await db
|
|
||||||
.update(userReputationsTable)
|
|
||||||
.set({
|
|
||||||
clean_message_streak: newStreak,
|
|
||||||
trust_score: newScore,
|
|
||||||
updated_at: Date.now(),
|
|
||||||
})
|
|
||||||
.where(eq(userReputationsTable.user_id, userId));
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
{ userId, previousScore: rep.trust_score, newScore, newStreak },
|
|
||||||
"Clean message recorded, reputation updated",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply an infraction penalty to a user.
|
|
||||||
*
|
|
||||||
* Fairness rules:
|
|
||||||
* - First offense ever → penalty halved (leniency, rounded up).
|
|
||||||
* - Repeat offense within the 7-day window → ×1.5 (escalation).
|
|
||||||
* - Severity floor prevents minor infractions from zeroing a user.
|
|
||||||
* - Streak resets — trust must be re-earned through clean behavior.
|
|
||||||
*/
|
|
||||||
export async function recordInfraction(
|
|
||||||
userId: string,
|
|
||||||
guildId: string,
|
|
||||||
severity: "low" | "medium" | "high" | "critical",
|
|
||||||
): Promise<void> {
|
|
||||||
const rep = await initializeUserReputation(userId, guildId);
|
|
||||||
const db = getDatabase();
|
|
||||||
|
|
||||||
const outcome = computeInfractionPenalty({
|
|
||||||
totalInfractions: rep.total_infractions,
|
|
||||||
lastInfractionAt: rep.last_infraction_at,
|
|
||||||
severity,
|
|
||||||
});
|
|
||||||
const { penalty } = outcome;
|
|
||||||
|
|
||||||
const floor = INFRACTION_FLOORS[severity];
|
|
||||||
const newScore = Math.max(floor, clampTrust(rep.trust_score - penalty));
|
|
||||||
|
|
||||||
await db
|
|
||||||
.update(userReputationsTable)
|
|
||||||
.set({
|
|
||||||
trust_score: newScore,
|
|
||||||
clean_message_streak: 0, // Reset streak on infraction
|
|
||||||
total_infractions: rep.total_infractions + 1,
|
|
||||||
last_infraction_at: Date.now(),
|
|
||||||
updated_at: Date.now(),
|
|
||||||
})
|
|
||||||
.where(eq(userReputationsTable.user_id, userId));
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
userId,
|
|
||||||
severity,
|
|
||||||
basePenalty: INFRACTION_PENALTIES[severity],
|
|
||||||
penalty,
|
|
||||||
appliedRules: outcome.appliedRules,
|
|
||||||
previousScore: rep.trust_score,
|
|
||||||
newScore,
|
|
||||||
floor,
|
|
||||||
totalInfractions: rep.total_infractions + 1,
|
|
||||||
},
|
|
||||||
"Infraction recorded",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetch a user's past N flagged messages for context injection.
|
|
||||||
*/
|
|
||||||
export async function getUserRecentInfractions(
|
|
||||||
userId: string,
|
|
||||||
limit: number = 3,
|
|
||||||
) {
|
|
||||||
const db = getDatabase();
|
|
||||||
return await db
|
|
||||||
.select({
|
|
||||||
content: messagesTable.content,
|
|
||||||
flags: messagesTable.ai_moderation_flags,
|
|
||||||
severity: messagesTable.ai_severity,
|
|
||||||
created_at: messagesTable.created_at,
|
|
||||||
})
|
|
||||||
.from(messagesTable)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(messagesTable.user_id, userId),
|
|
||||||
eq(messagesTable.ai_status, "flagged"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.orderBy(desc(messagesTable.created_at))
|
|
||||||
.limit(limit);
|
|
||||||
}
|
|
||||||
@@ -222,7 +222,8 @@ export const configSchema = z
|
|||||||
AI_GLOSSARY_MAX_TERMS: z.coerce.number().int().min(1).max(20).default(6),
|
AI_GLOSSARY_MAX_TERMS: z.coerce.number().int().min(1).max(20).default(6),
|
||||||
// Per-user personal profile summaries (userProfileLearner). Disabled by
|
// Per-user personal profile summaries (userProfileLearner). Disabled by
|
||||||
// default: profiles bloat the analysis context and add LLM/DB cost for
|
// default: profiles bloat the analysis context and add LLM/DB cost for
|
||||||
// little moderation signal — only <user_reputation> history is injected.
|
// little moderation signal — user history context (last flagged messages)
|
||||||
|
// is injected via <user_history> instead of a numeric trust score.
|
||||||
AI_USER_PROFILE_LEARNING_ENABLED: z
|
AI_USER_PROFILE_LEARNING_ENABLED: z
|
||||||
.string()
|
.string()
|
||||||
.optional()
|
.optional()
|
||||||
|
|||||||
@@ -337,32 +337,6 @@ export const pgUserProfilesTable = pgTable(
|
|||||||
|
|
||||||
export const userProfilesTable = pgUserProfilesTable;
|
export const userProfilesTable = pgUserProfilesTable;
|
||||||
|
|
||||||
/**
|
|
||||||
* User Reputations Table (PostgreSQL)
|
|
||||||
* Tracks user trust score and infractions to provide context to AI.
|
|
||||||
*/
|
|
||||||
export const pgUserReputationsTable = pgTable(
|
|
||||||
"user_reputations",
|
|
||||||
{
|
|
||||||
user_id: pgText("user_id").primaryKey(),
|
|
||||||
guild_id: pgText("guild_id").notNull(),
|
|
||||||
trust_score: pgInteger("trust_score").notNull().default(50),
|
|
||||||
clean_message_streak: pgInteger("clean_message_streak")
|
|
||||||
.notNull()
|
|
||||||
.default(0),
|
|
||||||
total_infractions: pgInteger("total_infractions").notNull().default(0),
|
|
||||||
last_infraction_at: pgBigint("last_infraction_at", { mode: "number" }),
|
|
||||||
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
|
|
||||||
updated_at: pgBigint("updated_at", { mode: "number" }).notNull(),
|
|
||||||
},
|
|
||||||
(table) => ({
|
|
||||||
guildIdx: pgIndex("idx_user_reputations_guild_id").on(table.guild_id),
|
|
||||||
scoreIdx: pgIndex("idx_user_reputations_trust_score").on(table.trust_score),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
export const userReputationsTable = pgUserReputationsTable;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Channel Cultures Table (PostgreSQL)
|
* Channel Cultures Table (PostgreSQL)
|
||||||
* Stores AI-generated summaries of channel norms and slang to inject as context.
|
* Stores AI-generated summaries of channel norms and slang to inject as context.
|
||||||
@@ -592,10 +566,6 @@ export type AIAnalysisRunInsert = typeof aiAnalysisRunsTable.$inferInsert;
|
|||||||
export type UserProfile = typeof userProfilesTable.$inferSelect;
|
export type UserProfile = typeof userProfilesTable.$inferSelect;
|
||||||
export type UserProfileInsert = typeof userProfilesTable.$inferInsert;
|
export type UserProfileInsert = typeof userProfilesTable.$inferInsert;
|
||||||
|
|
||||||
// User Reputations
|
|
||||||
export type UserReputation = typeof userReputationsTable.$inferSelect;
|
|
||||||
export type UserReputationInsert = typeof userReputationsTable.$inferInsert;
|
|
||||||
|
|
||||||
// Channel Cultures
|
// Channel Cultures
|
||||||
export type ChannelCulture = typeof channelCulturesTable.$inferSelect;
|
export type ChannelCulture = typeof channelCulturesTable.$inferSelect;
|
||||||
export type ChannelCultureInsert = typeof channelCulturesTable.$inferInsert;
|
export type ChannelCultureInsert = typeof channelCulturesTable.$inferInsert;
|
||||||
|
|||||||
@@ -2,26 +2,17 @@ import {
|
|||||||
pgAIAnalysisRunsTable,
|
pgAIAnalysisRunsTable,
|
||||||
pgChannelCulturesTable,
|
pgChannelCulturesTable,
|
||||||
pgUserProfilesTable,
|
pgUserProfilesTable,
|
||||||
pgUserReputationsTable,
|
|
||||||
} from "../../../shared/index.js";
|
} from "../../../shared/index.js";
|
||||||
|
|
||||||
// Re-export shared tables
|
// Re-export shared tables
|
||||||
export {
|
export { pgAIAnalysisRunsTable, pgChannelCulturesTable, pgUserProfilesTable };
|
||||||
pgAIAnalysisRunsTable,
|
|
||||||
pgChannelCulturesTable,
|
|
||||||
pgUserProfilesTable,
|
|
||||||
pgUserReputationsTable,
|
|
||||||
};
|
|
||||||
export const aiAnalysisRunsTable = pgAIAnalysisRunsTable;
|
export const aiAnalysisRunsTable = pgAIAnalysisRunsTable;
|
||||||
export const channelCulturesTable = pgChannelCulturesTable;
|
export const channelCulturesTable = pgChannelCulturesTable;
|
||||||
export const userProfilesTable = pgUserProfilesTable;
|
export const userProfilesTable = pgUserProfilesTable;
|
||||||
export const userReputationsTable = pgUserReputationsTable;
|
|
||||||
|
|
||||||
// Types
|
// Types
|
||||||
export type AIAnalysisRun = typeof aiAnalysisRunsTable.$inferSelect;
|
export type AIAnalysisRun = typeof aiAnalysisRunsTable.$inferSelect;
|
||||||
export type AIAnalysisRunInsert = typeof aiAnalysisRunsTable.$inferInsert;
|
export type AIAnalysisRunInsert = typeof aiAnalysisRunsTable.$inferInsert;
|
||||||
export type UserReputation = typeof userReputationsTable.$inferSelect;
|
|
||||||
export type UserReputationInsert = typeof userReputationsTable.$inferInsert;
|
|
||||||
export type ChannelCulture = typeof channelCulturesTable.$inferSelect;
|
export type ChannelCulture = typeof channelCulturesTable.$inferSelect;
|
||||||
export type ChannelCultureInsert = typeof channelCulturesTable.$inferInsert;
|
export type ChannelCultureInsert = typeof channelCulturesTable.$inferInsert;
|
||||||
export type UserProfile = typeof userProfilesTable.$inferSelect;
|
export type UserProfile = typeof userProfilesTable.$inferSelect;
|
||||||
|
|||||||
@@ -1,12 +1,11 @@
|
|||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
// Context enrichment builders — rich <user_reputation> attrs, <user_history>,
|
// Context enrichment builders — <user_history>, <user_profiles> as_of,
|
||||||
// <user_profiles> as_of, bot/edited detection (pure, no DB)
|
// bot/edited detection (pure, no DB)
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
buildUserHistoryXml,
|
buildUserHistoryXml,
|
||||||
buildUserProfilesBlock,
|
buildUserProfilesBlock,
|
||||||
formatReputationAttrs,
|
|
||||||
resolveIsBot,
|
resolveIsBot,
|
||||||
resolveIsEdited,
|
resolveIsEdited,
|
||||||
} from "../src/modules/ai-moderation/moderationBuilders.js";
|
} from "../src/modules/ai-moderation/moderationBuilders.js";
|
||||||
@@ -42,72 +41,6 @@ function msg(overrides: Partial<MessageRecord> = {}): MessageRecord {
|
|||||||
|
|
||||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
describe("formatReputationAttrs — rich reputation signal", () => {
|
|
||||||
it("emits trust, infraction count and clean streak", () => {
|
|
||||||
const attrs = formatReputationAttrs({
|
|
||||||
trust_score: 62,
|
|
||||||
total_infractions: 3,
|
|
||||||
clean_message_streak: 45,
|
|
||||||
last_infraction_at: null,
|
|
||||||
});
|
|
||||||
expect(attrs).toContain('trust_score="62"');
|
|
||||||
expect(attrs).toContain('total_infractions="3"');
|
|
||||||
expect(attrs).toContain('clean_streak="45"');
|
|
||||||
});
|
|
||||||
|
|
||||||
it("derives last_offense_days_ago and marks repeat offenders (7-day window)", () => {
|
|
||||||
const attrs = formatReputationAttrs(
|
|
||||||
{
|
|
||||||
trust_score: 50,
|
|
||||||
total_infractions: 2,
|
|
||||||
clean_message_streak: 0,
|
|
||||||
last_infraction_at: NOW - 2 * DAY_MS,
|
|
||||||
},
|
|
||||||
NOW,
|
|
||||||
);
|
|
||||||
expect(attrs).toContain('last_offense_days_ago="2"');
|
|
||||||
expect(attrs).toContain('repeat_offender="true"');
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does NOT mark repeat offender when the last offense is older than 7 days", () => {
|
|
||||||
const attrs = formatReputationAttrs(
|
|
||||||
{
|
|
||||||
trust_score: 50,
|
|
||||||
total_infractions: 2,
|
|
||||||
clean_message_streak: 10,
|
|
||||||
last_infraction_at: NOW - 30 * DAY_MS,
|
|
||||||
},
|
|
||||||
NOW,
|
|
||||||
);
|
|
||||||
expect(attrs).toContain('last_offense_days_ago="30"');
|
|
||||||
expect(attrs).not.toContain("repeat_offender");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("omits offense-derived attrs when the user has no recorded infraction date", () => {
|
|
||||||
const attrs = formatReputationAttrs({
|
|
||||||
trust_score: 85,
|
|
||||||
total_infractions: 0,
|
|
||||||
clean_message_streak: 120,
|
|
||||||
last_infraction_at: null,
|
|
||||||
});
|
|
||||||
expect(attrs).not.toContain("last_offense_days_ago");
|
|
||||||
expect(attrs).not.toContain("repeat_offender");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clamps a future/skewed timestamp to days_ago=0", () => {
|
|
||||||
const attrs = formatReputationAttrs(
|
|
||||||
{
|
|
||||||
trust_score: 50,
|
|
||||||
total_infractions: 1,
|
|
||||||
clean_message_streak: 0,
|
|
||||||
last_infraction_at: NOW + 5 * DAY_MS,
|
|
||||||
},
|
|
||||||
NOW,
|
|
||||||
);
|
|
||||||
expect(attrs).toContain('last_offense_days_ago="0"');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("buildUserHistoryXml — last flagged messages for repeat offenders", () => {
|
describe("buildUserHistoryXml — last flagged messages for repeat offenders", () => {
|
||||||
it("returns empty when there is no real history", () => {
|
it("returns empty when there is no real history", () => {
|
||||||
expect(buildUserHistoryXml([])).toBe("");
|
expect(buildUserHistoryXml([])).toBe("");
|
||||||
|
|||||||
@@ -1,93 +0,0 @@
|
|||||||
// ═══════════════════════════════════════════════════════════════════════════
|
|
||||||
// Trust model v2 — pure math tests (no DB required)
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
|
||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import {
|
|
||||||
computeCleanTrustGain,
|
|
||||||
computeInfractionPenalty,
|
|
||||||
INFRACTION_FLOORS,
|
|
||||||
INFRACTION_PENALTIES,
|
|
||||||
TRUST_DEFAULTS,
|
|
||||||
} from "../src/modules/ai-moderation/userReputationStore.js";
|
|
||||||
|
|
||||||
describe("computeCleanTrustGain — trust CAN rise", () => {
|
|
||||||
it("grants +1 every CLEAN_MESSAGES_PER_POINT clean messages", () => {
|
|
||||||
const before = computeCleanTrustGain(14);
|
|
||||||
expect(before.newStreak).toBe(15);
|
|
||||||
expect(before.trustGain).toBe(1);
|
|
||||||
|
|
||||||
const after = computeCleanTrustGain(15);
|
|
||||||
expect(after.newStreak).toBe(16);
|
|
||||||
expect(after.trustGain).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps compounding past the threshold (no wasted progress)", () => {
|
|
||||||
expect(computeCleanTrustGain(29).trustGain).toBe(1);
|
|
||||||
expect(computeCleanTrustGain(44).trustGain).toBe(1);
|
|
||||||
// 45 clean messages from a fresh start → 3 points of recovery
|
|
||||||
let gain = 0;
|
|
||||||
let streak = 0;
|
|
||||||
for (let i = 0; i < 45; i++) {
|
|
||||||
const r = computeCleanTrustGain(streak);
|
|
||||||
streak = r.newStreak;
|
|
||||||
gain += r.trustGain;
|
|
||||||
}
|
|
||||||
expect(gain).toBe(3);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("computeInfractionPenalty — fair and escalating", () => {
|
|
||||||
const NOW = Date.now();
|
|
||||||
|
|
||||||
it("applies base penalty for a repeat offender outside the window", () => {
|
|
||||||
const r = computeInfractionPenalty({
|
|
||||||
totalInfractions: 3,
|
|
||||||
lastInfractionAt: NOW - TRUST_DEFAULTS.REPEAT_OFFENSE_WINDOW_MS - 1000,
|
|
||||||
severity: "medium",
|
|
||||||
now: NOW,
|
|
||||||
});
|
|
||||||
expect(r.penalty).toBe(INFRACTION_PENALTIES.medium); // 6
|
|
||||||
expect(r.appliedRules.firstOffense).toBe(false);
|
|
||||||
expect(r.appliedRules.repeatEscalation).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("halves the penalty for a first offense (leniency)", () => {
|
|
||||||
const r = computeInfractionPenalty({
|
|
||||||
totalInfractions: 0,
|
|
||||||
lastInfractionAt: null,
|
|
||||||
severity: "high",
|
|
||||||
now: NOW,
|
|
||||||
});
|
|
||||||
expect(r.penalty).toBe(Math.ceil(INFRACTION_PENALTIES.high / 2)); // 6
|
|
||||||
expect(r.appliedRules.firstOffense).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("escalates ×1.5 for a repeat offense within 7 days", () => {
|
|
||||||
const r = computeInfractionPenalty({
|
|
||||||
totalInfractions: 2,
|
|
||||||
lastInfractionAt: NOW - 60 * 60 * 1000, // 1h ago
|
|
||||||
severity: "medium",
|
|
||||||
now: NOW,
|
|
||||||
});
|
|
||||||
expect(r.penalty).toBe(Math.ceil(INFRACTION_PENALTIES.medium * 1.5)); // 9
|
|
||||||
expect(r.appliedRules.repeatEscalation).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("critical first offense still hurts but is halved", () => {
|
|
||||||
const r = computeInfractionPenalty({
|
|
||||||
totalInfractions: 0,
|
|
||||||
lastInfractionAt: null,
|
|
||||||
severity: "critical",
|
|
||||||
now: NOW,
|
|
||||||
});
|
|
||||||
expect(r.penalty).toBe(Math.ceil(INFRACTION_PENALTIES.critical / 2)); // 13
|
|
||||||
});
|
|
||||||
|
|
||||||
it("severity floors prevent minor offenses from zeroing a user", () => {
|
|
||||||
expect(INFRACTION_FLOORS.low).toBeGreaterThan(0);
|
|
||||||
expect(INFRACTION_FLOORS.medium).toBeGreaterThan(0);
|
|
||||||
// high/critical can still reach zero — severe behavior has consequences
|
|
||||||
expect(INFRACTION_FLOORS.high).toBe(0);
|
|
||||||
expect(INFRACTION_FLOORS.critical).toBe(0);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
Reference in New Issue
Block a user