feat(analytics): add daily trend and activity heatmap endpoints, and implement corresponding frontend components
- Added new API endpoints for daily trend data and activity heatmap in analyticsRoutes.ts. - Created new frontend components: ActivityChart, ControlBar, Heatmap, SummaryCards, TopicList, TrendChart, UserTable, and ViolatorTable for displaying analytics data. - Implemented loading and empty states in the new components. - Enhanced the existing moderation tests with remote fallback handling for Indonesian text normalization.
This commit is contained in:
MythEclipse
@@ -1,6 +1,6 @@
|
||||
import { config } from "../config.js";
|
||||
import { executeAll, executeGet } from "../database/drizzle.js";
|
||||
import { createChildLogger } from "../logger.js";
|
||||
import { config } from "../config.js";
|
||||
import type { MessageRecord } from "./types.js";
|
||||
|
||||
const logger = createChildLogger("analytics-store");
|
||||
@@ -130,15 +130,19 @@ export async function getHourlyStats(input: {
|
||||
GROUP BY (created_at / 3600000)
|
||||
ORDER BY hour ASC
|
||||
`,
|
||||
channelId
|
||||
? [guildId, since, channelId, channelId]
|
||||
: [guildId, since],
|
||||
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||
);
|
||||
|
||||
// Initialize all hour buckets (fill gaps with zeros)
|
||||
const buckets = new Map<
|
||||
string,
|
||||
{ count: number; clean: number; warned: number; flagged: number; error: number }
|
||||
{
|
||||
count: number;
|
||||
clean: number;
|
||||
warned: number;
|
||||
flagged: number;
|
||||
error: number;
|
||||
}
|
||||
>();
|
||||
|
||||
for (let h = 0; h < hours; h++) {
|
||||
@@ -178,25 +182,156 @@ export async function getHourlyStats(input: {
|
||||
// ── Topic Trends ───────────────────────────────────────────────────────
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
"yang", "dan", "itu", "ini", "dengan", "akan", "pada", "dari", "di", "ke",
|
||||
"untuk", "tidak", "ada", "juga", "sudah", "saya", "kamu", "dia", "mereka",
|
||||
"kami", "aku", "lo", "lu", "gua", "gue", "org", "orang", "aja", "sama",
|
||||
"kalo", "kalau", "bisa", "karena", "gak", "nggak", "ga", "tak", "belum",
|
||||
"udah", "dah", "lah", "kah", "pun", "nih", "tuh", "deh", "dong", "si",
|
||||
"nya", "kan", "ya", "yah", "yuk", "kok", "loh", "nah", "wow", "eh",
|
||||
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
|
||||
"have", "has", "had", "having", "do", "does", "did", "doing",
|
||||
"will", "would", "could", "should", "may", "might", "must", "shall",
|
||||
"i", "you", "he", "she", "it", "we", "they",
|
||||
"me", "him", "her", "us", "them",
|
||||
"my", "your", "his", "its", "our", "their",
|
||||
"and", "but", "or", "nor", "not", "so", "yet", "for", "if",
|
||||
"to", "of", "in", "on", "at", "by", "as", "with",
|
||||
"about", "just", "then", "now", "here", "there",
|
||||
"when", "where", "why", "how",
|
||||
"all", "both", "each", "few", "more", "most", "other",
|
||||
"some", "such", "only", "own", "same", "too", "very",
|
||||
"can", "go", "ok", "okay", "yeah", "yes", "no",
|
||||
"yang",
|
||||
"dan",
|
||||
"itu",
|
||||
"ini",
|
||||
"dengan",
|
||||
"akan",
|
||||
"pada",
|
||||
"dari",
|
||||
"di",
|
||||
"ke",
|
||||
"untuk",
|
||||
"tidak",
|
||||
"ada",
|
||||
"juga",
|
||||
"sudah",
|
||||
"saya",
|
||||
"kamu",
|
||||
"dia",
|
||||
"mereka",
|
||||
"kami",
|
||||
"aku",
|
||||
"lo",
|
||||
"lu",
|
||||
"gua",
|
||||
"gue",
|
||||
"org",
|
||||
"orang",
|
||||
"aja",
|
||||
"sama",
|
||||
"kalo",
|
||||
"kalau",
|
||||
"bisa",
|
||||
"karena",
|
||||
"gak",
|
||||
"nggak",
|
||||
"ga",
|
||||
"tak",
|
||||
"belum",
|
||||
"udah",
|
||||
"dah",
|
||||
"lah",
|
||||
"kah",
|
||||
"pun",
|
||||
"nih",
|
||||
"tuh",
|
||||
"deh",
|
||||
"dong",
|
||||
"si",
|
||||
"nya",
|
||||
"kan",
|
||||
"ya",
|
||||
"yah",
|
||||
"yuk",
|
||||
"kok",
|
||||
"loh",
|
||||
"nah",
|
||||
"wow",
|
||||
"eh",
|
||||
"the",
|
||||
"a",
|
||||
"an",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"having",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"doing",
|
||||
"will",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"may",
|
||||
"might",
|
||||
"must",
|
||||
"shall",
|
||||
"i",
|
||||
"you",
|
||||
"he",
|
||||
"she",
|
||||
"it",
|
||||
"we",
|
||||
"they",
|
||||
"me",
|
||||
"him",
|
||||
"her",
|
||||
"us",
|
||||
"them",
|
||||
"my",
|
||||
"your",
|
||||
"his",
|
||||
"its",
|
||||
"our",
|
||||
"their",
|
||||
"and",
|
||||
"but",
|
||||
"or",
|
||||
"nor",
|
||||
"not",
|
||||
"so",
|
||||
"yet",
|
||||
"for",
|
||||
"if",
|
||||
"to",
|
||||
"of",
|
||||
"in",
|
||||
"on",
|
||||
"at",
|
||||
"by",
|
||||
"as",
|
||||
"with",
|
||||
"about",
|
||||
"just",
|
||||
"then",
|
||||
"now",
|
||||
"here",
|
||||
"there",
|
||||
"when",
|
||||
"where",
|
||||
"why",
|
||||
"how",
|
||||
"all",
|
||||
"both",
|
||||
"each",
|
||||
"few",
|
||||
"more",
|
||||
"most",
|
||||
"other",
|
||||
"some",
|
||||
"such",
|
||||
"only",
|
||||
"own",
|
||||
"same",
|
||||
"too",
|
||||
"very",
|
||||
"can",
|
||||
"go",
|
||||
"ok",
|
||||
"okay",
|
||||
"yeah",
|
||||
"yes",
|
||||
"no",
|
||||
]);
|
||||
|
||||
function extractTopics(messages: MessageRecord[], topN = 15): TopicTrend[] {
|
||||
@@ -232,7 +367,10 @@ function extractTopics(messages: MessageRecord[], topN = 15): TopicTrend[] {
|
||||
existing.count++;
|
||||
existing.score += msg.ai_moderation_score || 0;
|
||||
} else {
|
||||
topicScores.set(cat, { count: 1, score: msg.ai_moderation_score || 0 });
|
||||
topicScores.set(cat, {
|
||||
count: 1,
|
||||
score: msg.ai_moderation_score || 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -267,7 +405,11 @@ function extractTopics(messages: MessageRecord[], topN = 15): TopicTrend[] {
|
||||
|
||||
for (const [word, count] of sortedWords) {
|
||||
if (!topicScores.has(word)) {
|
||||
results.push({ topic: word, count, score: flaggedWordFreq.get(word) || 0 });
|
||||
results.push({
|
||||
topic: word,
|
||||
count,
|
||||
score: flaggedWordFreq.get(word) || 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -289,7 +431,7 @@ export async function getTopicTrends(input: {
|
||||
|
||||
// Only fetch messages that have ai_analysis (the ones that actually have topics)
|
||||
// This dramatically reduces rows for large guilds
|
||||
const rows = await executeAll(
|
||||
const rows = (await executeAll(
|
||||
`
|
||||
SELECT
|
||||
id, content, ai_status, ai_analysis, ai_moderation_score,
|
||||
@@ -303,10 +445,8 @@ export async function getTopicTrends(input: {
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 2000
|
||||
`,
|
||||
channelId
|
||||
? [guildId, since, channelId, channelId]
|
||||
: [guildId, since],
|
||||
) as MessageRecord[];
|
||||
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||
)) as MessageRecord[];
|
||||
|
||||
const result = extractTopics(rows);
|
||||
setCache(cacheKey, result, TOPIC_CACHE_TTL_MS);
|
||||
@@ -329,7 +469,12 @@ export async function getUserLeaderboard(input: {
|
||||
limit?: number;
|
||||
}): Promise<UserStat[]> {
|
||||
const { guildId, channelId, hours = 24, limit = 20 } = input;
|
||||
const cacheKey = makeCacheKey("leaderboard", { guildId, channelId, hours, limit });
|
||||
const cacheKey = makeCacheKey("leaderboard", {
|
||||
guildId,
|
||||
channelId,
|
||||
hours,
|
||||
limit,
|
||||
});
|
||||
const cached = getCached<UserStat[]>(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
@@ -408,9 +553,7 @@ export async function getModerationStats(input: {
|
||||
AND deleted_at IS NULL
|
||||
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||
`,
|
||||
channelId
|
||||
? [guildId, since, channelId, channelId]
|
||||
: [guildId, since],
|
||||
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||
);
|
||||
|
||||
const result: ModerationBreakdown = row
|
||||
@@ -423,7 +566,15 @@ export async function getModerationStats(input: {
|
||||
pending: row.pending ?? 0,
|
||||
average_score: row.average_score ?? 0,
|
||||
}
|
||||
: { total: 0, clean: 0, warned: 0, flagged: 0, error: 0, pending: 0, average_score: 0 };
|
||||
: {
|
||||
total: 0,
|
||||
clean: 0,
|
||||
warned: 0,
|
||||
flagged: 0,
|
||||
error: 0,
|
||||
pending: 0,
|
||||
average_score: 0,
|
||||
};
|
||||
|
||||
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||
return result;
|
||||
@@ -432,7 +583,15 @@ export async function getModerationStats(input: {
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to get moderation stats",
|
||||
);
|
||||
return { total: 0, clean: 0, warned: 0, flagged: 0, error: 0, pending: 0, average_score: 0 };
|
||||
return {
|
||||
total: 0,
|
||||
clean: 0,
|
||||
warned: 0,
|
||||
flagged: 0,
|
||||
error: 0,
|
||||
pending: 0,
|
||||
average_score: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -493,7 +652,12 @@ export async function getTopViolators(input: {
|
||||
limit?: number;
|
||||
}): Promise<ViolatorStat[]> {
|
||||
const { guildId, channelId, hours = 24, limit = 20 } = input;
|
||||
const cacheKey = makeCacheKey("violators", { guildId, channelId, hours, limit });
|
||||
const cacheKey = makeCacheKey("violators", {
|
||||
guildId,
|
||||
channelId,
|
||||
hours,
|
||||
limit,
|
||||
});
|
||||
const cached = getCached<ViolatorStat[]>(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
@@ -551,6 +715,192 @@ export async function getTopViolators(input: {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Daily Trend (for multi-day line chart) ────────────────────────────
|
||||
|
||||
export interface TrendBucket {
|
||||
date: string;
|
||||
count: number;
|
||||
clean: number;
|
||||
warned: number;
|
||||
flagged: number;
|
||||
error: number;
|
||||
}
|
||||
|
||||
export async function getDailyTrend(input: {
|
||||
guildId: string;
|
||||
channelId?: string;
|
||||
hours?: number;
|
||||
}): Promise<TrendBucket[]> {
|
||||
const { guildId, channelId, hours = 168 } = input;
|
||||
const cacheKey = makeCacheKey("daily_trend", { guildId, channelId, hours });
|
||||
const cached = getCached<TrendBucket[]>(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
try {
|
||||
const since = Date.now() - hours * 3600_000;
|
||||
const isPg = config.DATABASE_TYPE === "postgres";
|
||||
|
||||
const dateExpr = isPg
|
||||
? `to_char(date_trunc('day', to_timestamp(created_at / 1000)), 'YYYY-MM-DD') as date`
|
||||
: `date(created_at / 1000, 'unixepoch') as date`;
|
||||
|
||||
const rows = await executeAll(
|
||||
`
|
||||
SELECT
|
||||
${dateExpr},
|
||||
count(*) as count,
|
||||
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||
count(case when ai_status = 'flagged' then 1 end) as flagged,
|
||||
count(case when ai_status = 'error' then 1 end) as error
|
||||
FROM messages
|
||||
WHERE guild_id = ?
|
||||
AND created_at >= ?
|
||||
AND deleted_at IS NULL
|
||||
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||
GROUP BY date(created_at / 1000, 'unixepoch')
|
||||
ORDER BY date ASC
|
||||
`,
|
||||
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||
);
|
||||
|
||||
// Initialize all day buckets (fill gaps with zeros)
|
||||
const buckets = new Map<
|
||||
string,
|
||||
{
|
||||
count: number;
|
||||
clean: number;
|
||||
warned: number;
|
||||
flagged: number;
|
||||
error: number;
|
||||
}
|
||||
>();
|
||||
const msPerDay = 86400_000;
|
||||
const startDay = Math.floor(since / msPerDay) * msPerDay;
|
||||
const endDay = Math.floor(Date.now() / msPerDay) * msPerDay;
|
||||
|
||||
for (let d = startDay; d <= endDay; d += msPerDay) {
|
||||
const key = new Date(d).toISOString().slice(0, 10);
|
||||
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
const bucket = buckets.get(row.date);
|
||||
if (!bucket) continue;
|
||||
bucket.count = row.count;
|
||||
bucket.clean = row.clean;
|
||||
bucket.warned = row.warned;
|
||||
bucket.flagged = row.flagged;
|
||||
bucket.error = row.error;
|
||||
}
|
||||
|
||||
const result = Array.from(buckets.entries())
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([date, data]) => ({ date, ...data }));
|
||||
|
||||
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to get daily trend",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Activity Heatmap (day-of-week × hour-of-day) ──────────────────────
|
||||
|
||||
export interface HeatmapCell {
|
||||
dayOfWeek: number; // 0=Senin, 6=Minggu
|
||||
hour: number; // 0-23
|
||||
count: number;
|
||||
clean: number;
|
||||
warned: number;
|
||||
flagged: number;
|
||||
}
|
||||
|
||||
export async function getActivityHeatmap(input: {
|
||||
guildId: string;
|
||||
channelId?: string;
|
||||
hours?: number;
|
||||
}): Promise<HeatmapCell[]> {
|
||||
const { guildId, channelId, hours = 168 } = input;
|
||||
const cacheKey = makeCacheKey("heatmap", { guildId, channelId, hours });
|
||||
const cached = getCached<HeatmapCell[]>(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
try {
|
||||
const since = Date.now() - hours * 3600_000;
|
||||
const isPg = config.DATABASE_TYPE === "postgres";
|
||||
|
||||
// SQLite: cast to int for modulo; Postgres: use extract()
|
||||
const dayExpr = isPg
|
||||
? `(extract(isodow from to_timestamp(created_at / 1000)) % 7)::int as day_of_week`
|
||||
: `(cast((created_at / 86400000) as integer) % 7) as day_of_week`;
|
||||
const hourExpr = isPg
|
||||
? `extract(hour from to_timestamp(created_at / 1000))::int as hour`
|
||||
: `(cast((created_at / 3600000) as integer) % 24) as hour`;
|
||||
|
||||
const rows = await executeAll(
|
||||
`
|
||||
SELECT
|
||||
${dayExpr},
|
||||
${hourExpr},
|
||||
count(*) as count,
|
||||
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||
count(case when ai_status = 'flagged' then 1 end) as flagged
|
||||
FROM messages
|
||||
WHERE guild_id = ?
|
||||
AND created_at >= ?
|
||||
AND deleted_at IS NULL
|
||||
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||
GROUP BY day_of_week, hour
|
||||
ORDER BY day_of_week, hour
|
||||
`,
|
||||
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
|
||||
);
|
||||
|
||||
// Initialize all 7×24 cells with zeros
|
||||
const cells = new Map<
|
||||
string,
|
||||
{ count: number; clean: number; warned: number; flagged: number }
|
||||
>();
|
||||
for (let d = 0; d < 7; d++) {
|
||||
for (let h = 0; h < 24; h++) {
|
||||
cells.set(`${d}-${h}`, { count: 0, clean: 0, warned: 0, flagged: 0 });
|
||||
}
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
const key = `${row.day_of_week}-${row.hour}`;
|
||||
const cell = cells.get(key);
|
||||
if (!cell) continue;
|
||||
cell.count = row.count;
|
||||
cell.clean = row.clean;
|
||||
cell.warned = row.warned;
|
||||
cell.flagged = row.flagged;
|
||||
}
|
||||
|
||||
const result = Array.from(cells.entries())
|
||||
.map(([key, data]) => {
|
||||
const [dayOfWeek, hour] = key.split("-").map(Number);
|
||||
return { dayOfWeek, hour, ...data };
|
||||
})
|
||||
.sort((a, b) => a.dayOfWeek - b.dayOfWeek || a.hour - b.hour);
|
||||
|
||||
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"Failed to get activity heatmap",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Cache Invalidation (called when new messages arrive) ───────────────
|
||||
|
||||
export function invalidateAnalyticsCache(guildId: string): void {
|
||||
@@ -574,13 +924,15 @@ export async function getAnalyticsOverview(input: {
|
||||
const now = Date.now();
|
||||
const since = now - hours * 3600_000;
|
||||
|
||||
const [messages, hourly, topics, topUsers, totalChannels] = await Promise.all([
|
||||
getModerationStats(input),
|
||||
getHourlyStats(input),
|
||||
getTopicTrends(input),
|
||||
getUserLeaderboard(input),
|
||||
getActiveChannelCount({ guildId, hours }),
|
||||
]);
|
||||
const [messages, hourly, topics, topUsers, totalChannels] = await Promise.all(
|
||||
[
|
||||
getModerationStats(input),
|
||||
getHourlyStats(input),
|
||||
getTopicTrends(input),
|
||||
getUserLeaderboard(input),
|
||||
getActiveChannelCount({ guildId, hours }),
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
period: { start: since, end: now },
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import axios from "axios";
|
||||
import OpenAI from "openai";
|
||||
import { config } from "../config.js";
|
||||
import { INDONESIAN_SLANG_LEXICON } from "./resources/indonesianSlangLexicon.js";
|
||||
import { createChildLogger } from "../logger.js";
|
||||
import { retryWithBackoff } from "../retry.js";
|
||||
|
||||
const log = createChildLogger("indonesianTextNormalizer");
|
||||
|
||||
@@ -36,6 +38,46 @@ const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
|
||||
insult: "harassment",
|
||||
};
|
||||
|
||||
const VALID_PRIMARY_AI_FLAGS = new Set([
|
||||
"spam",
|
||||
"hate_speech",
|
||||
"sara",
|
||||
"hoaks",
|
||||
"harassment",
|
||||
"vulgar_language",
|
||||
"sexual_content",
|
||||
"sexual_deviation",
|
||||
"violence",
|
||||
"self_harm",
|
||||
"doxxing",
|
||||
"scam",
|
||||
"misinformation",
|
||||
"nsfw_image",
|
||||
"gore_image",
|
||||
"illegal_content",
|
||||
"gambling",
|
||||
"drugs",
|
||||
"child_safety",
|
||||
"financial_scam",
|
||||
"religious_insult",
|
||||
"self_promo",
|
||||
]);
|
||||
|
||||
const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000;
|
||||
const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 60 * 1000;
|
||||
const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 30 * 1000;
|
||||
|
||||
interface BadwordCacheEntry {
|
||||
value: string[];
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
const badwordCache = new Map<string, BadwordCacheEntry>();
|
||||
const inFlightBadwordLookups = new Map<string, Promise<string[]>>();
|
||||
let nemotronUnavailableUntil = 0;
|
||||
let primaryAiUnavailableUntil = 0;
|
||||
let primaryModerationClient: OpenAI | null = null;
|
||||
|
||||
export interface ModerationTextEvidence {
|
||||
raw: string;
|
||||
normalized: string;
|
||||
@@ -159,6 +201,174 @@ function detectLocalBadwords(text: string): string[] {
|
||||
return Array.from(new Set(hits));
|
||||
}
|
||||
|
||||
function normalizeBadwordCacheKey(text: string): string {
|
||||
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
||||
}
|
||||
|
||||
function getCachedBadwords(key: string): string[] | null {
|
||||
const entry = badwordCache.get(key);
|
||||
if (!entry) return null;
|
||||
if (entry.expiresAt <= Date.now()) {
|
||||
badwordCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
return [...entry.value];
|
||||
}
|
||||
|
||||
function setCachedBadwords(key: string, value: string[]): void {
|
||||
badwordCache.set(key, {
|
||||
value: [...new Set(value)],
|
||||
expiresAt: Date.now() + BADWORD_CACHE_TTL_MS,
|
||||
});
|
||||
|
||||
if (badwordCache.size > 500) {
|
||||
const now = Date.now();
|
||||
for (const [cacheKey, entry] of badwordCache) {
|
||||
if (entry.expiresAt <= now) {
|
||||
badwordCache.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (badwordCache.size > 500) {
|
||||
const oldestKeys = Array.from(badwordCache.entries())
|
||||
.sort((a, b) => a[1].expiresAt - b[1].expiresAt)
|
||||
.slice(0, badwordCache.size - 500)
|
||||
.map(([cacheKey]) => cacheKey);
|
||||
for (const cacheKey of oldestKeys) {
|
||||
badwordCache.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getPrimaryModerationClient(): OpenAI | null {
|
||||
if (!config.AI_LLM_API_KEY) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!primaryModerationClient) {
|
||||
primaryModerationClient = new OpenAI({
|
||||
apiKey: config.AI_LLM_API_KEY,
|
||||
baseURL: config.AI_LLM_BASE_URL,
|
||||
maxRetries: 0,
|
||||
timeout: 15000,
|
||||
});
|
||||
}
|
||||
|
||||
return primaryModerationClient;
|
||||
}
|
||||
|
||||
function normalizePrimaryAiFlag(value: string): string | null {
|
||||
const lower = value.trim().toLowerCase().replace(/[\s-]+/g, "_");
|
||||
if (!lower) return null;
|
||||
|
||||
if (VALID_PRIMARY_AI_FLAGS.has(lower)) {
|
||||
return lower;
|
||||
}
|
||||
|
||||
return CATEGORY_TO_BADWORD_LABEL[lower] ?? null;
|
||||
}
|
||||
|
||||
function extractFlagsFromPrimaryAiContent(content: string): string[] {
|
||||
const flags = new Set<string>();
|
||||
let parsed: unknown;
|
||||
|
||||
try {
|
||||
parsed = JSON.parse(content);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
|
||||
const addValue = (value: unknown) => {
|
||||
if (typeof value !== "string") return;
|
||||
const normalized = normalizePrimaryAiFlag(value);
|
||||
if (normalized) flags.add(normalized);
|
||||
};
|
||||
|
||||
if (Array.isArray(parsed)) {
|
||||
for (const item of parsed) {
|
||||
addValue(item);
|
||||
}
|
||||
} else if (parsed && typeof parsed === "object") {
|
||||
const candidate = parsed as Record<string, unknown>;
|
||||
for (const key of ["flags", "categories", "badwords"]) {
|
||||
const value = candidate[key];
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) addValue(item);
|
||||
} else {
|
||||
addValue(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (flags.size > 0) {
|
||||
return Array.from(flags);
|
||||
}
|
||||
|
||||
const lowerContent = content.toLowerCase();
|
||||
for (const flag of VALID_PRIMARY_AI_FLAGS) {
|
||||
if (lowerContent.includes(flag)) {
|
||||
flags.add(flag);
|
||||
}
|
||||
}
|
||||
|
||||
for (const category of Object.keys(CATEGORY_TO_BADWORD_LABEL)) {
|
||||
if (lowerContent.includes(category)) {
|
||||
const mapped = CATEGORY_TO_BADWORD_LABEL[category];
|
||||
if (mapped) flags.add(mapped);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(flags);
|
||||
}
|
||||
|
||||
async function callPrimaryAiModeration(text: string): Promise<string[]> {
|
||||
const client = getPrimaryModerationClient();
|
||||
if (!client) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const completion = await retryWithBackoff(
|
||||
async () => {
|
||||
return client.chat.completions.create({
|
||||
model: config.AI_LLM_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " +
|
||||
"Balas hanya JSON object dengan format {\"flags\":[...]} dan gunakan hanya flag valid ini: " +
|
||||
Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") +
|
||||
". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " +
|
||||
text,
|
||||
},
|
||||
],
|
||||
temperature: 0.1,
|
||||
top_p: 0.9,
|
||||
max_tokens: 200,
|
||||
stream: false,
|
||||
response_format: { type: "json_object" },
|
||||
chat_template_kwargs: { enable_thinking: false },
|
||||
reasoning_budget: 0,
|
||||
} as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
|
||||
},
|
||||
{
|
||||
retries: 1,
|
||||
minTimeout: 500,
|
||||
maxTimeout: 2000,
|
||||
factor: 2,
|
||||
logger: log,
|
||||
},
|
||||
);
|
||||
|
||||
const content = completion.choices[0]?.message?.content?.trim();
|
||||
if (!content) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return extractFlagsFromPrimaryAiContent(content);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NVIDIA Nemotron-3 Content Safety API
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -236,25 +446,81 @@ async function callNemotronContentSafety(text: string): Promise<string[]> {
|
||||
export async function detectIndonesianBadwords(
|
||||
text: string,
|
||||
): Promise<string[]> {
|
||||
// Always run local detection first (fast, no network dependency)
|
||||
const localHits = detectLocalBadwords(text);
|
||||
|
||||
// Try NVIDIA API if key is configured
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (apiKey) {
|
||||
try {
|
||||
const apiCategories = await callNemotronContentSafety(text);
|
||||
const allHits = Array.from(new Set([...localHits, ...apiCategories]));
|
||||
return allHits;
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
{ error },
|
||||
"NVIDIA Nemotron API call failed, falling back to local detection",
|
||||
);
|
||||
}
|
||||
const cacheKey = normalizeBadwordCacheKey(text);
|
||||
const cached = getCachedBadwords(cacheKey);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
return localHits;
|
||||
const inFlight = inFlightBadwordLookups.get(cacheKey);
|
||||
if (inFlight) {
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
const lookupPromise = (async () => {
|
||||
// Always run local detection first (fast, no network dependency)
|
||||
const localHits = detectLocalBadwords(text);
|
||||
|
||||
// If we already have explicit local badword hits, avoid unnecessary API calls.
|
||||
if (localHits.length > 0) {
|
||||
setCachedBadwords(cacheKey, localHits);
|
||||
return localHits;
|
||||
}
|
||||
|
||||
const hits = new Set<string>(localHits);
|
||||
|
||||
// Try NVIDIA API if key is configured and it is not rate limited.
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (apiKey && Date.now() >= nemotronUnavailableUntil) {
|
||||
try {
|
||||
const apiCategories = await callNemotronContentSafety(text);
|
||||
for (const hit of apiCategories) {
|
||||
hits.add(hit);
|
||||
}
|
||||
} catch (error) {
|
||||
const status = axios.isAxiosError(error) ? error.response?.status : null;
|
||||
if (status === 429) {
|
||||
nemotronUnavailableUntil = Date.now() + NEMOTRON_RATE_LIMIT_COOLDOWN_MS;
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"NVIDIA Nemotron API call failed, falling back to primary AI then local detection",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Try the main AI model next, mirroring the image-analysis fallback path.
|
||||
if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
|
||||
try {
|
||||
const primaryHits = await callPrimaryAiModeration(text);
|
||||
for (const hit of primaryHits) {
|
||||
hits.add(hit);
|
||||
}
|
||||
} catch (error) {
|
||||
const status = axios.isAxiosError(error) ? error.response?.status : null;
|
||||
if (status === 429) {
|
||||
primaryAiUnavailableUntil =
|
||||
Date.now() + PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS;
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"Primary AI badword detection failed, falling back to local detection",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const finalHits = Array.from(hits);
|
||||
setCachedBadwords(cacheKey, finalHits);
|
||||
return finalHits;
|
||||
})();
|
||||
|
||||
inFlightBadwordLookups.set(cacheKey, lookupPromise);
|
||||
|
||||
try {
|
||||
return await lookupPromise;
|
||||
} finally {
|
||||
inFlightBadwordLookups.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -2,11 +2,13 @@ import type { Router } from "express";
|
||||
import express from "express";
|
||||
import { AppError } from "../errors.js";
|
||||
import {
|
||||
getActivityHeatmap,
|
||||
getAnalyticsOverview,
|
||||
getDailyTrend,
|
||||
getHourlyStats,
|
||||
getModerationStats,
|
||||
getTopViolators,
|
||||
getTopicTrends,
|
||||
getTopViolators,
|
||||
getUserLeaderboard,
|
||||
} from "../moderation/analyticsStore.js";
|
||||
|
||||
@@ -211,5 +213,69 @@ export function createAnalyticsRoutes(): Router {
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/analytics/trend - Daily trend data (for line chart)
|
||||
// Query params: guildId (required), channelId, hours (default 168)
|
||||
router.get("/analytics/trend", async (req, res, next) => {
|
||||
try {
|
||||
const { guildId, channelId, hours } = req.query as {
|
||||
guildId?: string;
|
||||
channelId?: string;
|
||||
hours?: string;
|
||||
};
|
||||
|
||||
if (!guildId) {
|
||||
throw new AppError(
|
||||
"guildId query parameter is required",
|
||||
"MISSING_GUILD_ID",
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const hoursNum = hours ? Math.min(parseInt(hours) || 168, 720) : 168;
|
||||
|
||||
const trend = await getDailyTrend({
|
||||
guildId,
|
||||
channelId,
|
||||
hours: hoursNum,
|
||||
});
|
||||
|
||||
res.json(trend);
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/analytics/heatmap - Activity heatmap (day × hour)
|
||||
// Query params: guildId (required), channelId, hours (default 168)
|
||||
router.get("/analytics/heatmap", async (req, res, next) => {
|
||||
try {
|
||||
const { guildId, channelId, hours } = req.query as {
|
||||
guildId?: string;
|
||||
channelId?: string;
|
||||
hours?: string;
|
||||
};
|
||||
|
||||
if (!guildId) {
|
||||
throw new AppError(
|
||||
"guildId query parameter is required",
|
||||
"MISSING_GUILD_ID",
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const hoursNum = hours ? Math.min(parseInt(hours) || 168, 720) : 168;
|
||||
|
||||
const heatmap = await getActivityHeatmap({
|
||||
guildId,
|
||||
channelId,
|
||||
hours: hoursNum,
|
||||
});
|
||||
|
||||
res.json(heatmap);
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user