From de0c7927c92dc96c9fee7f4416ed5c7a7fb588c3 Mon Sep 17 00:00:00 2001
From: MythEclipse <asepharyanalm001@gmail.com>
Date: Wed, 3 Jun 2026 19:37:16 +0700
Subject: [PATCH] refactor(analytics): update topic analysis to use word
 frequency

Replaces the previous AI-category based topic analysis with a word-based
frequency analysis using a Common Table Expression (CTE).

- Implements `word_list` CTE to split message content into individual words.
- Adds regex filtering to exclude URLs, Discord stickers, and emojis.
- Implements a stop-word filter to remove common Indonesian and English
  conjunctions, pronouns, and prepositions.
- Filters out words shorter than 3 characters.
- Updates the aggregation to group by word and limit results to the top 10.
---
 .../modules/analytics/analytics.repository.ts | 41 ++++++++++++++++---
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/services/backend/src/modules/analytics/analytics.repository.ts b/services/backend/src/modules/analytics/analytics.repository.ts
index cb6f17b..f5a224e 100644
--- a/services/backend/src/modules/analytics/analytics.repository.ts
+++ b/services/backend/src/modules/analytics/analytics.repository.ts
@@ -317,16 +317,45 @@ export class AnalyticsRepository {
 
     const { rows } = await pool.query(
       `
+        WITH word_list AS (
+          SELECT
+            LOWER(TRIM(BOTH '.,!?;:\"'()[]{}' FROM word)) AS word,
+            ai_moderation_score
+          FROM messages,
+          LATERAL regexp_split_to_table(content, E'\\\\s+') AS word
+          ${filter.where}
+            AND content IS NOT NULL
+            AND content != ''
+            AND LENGTH(TRIM(BOTH '.,!?;:\"()[]{}' FROM word)) >= 3
+            -- Skip sticker & custom emoji (<:name:id> or <a:name:id>)
+            AND word !~ '^<a?:.+:\\d+>$'
+            -- Skip URLs
+            AND word !~ '^https?://'
+            AND word !~ '^discord\\.(gg|app|com)'
+            -- Skip common Discord embed artifacts
+            AND word !~ '^cdn\\.discord'
+        )
         SELECT
-          TRIM(UNNEST(STRING_TO_ARRAY(ai_categories, ','))) AS topic,
+          word AS topic,
           COUNT(*)::int AS count,
           COALESCE(AVG(ai_moderation_score), 0)::real AS score
-        FROM messages
-        ${filter.where}
-          AND ai_categories IS NOT NULL
-          AND ai_categories != ''
-        GROUP BY topic
+        FROM word_list
+        WHERE word NOT IN (
+          'yang', 'dan', 'di', 'ke', 'dari', 'dengan', 'untuk', 'pada', 'ini', 'itu',
+          'ada', 'akan', 'telah', 'sudah', 'bisa', 'dapat', 'tidak', 'nggak', 'enggak',
+          'gak', 'gk', 'ga', 'aku', 'saya', 'kamu', 'dia', 'kami', 'kita', 'mereka',
+          'iya', 'ya', 'yah', 'oh', 'ah', 'eh', 'lah', 'pun', 'juga', 'masih',
+          'saja', 'hanya', 'sama', 'atau', 'tapi', 'namun', 'sedang', 'sangat',
+          'begitu', 'karena', 'sebab', 'kalau', 'jika', 'maka', 'lalu', 'setelah',
+          'seperti', 'antara', 'oleh', 'sebagai', 'secara', 'melalui', 'dalam',
+          'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'has',
+          'was', 'were', 'been', 'like', 'just', 'that', 'this', 'with', 'your',
+          'from', 'they', 'have', 'what', 'when', 'where', 'which', 'their',
+          'about', 'would', 'could', 'should', 'very', 'also', 'than', 'then'
+        )
+        GROUP BY word
         ORDER BY count DESC
+        LIMIT 10
       `,
       filter.params,
     );