diff --git a/services/backend/src/modules/health/moderationMetrics.ts b/services/backend/src/modules/health/moderationMetrics.ts index 3e0a0a4..619e621 100644 --- a/services/backend/src/modules/health/moderationMetrics.ts +++ b/services/backend/src/modules/health/moderationMetrics.ts @@ -67,9 +67,9 @@ export const moderationErrors = new Counter({ labelNames: ["type"] as const, }); -export const searxngCalls = new Counter({ - name: "moderation_searxng_calls_total", - help: "SearXNG search calls", +export const webSearchCalls = new Counter({ + name: "moderation_websearch_calls_total", + help: "Wikipedia web-search calls", labelNames: ["status"] as const, }); diff --git a/services/discord-gateway/pnpm-lock.yaml b/services/discord-gateway/pnpm-lock.yaml index c3c5688..e744154 100644 --- a/services/discord-gateway/pnpm-lock.yaml +++ b/services/discord-gateway/pnpm-lock.yaml @@ -29,9 +29,6 @@ importers: drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@types/pg@8.20.0)(pg@8.22.0) - imghash: - specifier: ^1.1.4 - version: 1.1.4 ioredis: specifier: ^5.11.0 version: 5.11.1(supports-color@7.2.0) @@ -80,7 +77,7 @@ importers: devDependencies: '@biomejs/biome': specifier: latest - version: 2.5.7 + version: 2.5.8 '@types/node': specifier: ^25.9.0 version: 25.9.5 @@ -105,86 +102,63 @@ importers: packages: - '@biomejs/biome@2.5.7': - resolution: {integrity: sha512-zr8K/DcY5tYsQOQwqMJ0AWElo6QgmgNI7idXgXLhevVszlt8RGVpesEJPqx3ThazLaOwjJ5Y8fz3BtH5fGZNsw==} + '@biomejs/biome@2.5.8': + resolution: {integrity: sha512-aeAeeJB9fSDc7Gq+2GqpQxA0qBj6gj1k2R6L1cYqGePKP/baIq1WX8y6B+D+nRsO5ViQL22K/8IwbqERW0q1nw==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-darwin-arm64@2.5.7': - resolution: {integrity: sha512-vxo/Ls3/PYdQWyLhYYcgMOCzQypAjcY+iihS8M0wW03l16TCLW4zqZzGo75gm1VdCMj38hTVZ31KBWrZ4G9dJw==} + '@biomejs/cli-darwin-arm64@2.5.8': + resolution: {integrity: sha512-mk1QON9PHllvvLN5gU3f4rMxeh4syK5p9OvKyWH6/W8ueh04uaC8TUXXByhGufWf/y5mQc03ZLM45zU+cmqMjA==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.5.7': - resolution: {integrity: sha512-Cd3Ga61amT/Yl/0x8elP5hhGYaFy4bw6WuysTgf7oo8TA5tJ5A1k+DkVoJ2BHbTVil51gTX9VPzArnrlLJ3Kyg==} + '@biomejs/cli-darwin-x64@2.5.8': + resolution: {integrity: sha512-bsGwFMBNyHPyiLSsQcZJxdoRrg1V4JL+d7wEsvUBczlP9U9lwM+7mzQHxI4o1mhBsTmdOBbAb6fHU3Z3snN45w==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.5.7': - resolution: {integrity: sha512-xPI5yB6XlpDbNkS+bm1t42olw5c4l3UrlOmLg7KtLJvjvkNF/1V4tnUgfkylGIeb3u/T+BzMGYqgQhzjAoJzuQ==} + '@biomejs/cli-linux-arm64-musl@2.5.8': + resolution: {integrity: sha512-VcJNbstduTHx83NGAdhp78/JOcP45BZHXL7yNsfI1uGzdUgegAz2s+mSoT7wK6PBNzLoqG0zDOXaz/RQYVtSiw==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [musl] - '@biomejs/cli-linux-arm64@2.5.7': - resolution: {integrity: sha512-rR2QE0yF2GYSuYuKIa7pKvODGJqnOH+2eDREAM8wV+mWKSkMQKdAp4zXEZfTaxY8PMoNONnpgSWcBCyLDPDOKg==} + '@biomejs/cli-linux-arm64@2.5.8': + resolution: {integrity: sha512-XmFiA0WPYFC+uiUDC8WRFzAIH9bo7vwQLav38Uoq4ETC+T/+uBi0TsYGJECkugY3r8USl3jc+Ae2/irAF6F2lQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [glibc] - '@biomejs/cli-linux-x64-musl@2.5.7': - resolution: {integrity: sha512-rE5VZi+qtmPgQH+l7jVxYoZ18b/TiHEhulhMpjmCZH1PltSbjRcxNWywC3HZ9tYottG7ORkeTtoscBilKSBm0g==} + '@biomejs/cli-linux-x64-musl@2.5.8': + resolution: {integrity: sha512-kKmiyokeISRGq2FLwvr+TzsgBusfxaZ0FZNLcOYOpCK/78tRrEjeEBLvq3xLZMpqbANgJdRPI7vZX8ZL37u9/w==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [musl] - '@biomejs/cli-linux-x64@2.5.7': - resolution: {integrity: sha512-FQgqJhscrqJUFptGaRSUJWlXAExwWcDwLuK49dvKfkQ1bB5SEEyFssnsxQY83Xm6jR0EbbX3+8+D5bfvYqUG2Q==} + '@biomejs/cli-linux-x64@2.5.8': + resolution: {integrity: sha512-S5wcm9OBDvLHodD4PUaN488hCpco9QD/9ZxuYJiw4euWtr/oQvLR72z2ixItH8Wd5BCm6FZaeb+YNvOoM1xHtQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [glibc] - '@biomejs/cli-win32-arm64@2.5.7': - resolution: {integrity: sha512-Oq4x0CCwP4jirrcTywXs5kOGZ4v5vuEP+gWrbtjApOA2CL9F3F9GlIdQIci8AKSCa/zURanMRpX/4wQ7Am6hHg==} + '@biomejs/cli-win32-arm64@2.5.8': + resolution: {integrity: sha512-nILH0mzm3Hi3iEdd7o7GpB8kBR/mSQwfQG/tyBqyNrY2GFtcgwfV9nV8xLmbtUpMNY/Oi0Ml1XgfR4flOdq+AA==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.5.7': - resolution: {integrity: sha512-V+0wu/nrj2S+MhP4EQ0uHNolP0IALEsz45pg0WoKkHfDeh0+ItHwP/p7bX5RPoMOl9NkpHYWdYPhIcy2mACHvQ==} + '@biomejs/cli-win32-x64@2.5.8': + resolution: {integrity: sha512-I2czzXTY61f3nFJxXoMDq80t7MivxDEnCjE+8sDKoFfcKMaoQdkqhIFQ3KyY0XLzeSpUBYeNAXgD+iOV/BU0VA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] - '@canvas/image-data@1.1.0': - resolution: {integrity: sha512-QdObRRjRbcXGmM1tmJ+MrHcaz1MftF2+W7YI+MsphnsCrmtyfS0d5qJbk0MeSbUeyM/jCb0hmnkXPsy026L7dA==} - - '@canvas/image@2.0.0': - resolution: {integrity: sha512-DQKEftZ5M4eM8Rzhv8FFZGdOsO8bVjzPC/9pBF+P//HjQM/Xf+PvQRTv97mVpsHt+Pejs1V7ooK6xf+dYgDNpw==} - engines: {node: '>=10'} - - '@cwasm/jpeg-turbo@0.1.3': - resolution: {integrity: sha512-FkZxwwC6r4zhzlqM0nYGaMj/MDSrZPxLOdPdM6ySlgsMfOpNAZcLQkpNF4jP+DmsuUvRoeUD0YSMBvg3jYfK6w==} - - '@cwasm/lodepng@0.1.9': - resolution: {integrity: sha512-vb2H7/jTxnqJi7hHiEgtFm3smIqVpeY417vN+8cwsq3iTNrHzwnMzFXbeCf2H9Dl13Vh64qgUUcC2mxf6TPODA==} - engines: {node: '>=8.0.0'} - - '@cwasm/nsbmp@0.1.3': - resolution: {integrity: sha512-APiz9Rj2E049rBapTtwnCGqQeqJjmC85busDQ44UCaujU+LoggUm06NuS1WIBAZcDMRbJOgCFcRWc0tMT2kpfg==} - - '@cwasm/nsgif@0.1.2': - resolution: {integrity: sha512-LOD5HlL0O5jpnIAl+dLSZcB3v0RBNBjtoaymdCEPe2kyKzaP20BF+jy/QUyOZogQsgMVjusZES3tgwwoiiJ2rA==} - - '@cwasm/webp@0.1.5': - resolution: {integrity: sha512-ceIZQkyxK+s7mmItNcWqqHdOBiJAxYxTnrnPNgUNjldB1M9j+Bp/3eVIVwC8rUFyN/zoFwuT0331pyY3ackaNA==} - '@discordjs/builders@1.14.1': resolution: {integrity: sha512-gSKkhXLqs96TCzk66VZuHHl8z2bQMJFGwrXC0f33ngK+FLNau4hU1PYny3DNJfNdSH+gVMzE85/d5FQ2BpcNwQ==} engines: {node: '>=16.11.0'} @@ -1288,9 +1262,6 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - blockhash-core@0.1.0: - resolution: {integrity: sha512-Cv7BgBo0jjVPaeuel4cvxf9LqIGsYNIPz9DAGvvrF9LRlEq9Q3HXu+S8bklPCae0sCxAXic4HGMoImf3FeO3Nw==} - brace-expansion@1.1.18: resolution: {integrity: sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==} @@ -1377,18 +1348,6 @@ packages: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} - decode-bmp@0.2.1: - resolution: {integrity: sha512-NiOaGe+GN0KJqi2STf24hfMkFitDUaIoUU3eKvP/wAbLe8o6FuW5n/x7MHPR0HKvBokp6MQY/j7w8lewEeVCIA==} - engines: {node: '>=8.6.0'} - - decode-ico@0.4.1: - resolution: {integrity: sha512-69NZfbKIzux1vBOd31al3XnMnH+2mqDhEgLdpygErm4d60N+UwA5Sq5WFjmEDQzumgB9fElojGwWG0vybVfFmA==} - engines: {node: '>=8.6'} - - decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -1566,15 +1525,6 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} - fast-base64-decode@1.0.0: - resolution: {integrity: sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q==} - - fast-base64-encode@1.0.0: - resolution: {integrity: sha512-z2XCzVK4fde2cuTEHu2QGkLD6BPtJNKJPn0Z7oINvmhq/quUuIIVPYKUdN0gYeZqOyurjJjBH/bUzK5gafyHvw==} - - fast-base64-length@1.0.0: - resolution: {integrity: sha512-MV+/ioblHx6SMjc/1l4EAnRJyAku6+6DxZ6RW0FoFCF1Aol/Ldb6FqwE3Kn3Ju1aam2m1KCIVoCljhgcG+Umzg==} - fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1590,10 +1540,6 @@ packages: fetch-cookie@3.2.0: resolution: {integrity: sha512-n61pQIxP25C6DRhcJxn7BDzgHP/+S56Urowb5WFxtcRMpU6drqXD90xjyAsVQYsNSNNVbaCcYY1DuHsdkZLuiA==} - file-type@10.11.0: - resolution: {integrity: sha512-uzk64HRpUZyTGZtVuvrjP0FYxzQrBf4rojot6J65YMEbwBLB0CWm0CLojVpwpmFmxcE/lkvYICgfcGozbBq6rw==} - engines: {node: '>=6'} - find-process@2.1.1: resolution: {integrity: sha512-SrQDx3QhlmHM90iqn9rdjCQcw/T+WlpOkHFsjoRgB+zTpDfltNA1VSNYeYELwhUTJy12UFxqjWhmhOrJc+o4sA==} hasBin: true @@ -1684,14 +1630,6 @@ packages: ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - image-type@4.1.0: - resolution: {integrity: sha512-CFJMJ8QK8lJvRlTCEgarL4ro6hfDQKif2HjSvYCdQZESaIPV4v9imrf7BQHK+sQeTeNeMpWciR9hyC/g8ybXEg==} - engines: {node: '>=6'} - - imghash@1.1.4: - resolution: {integrity: sha512-atRI6S7rwrdZlr8pBsuk0NfBfeuPzyna98ugz2Sodj1P8SOZJLVYuqWnUrUFg2YnFTxtdbdSq4Ucn1osLFQ+qA==} - engines: {node: '>=20'} - inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -1711,9 +1649,6 @@ packages: resolution: {integrity: sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==} engines: {node: '>=16'} - jpeg-js@0.4.4: - resolution: {integrity: sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==} - libsodium-wrappers@0.8.4: resolution: {integrity: sha512-mu8aAWucZjTB5O/BtGXtW4e1agy7uHxNYG7zPthmmD1jU43LCDmSWZLN4JhflbdPXj3yDO4lxM1O9hLDgIOXDw==} @@ -1831,10 +1766,6 @@ packages: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} - mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} @@ -2139,12 +2070,6 @@ packages: signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} - simple-concat@1.0.1: - resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - - simple-get@4.0.1: - resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} - sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -2231,9 +2156,6 @@ packages: resolution: {integrity: sha512-3kZ8wQQ/k5DrChD4X4FVvr2D7E5uoRgAqkPyLpSCGUvqOvqu+JEdr3mwMUaVWb+vMHZaKhF5fp2PBigKsui7hA==} hasBin: true - to-data-view@1.1.0: - resolution: {integrity: sha512-1eAdufMg6mwgmlojAx3QeMnzB/BTVp7Tbndi3U7ftcT2zCZadjxkkmLmd97zmaxWi+sgGcgWrokmpEoy0Dn0vQ==} - tough-cookie@5.1.2: resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==} engines: {node: '>=16'} @@ -2426,77 +2348,41 @@ packages: snapshots: - '@biomejs/biome@2.5.7': + '@biomejs/biome@2.5.8': optionalDependencies: - '@biomejs/cli-darwin-arm64': 2.5.7 - '@biomejs/cli-darwin-x64': 2.5.7 - '@biomejs/cli-linux-arm64': 2.5.7 - '@biomejs/cli-linux-arm64-musl': 2.5.7 - '@biomejs/cli-linux-x64': 2.5.7 - '@biomejs/cli-linux-x64-musl': 2.5.7 - '@biomejs/cli-win32-arm64': 2.5.7 - '@biomejs/cli-win32-x64': 2.5.7 + '@biomejs/cli-darwin-arm64': 2.5.8 + '@biomejs/cli-darwin-x64': 2.5.8 + '@biomejs/cli-linux-arm64': 2.5.8 + '@biomejs/cli-linux-arm64-musl': 2.5.8 + '@biomejs/cli-linux-x64': 2.5.8 + '@biomejs/cli-linux-x64-musl': 2.5.8 + '@biomejs/cli-win32-arm64': 2.5.8 + '@biomejs/cli-win32-x64': 2.5.8 - '@biomejs/cli-darwin-arm64@2.5.7': + '@biomejs/cli-darwin-arm64@2.5.8': optional: true - '@biomejs/cli-darwin-x64@2.5.7': + '@biomejs/cli-darwin-x64@2.5.8': optional: true - '@biomejs/cli-linux-arm64-musl@2.5.7': + '@biomejs/cli-linux-arm64-musl@2.5.8': optional: true - '@biomejs/cli-linux-arm64@2.5.7': + '@biomejs/cli-linux-arm64@2.5.8': optional: true - '@biomejs/cli-linux-x64-musl@2.5.7': + '@biomejs/cli-linux-x64-musl@2.5.8': optional: true - '@biomejs/cli-linux-x64@2.5.7': + '@biomejs/cli-linux-x64@2.5.8': optional: true - '@biomejs/cli-win32-arm64@2.5.7': + '@biomejs/cli-win32-arm64@2.5.8': optional: true - '@biomejs/cli-win32-x64@2.5.7': + '@biomejs/cli-win32-x64@2.5.8': optional: true - '@canvas/image-data@1.1.0': {} - - '@canvas/image@2.0.0': - dependencies: - '@canvas/image-data': 1.1.0 - '@cwasm/jpeg-turbo': 0.1.3 - '@cwasm/lodepng': 0.1.9 - '@cwasm/nsbmp': 0.1.3 - '@cwasm/nsgif': 0.1.2 - '@cwasm/webp': 0.1.5 - decode-ico: 0.4.1 - fast-base64-decode: 1.0.0 - fast-base64-encode: 1.0.0 - fast-base64-length: 1.0.0 - simple-get: 4.0.1 - - '@cwasm/jpeg-turbo@0.1.3': - dependencies: - '@canvas/image-data': 1.1.0 - - '@cwasm/lodepng@0.1.9': - dependencies: - '@canvas/image-data': 1.1.0 - - '@cwasm/nsbmp@0.1.3': - dependencies: - '@canvas/image-data': 1.1.0 - - '@cwasm/nsgif@0.1.2': - dependencies: - '@canvas/image-data': 1.1.0 - - '@cwasm/webp@0.1.5': - dependencies: - '@canvas/image-data': 1.1.0 - '@discordjs/builders@1.14.1': dependencies: '@discordjs/formatters': 0.6.2 @@ -3280,8 +3166,6 @@ snapshots: base64-js@1.5.1: {} - blockhash-core@0.1.0: {} - brace-expansion@1.1.18: dependencies: balanced-match: 1.0.2 @@ -3352,21 +3236,6 @@ snapshots: decamelize@1.2.0: {} - decode-bmp@0.2.1: - dependencies: - '@canvas/image-data': 1.1.0 - to-data-view: 1.1.0 - - decode-ico@0.4.1: - dependencies: - '@canvas/image-data': 1.1.0 - decode-bmp: 0.2.1 - to-data-view: 1.1.0 - - decompress-response@6.0.0: - dependencies: - mimic-response: 3.1.0 - delayed-stream@1.0.0: {} delegates@1.0.0: {} @@ -3535,12 +3404,6 @@ snapshots: expect-type@1.4.0: {} - fast-base64-decode@1.0.0: {} - - fast-base64-encode@1.0.0: {} - - fast-base64-length@1.0.0: {} - fast-deep-equal@3.1.3: {} fdir@6.5.0(picomatch@4.0.5): @@ -3552,8 +3415,6 @@ snapshots: set-cookie-parser: 2.7.2 tough-cookie: 6.0.2 - file-type@10.11.0: {} - find-process@2.1.1: dependencies: chalk: 4.1.2 @@ -3658,17 +3519,6 @@ snapshots: ieee754@1.2.1: {} - image-type@4.1.0: - dependencies: - file-type: 10.11.0 - - imghash@1.1.4: - dependencies: - '@canvas/image': 2.0.0 - blockhash-core: 0.1.0 - image-type: 4.1.0 - jpeg-js: 0.4.4 - inflight@1.0.6: dependencies: once: 1.4.0 @@ -3692,8 +3542,6 @@ snapshots: is-network-error@1.3.2: {} - jpeg-js@0.4.4: {} - libsodium-wrappers@0.8.4: dependencies: libsodium: 0.8.4 @@ -3780,8 +3628,6 @@ snapshots: dependencies: mime-db: 1.52.0 - mimic-response@3.1.0: {} - minimatch@3.1.5: dependencies: brace-expansion: 1.1.18 @@ -4061,14 +3907,6 @@ snapshots: signal-exit@3.0.7: {} - simple-concat@1.0.1: {} - - simple-get@4.0.1: - dependencies: - decompress-response: 6.0.0 - once: 1.4.0 - simple-concat: 1.0.1 - sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -4148,8 +3986,6 @@ snapshots: dependencies: tldts-core: 7.4.9 - to-data-view@1.1.0: {} - tough-cookie@5.1.2: dependencies: tldts: 6.1.86 diff --git a/services/discord-gateway/src/modules/ai-moderation/cacheStore.ts b/services/discord-gateway/src/modules/ai-moderation/cacheStore.ts new file mode 100644 index 0000000..dce4c8d --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/cacheStore.ts @@ -0,0 +1,78 @@ +/** + * cacheStore.ts + * + * Shared Redis cache used by the AI-moderation modules (term glossary, etc.). + * + * Extracted when SearXNG was removed (replaced by the Wikipedia adapter in + * wikipediaClient.ts). The cache was never SearXNG-specific — it is a generic + * namespaced key/value store with graceful degradation when Redis is + * unavailable. Other modules import `makeCacheKey`, `cacheGet`, `cacheSet`, + * and `initCacheStore` instead of reaching into a search module. + */ + +import Redis from "ioredis"; +import { createChildLogger } from "@/shared/logger/index"; + +const log = createChildLogger("cache-store"); + +const CACHE_PREFIX = "gmw:"; +const CACHE_TTL = 86400; // 24 hours (used as a sane default) + +let redis: Redis | null = null; + +/** + * Initialize the shared Redis connection for the moderation cache. + * Safe to call multiple times — only creates one connection. + * Degrades gracefully to `null` (no-cache) when Redis is unavailable. + */ +export function initCacheStore(redisUrl: string): void { + if (redis) return; + redis = new Redis(redisUrl, { + maxRetriesPerRequest: 3, + retryStrategy(times) { + const delay = Math.min(times * 200, 2000); + return delay; + }, + lazyConnect: true, + enableReadyCheck: false, + }); + redis.on("error", (err) => { + log.warn({ err: err.message }, "Cache Redis error"); + }); + redis.connect().catch(() => { + log.warn("Cache Redis unavailable — falling back to no-cache"); + redis = null; + }); + log.info("Cache Redis initialized"); +} + +/** Exposes the shared Redis connection; null when Redis is unavailable. */ +export function getCacheRedis(): Redis | null { + return redis; +} + +/** Builds a namespaced cache key (shared across modules). */ +export function makeCacheKey(namespace: string, key: string): string { + return `${CACHE_PREFIX}${namespace}:${key.toLowerCase().trim()}`; +} + +/** Reads a value from the cache; null on miss/unavailable. */ +export async function cacheGet(key: string): Promise { + if (!redis) return null; + try { + return await redis.get(key); + } catch { + return null; + } +} + +/** Writes a value to the cache, fire-and-forget. */ +export function cacheSet(key: string, value: string, ttlSeconds: number): void { + if (!redis) return; + redis.setex(key, ttlSeconds, value).catch(() => { + // Cache write failed silently + }); +} + +/** Default TTL (exposed for callers that want the standard window). */ +export const DEFAULT_CACHE_TTL = CACHE_TTL; diff --git a/services/discord-gateway/src/modules/ai-moderation/moderationOrchestrator.ts b/services/discord-gateway/src/modules/ai-moderation/moderationOrchestrator.ts index 404efe6..257366b 100644 --- a/services/discord-gateway/src/modules/ai-moderation/moderationOrchestrator.ts +++ b/services/discord-gateway/src/modules/ai-moderation/moderationOrchestrator.ts @@ -12,12 +12,12 @@ import type { AttachmentRecord, MessageRecord, } from "../message-capture/types.js"; +import { initCacheStore } from "./cacheStore.js"; import { embedTexts, isEmbeddingEnabled } from "./embeddingClient.js"; import { hasMediaContent } from "./mediaAnalysisClient.js"; import { runMediaBatch } from "./mediaBatchProcessor.js"; import { isQdrantConfigured, searchQdrantBatch } from "./qdrantClient.js"; import { logCacheEvent } from "./responseLogger.js"; -import { initSearxngCache } from "./searxngSearch.js"; import { runTextOnlyBatch } from "./textBatchProcessor.js"; import { findSimilarTextModeration, @@ -70,7 +70,7 @@ export async function runModerationAnalysis( ): Promise { const { targets, contextBlock, attachments } = input; - initSearxngCache(config.REDIS_URL); + initCacheStore(config.REDIS_URL); if (!targets.length) throw new Error("No targets provided for analysis"); // ── Phase 1: exact-hash cache (per conversation context) ──────────────── diff --git a/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts b/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts deleted file mode 100644 index 22f7d98..0000000 --- a/services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts +++ /dev/null @@ -1,262 +0,0 @@ -import Redis from "ioredis"; -import { createChildLogger } from "@/shared/logger/index"; -import { createAbortControllerWithTimeout } from "@/shared/utils/index"; -import { config } from "../../shared/config/config.js"; - -const log = createChildLogger("searxng-search"); - -const SEARXNG_BASE_URL = config.SEARXNG_BASE_URL; -const MAX_RESULTS = 3; -const TIMEOUT_MS = 8000; -const CACHE_TTL = 86400; // 24 hours -const CACHE_PREFIX = "searxng:"; - -let redis: Redis | null = null; - -/** - * Exposes the shared SearXNG Redis connection so other modules (e.g. the - * term glossary) reuse the same connection and cache prefix instead of - * opening their own. Returns null when Redis is unavailable. - */ -export function getSearxngRedis(): Redis | null { - return redis; -} - -/** Builds a namespaced SearXNG cache key (shared across modules). */ -export function makeSearxngCacheKey(namespace: string, key: string): string { - return `${CACHE_PREFIX}${namespace}:${key.toLowerCase().trim()}`; -} - -/** Reads a value from the SearXNG Redis cache; null on miss/unavailable. */ -export async function searxngCacheGet(key: string): Promise { - if (!redis) return null; - try { - return await redis.get(key); - } catch { - return null; - } -} - -/** Writes a value to the SearXNG Redis cache, fire-and-forget. */ -export function searxngCacheSet( - key: string, - value: string, - ttlSeconds: number, -): void { - if (!redis) return; - redis.setex(key, ttlSeconds, value).catch(() => { - // Cache write failed silently - }); -} - -/** - * Initialize Redis connection for SearXNG cache. - * Safe to call multiple times — only creates one connection. - */ -export function initSearxngCache(redisUrl: string): void { - if (redis) return; - // Dedicated Redis connection needed because: this connection serves as an - // optional cache for SearXNG web search results with graceful degradation - // when Redis is unavailable (lazyConnect + null-assignment on failure). - // It uses custom retry strategy and must not block or break the main event - // pipeline if the cache is down. - redis = new Redis(redisUrl, { - maxRetriesPerRequest: 3, - retryStrategy(times) { - const delay = Math.min(times * 200, 2000); - return delay; - }, - lazyConnect: true, - enableReadyCheck: false, - }); - redis.on("error", (err) => { - log.warn({ err: err.message }, "SearXNG Redis cache error"); - }); - redis.connect().catch(() => { - log.warn("SearXNG Redis cache unavailable — falling back to no-cache"); - redis = null; - }); - log.info("SearXNG Redis cache initialized"); -} - -export interface SearxngResult { - title: string; - url: string; - snippet: string; -} - -/** - * Search SearXNG for a query and return structured results. - * Uses Redis cache when available — same query within 24h returns cached results. - * - * @param engines Optional comma-separated SearXNG engine list to constrain - * the search (e.g. "wikipedia"). When set, results are cached under a - * separate cache namespace so engine-specific results never collide. - */ -export async function searchSearxng( - query: string, - category: "general" | "news" | "science" = "general", - engines?: string, - timeoutMs: number = TIMEOUT_MS, -): Promise { - const engineNs = engines ? `eng:${engines}` : "auto"; - const cacheKey = makeSearxngCacheKey(`${category}:${engineNs}`, query); - - // Try cache first - if (redis) { - try { - const cached = await redis.get(cacheKey); - if (cached) { - log.debug({ query, category, engines }, "SearXNG cache HIT"); - return JSON.parse(cached) as SearxngResult[]; - } - } catch { - // Cache read failed, continue to API - } - } - - // Cache miss — hit SearXNG API - try { - const engineParam = engines - ? `&engines=${encodeURIComponent(engines)}` - : ""; - const url = `${SEARXNG_BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&language=id&categories=${category}${engineParam}`; - const { controller, clear } = createAbortControllerWithTimeout(timeoutMs); - - try { - const response = await fetch(url, { - signal: controller.signal, - headers: { - Accept: "application/json", - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - }, - }); - - if (!response.ok) { - log.warn({ status: response.status, query }, "SearXNG search failed"); - return []; - } - - const data = (await response.json()) as { - results?: Array<{ title?: string; url?: string; content?: string }>; - }; - const results = data.results ?? []; - const mapped = results.slice(0, MAX_RESULTS).map((r) => ({ - title: r.title ?? "", - url: r.url ?? "", - snippet: (r.content ?? "").slice(0, 500), - })); - - // Store in cache (fire and forget — don't block on write) - if (redis) { - redis.setex(cacheKey, CACHE_TTL, JSON.stringify(mapped)).catch(() => { - // Cache write failed silently - }); - } - - log.debug( - { query, category, resultCount: mapped.length }, - "SearXNG search OK", - ); - return mapped; - } finally { - clear(); - } - } catch (err) { - log.warn( - { error: err instanceof Error ? err.message : String(err), query }, - "SearXNG search error", - ); - return []; - } -} - -/** - * Extract meaningful search queries from message content. - * Uses multiple strategies to find terms worth searching. - * Returns up to 3 clean queries. - */ -export function extractSearchQueries(content: string): string[] { - const queries = new Set(); - - // 1. Quoted phrases (explicit user intent) - const quotedPhrases = content.match(/"([^"]+)"|'([^']+)'/g); - if (quotedPhrases) { - for (const phrase of quotedPhrases) { - const clean = phrase.replace(/["']/g, "").trim(); - if (clean.length >= 3) queries.add(clean); - } - } - - // 2. "nonton X" pattern — extract the title - const nontonMatch = content.match( - /\b(nonton|tonton|rekomen|cari|search|google)\s+(.+?)(?:\s+(?:anime|kartun|film|movie|series|serial))?\s*[!?.]*$/i, - ); - if (nontonMatch) { - const title = nontonMatch[2].trim(); - if (title.length >= 2 && title.length <= 80) { - queries.add(title); - } - } - - // 3. "X anime/film" pattern — title before category - const titleBeforeCategory = content.match( - /\b(\w[\w\s]{2,40})\s+(?:anime|kartun|film|movie|series|serial)\b/i, - ); - if (titleBeforeCategory) { - const title = titleBeforeCategory[1].trim(); - if ( - title.length >= 3 && - !/^(yang|yang|sama|dari|untuk|ini|itu|ada)$/i.test(title) - ) { - queries.add(title); - } - } - - // 4. Standalone proper nouns (2+ words, capitalized) that look like titles - const properNouns = content.match( - /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,4})\b/g, - ); - if (properNouns) { - for (const noun of properNouns) { - // Skip common non-title proper nouns - const skip = - /^(Discord|YouTube|Google|Facebook|Instagram|Twitter|Github|ChatGPT|OpenAI|Claude|Telegram|WhatsApp|TikTok|Netflix|Spotify|Steam|Instagram)$/i; - if (!skip.test(noun) && noun.length >= 5) { - queries.add(noun); - } - } - } - - // 5. Terms that suggest research intent - const researchTerms = content.match( - /\b(apa\s+(?:itu|sih)|what\s+is|siapa\s+itu|who\s+is|arti|meaning|definisi|definition)\s+(.{3,60})/i, - ); - if (researchTerms) { - const term = researchTerms[2].trim().replace(/[?!.]+$/, ""); - if (term.length >= 3) queries.add(term); - } - - return Array.from(queries).slice(0, 3); -} - -/** - * Format SearXNG results as XML for LLM context. - */ -export function formatSearchResults(results: SearxngResult[]): string { - if (results.length === 0) return ""; - const lines = results.map( - (r) => - ` ${escapeXml(r.snippet)}`, - ); - return `\n${lines.join("\n")}\n`; -} - -function escapeXml(str: string): string { - return str - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """); -} diff --git a/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts b/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts index 1dd4bbe..a8a0a86 100644 --- a/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts +++ b/services/discord-gateway/src/modules/ai-moderation/termGlossary.ts @@ -10,18 +10,18 @@ * wording (false negative on an unknown vulgar/slang term). * * Solution: extract candidate "unknown-looking" words from message content, - * look each one up on Wikipedia via SearXNG, and inject the definitions into - * the LLM prompt as a `` block so verdicts are based on facts - * instead of guesses. + * look each one up on Wikipedia via the Wikipedia REST/Action APIs, and inject + * the definitions into the LLM prompt as a `` block so verdicts + * are based on facts instead of guesses. * * Cost control & persistence: * - successfully resolved definitions are PERSISTED PERMANENTLY in Postgres * (`term_glossary_cache`) — definitions rarely change, so a resolved term * is never searched again; only misses stay ephemeral (Redis/LRU, 1h); - * - in-memory LRU + Redis (shared with the SearXNG cache) sit in front of + * - in-memory LRU + Redis (shared cache store) sit in front of * the DB as fast read caches, so repeat lookups are effectively free; * - lookups per batch are bounded (AI_GLOSSARY_MAX_TERMS); - * - live SearXNG calls are rate-limit aware: concurrency 2 + stagger, retry + * - live Wikipedia calls are rate-limit aware: concurrency 2 + stagger, retry * once on empty results, and misses cached for only 1h so a limiter/ * network blip is not treated as a permanent miss; * - only results that read like actual definitions are accepted (Wikipedia @@ -35,17 +35,13 @@ import pLimit from "p-limit"; import { createChildLogger } from "@/shared/logger/index"; import { delay } from "@/shared/utils/index"; import { config } from "../../shared/config/config.js"; +import { cacheGet, cacheSet, makeCacheKey } from "./cacheStore.js"; import { escapeXml } from "./moderationBuilders.js"; -import { - makeSearxngCacheKey, - searchSearxng, - searxngCacheGet, - searxngCacheSet, -} from "./searxngSearch.js"; import { getTermDefinitionFromDb, setTermDefinitionInDb, } from "./termGlossaryStore.js"; +import { wikipediaSummary } from "./wikipediaClient.js"; const log = createChildLogger("term-glossary"); @@ -65,16 +61,13 @@ const MISS_TTL_SECONDS = 60 * 60; const MISS_TTL_MS = MISS_TTL_SECONDS * 1000; /** Sentinel stored in caches for "term has no resolvable definition". */ const EMPTY_SENTINEL = "__not_found__"; -/** Per-search timeout — keep glossary lookups snappy even on a slow SearXNG. */ -const GLOSSARY_SEARCH_TIMEOUT_MS = 5000; /** Delay before retrying a search that returned zero results. */ const RETRY_DELAY_MS = 350; /** Max definition snippet length kept in the prompt. */ const MAX_DEFINITION_CHARS = 300; /** - * SearXNG rate-limits aggressive parallel bursts (returns 200 with empty - * results). Never fire all terms at once — cap live searches at 2 concurrent - * and stagger the start times slightly. + * Wikipedia can be flaky under aggressive parallel bursts. Never fire all + * terms at once — cap live lookups at 2 concurrent and stagger the start times. */ const LIVE_SEARCH_CONCURRENCY = 2; const LIVE_SEARCH_STAGGER_MS = 250; @@ -90,7 +83,7 @@ const termLru = new LRUCache({ ttl: 24 * 60 * 60 * 1000, }); -/** Serializes live SearXNG lookups (rate-limit aware) with a small stagger. */ +/** Serializes live Wikipedia lookups (rate-limit aware) with a small stagger. */ const liveSearchLimit = pLimit(LIVE_SEARCH_CONCURRENCY); let lastLiveSearchAt = 0; async function acquireLiveSlot(): Promise { @@ -274,58 +267,8 @@ export interface TermDefinition { sourceUrl: string; } -/** Definition-like markers for accepting a non-Wikipedia search result. */ -const DEF_MARKERS = - /adalah|merupakan|istilah (?:untuk|yang|yg)|artinya|sebutan|berarti|refers? to|known as|also called|short for|a term (?:for|used)|istilah dalam|kata (?:asing|serapan)? ?untuk/i; - -/** True when the term appears in the result text (or a 4+ char word in the - * result is part of the term). Lenient — "kafircel" matches a "Kafir" - * article via substring, while a Google-Translate homepage snippet does not. */ -function hasTermOverlap(term: string, title: string, snippet: string): boolean { - const termLower = term.toLowerCase(); - const text = `${title} ${snippet}`.toLowerCase(); - if (text.includes(termLower)) return true; - const words = text.match(/[a-z0-9]{4,}/gi) ?? []; - return words.some((w) => termLower.includes(w)); -} - -/** Quality gate: is this result good enough to quote as a definition? */ -function isUsableDefinition( - r: { title: string; url: string; snippet: string }, - term: string, - isWiki: boolean, -): boolean { - const text = `${r.title} ${r.snippet}`; - // Wikipedia disambiguation pages are not definitions - if (/disambiguasi|disambiguation/i.test(text)) return false; - if ((r.snippet ?? "").trim().length < 25) return false; - if (!hasTermOverlap(term, r.title, r.snippet)) return false; - // Wikipedia articles are accepted with just the overlap+length gate; - // everything else must read like an actual definition, not an ad, - // a translate homepage, or a navigation blurb. - if (isWiki) return true; - return DEF_MARKERS.test(r.snippet); -} - -/** Picks the best definition from search results, preferring a genuine - * Wikipedia article; otherwise the first result that reads like a - * definition. Returns null when nothing qualifies. */ -function pickDefinition( - results: Array<{ title: string; url: string; snippet: string }>, - term: string, -): TermDefinition | null { - const wiki = results.find((r) => /wikipedia\.org/i.test(r.url)); - const best = wiki && isUsableDefinition(wiki, term, true) ? wiki : null; - if (!best) { - for (const r of results) { - if (isUsableDefinition(r, term, false)) { - return buildDefinition(r, term); - } - } - return null; - } - return buildDefinition(best, term); -} +/** Per-search timeout — keep glossary lookups snappy even on a slow Wikipedia. */ +const GLOSSARY_SEARCH_TIMEOUT_MS = 5000; function buildDefinition( best: { title: string; url: string; snippet: string }, @@ -339,7 +282,7 @@ function buildDefinition( return { term, definition, sourceUrl: best.url }; } -/** Live (network) lookup — runs under the shared SearXNG rate-limit gate. */ +/** Live (network) lookup — runs under the shared Wikipedia rate-limit gate. */ async function fetchDefinitionLive( term: string, key: string, @@ -348,31 +291,21 @@ async function fetchDefinitionLive( return liveSearchLimit(async () => { await acquireLiveSlot(); try { - let results = await searchSearxng( - key, - "general", - undefined, - GLOSSARY_SEARCH_TIMEOUT_MS, - ); - let def = pickDefinition(results, term); - // Zero results is usually the limiter kicking in, not a real miss — - // retry once. Results-but-unusable = genuine miss, no retry. - if (!def && results.length === 0) { + let result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS); + let def = result ? buildDefinition(result, term) : null; + // Zero result is usually the limiter/network blip, not a real miss — + // retry once. Result-but-unusable = genuine miss, no retry. + if (!def) { await delay(RETRY_DELAY_MS); - results = await searchSearxng( - key, - "general", - undefined, - GLOSSARY_SEARCH_TIMEOUT_MS, - ); - def = pickDefinition(results, term); + result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS); + def = result ? buildDefinition(result, term) : null; } if (def) { // Persist permanently (definitions rarely change) — best-effort, // then warm the fast caches. void setTermDefinitionInDb(key, def.definition, def.sourceUrl); - searxngCacheSet( + cacheSet( cacheKey, JSON.stringify({ definition: def.definition, @@ -393,13 +326,13 @@ async function fetchDefinitionLive( // No definition — cache the miss with a SHORT TTL so a transient // limiter/network failure is retried on a later batch. - searxngCacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS); + cacheSet(cacheKey, EMPTY_SENTINEL, MISS_TTL_SECONDS); termLru.set(key, NOT_FOUND, { ttl: MISS_TTL_MS }); return null; }); } -/** Resolve one term: LRU → Redis → Postgres (permanent) → live SearXNG +/** Resolve one term: LRU → Redis → Postgres (permanent) → live Wikipedia * (rate-limited). The fast caches sit in front of the DB; the DB is the * source of truth for successfully resolved definitions. */ async function resolveTerm(term: string): Promise { @@ -412,8 +345,8 @@ async function resolveTerm(term: string): Promise { // 2. Redis — shared across processes/workers. A miss sentinel here is NOT // a definitive answer: it may predate a permanent DB entry written by // another process, so we keep going and let the DB decide. - const cacheKey = makeSearxngCacheKey("def", key); - const cached = await searxngCacheGet(cacheKey); + const cacheKey = makeCacheKey("def", key); + const cached = await cacheGet(cacheKey); let redisMiss = false; if (cached !== null) { if (cached === EMPTY_SENTINEL) { @@ -449,7 +382,7 @@ async function resolveTerm(term: string): Promise { sourceUrl: dbDef.sourceUrl, }; termLru.set(key, def); - searxngCacheSet( + cacheSet( cacheKey, JSON.stringify({ definition: def.definition, sourceUrl: def.sourceUrl }), DEF_TTL_SECONDS, diff --git a/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts b/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts index fb87f6b..2a5a219 100644 --- a/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts +++ b/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts @@ -1,7 +1,7 @@ /** * textBatchProcessor.ts * - * Processes text-only moderation batches — fetches URL content, runs SearXNG + * Processes text-only moderation batches — fetches URL content, runs Wikipedia * searches, deduplicates short messages, splits into sub-batches, and calls * the LLM for analysis. Extracted from moderationOrchestrator.ts. */ @@ -28,15 +28,15 @@ import { } from "./moderationBuilders.js"; import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js"; import { logModerationAnalysis } from "./responseLogger.js"; -import { - extractSearchQueries, - formatSearchResults, - searchSearxng, -} from "./searxngSearch.js"; import { buildTermGlossaryBlock } from "./termGlossary.js"; import { getRecentCorrectedModerations } from "./textCacheStore.js"; import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; import type { MessageImagePart } from "./visionAnalyzer.js"; +import { + extractSearchQueries, + formatSearchResults, + wikipediaSearch, +} from "./wikipediaClient.js"; const log = createChildLogger("textBatchProcessor"); @@ -117,7 +117,7 @@ export async function runTextOnlyBatch( return { text: textMap, image: imageMap, title: titleMap }; })(); - const searxngPromise = (async () => { + const webSearchPromise = (async () => { const queries = new Set(); for (const msg of targets) { for (const q of extractSearchQueries(msg.edited_content ?? msg.content)) @@ -126,7 +126,7 @@ export async function runTextOnlyBatch( if (queries.size === 0) return new Map(); const queryArr = Array.from(queries).slice(0, 3); const results = await Promise.allSettled( - queryArr.map((q) => searchSearxng(q)), + queryArr.map((q) => wikipediaSearch(q)), ); const map = new Map(); for (let i = 0; i < queryArr.length; i++) { @@ -138,15 +138,13 @@ export async function runTextOnlyBatch( })(); // Term glossary — per-word Wikipedia lookups for words the LLM may not - // know (slang, jargon, regional language). Cached in Redis + in-memory, so - // repeat terms resolve instantly and only genuinely new words hit SearXNG. const glossaryPromise = buildTermGlossaryBlock( targets.map((msg) => getAnalysisContent(msg)), ).catch(() => ""); - const [urlFetchMaps, searxngResults, glossaryBlock] = await Promise.all([ + const [urlFetchMaps, webSearchResults, glossaryBlock] = await Promise.all([ urlFetchPromise, - searxngPromise, + webSearchPromise, glossaryPromise, ]); const urlFetchMap = urlFetchMaps.text; @@ -308,9 +306,9 @@ export async function runTextOnlyBatch( ) ).join("\n"); - const searxngBlock = - searxngResults.size > 0 - ? `\n${Array.from(searxngResults.entries()) + const webSearchBlock = + webSearchResults.size > 0 + ? `\n${Array.from(webSearchResults.entries()) .map( ([q, xml]) => ` \n${xml} `, @@ -323,7 +321,7 @@ export async function runTextOnlyBatch( // profile descriptions are intentionally omitted (see above). const userBlocks = [ contextBlock?.trimEnd() ?? "", - searxngBlock, + webSearchBlock, glossaryBlock, `\n${messagesBlock}\n`, ].filter((b) => b.trim().length > 0); diff --git a/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts b/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts index 19b14ca..3ef27fc 100644 --- a/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts +++ b/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts @@ -76,13 +76,13 @@ import { buildStickerTextOnlyWarning, buildStickerVisionPrompt, } from "./moderationPrompt.js"; +import { buildTermGlossaryBlock } from "./termGlossary.js"; +import { extractUrlsFromText } from "./urlFetcher.js"; import { extractSearchQueries, formatSearchResults, - searchSearxng, -} from "./searxngSearch.js"; -import { buildTermGlossaryBlock } from "./termGlossary.js"; -import { extractUrlsFromText } from "./urlFetcher.js"; + wikipediaSearch, +} from "./wikipediaClient.js"; // --------------------------------------------------------------------------- // Types @@ -354,12 +354,12 @@ export async function prepareMediaMessage( ), ); - // SearXNG - let searxngXml = ""; + // Wikipedia web search (context enrichment) + let webSearchXml = ""; const queries = extractSearchQueries(content); if (queries.length > 0) { const results = await Promise.allSettled( - queries.map((q) => searchSearxng(q)), + queries.map((q) => wikipediaSearch(q)), ); const parts: string[] = []; for (let i = 0; i < results.length; i++) { @@ -368,7 +368,7 @@ export async function prepareMediaMessage( parts.push(formatSearchResults(r.value)); } if (parts.length > 0) - searxngXml = `\n\n${parts.join("\n")}\n`; + webSearchXml = `\n\n${parts.join("\n")}\n`; } // Term glossary — cached per-word Wikipedia definitions for words the LLM @@ -403,6 +403,6 @@ export async function prepareMediaMessage( // still tracked in the DB for enforcement, just not shown to the LLM. const isBot = resolveIsBot(target); const isEdited = resolveIsEdited(target); - const messageBlock = `\n ${refXml ? `\n ${refXml}` : ""}\n ${escapeXml(truncateForAi(content))}${mediaContext ? ` ${escapeXml(mediaContext)}` : ""}${webContext}${mediaAnalysisContext}${searxngXml}${glossaryCtx}\n`; + const messageBlock = `\n ${refXml ? `\n ${refXml}` : ""}\n ${escapeXml(truncateForAi(content))}${mediaContext ? ` ${escapeXml(mediaContext)}` : ""}${webContext}${mediaAnalysisContext}${webSearchXml}${glossaryCtx}\n`; return { targetId, messageBlock }; } diff --git a/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts b/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts new file mode 100644 index 0000000..ba67b2c --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/wikipediaClient.ts @@ -0,0 +1,260 @@ +/** + * wikipediaClient.ts + * + * Wikipedia adapter for AI analysis context enrichment. + * + * Replaces the old SearXNG web-search dependency (removed). Instead of a + * meta-search instance, we talk to the public Wikipedia REST + Action APIs + * directly with native `fetch` — no extra npm dependency, full control, and + * a stable, well-documented endpoint. + * + * Layer exposed to the moderation pipeline: + * WikipediaClient + * ├── search() → list(query) (Action API: opensearch-like) + * ├── getSummary() → summary(title) (REST summary endpoint) + * └── (page content) → page(title) [reserved] + * + * The functions below are thin wrappers matching the old consumer surface so + * call sites change as little as possible. + */ + +import { createChildLogger } from "@/shared/logger/index"; +import { createAbortControllerWithTimeout } from "@/shared/utils/index"; +import { config } from "../../shared/config/config.js"; + +const log = createChildLogger("wikipedia-client"); + +const WIKIPEDIA_LANG = config.WIKIPEDIA_LANG.toLowerCase(); +const MAX_RESULTS = 3; +const DEFAULT_TIMEOUT_MS = config.WIKIPEDIA_TIMEOUT_MS; + +/** Canonical article URL for a title in the active wiki language. */ +export function wikipediaPageUrl(title: string): string { + return `https://${WIKIPEDIA_LANG}.wikipedia.org/wiki/${encodeURIComponent( + title.trim().replace(/ /g, "_"), + )}`; +} + +export interface SearchResult { + title: string; + url: string; + snippet: string; +} + +function buildUserAgent(): string { + return "GMWBeta/1.0 (https://github.com/asepharyana; Discord moderation bot)"; +} + +function stripHtml(snippet: string): string { + return snippet + .replace(/<[^>]+>/g, "") + .replace(/"/g, '"') + .replace(/&/g, "&") + .replace(/'/g, "'") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Search Wikipedia for a query and return up to MAX_RESULTS structured hits. + * Uses the Action API `list=search` (srsearch) which is stable and returns + * title + HTML snippet. Graceful: returns [] on any failure. + */ +export async function wikipediaSearch( + query: string, + timeoutMs: number = DEFAULT_TIMEOUT_MS, +): Promise { + const q = query.trim(); + if (!q) return []; + + const params = new URLSearchParams({ + action: "query", + list: "search", + srsearch: q, + srlimit: String(MAX_RESULTS), + format: "json", + origin: "*", + }); + const { controller, clear } = createAbortControllerWithTimeout(timeoutMs); + + try { + const res = await fetch( + `https://${WIKIPEDIA_LANG}.wikipedia.org/w/api.php?${params.toString()}`, + { + signal: controller.signal, + headers: { + Accept: "application/json", + "User-Agent": buildUserAgent(), + }, + }, + ); + + if (!res.ok) { + log.warn({ status: res.status, query: q }, "Wikipedia search failed"); + return []; + } + + const data = (await res.json()) as { + query?: { search?: Array<{ title: string; snippet?: string }> }; + }; + const hits = data.query?.search ?? []; + const mapped = hits.slice(0, MAX_RESULTS).map((h) => ({ + title: h.title, + url: wikipediaPageUrl(h.title), + snippet: stripHtml(h.snippet ?? "").slice(0, 500), + })); + + log.debug({ query: q, resultCount: mapped.length }, "Wikipedia search OK"); + return mapped; + } catch (err) { + log.warn( + { error: err instanceof Error ? err.message : String(err), query: q }, + "Wikipedia search error", + ); + return []; + } finally { + clear(); + } +} + +/** + * Fetch the lead summary of a specific Wikipedia article via the REST + * summary endpoint. Returns null when the article is missing or the request + * fails. Useful for the term glossary's direct lookups. + */ +export async function wikipediaSummary( + title: string, + timeoutMs: number = DEFAULT_TIMEOUT_MS, +): Promise { + const t = title.trim(); + if (!t) return null; + + const { controller, clear } = createAbortControllerWithTimeout(timeoutMs); + try { + const res = await fetch( + `https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent( + t.replace(/ /g, "_"), + )}`, + { + signal: controller.signal, + headers: { + Accept: "application/json", + "User-Agent": buildUserAgent(), + }, + }, + ); + + if (!res.ok) return null; + const data = (await res.json()) as { + title?: string; + extract?: string; + content_urls?: { desktop?: { page?: string } }; + }; + if (!data.extract) return null; + + return { + title: data.title ?? t, + url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t), + snippet: data.extract.slice(0, 500), + }; + } catch (err) { + log.warn( + { error: err instanceof Error ? err.message : String(err), title: t }, + "Wikipedia summary error", + ); + return null; + } finally { + clear(); + } +} + +/** + * Extract meaningful search queries from message content. + * Uses multiple strategies to find terms worth searching. + * Returns up to 3 clean queries. + */ +export function extractSearchQueries(content: string): string[] { + const queries = new Set(); + + // 1. Quoted phrases (explicit user intent) + const quotedPhrases = content.match(/"([^"]+)"|'([^']+)'/g); + if (quotedPhrases) { + for (const phrase of quotedPhrases) { + const clean = phrase.replace(/["']/g, "").trim(); + if (clean.length >= 3) queries.add(clean); + } + } + + // 2. "nonton X" pattern — extract the title + const nontonMatch = content.match( + /\b(nonton|tonton|rekomen|cari|search|google)\s+(.+?)(?:\s+(?:anime|kartun|film|movie|series|serial))?\s*[!?.]*$/i, + ); + if (nontonMatch) { + const title = nontonMatch[2].trim(); + if (title.length >= 2 && title.length <= 80) { + queries.add(title); + } + } + + // 3. "X anime/film" pattern — title before category + const titleBeforeCategory = content.match( + /\b(\w[\w\s]{2,40})\s+(?:anime|kartun|film|movie|series|serial)\b/i, + ); + if (titleBeforeCategory) { + const title = titleBeforeCategory[1].trim(); + if ( + title.length >= 3 && + !/^(yang|yang|sama|dari|untuk|ini|itu|ada)$/i.test(title) + ) { + queries.add(title); + } + } + + // 4. Standalone proper nouns (2+ words, capitalized) that look like titles + const properNouns = content.match( + /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,4})\b/g, + ); + if (properNouns) { + for (const noun of properNouns) { + // Skip common non-title proper nouns + const skip = + /^(Discord|YouTube|Google|Facebook|Instagram|Twitter|Github|ChatGPT|OpenAI|Claude|Telegram|WhatsApp|TikTok|Netflix|Spotify|Steam|Instagram)$/i; + if (!skip.test(noun) && noun.length >= 5) { + queries.add(noun); + } + } + } + + // 5. Terms that suggest research intent + const researchTerms = content.match( + /\b(apa\s+(?:itu|sih)|what\s+is|siapa\s+itu|who\s+is|arti|meaning|definisi|definition)\s+(.{3,60})/i, + ); + if (researchTerms) { + const term = researchTerms[2].trim().replace(/[?!.]+$/, ""); + if (term.length >= 3) queries.add(term); + } + + return Array.from(queries).slice(0, 3); +} + +/** + * Format Wikipedia results as XML for LLM context. + */ +export function formatSearchResults(results: SearchResult[]): string { + if (results.length === 0) return ""; + const lines = results.map( + (r) => + ` ${escapeXml(r.snippet)}`, + ); + return `\n${lines.join("\n")}\n`; +} + +function escapeXml(str: string): string { + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} diff --git a/services/discord-gateway/src/shared/config/index.ts b/services/discord-gateway/src/shared/config/index.ts index c15c7c0..133ff9b 100644 --- a/services/discord-gateway/src/shared/config/index.ts +++ b/services/discord-gateway/src/shared/config/index.ts @@ -104,10 +104,12 @@ export const configSchema = z // ── Redis ──────────────────────────────────────────────────────────── REDIS_URL: z.string().default("redis://localhost:6379"), - // ── SearXNG ─────────────────────────────────────────────────────────── - // Instance for web search + term glossary lookups. Override when the - // default instance is down/rate-limited. - SEARXNG_BASE_URL: z.string().url().default("https://searxng.imrnes.team"), + // ── Wikipedia (web-search / glossary source) ───────────────────────── + // Native fetch to Wikipedia REST + Action APIs — no SearXNG dependency. + // Language for summaries/search (e.g. "id", "en"). + WIKIPEDIA_LANG: z.string().min(1).default("id"), + // Per-request timeout (ms) for Wikipedia API calls. + WIKIPEDIA_TIMEOUT_MS: z.coerce.number().positive().default(8000), // ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ──── VOICE_PCM_WS_ENABLED: z .string()