MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add installation script for yt-dlp and update package.json

Browse Source
This commit is contained in:
MythEclipse
2026-05-15 21:40:20 +07:00
parent 119258c2b0
commit 6ac4a5c11a
6 changed files with 200 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+15 -2
View File
@@ -8,7 +8,8 @@ Stack utama: Node.js, pnpm, TypeScript, `discord.js-selfbot-v13`, `@discordjs/vo
- Node.js versi modern yang kompatibel dengan TypeScript dan Vite.
- pnpm 10.x. Repo ini dipin ke `pnpm@10.25.0`.
- FFmpeg tersedia di `PATH` untuk proses muxing audio.
- FFmpeg tersedia di `PATH` untuk proses muxing audio dan playback media.
- `yt-dlp` tersedia di `PATH` untuk resolve audio YouTube, search result YouTube, dan Spotify track.
- Native audio dependencies dapat dibuild di mesin lokal (`@discordjs/opus`, `better-sqlite3`, `sodium-native`).
Install FFmpeg:
@@ -21,6 +22,14 @@ sudo apt install ffmpeg
sudo pacman -S ffmpeg
```
Install `yt-dlp`:
```bash
pnpm run install:yt-dlp
```
Script installer akan memakai package manager yang tersedia (`pacman`, `apt-get`, `dnf`, `brew`) atau fallback ke `pipx`/`pip`.
## Setup
```bash
@@ -76,6 +85,9 @@ pnpm run test
# Build frontend + TypeScript
pnpm run build
# Install external yt-dlp CLI for YouTube/search/Spotify track playback
pnpm run install:yt-dlp
```
## Database
@@ -104,7 +116,8 @@ pnpm run db:studio
- Attachment capture dan upload ke endpoint Picser.
- SQLite/PostgreSQL via Drizzle ORM.
- REST API dan WebSocket untuk dashboard.
- Dashboard React untuk pesan, gambar, voice, dan moderation review.
- Dashboard React untuk pesan, gambar, voice, media playback, dan moderation review.
- Media playback dari direct URL, file lokal, YouTube URL, search terms, dan Spotify track URL.
- Metrics Prometheus di endpoint server.
- Retry dengan backoff untuk operasi eksternal.
- AI moderation analysis opsional via konfigurasi `AI_*`.
package.json
+2 -1
View File
@@ -18,7 +18,8 @@
"db:generate": "drizzle-kit generate",
"db:migrate": "drizzle-kit migrate",
"db:migrate:programmatic": "tsx src/database/migrate.ts",
"db:studio": "drizzle-kit studio"
"db:studio": "drizzle-kit studio",
"install:yt-dlp": "sh scripts/install-yt-dlp.sh"
},
"dependencies": {
"@dank074/discord-video-stream": "workspace:*",
public/index.html
+3 -3
View File
File diff suppressed because one or more lines are too long
scripts/install-yt-dlp.sh
Executable
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env sh
set -eu
if command -v yt-dlp >/dev/null 2>&1; then
echo "yt-dlp already installed: $(command -v yt-dlp)"
yt-dlp --version
exit 0
fi
if command -v pacman >/dev/null 2>&1; then
sudo pacman -S --needed yt-dlp
elif command -v apt-get >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install -y yt-dlp
elif command -v dnf >/dev/null 2>&1; then
sudo dnf install -y yt-dlp
elif command -v brew >/dev/null 2>&1; then
brew install yt-dlp
elif command -v pipx >/dev/null 2>&1; then
pipx install yt-dlp
elif command -v python3 >/dev/null 2>&1; then
python3 -m pip install --user --upgrade yt-dlp
else
echo "Could not find pacman, apt-get, dnf, brew, pipx, or python3 to install yt-dlp." >&2
exit 1
fi
if ! command -v yt-dlp >/dev/null 2>&1; then
echo "yt-dlp installed but is not on PATH. Restart your shell or add the installer bin directory to PATH." >&2
exit 1
fi
echo "yt-dlp installed: $(command -v yt-dlp)"
yt-dlp --version
src/moderation/aiAnalysisWorker.ts
+94
View File
@@ -0,0 +1,94 @@
import { parentPort } from "node:worker_threads";
import { buildConversationPromptMessages } from "./conversationContext";
import { runModerationAnalysis } from "./llmModerationClient";
import {
getConversationContextBefore,
updateMessageAIAnalysis,
} from "./messageStore";
import type { MessageRecord } from "./types";
const MAX_CONTEXT_TOKENS = 8000;
interface AnalysisWorkerRequest {
conversationKey: string;
messages: MessageRecord[];
}
type AnalysisWorkerResponse =
| {
ok: true;
conversationKey: string;
rows: MessageRecord[];
}
| {
ok: false;
conversationKey: string;
rows: MessageRecord[];
error: string;
};
async function processAnalysisRequest({
conversationKey,
messages,
}: AnalysisWorkerRequest): Promise<AnalysisWorkerResponse> {
try {
const firstMessage = messages[0];
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
const contextBefore = await getConversationContextBefore({
channelId: firstMessage.channel_id,
threadId: firstMessage.thread_id,
beforeCreatedAt: firstMessage.created_at,
limit: 20,
});
const promptMessages = buildConversationPromptMessages({
contextBefore,
targets: messages,
maxTokens: MAX_CONTEXT_TOKENS,
});
const result = await runModerationAnalysis({
targets: messages,
contextText: promptMessages.join("\n"),
});
const rows: MessageRecord[] = [];
for (const analysisResult of result.results) {
const row = await updateMessageAIAnalysis(analysisResult.messageId, {
status: analysisResult.status,
flags: JSON.stringify(analysisResult.flags),
score: analysisResult.score,
raw: JSON.stringify(result.raw),
analysis: analysisResult.analysis,
analyzedAt: Date.now(),
error: null,
});
if (row) rows.push(row);
}
return { ok: true, conversationKey, rows };
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
const rows: MessageRecord[] = [];
for (const msg of messages) {
const row = await updateMessageAIAnalysis(msg.id, {
status: "error",
flags: null,
score: null,
raw: null,
analysis: null,
analyzedAt: Date.now(),
error: errorMessage,
});
if (row) rows.push(row);
}
return { ok: false, conversationKey, rows, error: errorMessage };
}
}
parentPort?.on("message", async (request: AnalysisWorkerRequest) => {
parentPort?.postMessage(await processAnalysisRequest(request));
});
src/moderation/aiAnalyzer.ts
+52 -59
View File
@@ -1,9 +1,7 @@
import { Worker } from "node:worker_threads";
import { config } from "../config";
import { createChildLogger } from "../logger";
import { buildConversationPromptMessages } from "./conversationContext";
import { runModerationAnalysis } from "./llmModerationClient";
import {
getConversationContextBefore,
getMessageById,
getPendingConversationKeys,
getPendingMessagesByConversation,
@@ -38,9 +36,15 @@ const MAX_ACTIVE_REQUESTS = 1;
const DEBOUNCE_MS = 1500;
const RECOVERY_INTERVAL_MS = 15000;
const ERROR_COOLDOWN_MS = 30000;
const MAX_CONTEXT_TOKENS = 8000;
const MAX_BATCH_SIZE = 25;
interface AnalysisWorkerResponse {
ok: boolean;
conversationKey: string;
rows: MessageRecord[];
error?: string;
}
/**
* Gets the conversation key for a message (thread_id or channel_id)
*/
@@ -86,53 +90,25 @@ async function processBatch(
activeRequests++;
conversationProcessing.add(conversationKey);
try {
// Get context before the first message
const firstMessage = messages[0];
const contextBefore = await getConversationContextBefore({
channelId: firstMessage.channel_id,
threadId: firstMessage.thread_id,
beforeCreatedAt: firstMessage.created_at,
limit: 20,
});
const result = await runAnalysisInWorker(conversationKey, messages);
// Build prompt with context
const promptMessages = buildConversationPromptMessages({
contextBefore,
targets: messages,
maxTokens: MAX_CONTEXT_TOKENS,
});
const contextText = promptMessages.join("\n");
// Run moderation analysis
const result = await runModerationAnalysis({
targets: messages,
contextText,
});
// Store results
const analyzedRows: MessageRecord[] = [];
for (const analysisResult of result.results) {
const row = await updateMessageAIAnalysis(analysisResult.messageId, {
status: analysisResult.status,
flags: JSON.stringify(analysisResult.flags),
score: analysisResult.score,
raw: JSON.stringify(result.raw),
analysis: analysisResult.analysis,
analyzedAt: Date.now(),
error: null,
});
if (row) {
analyzedRows.push(row);
}
}
// Broadcast analyzed messages
for (const row of analyzedRows) {
for (const row of result.rows) {
getModerationBroadcaster()?.messageAnalyzed(row);
}
// Clear error cooldown on success
if (!result.ok) {
lastError = result.error ?? "Analysis worker failed";
conversationErrorCooldown.set(
conversationKey,
Date.now() + ERROR_COOLDOWN_MS,
);
logger.error(
{ conversationKey, error: lastError },
"Batch analysis failed",
);
return;
}
conversationErrorCooldown.delete(conversationKey);
logger.info(
@@ -141,13 +117,15 @@ async function processBatch(
);
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
conversationErrorCooldown.set(
conversationKey,
Date.now() + ERROR_COOLDOWN_MS,
);
logger.error(
{ conversationKey, error: lastError },
"Batch analysis failed",
"Analysis worker failed",
);
// Mark all messages in batch as error
for (const msg of messages) {
const row = await updateMessageAIAnalysis(msg.id, {
status: "error",
@@ -158,22 +136,37 @@ async function processBatch(
analyzedAt: Date.now(),
error: lastError,
});
if (row) {
getModerationBroadcaster()?.messageAnalyzed(row);
if (row) getModerationBroadcaster()?.messageAnalyzed(row);
}
}
// Set error cooldown for this conversation
conversationErrorCooldown.set(
conversationKey,
Date.now() + ERROR_COOLDOWN_MS,
);
} finally {
activeRequests--;
conversationProcessing.delete(conversationKey);
}
}
async function runAnalysisInWorker(
conversationKey: string,
messages: MessageRecord[],
): Promise<AnalysisWorkerResponse> {
return new Promise((resolve, reject) => {
const worker = new Worker(new URL("./aiAnalysisWorker.ts", import.meta.url));
worker.once("message", (response: AnalysisWorkerResponse) => {
worker.terminate().catch((error) => {
logger.warn({ error }, "Failed to terminate analysis worker");
});
resolve(response);
});
worker.once("error", reject);
worker.once("exit", (code) => {
if (code !== 0) {
reject(new Error(`Analysis worker exited with code ${code}`));
}
});
worker.postMessage({ conversationKey, messages });
});
}
/**
* Debounced analysis trigger for a conversation
*/