MythEclipse
8f6a35f591
- Added configuration options for NVIDIA Nemotron API key, model, and base URL. - Refactored badword detection to utilize NVIDIA API, with a fallback to a local badword list. - Updated moderation functions to handle asynchronous operations for text evidence generation. - Removed dependency on the `indonesian-badwords` package and implemented custom detection logic. - Enhanced tests to accommodate asynchronous behavior and validate new detection methods.
73 lines
2.4 KiB
TypeScript
73 lines
2.4 KiB
TypeScript
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
|
import { formatMediaEvidenceForPrompt } from "./messageMetadata.js";
|
|
import type { MessageRecord } from "./types.js";
|
|
|
|
export interface ConversationContextInput {
|
|
contextBefore: MessageRecord[];
|
|
targets: MessageRecord[];
|
|
maxTokens: number;
|
|
}
|
|
|
|
/**
|
|
* Formats a timestamp to ISO 8601 string
|
|
*/
|
|
function formatTimestamp(ms: number): string {
|
|
return new Date(ms).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Estimates token count for a string (pessimistic approximation for Indonesian slang & JSON overhead)
|
|
*/
|
|
export function estimateTokens(text: string): number {
|
|
return Math.ceil(text.length / 3) + 15;
|
|
}
|
|
|
|
/**
|
|
* Formats a single message for context or target display
|
|
*/
|
|
export async function formatMessageForPrompt(
|
|
msg: MessageRecord,
|
|
label: "context" | "target",
|
|
): Promise<string> {
|
|
const content = msg.edited_content ?? msg.content;
|
|
const timestamp = formatTimestamp(msg.created_at);
|
|
const textEvidence = await formatModerationTextEvidenceForPrompt(content);
|
|
const textSuffix = textEvidence ? ` ${textEvidence}` : "";
|
|
const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata);
|
|
const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : "";
|
|
return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${textSuffix}${mediaSuffix}`;
|
|
}
|
|
|
|
/**
|
|
* Builds conversation historical context without including targets.
|
|
* Calculates how much token budget targets use, and fills the rest with context.
|
|
*/
|
|
export async function buildConversationContext(
|
|
input: ConversationContextInput,
|
|
): Promise<string[]> {
|
|
const { contextBefore, targets, maxTokens } = input;
|
|
|
|
// Calculate tokens used by targets (parallel)
|
|
const targetLines = await Promise.all(
|
|
targets.map((msg) => formatMessageForPrompt(msg, "target")),
|
|
);
|
|
let usedTokens = targetLines.reduce((sum, line) => sum + estimateTokens(line), 0);
|
|
|
|
const selectedContextLines: string[] = [];
|
|
|
|
// Go backwards through context, taking most recent first
|
|
for (let i = contextBefore.length - 1; i >= 0; i--) {
|
|
const msg = contextBefore[i];
|
|
const line = await formatMessageForPrompt(msg, "context");
|
|
const lineTokens = estimateTokens(line);
|
|
|
|
if (usedTokens + lineTokens <= maxTokens) {
|
|
// Unshift so oldest context is first in the array
|
|
selectedContextLines.unshift(line);
|
|
usedTokens += lineTokens;
|
|
}
|
|
}
|
|
|
|
return selectedContextLines;
|
|
}
|