MythEclipse
f224be2a66
- Updated test files to use a separate test database configuration. - Introduced a new helper module for managing test database operations. - Added a setup file to configure the environment for tests. - Created new database migration scripts to optimize message indexing. - Added a sample environment file for test database configuration.
78 lines
2.5 KiB
TypeScript
78 lines
2.5 KiB
TypeScript
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
|
import { formatMediaEvidenceForPrompt } from "./messageMetadata.js";
|
|
import type { MessageRecord } from "./types.js";
|
|
|
|
export interface ConversationContextInput {
|
|
contextBefore: MessageRecord[];
|
|
targets: MessageRecord[];
|
|
maxTokens: number;
|
|
}
|
|
|
|
/**
|
|
* Formats a timestamp to ISO 8601 string
|
|
*/
|
|
function formatTimestamp(ms: number): string {
|
|
return new Date(ms).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Estimates token count for a string (pessimistic approximation for Indonesian slang & JSON overhead)
|
|
*/
|
|
export function estimateTokens(text: string): number {
|
|
return Math.ceil(text.length / 3) + 15;
|
|
}
|
|
|
|
/**
|
|
* Formats a single message for context or target display
|
|
*/
|
|
export async function formatMessageForPrompt(
|
|
msg: MessageRecord,
|
|
label: "context" | "target",
|
|
): Promise<string> {
|
|
const content = msg.edited_content ?? msg.content;
|
|
const timestamp = formatTimestamp(msg.created_at);
|
|
const textEvidence = await formatModerationTextEvidenceForPrompt(content);
|
|
const textSuffix = textEvidence ? ` ${textEvidence}` : "";
|
|
const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata);
|
|
const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : "";
|
|
return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${textSuffix}${mediaSuffix}`;
|
|
}
|
|
|
|
/**
|
|
* Builds conversation historical context without including targets.
|
|
* Calculates how much token budget targets use, and fills the rest with context.
|
|
*/
|
|
export async function buildConversationContext(
|
|
input: ConversationContextInput,
|
|
): Promise<string[]> {
|
|
const { contextBefore, targets, maxTokens } = input;
|
|
|
|
// Calculate tokens used by targets (parallel)
|
|
const targetLines = await Promise.all(
|
|
targets.map((msg) => formatMessageForPrompt(msg, "target")),
|
|
);
|
|
let usedTokens = targetLines.reduce(
|
|
(sum, line) => sum + estimateTokens(line),
|
|
0,
|
|
);
|
|
|
|
const contextLines = await Promise.all(
|
|
contextBefore.map((msg) => formatMessageForPrompt(msg, "context")),
|
|
);
|
|
const selectedContextLines: string[] = [];
|
|
|
|
// Go backwards through context, taking most recent first
|
|
for (let i = contextLines.length - 1; i >= 0; i--) {
|
|
const line = contextLines[i];
|
|
const lineTokens = estimateTokens(line);
|
|
|
|
if (usedTokens + lineTokens <= maxTokens) {
|
|
// Unshift so oldest context is first in the array
|
|
selectedContextLines.unshift(line);
|
|
usedTokens += lineTokens;
|
|
}
|
|
}
|
|
|
|
return selectedContextLines;
|
|
}
|