refactor: remove unused text analysis module and integrate Qdrant enhancements
Build & Deploy (Nix) / build-and-deploy (backend) (push) Successful in 2m30s
Build & Deploy (Nix) / build-and-deploy (discord-gateway) (push) Successful in 3m7s
Build & Deploy (Nix) / build-and-deploy (proxy) (push) Successful in 3m20s

- Deleted the text analysis prompt constants and helpers as they are no longer needed.
- Added batch search functionality for Qdrant to optimize vector searches.
- Implemented methods for deleting expired Qdrant points and invalidating cache based on content hash.
- Updated text batch processor to use new timeout configurations and modified content building for moderation prompts.
- Enhanced text cache store to support new Qdrant integration and improved cache invalidation logic.
- Introduced a new user reputation model with a more nuanced trust scoring system, including penalties and rewards for user behavior.
- Added unit tests for the new trust model to ensure correctness of penalty and trust gain calculations.
- Updated configuration schema to reflect new timeout settings and removed deprecated OpenAI moderation keys.
This commit is contained in:
Developer
2026-07-31 23:09:00 +07:00
parent fc475dfbb7
commit 6df4f306dd
16 changed files with 893 additions and 802 deletions
@@ -22,6 +22,9 @@ export interface QdrantVerdictPayload {
flags: string; // JSON string of the full moderation result
analyzed_at: number;
expires_at: number;
/** Bare content hash (16 hex chars) — enables content-based invalidation
* regardless of the (context-scoped) point id. */
content_hash?: string;
}
function baseUrl(): string {
@@ -215,6 +218,150 @@ export async function searchQdrant(
}
}
/**
* Batch search: one HTTP round-trip for N vectors (Qdrant
* `/points/search/batch`). Result is index-aligned with `vectors` — each
* entry is the top hits for that vector (or [] on per-vector failure).
* Used by the orchestrator to avoid N sequential embed→search round-trips.
*/
export async function searchQdrantBatch(
vectors: number[][],
limit: number,
scoreThreshold: number,
): Promise<QdrantSearchHit[][]> {
if (vectors.length === 0) return [];
try {
const json = (await request(
"POST",
`/collections/${collectionName()}/points/search/batch`,
{
searches: vectors.map((vector) => ({
vector,
limit,
score_threshold: scoreThreshold,
with_payload: true,
filter: {
must: [
{
key: "expires_at",
range: { gte: Date.now() },
},
],
},
})),
},
)) as {
result?: Array<{
result?: Array<{
id?: number;
score?: number;
payload?: QdrantVerdictPayload;
}>;
}>;
};
return (json.result ?? []).map((entry) =>
(entry.result ?? [])
.filter((hit) => hit.payload?.flags)
.map((hit) => ({
cacheKey: `qdrant:${hit.id ?? "?"}`,
score: hit.score ?? 0,
payload: hit.payload as QdrantVerdictPayload,
})),
);
} catch (error) {
log.warn(
{ error: error instanceof Error ? error.message : String(error) },
"Qdrant batch search failed — semantic cache skipped",
);
return vectors.map(() => []);
}
}
/**
* Delete expired verdict points from the collection. Best-effort: 404
* (collection missing) and failures are swallowed — the periodic pruner
* just retries next sweep.
*/
export async function deleteExpiredQdrantPoints(): Promise<number> {
try {
const json = (await request(
"POST",
`/collections/${collectionName()}/points/delete`,
{
filter: {
must: [
{
key: "expires_at",
range: { lt: Date.now() },
},
],
},
},
)) as { result?: { deleted?: number } | null };
return json.result?.deleted ?? 0;
} catch (error) {
if (error instanceof Error && error.message.includes("-> 404")) {
log.debug({}, "Qdrant collection absent — nothing to prune");
} else {
log.warn(
{ error: error instanceof Error ? error.message : String(error) },
"Qdrant expired-point prune failed",
);
}
return 0;
}
}
/**
* Delete the verdict point for an exact cache key (used by cache
* invalidation when a moderator corrects a verdict).
*/
export async function deleteQdrantPoint(cacheKey: string): Promise<boolean> {
try {
await request("POST", `/collections/${collectionName()}/points/delete`, {
points: [qdrantPointId(cacheKey)],
});
return true;
} catch (error) {
log.warn(
{ error: error instanceof Error ? error.message : String(error) },
"Qdrant point delete failed",
);
return false;
}
}
/**
* Delete all verdict points whose payload carries a given bare content hash.
* Used by cache invalidation for corrected verdicts — matches context-scoped
* points that share the same content regardless of their point ids.
*/
export async function deleteQdrantPointsByContentHash(
bareHash: string,
): Promise<boolean> {
try {
await request("POST", `/collections/${collectionName()}/points/delete`, {
filter: {
must: [
{
key: "content_hash",
match: { value: bareHash },
},
],
},
});
return true;
} catch (error) {
log.warn(
{ error: error instanceof Error ? error.message : String(error) },
"Qdrant content-hash point delete failed",
);
return false;
}
}
/** True when Qdrant is configured (non-empty URL). */
export function isQdrantConfigured(): boolean {
return Boolean(config.QDRANT_URL);