From f224be2a66a5fcdaf05a23165f492bf97c4bde07 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Sun, 31 May 2026 16:32:38 +0700 Subject: [PATCH] Refactor test database setup and add migrations - Updated test files to use a separate test database configuration. - Introduced a new helper module for managing test database operations. - Added a setup file to configure the environment for tests. - Created new database migration scripts to optimize message indexing. - Added a sample environment file for test database configuration. --- .env.test.example | 6 + .../0005_optimize-message-index.sql | 3 + drizzle/migrations/meta/0005_snapshot.json | 1456 +++++++++++++++++ drizzle/migrations/meta/_journal.json | 7 + src/database/schema.ts | 12 + src/moderation/conversationContext.ts | 8 +- src/moderation/messageCapture.ts | 17 +- src/moderation/messageStore.ts | 39 +- src/recorder/uploader.ts | 4 +- tests/database.test.ts | 5 + tests/helpers/testDatabase.ts | 64 + tests/moderation/llmModerationClient.test.ts | 34 +- tests/moderation/messageCapture.test.ts | 23 +- tests/moderation/messageStoreQueries.test.ts | 23 +- tests/setup.ts | 7 + vitest.config.ts | 1 + 16 files changed, 1639 insertions(+), 70 deletions(-) create mode 100644 .env.test.example create mode 100644 drizzle/migrations/0005_optimize-message-index.sql create mode 100644 drizzle/migrations/meta/0005_snapshot.json create mode 100644 tests/helpers/testDatabase.ts create mode 100644 tests/setup.ts diff --git a/.env.test.example b/.env.test.example new file mode 100644 index 0000000..3a29b37 --- /dev/null +++ b/.env.test.example @@ -0,0 +1,6 @@ +NODE_ENV=test +# Use a separate database/data area for tests. It may be on the same PostgreSQL host, +# but the database name must clearly be a test database so destructive test setup +# cannot touch production data. +TEST_DATABASE_URL=postgres://root:root@100.108.1.124:5432/hub_test +DATABASE_URL=postgres://root:root@100.108.1.124:5432/hub_test diff --git a/drizzle/migrations/0005_optimize-message-index.sql b/drizzle/migrations/0005_optimize-message-index.sql new file mode 100644 index 0000000..c656ca8 --- /dev/null +++ b/drizzle/migrations/0005_optimize-message-index.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS "idx_messages_guild_created_deleted" ON "messages" USING btree ("guild_id","created_at","deleted_at","id");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_messages_channel_ai_status_created" ON "messages" USING btree ("channel_id","ai_status","created_at","id");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_messages_thread_ai_status_created" ON "messages" USING btree ("thread_id","ai_status","created_at","id"); diff --git a/drizzle/migrations/meta/0005_snapshot.json b/drizzle/migrations/meta/0005_snapshot.json new file mode 100644 index 0000000..f2cb42f --- /dev/null +++ b/drizzle/migrations/meta/0005_snapshot.json @@ -0,0 +1,1456 @@ +{ + "id": "5f2738a8-e164-4648-91a3-393be1e1b69d", + "prevId": "a4dd8687-4a2c-4520-9cc5-03aa1774e383", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_analysis_runs": { + "name": "ai_analysis_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "conversation_key": { + "name": "conversation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_message_ids": { + "name": "target_message_ids", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_tokens_estimate": { + "name": "request_tokens_estimate", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "response_raw": { + "name": "response_raw", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_ai_analysis_runs_conversation_key": { + "name": "idx_ai_analysis_runs_conversation_key", + "columns": [ + { + "expression": "conversation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_ai_analysis_runs_status": { + "name": "idx_ai_analysis_runs_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_ai_analysis_runs_created_at": { + "name": "idx_ai_analysis_runs_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "guild_id": { + "name": "guild_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "discord_url": { + "name": "discord_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_url": { + "name": "uploaded_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upload_status": { + "name": "upload_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "upload_error": { + "name": "upload_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_attachments_channel": { + "name": "idx_attachments_channel", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_attachments_message": { + "name": "idx_attachments_message", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_attachments_status": { + "name": "idx_attachments_status", + "columns": [ + { + "expression": "upload_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_attachments_channel_created": { + "name": "idx_attachments_channel_created", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_attachments_thread_created": { + "name": "idx_attachments_thread_created", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fk_attachments_message_id": { + "name": "fk_attachments_message_id", + "tableFrom": "attachments", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_reviews": { + "name": "message_reviews", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "guild_id": { + "name": "guild_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reviewer_id": { + "name": "reviewer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_message_reviews_message_id": { + "name": "idx_message_reviews_message_id", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_reviews_status": { + "name": "idx_message_reviews_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_reviews_created_at": { + "name": "idx_message_reviews_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_reviews_guild_status": { + "name": "idx_message_reviews_guild_status", + "columns": [ + { + "expression": "guild_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.messages": { + "name": "messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "guild_id": { + "name": "guild_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "edited_content": { + "name": "edited_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "edited_at": { + "name": "edited_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text'" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_status": { + "name": "ai_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "ai_moderation_flags": { + "name": "ai_moderation_flags", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_moderation_score": { + "name": "ai_moderation_score", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "ai_analysis": { + "name": "ai_analysis", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_categories": { + "name": "ai_categories", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_severity": { + "name": "ai_severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_confidence": { + "name": "ai_confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "ai_recommended_action": { + "name": "ai_recommended_action", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_analyzed_at": { + "name": "ai_analyzed_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "ai_error": { + "name": "ai_error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_messages_channel": { + "name": "idx_messages_channel", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_user": { + "name": "idx_messages_user", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_created": { + "name": "idx_messages_created", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_thread": { + "name": "idx_messages_thread", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_channel_created": { + "name": "idx_messages_channel_created", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_thread_created": { + "name": "idx_messages_thread_created", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_ai_status_created": { + "name": "idx_messages_ai_status_created", + "columns": [ + { + "expression": "ai_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_guild_ai_status_created": { + "name": "idx_messages_guild_ai_status_created", + "columns": [ + { + "expression": "guild_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ai_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_guild_created_deleted": { + "name": "idx_messages_guild_created_deleted", + "columns": [ + { + "expression": "guild_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_channel_ai_status_created": { + "name": "idx_messages_channel_ai_status_created", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ai_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_messages_thread_ai_status_created": { + "name": "idx_messages_thread_ai_status_created", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ai_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.moderation_actions": { + "name": "moderation_actions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "guild_id": { + "name": "guild_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "executed_by": { + "name": "executed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "executed_at": { + "name": "executed_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_moderation_actions_message_id": { + "name": "idx_moderation_actions_message_id", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_moderation_actions_user_id": { + "name": "idx_moderation_actions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_moderation_actions_status": { + "name": "idx_moderation_actions_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_moderation_actions_guild_status": { + "name": "idx_moderation_actions_guild_status", + "columns": [ + { + "expression": "guild_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.muxer_jobs": { + "name": "muxer_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "maxAttempts": { + "name": "maxAttempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "createdAt": { + "name": "createdAt", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updatedAt": { + "name": "updatedAt", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_muxer_jobs_status": { + "name": "idx_muxer_jobs_status", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_muxer_jobs_createdAt": { + "name": "idx_muxer_jobs_createdAt", + "columns": [ + { + "expression": "createdAt", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_policies": { + "name": "retention_policies", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "guild_id": { + "name": "guild_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "retention_days": { + "name": "retention_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 90 + }, + "apply_to_media": { + "name": "apply_to_media", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "apply_to_voice": { + "name": "apply_to_voice", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_retention_policies_guild_id": { + "name": "idx_retention_policies_guild_id", + "columns": [ + { + "expression": "guild_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_retention_policies_enabled": { + "name": "idx_retention_policies_enabled", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ui_state": { + "name": "ui_state", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.voice_recordings": { + "name": "voice_recordings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "guild_id": { + "name": "guild_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "channel_name": { + "name": "channel_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upload_status": { + "name": "upload_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "upload_error": { + "name": "upload_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_voice_recordings_user_id": { + "name": "idx_voice_recordings_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_voice_recordings_channel_id": { + "name": "idx_voice_recordings_channel_id", + "columns": [ + { + "expression": "channel_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_voice_recordings_created_at": { + "name": "idx_voice_recordings_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/migrations/meta/_journal.json b/drizzle/migrations/meta/_journal.json index 4624cb3..47a085b 100644 --- a/drizzle/migrations/meta/_journal.json +++ b/drizzle/migrations/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1780079000000, "tag": "0003_ai_moderation_review_guardrails", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1780218363790, + "tag": "0005_optimize-message-index", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/database/schema.ts b/src/database/schema.ts index e85133a..e04c374 100644 --- a/src/database/schema.ts +++ b/src/database/schema.ts @@ -106,6 +106,18 @@ export const pgMessagesTable = pgTable( table.created_at, table.id, ), + guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on( + table.guild_id, + table.created_at, + table.deleted_at, + table.id, + ), + channelAiStatusCreatedIdx: pgIndex( + "idx_messages_channel_ai_status_created", + ).on(table.channel_id, table.ai_status, table.created_at, table.id), + threadAiStatusCreatedIdx: pgIndex( + "idx_messages_thread_ai_status_created", + ).on(table.thread_id, table.ai_status, table.created_at, table.id), }), ); diff --git a/src/moderation/conversationContext.ts b/src/moderation/conversationContext.ts index 42fddc8..24b8cab 100644 --- a/src/moderation/conversationContext.ts +++ b/src/moderation/conversationContext.ts @@ -56,12 +56,14 @@ export async function buildConversationContext( 0, ); + const contextLines = await Promise.all( + contextBefore.map((msg) => formatMessageForPrompt(msg, "context")), + ); const selectedContextLines: string[] = []; // Go backwards through context, taking most recent first - for (let i = contextBefore.length - 1; i >= 0; i--) { - const msg = contextBefore[i]; - const line = await formatMessageForPrompt(msg, "context"); + for (let i = contextLines.length - 1; i >= 0; i--) { + const line = contextLines[i]; const lineTokens = estimateTokens(line); if (usedTokens + lineTokens <= maxTokens) { diff --git a/src/moderation/messageCapture.ts b/src/moderation/messageCapture.ts index 947fefe..d236cdf 100644 --- a/src/moderation/messageCapture.ts +++ b/src/moderation/messageCapture.ts @@ -178,14 +178,27 @@ export async function captureMessage( // Queue analysis after attachment uploads settle so AI uses stable tele URLs. if (!isBacklog) { if (attachmentUploadTasks.length > 0) { - setTimeout(() => queueMessageAnalysis(message.id), 30000); + let analysisQueued = false; + let fallbackTimer: NodeJS.Timeout | null = null; + const queueAnalysisOnce = () => { + if (analysisQueued) return; + analysisQueued = true; + if (fallbackTimer) { + clearTimeout(fallbackTimer); + fallbackTimer = null; + } + queueMessageAnalysis(message.id); + }; + + fallbackTimer = setTimeout(queueAnalysisOnce, 30000); Promise.allSettled(attachmentUploadTasks) - .then(() => queueMessageAnalysis(message.id)) + .then(queueAnalysisOnce) .catch((err) => { logger.error( { messageId: message.id, error: err }, "Failed to queue message analysis after attachment upload", ); + queueAnalysisOnce(); }); } else { queueMessageAnalysis(message.id); diff --git a/src/moderation/messageStore.ts b/src/moderation/messageStore.ts index 990187a..c48620b 100644 --- a/src/moderation/messageStore.ts +++ b/src/moderation/messageStore.ts @@ -48,6 +48,7 @@ interface MessageDatabase { selectDistinct(...args: unknown[]): QueryBuilder; insert(...args: unknown[]): QueryBuilder; update(...args: unknown[]): QueryBuilder; + transaction(callback: (tx: MessageDatabase) => Promise): Promise; } function db(): MessageDatabase { @@ -457,28 +458,28 @@ export async function updateMessagesAIAnalysisBulk( ): Promise { if (updates.length === 0) return []; try { - // Use raw SQL batch UPDATE instead of Promise.all per-message queries - // (P2: reduce N*2 queries → 2 queries total) const database = db(); const now = Date.now(); - for (const { messageId, result } of updates) { - await database - .update(messagesTable) - .set({ - ai_status: result.status, - ai_moderation_flags: result.flags ?? null, - ai_moderation_score: result.score ?? null, - ai_analysis: result.analysis ?? null, - ai_categories: stringifyAIList(result.categories), - ai_severity: result.severity ?? null, - ai_confidence: result.confidence ?? result.score ?? null, - ai_recommended_action: result.recommendedAction ?? null, - ai_analyzed_at: result.analyzedAt ?? now, - ai_error: result.error ?? null, - }) - .where(eq(messagesTable.id, messageId)); - } + await database.transaction(async (tx) => { + for (const { messageId, result } of updates) { + await tx + .update(messagesTable) + .set({ + ai_status: result.status, + ai_moderation_flags: result.flags ?? null, + ai_moderation_score: result.score ?? null, + ai_analysis: result.analysis ?? null, + ai_categories: stringifyAIList(result.categories), + ai_severity: result.severity ?? null, + ai_confidence: result.confidence ?? result.score ?? null, + ai_recommended_action: result.recommendedAction ?? null, + ai_analyzed_at: result.analyzedAt ?? now, + ai_error: result.error ?? null, + }) + .where(eq(messagesTable.id, messageId)); + } + }); // Fetch all updated messages in a single query const ids = updates.map(({ messageId }) => messageId); diff --git a/src/recorder/uploader.ts b/src/recorder/uploader.ts index 3751874..ba5e2de 100644 --- a/src/recorder/uploader.ts +++ b/src/recorder/uploader.ts @@ -38,7 +38,7 @@ export async function uploadRecordingSegment(input: { try { // 1. Get file size and insert initial pending state to DB - const stats = fs.statSync(oggPath); + const stats = await fs.promises.stat(oggPath); await insertVoiceRecording({ id, user_id: userId, @@ -54,7 +54,7 @@ export async function uploadRecordingSegment(input: { }); // 2. Perform async upload with retry logic - const fileBuffer = fs.readFileSync(oggPath); + const fileBuffer = await fs.promises.readFile(oggPath); const uploadResult = await uploadToTele({ buffer: fileBuffer, filename: fileName, diff --git a/tests/database.test.ts b/tests/database.test.ts index ac1a62d..94dfa8e 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -1,5 +1,6 @@ import process from "node:process"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { assertSafeTestDatabaseUrl } from "./helpers/testDatabase"; const originalEnv = process.env; @@ -14,6 +15,10 @@ describe("Drizzle ORM Database", () => { DISCORD_TOKEN: "test-token", NODE_ENV: "test", }; + if (originalEnv.TEST_DATABASE_URL) { + process.env.DATABASE_URL = originalEnv.TEST_DATABASE_URL; + } + assertSafeTestDatabaseUrl(); // Reset modules to pick up new environment vi.resetModules(); diff --git a/tests/helpers/testDatabase.ts b/tests/helpers/testDatabase.ts new file mode 100644 index 0000000..b54686d --- /dev/null +++ b/tests/helpers/testDatabase.ts @@ -0,0 +1,64 @@ +import process from "node:process"; +import { + getDatabase, + initializeDatabase, +} from "../../src/database/drizzle"; + +interface RunnableDatabase { + run(sql: string): Promise; +} + +const SAFE_TEST_DATABASE_NAME = /(^|[_-])(test|testing)([_-]|$)|gmw_test/i; + +function getDatabaseNameFromUrl(databaseUrl: string): string { + try { + const parsed = new URL(databaseUrl); + return parsed.pathname.replace(/^\//, ""); + } catch { + return ""; + } +} + +function getConfiguredDatabaseName(): string { + if (process.env.DATABASE_URL) { + return getDatabaseNameFromUrl(process.env.DATABASE_URL); + } + return process.env.POSTGRES_DB ?? ""; +} + +export function assertSafeTestDatabaseUrl(): void { + if (process.env.NODE_ENV !== "test") { + throw new Error( + `Refusing to run destructive database test outside NODE_ENV=test (got ${process.env.NODE_ENV ?? "unset"})`, + ); + } + + if (process.env.TEST_DATABASE_URL) { + process.env.DATABASE_URL = process.env.TEST_DATABASE_URL; + } + + const databaseName = getConfiguredDatabaseName(); + if (!SAFE_TEST_DATABASE_NAME.test(databaseName)) { + throw new Error( + `Refusing to run destructive database test against non-test database "${databaseName || "unknown"}". Set TEST_DATABASE_URL or DATABASE_URL to a database whose name contains "test" (for example hub_test).`, + ); + } +} + +export async function initializeTestDatabase() { + assertSafeTestDatabaseUrl(); + return initializeDatabase(); +} + +export function getTestDatabase(): RunnableDatabase { + assertSafeTestDatabaseUrl(); + return getDatabase() as unknown as RunnableDatabase; +} + +export async function clearTestTables(...tableNames: string[]): Promise { + assertSafeTestDatabaseUrl(); + const db = getTestDatabase(); + for (const tableName of tableNames) { + await db.run(`DELETE FROM "${tableName}"`); + } +} diff --git a/tests/moderation/llmModerationClient.test.ts b/tests/moderation/llmModerationClient.test.ts index a9f609c..bfa1b3f 100644 --- a/tests/moderation/llmModerationClient.test.ts +++ b/tests/moderation/llmModerationClient.test.ts @@ -686,9 +686,7 @@ describe("runModerationAnalysis", () => { expect(secondRequestBody.messages[0].content).toContain( "RESPON SEBELUMNYA GAGAL VALIDASI", ); - expect(secondRequestBody.messages[0].content).toContain( - "Invalid option", - ); + expect(secondRequestBody.messages[0].content).toContain("Invalid option"); expect(secondRequestBody.messages[0].content).toContain( "Coba lagi dengan output JSON yang benar", ); @@ -753,8 +751,8 @@ describe("runModerationAnalysis", () => { arrayBuffer: async () => { // Minimal valid PNG bytes (8-byte signature) const png = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, - 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, + 0x0d, 0x49, 0x48, 0x44, 0x52, ]); return png.buffer.slice( png.byteOffset, @@ -932,7 +930,8 @@ describe("runModerationAnalysis", () => { status: "warn", flags: ["harassment"], score: 0.65, - analysis: "Teks mengandung unsur harassment dan memerlukan tindakan lebih lanjut.", + analysis: + "Teks mengandung unsur harassment dan memerlukan tindakan lebih lanjut.", }, ], }), @@ -941,7 +940,22 @@ describe("runModerationAnalysis", () => { ], }; - const imageBytes = Buffer.from("realistic-image-bytes"); + const imageBytes = Buffer.from([ + 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, + ]); + const toBody = (buffer: Buffer) => { + let done = false; + return { + getReader: () => ({ + read: async () => { + if (done) return { done: true, value: undefined }; + done = true; + return { done: false, value: new Uint8Array(buffer) }; + }, + cancel: vi.fn(), + }), + }; + }; global.fetch = vi.fn().mockImplementation((url: string) => { if ( url === "https://httpbin.org/image/png" || @@ -949,11 +963,7 @@ describe("runModerationAnalysis", () => { ) { return Promise.resolve({ ok: true, - arrayBuffer: async () => - imageBytes.buffer.slice( - imageBytes.byteOffset, - imageBytes.byteOffset + imageBytes.byteLength, - ), + body: toBody(imageBytes), }); } diff --git a/tests/moderation/messageCapture.test.ts b/tests/moderation/messageCapture.test.ts index 3e40bdd..4fa90c8 100644 --- a/tests/moderation/messageCapture.test.ts +++ b/tests/moderation/messageCapture.test.ts @@ -7,11 +7,12 @@ import { it, vi, } from "vitest"; +import { closeDatabase } from "../../src/database/drizzle"; import { - closeDatabase, - getDatabase, - initializeDatabase, -} from "../../src/database/drizzle"; + clearTestTables, + getTestDatabase, + initializeTestDatabase, +} from "../helpers/testDatabase"; import { captureMessage } from "../../src/moderation/messageCapture"; import type { ModerationBroadcaster } from "../../src/moderation/types"; @@ -22,14 +23,6 @@ type ModerationTestGlobal = typeof globalThis & { moderationBroadcaster?: Partial; }; -interface TestDatabase { - run(sql: string): Promise; -} - -function getTestDatabase(): TestDatabase { - return getDatabase() as unknown as TestDatabase; -} - vi.mock("../../src/moderation/aiAnalyzer", () => ({ queueMessageAnalysis: (id: string) => queueMessageAnalysis(id), })); @@ -118,15 +111,13 @@ async function createTables() { describe("captureMessage", () => { beforeAll(async () => { - await initializeDatabase(); + await initializeTestDatabase(); await createTables(); }); beforeEach(async () => { queueMessageAnalysis.mockClear(); - const db = getTestDatabase(); - await db.run(`DELETE FROM "attachments"`); - await db.run(`DELETE FROM "messages"`); + await clearTestTables("attachments", "messages"); delete (globalThis as ModerationTestGlobal).moderationBroadcaster; }); diff --git a/tests/moderation/messageStoreQueries.test.ts b/tests/moderation/messageStoreQueries.test.ts index cf3c155..6ebda25 100644 --- a/tests/moderation/messageStoreQueries.test.ts +++ b/tests/moderation/messageStoreQueries.test.ts @@ -1,9 +1,10 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { closeDatabase } from "../../src/database/drizzle"; import { - closeDatabase, - getDatabase, - initializeDatabase, -} from "../../src/database/drizzle"; + clearTestTables, + getTestDatabase, + initializeTestDatabase, +} from "../helpers/testDatabase"; import { createChildLogger } from "../../src/logger"; import { decodeCursor, @@ -18,14 +19,6 @@ import { } from "../../src/moderation/messageStore"; import type { MessageRecord } from "../../src/moderation/types"; -interface TestDatabase { - run(sql: string): Promise; -} - -function getTestDatabase(): TestDatabase { - return getDatabase() as unknown as TestDatabase; -} - const logger = createChildLogger("messageStoreQueries.test"); describe("message cursor helpers", () => { @@ -44,7 +37,7 @@ describe("message cursor helpers", () => { describe("message query integration tests", () => { beforeAll(async () => { - await initializeDatabase(); + await initializeTestDatabase(); // Create tables directly for isolated query integration tests const db = getTestDatabase(); try { @@ -110,9 +103,7 @@ describe("message query integration tests", () => { beforeEach(async () => { // Clear tables before each test try { - const db = getTestDatabase(); - await db.run(`DELETE FROM "attachments"`); - await db.run(`DELETE FROM "messages"`); + await clearTestTables("attachments", "messages"); } catch (error) { logger.debug({ error }, "Could not clear tables"); } diff --git a/tests/setup.ts b/tests/setup.ts new file mode 100644 index 0000000..271c2e5 --- /dev/null +++ b/tests/setup.ts @@ -0,0 +1,7 @@ +import process from "node:process"; + +process.env.NODE_ENV = "test"; + +if (process.env.TEST_DATABASE_URL) { + process.env.DATABASE_URL = process.env.TEST_DATABASE_URL; +} diff --git a/vitest.config.ts b/vitest.config.ts index 487aca9..24a7571 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,5 +8,6 @@ export default defineConfig({ environment: "node", fileParallelism: false, include: ["tests/**/*.test.ts"], + setupFiles: ["tests/setup.ts"], }, });