From deed7bdfb0a1b30697c7e6372e0c937826033ff6 Mon Sep 17 00:00:00 2001
From: asepharyana <superaseph@gmail.com>
Date: Thu, 24 Sep 2026 17:26:54 +0700
Subject: [PATCH] fix(gateway): destroy AI worker pools on graceful shutdown

Piscina worker threads outlive process.exit() and linger as orphaned
processes holding DB connections/locks after a deploy restart. Two live
gateways then fight over the same messages table rows (one claims
processing, the other reverts), which left messages stuck in
ai_status='processing' forever.

Destroy both worker pools before closing the DB, with a 5s fallback so
a hung vision job cannot block shutdown indefinitely.
---
 services/discord-gateway/src/app/shutdown.ts | 28 ++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/services/discord-gateway/src/app/shutdown.ts b/services/discord-gateway/src/app/shutdown.ts
index 6debd98a..5b411f2b 100644
--- a/services/discord-gateway/src/app/shutdown.ts
+++ b/services/discord-gateway/src/app/shutdown.ts
@@ -1,5 +1,9 @@
 import type { Client } from "discord.js-selfbot-v13";
 import type { createChildLogger } from "@/shared/logger/index";
+import {
+  mediaWorkerPool,
+  textWorkerPool,
+} from "../modules/ai-moderation/circuitBreaker.js";
 import type { CommandHandler } from "../modules/command-handler/commandHandler.js";
 import type { EventBroadcaster } from "../modules/event-broadcaster/index.js";
 import type { stopMetricsServer } from "../modules/gateway-metrics/index.js";
@@ -45,6 +49,30 @@ export function createGracefulShutdown(
       options.logger.info("Closing command handler...");
       await options.commandHandler.close();
 
+      // ½. Tear down AI-analysis worker pools BEFORE closing the DB.
+      // Piscina worker threads survive process.exit() as orphans otherwise —
+      // they keep holding DB connections/locks after the main process is gone.
+      // (Two live gateways fighting over the same rows was the root cause of
+      // messages stuck in ai_status='processing'.)
+      options.logger.info("Destroying AI worker pools...");
+      const destroyPool = (pool: { destroy: () => Promise<void> }) =>
+        Promise.race([
+          pool.destroy(),
+          new Promise<void>((resolve) =>
+            setTimeout(() => {
+              options.logger.warn(
+                "Timed out destroying worker pool; exiting anyway",
+              );
+              resolve();
+            }, 5000),
+          ),
+        ]);
+      await Promise.allSettled([
+        destroyPool(textWorkerPool),
+        destroyPool(mediaWorkerPool),
+      ]);
+      options.logger.info("AI worker pools destroyed");
+
       // 2. DB pool
       options.logger.info("Closing database...");
       await options.closeDatabase();