fix(ai-moderation): matikan hot requeue loop saat upload attachment in-flight
Batch race guard balikin {ok:true, rows:[]} tanpa sinyal saat semua target
masih upload-pending -> processor klasifikasi semua incomplete -> fanout ke
individual queue -> di situ requeue + reschedule 250ms -> balik ke batch:
hot loop ~300ms sepanjang upload (10 siklus/3 dtk di log prod 08:13).
Fix: worker batch kini return uploadPendingIds eksplisit; classifier pure
baru (partitionBatchOutcome) partisi completed/upload_pending/incomplete/
parse_failed/api_failed; target upload-pending DEFERRED dengan poll backoff
linear (AI_ANALYSIS_UPLOAD_POLL_MS 1500 base, cap AI_ANALYSIS_MAX_UPLOAD_POLL_MS
8000), tidak pernah masuk fanout; tail shouldScheduleNext tak menimpa defer.
Test: tests/batchOutcomeClassifier.test.ts (8 kasus, pure tanpa DB/Piscina).
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// partitionBatchOutcome — upload-pending defer vs fanout (2026-08-25)
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Bug history: the batch worker's race guard returned {ok:true, rows:[]} while
|
||||
// attachments were still uploading; every target was classified "incomplete",
|
||||
// fanned out to the individual queue, requeued there, rescheduled at 250ms —
|
||||
// a hot ~300ms loop for the whole upload duration (~10 cycles in 3s in prod).
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
computeUploadPollDelayMs,
|
||||
partitionBatchOutcome,
|
||||
} from "../src/modules/ai-moderation/batchOutcomeClassifier.js";
|
||||
|
||||
const msgs = (...ids: string[]) => ids.map((id) => ({ id }));
|
||||
|
||||
describe("partitionBatchOutcome", () => {
|
||||
it("marks ALL targets upload_pending when the full-batch guard fires", () => {
|
||||
const out = partitionBatchOutcome(msgs("a", "b"), {
|
||||
ok: true,
|
||||
rows: [],
|
||||
uploadPendingIds: ["a", "b"],
|
||||
});
|
||||
expect(out.get("a")).toBe("upload_pending");
|
||||
expect(out.get("b")).toBe("upload_pending");
|
||||
});
|
||||
|
||||
it("never classifies an explicit upload_pending id as incomplete", () => {
|
||||
// The regression this file exists for: uploadPendingIds must win over the
|
||||
// missing-row heuristic.
|
||||
const out = partitionBatchOutcome(msgs("a"), {
|
||||
ok: true,
|
||||
rows: [],
|
||||
uploadPendingIds: ["a"],
|
||||
});
|
||||
expect(out.get("a")).not.toBe("incomplete");
|
||||
expect(out.get("a")).toBe("upload_pending");
|
||||
});
|
||||
|
||||
it("partitions a mixed batch: completed + upload-pending + missing", () => {
|
||||
const out = partitionBatchOutcome(msgs("ok1", "up1", "gone1"), {
|
||||
ok: true,
|
||||
rows: [
|
||||
{ id: "ok1", ai_status: "clean" },
|
||||
// up1 has NO row but IS in uploadPendingIds -> deferred, not failed
|
||||
],
|
||||
uploadPendingIds: ["up1"],
|
||||
});
|
||||
expect(out.get("ok1")).toBe("completed");
|
||||
expect(out.get("up1")).toBe("upload_pending");
|
||||
expect(out.get("gone1")).toBe("incomplete"); // unexplained drop stays retryable
|
||||
});
|
||||
|
||||
it("routes flag-based failures to their buckets", () => {
|
||||
const out = partitionBatchOutcome(msgs("i", "p", "f"), {
|
||||
ok: true,
|
||||
rows: [
|
||||
{
|
||||
id: "i",
|
||||
ai_status: "error",
|
||||
ai_moderation_flags: JSON.stringify(["analysis_incomplete"]),
|
||||
},
|
||||
{
|
||||
id: "p",
|
||||
ai_status: "error",
|
||||
ai_moderation_flags: JSON.stringify(["analysis_parse_failed"]),
|
||||
},
|
||||
{
|
||||
id: "f",
|
||||
ai_status: "error",
|
||||
ai_moderation_flags: JSON.stringify(["analysis_api_failed"]),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(out.get("i")).toBe("incomplete");
|
||||
expect(out.get("p")).toBe("parse_failed");
|
||||
expect(out.get("f")).toBe("api_failed");
|
||||
});
|
||||
|
||||
it("treats error rows with no known flag as retryable incomplete", () => {
|
||||
const out = partitionBatchOutcome(msgs("x"), {
|
||||
ok: true,
|
||||
rows: [
|
||||
{ id: "x", ai_status: "error", ai_moderation_flags: '["weird_flag"]' },
|
||||
],
|
||||
});
|
||||
expect(out.get("x")).toBe("incomplete");
|
||||
});
|
||||
|
||||
it("accepts DB-round-trip MessageRecord shape (null ai_status)", () => {
|
||||
const out = partitionBatchOutcome([{ id: "r", ai_status: null }] as never, {
|
||||
ok: true,
|
||||
rows: [{ id: "r", ai_status: null }],
|
||||
});
|
||||
expect(out.get("r")).toBe("completed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeUploadPollDelayMs", () => {
|
||||
it("ramps linearly and respects the cap", () => {
|
||||
expect(computeUploadPollDelayMs(1, 1500, 8000)).toBe(1500);
|
||||
expect(computeUploadPollDelayMs(2, 1500, 8000)).toBe(3000);
|
||||
expect(computeUploadPollDelayMs(3, 1500, 8000)).toBe(4500);
|
||||
expect(computeUploadPollDelayMs(9, 1500, 8000)).toBe(8000); // capped
|
||||
});
|
||||
|
||||
it("is safe on degenerate input", () => {
|
||||
expect(computeUploadPollDelayMs(0, 1500, 8000)).toBe(1500);
|
||||
expect(computeUploadPollDelayMs(-5, 1000, 4000)).toBe(1000);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user