feat(vision): route multimodal analysis to dedicated NVIDIA direct endpoint

- config: add AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY (separate from text router)
- llmClient: llmVision() now calls dedicated vision endpoint when configured
  (axios POST to integrate.api.nvidia.com, model nvidia/nemotron-3-nano-omni-30b-a3b-reasoning,
  reasoning_budget 16384, non-stream), falls back to router combo otherwise
- keeps text/moderation on omniroute, vision on NVIDIA direct
This commit is contained in:
asepharyana
2026-08-15 14:31:53 +07:00
parent 589fd38fd8
commit bcb563ea7f
2 changed files with 83 additions and 1 deletions
@@ -283,6 +283,15 @@ export async function llmChat(
* Convenience for vision (image/sticker/emoji) analysis.
* Returns the raw completion content (trimmed) or null.
*
* When AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY are configured, vision is
* sent DIRECTLY to a dedicated multimodal endpoint (e.g. NVIDIA direct API),
* separate from the text/moderation router (omniroute/9router). This keeps
* image analysis on a vision-capable model while text moderation stays on the
* router's text combo.
*
* Otherwise it falls back to the shared AI_LLM_BASE_URL (router combo) using
* AI_LLM_VISION_MODEL.
*
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
* wraps this call in its own 3-attempt loop with exponential backoff.
* A second retry layer would multiply worst-case API calls (3×3=9/image).
@@ -291,6 +300,12 @@ export async function llmVision(
promptText: string,
imageUrl: { url: string },
): Promise<string | null> {
// ── Dedicated vision endpoint (NVIDIA direct, etc.) ──────────────────────
if (config.AI_LLM_VISION_BASE_URL && config.AI_LLM_VISION_API_KEY) {
return llmVisionDirect(promptText, imageUrl);
}
// ── Fallback: shared router combo ─────────────────────────────────────────
const completion = await llmChat({
messages: [
{
@@ -312,3 +327,64 @@ export async function llmVision(
if (!completion) return null;
return completion.choices[0]?.message?.content?.trim() ?? null;
}
import axios from "axios";
/**
* Direct vision call to a dedicated multimodal endpoint (NVIDIA integrate API).
* Model is fixed to the vision-capable one configured via AI_LLM_VISION_MODEL
* (default nvidia/nemotron-3-nano-omni-30b-a3b-reasoning). Uses reasoning_budget
* + non-streaming (axios JSON) — NVIDIA direct does not need the SSE streaming
* gymnastics the composite routers require.
*/
async function llmVisionDirect(
promptText: string,
imageUrl: { url: string },
): Promise<string | null> {
const visionModel = config.AI_LLM_VISION_MODEL || "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning";
try {
const response = await axios.post(
`${config.AI_LLM_VISION_BASE_URL}/chat/completions`,
{
messages: [
{
role: "user",
content: [
{ type: "text", text: promptText },
{ type: "image_url", image_url: imageUrl },
],
},
],
model: visionModel,
max_tokens: 65536,
reasoning_budget: 16384,
stream: false,
temperature: 0.6,
top_p: 0.95,
},
{
headers: {
Authorization: `Bearer ${config.AI_LLM_VISION_API_KEY}`,
Accept: "application/json",
"Content-Type": "application/json",
},
timeout: 120_000,
},
);
const content: string | undefined =
response.data?.choices?.[0]?.message?.content;
if (!content) return null;
return content.trim();
} catch (err: any) {
const status = err?.response?.status ?? "n/a";
const detail = err?.response?.data
? JSON.stringify(err.response.data).slice(0, 300)
: err?.message;
log.error(
{ status, detail, model: visionModel },
"Direct vision API call failed",
);
return null;
}
}
@@ -137,7 +137,13 @@ export const configSchema = z
.url()
.default("https://9router.asepharyana.my.id/v1"),
AI_LLM_MODEL: z.string().default("text"),
AI_LLM_VISION_MODEL: z.string().optional(),
AI_LLM_VISION_MODEL: z.string().default("multimodal"),
// Vision can be routed to a dedicated endpoint (e.g. NVIDIA direct) that is
// separate from the text/moderation router. When both are set, llmVision()
// calls the dedicated vision endpoint directly; otherwise it falls back to
// the shared AI_LLM_BASE_URL with AI_LLM_VISION_MODEL.
AI_LLM_VISION_BASE_URL: z.string().url().optional(),
AI_LLM_VISION_API_KEY: z.string().optional(),
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
AI_LLM_EMBEDDING_MIN_SIMILARITY: z.coerce
.number()