feat(vision): route multimodal analysis to dedicated NVIDIA direct endpoint
- config: add AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY (separate from text router) - llmClient: llmVision() now calls dedicated vision endpoint when configured (axios POST to integrate.api.nvidia.com, model nvidia/nemotron-3-nano-omni-30b-a3b-reasoning, reasoning_budget 16384, non-stream), falls back to router combo otherwise - keeps text/moderation on omniroute, vision on NVIDIA direct
This commit is contained in:
@@ -283,6 +283,15 @@ export async function llmChat(
|
||||
* Convenience for vision (image/sticker/emoji) analysis.
|
||||
* Returns the raw completion content (trimmed) or null.
|
||||
*
|
||||
* When AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY are configured, vision is
|
||||
* sent DIRECTLY to a dedicated multimodal endpoint (e.g. NVIDIA direct API),
|
||||
* separate from the text/moderation router (omniroute/9router). This keeps
|
||||
* image analysis on a vision-capable model while text moderation stays on the
|
||||
* router's text combo.
|
||||
*
|
||||
* Otherwise it falls back to the shared AI_LLM_BASE_URL (router combo) using
|
||||
* AI_LLM_VISION_MODEL.
|
||||
*
|
||||
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
|
||||
* wraps this call in its own 3-attempt loop with exponential backoff.
|
||||
* A second retry layer would multiply worst-case API calls (3×3=9/image).
|
||||
@@ -291,6 +300,12 @@ export async function llmVision(
|
||||
promptText: string,
|
||||
imageUrl: { url: string },
|
||||
): Promise<string | null> {
|
||||
// ── Dedicated vision endpoint (NVIDIA direct, etc.) ──────────────────────
|
||||
if (config.AI_LLM_VISION_BASE_URL && config.AI_LLM_VISION_API_KEY) {
|
||||
return llmVisionDirect(promptText, imageUrl);
|
||||
}
|
||||
|
||||
// ── Fallback: shared router combo ─────────────────────────────────────────
|
||||
const completion = await llmChat({
|
||||
messages: [
|
||||
{
|
||||
@@ -312,3 +327,64 @@ export async function llmVision(
|
||||
if (!completion) return null;
|
||||
return completion.choices[0]?.message?.content?.trim() ?? null;
|
||||
}
|
||||
|
||||
import axios from "axios";
|
||||
|
||||
/**
|
||||
* Direct vision call to a dedicated multimodal endpoint (NVIDIA integrate API).
|
||||
* Model is fixed to the vision-capable one configured via AI_LLM_VISION_MODEL
|
||||
* (default nvidia/nemotron-3-nano-omni-30b-a3b-reasoning). Uses reasoning_budget
|
||||
* + non-streaming (axios JSON) — NVIDIA direct does not need the SSE streaming
|
||||
* gymnastics the composite routers require.
|
||||
*/
|
||||
async function llmVisionDirect(
|
||||
promptText: string,
|
||||
imageUrl: { url: string },
|
||||
): Promise<string | null> {
|
||||
const visionModel = config.AI_LLM_VISION_MODEL || "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning";
|
||||
try {
|
||||
const response = await axios.post(
|
||||
`${config.AI_LLM_VISION_BASE_URL}/chat/completions`,
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: promptText },
|
||||
{ type: "image_url", image_url: imageUrl },
|
||||
],
|
||||
},
|
||||
],
|
||||
model: visionModel,
|
||||
max_tokens: 65536,
|
||||
reasoning_budget: 16384,
|
||||
stream: false,
|
||||
temperature: 0.6,
|
||||
top_p: 0.95,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.AI_LLM_VISION_API_KEY}`,
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
timeout: 120_000,
|
||||
},
|
||||
);
|
||||
|
||||
const content: string | undefined =
|
||||
response.data?.choices?.[0]?.message?.content;
|
||||
if (!content) return null;
|
||||
return content.trim();
|
||||
} catch (err: any) {
|
||||
const status = err?.response?.status ?? "n/a";
|
||||
const detail = err?.response?.data
|
||||
? JSON.stringify(err.response.data).slice(0, 300)
|
||||
: err?.message;
|
||||
log.error(
|
||||
{ status, detail, model: visionModel },
|
||||
"Direct vision API call failed",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,7 +137,13 @@ export const configSchema = z
|
||||
.url()
|
||||
.default("https://9router.asepharyana.my.id/v1"),
|
||||
AI_LLM_MODEL: z.string().default("text"),
|
||||
AI_LLM_VISION_MODEL: z.string().optional(),
|
||||
AI_LLM_VISION_MODEL: z.string().default("multimodal"),
|
||||
// Vision can be routed to a dedicated endpoint (e.g. NVIDIA direct) that is
|
||||
// separate from the text/moderation router. When both are set, llmVision()
|
||||
// calls the dedicated vision endpoint directly; otherwise it falls back to
|
||||
// the shared AI_LLM_BASE_URL with AI_LLM_VISION_MODEL.
|
||||
AI_LLM_VISION_BASE_URL: z.string().url().optional(),
|
||||
AI_LLM_VISION_API_KEY: z.string().optional(),
|
||||
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
|
||||
AI_LLM_EMBEDDING_MIN_SIMILARITY: z.coerce
|
||||
.number()
|
||||
|
||||
Reference in New Issue
Block a user