refactor(llm): unify vision routing with text moderation and remove dedicated endpoint
This commit is contained in:
@@ -325,14 +325,11 @@ export async function llmChat(
|
|||||||
* Convenience for vision (image/sticker/emoji) analysis.
|
* Convenience for vision (image/sticker/emoji) analysis.
|
||||||
* Returns the raw completion content (trimmed) or null.
|
* Returns the raw completion content (trimmed) or null.
|
||||||
*
|
*
|
||||||
* When AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY are configured, vision is
|
* Vision routes through the SAME router/base URL as text moderation
|
||||||
* sent DIRECTLY to a dedicated multimodal endpoint (e.g. NVIDIA direct API),
|
* (AI_LLM_BASE_URL) — the dedicated NVIDIA multimodal endpoint was removed.
|
||||||
* separate from the text/moderation router (omniroute/9router). This keeps
|
* It uses AI_LLM_VISION_MODEL (a different model alias from the text combo)
|
||||||
* image analysis on a vision-capable model while text moderation stays on the
|
* so image analysis stays on a vision-capable model. Thinking-disable from
|
||||||
* router's text combo.
|
* config.AI_LLM_DISABLE_THINKING applies automatically via buildLlmParams.
|
||||||
*
|
|
||||||
* Otherwise it falls back to the shared AI_LLM_BASE_URL (router combo) using
|
|
||||||
* AI_LLM_VISION_MODEL.
|
|
||||||
*
|
*
|
||||||
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
|
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
|
||||||
* wraps this call in its own 3-attempt loop with exponential backoff.
|
* wraps this call in its own 3-attempt loop with exponential backoff.
|
||||||
@@ -342,12 +339,6 @@ export async function llmVision(
|
|||||||
promptText: string,
|
promptText: string,
|
||||||
imageUrl: { url: string },
|
imageUrl: { url: string },
|
||||||
): Promise<string | null> {
|
): Promise<string | null> {
|
||||||
// ── Dedicated vision endpoint (NVIDIA direct, etc.) ──────────────────────
|
|
||||||
if (config.AI_LLM_VISION_BASE_URL && config.AI_LLM_VISION_API_KEY) {
|
|
||||||
return llmVisionDirect(promptText, imageUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Fallback: shared router combo ─────────────────────────────────────────
|
|
||||||
const completion = await llmChat({
|
const completion = await llmChat({
|
||||||
messages: [
|
messages: [
|
||||||
{
|
{
|
||||||
@@ -369,66 +360,3 @@ export async function llmVision(
|
|||||||
if (!completion) return null;
|
if (!completion) return null;
|
||||||
return completion.choices[0]?.message?.content?.trim() ?? null;
|
return completion.choices[0]?.message?.content?.trim() ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
import axios from "axios";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Direct vision call to a dedicated multimodal endpoint (NVIDIA integrate API).
|
|
||||||
* Model is fixed to the vision-capable one configured via AI_LLM_VISION_MODEL
|
|
||||||
* (default nvidia/nemotron-3-nano-omni-30b-a3b-reasoning). Uses reasoning_budget
|
|
||||||
* + non-streaming (axios JSON) — NVIDIA direct does not need the SSE streaming
|
|
||||||
* gymnastics the composite routers require.
|
|
||||||
*/
|
|
||||||
async function llmVisionDirect(
|
|
||||||
promptText: string,
|
|
||||||
imageUrl: { url: string },
|
|
||||||
): Promise<string | null> {
|
|
||||||
const visionModel =
|
|
||||||
config.AI_LLM_VISION_MODEL ||
|
|
||||||
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning";
|
|
||||||
try {
|
|
||||||
const response = await axios.post(
|
|
||||||
`${config.AI_LLM_VISION_BASE_URL}/chat/completions`,
|
|
||||||
{
|
|
||||||
messages: [
|
|
||||||
{
|
|
||||||
role: "user",
|
|
||||||
content: [
|
|
||||||
{ type: "text", text: promptText },
|
|
||||||
{ type: "image_url", image_url: imageUrl },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
model: visionModel,
|
|
||||||
max_tokens: 65536,
|
|
||||||
reasoning_budget: config.AI_LLM_DISABLE_THINKING ? 0 : 16384,
|
|
||||||
stream: false,
|
|
||||||
temperature: 0.6,
|
|
||||||
top_p: 0.95,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
headers: {
|
|
||||||
Authorization: `Bearer ${config.AI_LLM_VISION_API_KEY}`,
|
|
||||||
Accept: "application/json",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
timeout: 120_000,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
const content: string | undefined =
|
|
||||||
response.data?.choices?.[0]?.message?.content;
|
|
||||||
if (!content) return null;
|
|
||||||
return content.trim();
|
|
||||||
} catch (err: any) {
|
|
||||||
const status = err?.response?.status ?? "n/a";
|
|
||||||
const detail = err?.response?.data
|
|
||||||
? JSON.stringify(err.response.data).slice(0, 300)
|
|
||||||
: err?.message;
|
|
||||||
log.error(
|
|
||||||
{ status, detail, model: visionModel },
|
|
||||||
"Direct vision API call failed",
|
|
||||||
);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -144,13 +144,10 @@ export const configSchema = z
|
|||||||
.url()
|
.url()
|
||||||
.default("https://9router.asepharyana.my.id/v1"),
|
.default("https://9router.asepharyana.my.id/v1"),
|
||||||
AI_LLM_MODEL: z.string().default("text"),
|
AI_LLM_MODEL: z.string().default("text"),
|
||||||
|
// Vision uses the SAME router/base URL as text moderation
|
||||||
|
// (AI_LLM_BASE_URL) but a different model alias. The dedicated NVIDIA
|
||||||
|
// multimodal endpoint was removed.
|
||||||
AI_LLM_VISION_MODEL: z.string().default("multimodal"),
|
AI_LLM_VISION_MODEL: z.string().default("multimodal"),
|
||||||
// Vision can be routed to a dedicated endpoint (e.g. NVIDIA direct) that is
|
|
||||||
// separate from the text/moderation router. When both are set, llmVision()
|
|
||||||
// calls the dedicated vision endpoint directly; otherwise it falls back to
|
|
||||||
// the shared AI_LLM_BASE_URL with AI_LLM_VISION_MODEL.
|
|
||||||
AI_LLM_VISION_BASE_URL: z.string().url().optional(),
|
|
||||||
AI_LLM_VISION_API_KEY: z.string().optional(),
|
|
||||||
AI_LLM_DISABLE_THINKING: z
|
AI_LLM_DISABLE_THINKING: z
|
||||||
.string()
|
.string()
|
||||||
.default("true")
|
.default("true")
|
||||||
|
|||||||
Reference in New Issue
Block a user