From eec0cd508875b71d8b56588feaadb333a903451b Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Thu, 25 Jun 2026 21:25:02 +0700 Subject: [PATCH] feat: tambah DSML parser untuk konversi tool_calls DeepSeek ke format standar DeepSeek models mengembalikan tool calls dalam format DSML (DeepSeek Markup Language) di dalam text content, bukan sebagai JSON structured tool_calls. Ini menyebabkan tool calling gagal di Claude Code. Perubahan: - Buat src/lib/dsml-parser.ts: parser DSML berbasis regex dengan dukungan streaming (DSMLAccumulator), deteksi teks sebelum/sesudah DSML, dan parsing parameter JSON - Forward tools/tool_choice dari client ke backend di Anthropic & OpenAI paths - Konversi DSML ke tool_use content blocks (Anthropic) atau tool_calls array (OpenAI) di response non-streaming - Handle DSML di streaming: buffer text deltas, deteksi di akhir stream, emit tool_use/tool_calls events yang sesuai - Handle tool_result blocks dari Anthropic format di assistant messages - Tambah 29 tests untuk parser dan 18 tests untuk anthropic-proxy Fix: #285 tests pass, 0 fail Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/ai-proxy.ts | 108 ++++++++++- src/lib/anthropic-proxy.test.ts | 136 +++++++++++++- src/lib/anthropic-proxy.ts | 314 +++++++++++++++++++++++++++----- src/lib/dsml-parser.test.ts | 278 ++++++++++++++++++++++++++++ src/lib/dsml-parser.ts | 266 +++++++++++++++++++++++++++ 5 files changed, 1050 insertions(+), 52 deletions(-) create mode 100644 src/lib/dsml-parser.test.ts create mode 100644 src/lib/dsml-parser.ts diff --git a/src/lib/ai-proxy.ts b/src/lib/ai-proxy.ts index 1f48a82..7b0c55c 100644 --- a/src/lib/ai-proxy.ts +++ b/src/lib/ai-proxy.ts @@ -14,6 +14,7 @@ import type { ProxyPool, SessionProxyPool } from "./proxy-pool"; import { fetchWithRetry, fetchWithSessionRetry, SSELineBuffer, isDevMode, trackReader, releaseReader, wrapStreamWithCleanup, type FetchWithRetryResult } from "./fetch-utils"; import { getJwt, invalidateJwt } from "./mimo-auth"; +import { parseDSML, looksLikeDSML } from "./dsml-parser"; // --- Types ------------------------------------------------------------------- @@ -29,6 +30,8 @@ export interface OpenAIRequest { stop?: string | string[]; presence_penalty?: number; frequency_penalty?: number; + tools?: unknown[]; + tool_choice?: unknown; } export interface BackendConfig { @@ -175,7 +178,7 @@ function buildBackendRequest( req: OpenAIRequest, config: BackendConfig, ): { url: string; init: RequestInit & { proxy?: string } } { - const body = + const body: any = config.adaptRequest?.(req) ?? { model: req.model, messages: req.messages, @@ -186,6 +189,9 @@ function buildBackendRequest( stop: req.stop, }; + if (req.tools?.length) body.tools = req.tools; + if (req.tool_choice !== undefined) body.tool_choice = req.tool_choice; + const init: RequestInit & { proxy?: string } = { method: config.method ?? "POST", headers: config.headers, @@ -222,6 +228,37 @@ function parseJSONResponse( } // Default fallback -- assume raw text is the content + const parsedDSML = parseDSML(text); + + if (parsedDSML && parsedDSML.toolCalls.length > 0) { + // DSML detected — convert to structured tool_calls + return { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: req.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: parsedDSML.textBefore || null, + tool_calls: parsedDSML.toolCalls.map((tc) => ({ + id: `call_${crypto.randomUUID().slice(0, 12)}`, + type: "function", + function: { + name: tc.name, + arguments: JSON.stringify(tc.args), + }, + })), + }, + finish_reason: "tool_calls", + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }; + } + return { id: `chatcmpl-${Date.now()}`, object: "chat.completion", @@ -495,6 +532,10 @@ function transformStream( let keepaliveTimer: ReturnType | null = null; const KEEPALIVE_INTERVAL_MS = 15_000; + // DSML accumulation: buffer text deltas to detect DSML across chunks + let dsmlAccumulated = ""; + let dsmlDetecting = true; + function startKeepalive(controller: ReadableStreamDefaultController) { if (keepaliveTimer) return; keepaliveTimer = setInterval(() => { @@ -540,6 +581,45 @@ function transformStream( controller.enqueue(encoder.encode(remaining + "\n\n")); } } + + // Check accumulated text for DSML + if (dsmlDetecting) { + const parsed = parseDSML(dsmlAccumulated); + if (parsed && parsed.toolCalls.length > 0) { + // Emit tool_calls delta events + for (const tc of parsed.toolCalls) { + const toolCallsDelta = { + choices: [{ + index: 0, + delta: { + tool_calls: [{ + index: 0, + id: `call_${crypto.randomUUID().slice(0, 12)}`, + type: "function", + function: { + name: tc.name, + arguments: JSON.stringify(tc.args), + }, + }], + }, + finish_reason: "tool_calls", + }], + }; + controller.enqueue(encoder.encode(`data: ${JSON.stringify(toolCallsDelta)}\n\n`)); + } + + // Final finish event with tool_calls + const finishEvent = { + choices: [{ + index: 0, + delta: {}, + finish_reason: "tool_calls", + }], + }; + controller.enqueue(encoder.encode(`data: ${JSON.stringify(finishEvent)}\n\n`)); + } + } + controller.enqueue(encoder.encode("data: [DONE]\n\n")); controller.close(); return; @@ -549,14 +629,30 @@ function transformStream( const lines = lineBuffer.add(chunk); for (const line of lines) { + let outputLine = line; if (config.adaptStreamLine) { const adapted = config.adaptStreamLine(line, req); - if (adapted) { - controller.enqueue(encoder.encode(adapted + "\n\n")); - } - } else { - controller.enqueue(encoder.encode(line + "\n\n")); + if (!adapted) continue; + outputLine = adapted; } + + // Accumulate text content for DSML detection + if (dsmlDetecting && outputLine.startsWith("data: ")) { + try { + const data = JSON.parse(outputLine.slice(6)); + const content = data.choices?.[0]?.delta?.content; + if (typeof content === "string") { + dsmlAccumulated += content; + if (!looksLikeDSML(dsmlAccumulated) && dsmlAccumulated.length > 1000) { + dsmlDetecting = false; // Not DSML, stop checking + } + } + } catch { + // Not JSON, skip + } + } + + controller.enqueue(encoder.encode(outputLine + "\n\n")); } chunksProcessed++; diff --git a/src/lib/anthropic-proxy.test.ts b/src/lib/anthropic-proxy.test.ts index d080d77..72b1408 100644 --- a/src/lib/anthropic-proxy.test.ts +++ b/src/lib/anthropic-proxy.test.ts @@ -1,5 +1,5 @@ import { test, expect, describe } from "bun:test"; -import { anthropicToBackend } from "./anthropic-proxy"; +import { anthropicToBackend, backendToAnthropicResponse, generateToolUseId } from "./anthropic-proxy"; import type { BackendConfig } from "./ai-proxy"; const passthroughConfig: BackendConfig = { @@ -216,4 +216,138 @@ describe("anthropicToBackend", () => { expect(body.messages[0].role).toBe("system"); expect(body.messages[0].content).toBe("You are a helpful assistant."); }); + + test("should forward tools in request body", () => { + const result = anthropicToBackend( + { + model: "deepseek-v4-flash-free", + max_tokens: 100, + messages: [{ role: "user", content: "Hi" }], + tools: [{ name: "test_tool", input_schema: { type: "object" } }], + }, + passthroughConfig, + "deepseek-v4-flash-free", + ); + const body = result.body as any; + expect(body.tools).toBeDefined(); + expect(body.tools).toHaveLength(1); + expect(body.tools[0].name).toBe("test_tool"); + }); + + test("should forward tool_choice in request body", () => { + const result = anthropicToBackend( + { + model: "deepseek-v4-flash-free", + max_tokens: 100, + messages: [{ role: "user", content: "Hi" }], + tool_choice: { type: "tool", name: "test_tool" }, + }, + passthroughConfig, + "deepseek-v4-flash-free", + ); + const body = result.body as any; + expect(body.tool_choice).toBeDefined(); + expect(body.tool_choice.type).toBe("tool"); + }); +}); + +describe("backendToAnthropicResponse", () => { + test("should convert plain text to text content block", () => { + const raw = { + id: "test_1", + choices: [{ message: { content: "Hello world" }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 20 }, + }; + const result = backendToAnthropicResponse(raw, "deepseek-v4-flash-free"); + expect(result.content).toHaveLength(1); + expect(result.content[0].type).toBe("text"); + expect((result.content[0] as any).text).toBe("Hello world"); + expect(result.stop_reason).toBe("end_turn"); + }); + + test("should convert DSML to tool_use content blocks", () => { + const raw = { + id: "test_2", + choices: [{ + message: { content: ` + +ls -la + +` }, + finish_reason: "stop", + }], + usage: { prompt_tokens: 10, completion_tokens: 20 }, + }; + const result = backendToAnthropicResponse(raw, "deepseek-v4-flash-free"); + expect(result.content).toHaveLength(1); + expect(result.content[0].type).toBe("tool_use"); + const toolUse = result.content[0] as any; + expect(toolUse.name).toBe("Bash"); + expect(toolUse.input.command).toBe("ls -la"); + expect(toolUse.id).toBeTruthy(); + expect(result.stop_reason).toBe("tool_use"); + }); + + test("should convert text before DSML as separate text block", () => { + const raw = { + id: "test_3", + choices: [{ + message: { content: "Let me check.\n\n\n/tmp/test\n\n" }, + finish_reason: "stop", + }], + usage: { prompt_tokens: 5, completion_tokens: 15 }, + }; + const result = backendToAnthropicResponse(raw, "deepseek-v4-flash-free"); + expect(result.content).toHaveLength(2); + expect(result.content[0].type).toBe("text"); + expect(((result.content[0] as any).text).trim()).toBe("Let me check."); + expect(result.content[1].type).toBe("tool_use"); + expect((result.content[1] as any).name).toBe("Read"); + }); + + test("should handle multiple tool calls in DSML", () => { + const raw = { + id: "test_4", + choices: [{ + message: { content: ` + +ls + + +test.txt + +` }, + finish_reason: "stop", + }], + usage: { prompt_tokens: 5, completion_tokens: 15 }, + }; + const result = backendToAnthropicResponse(raw, "deepseek-v4-flash-free"); + expect(result.content).toHaveLength(2); + expect(result.content[0].type).toBe("tool_use"); + expect((result.content[0] as any).name).toBe("Bash"); + expect(result.content[1].type).toBe("tool_use"); + expect((result.content[1] as any).name).toBe("Read"); + }); + + test("should return plain text when no DSML in response", () => { + const raw = { + id: "test_5", + choices: [{ message: { content: "Hello" }, finish_reason: "stop" }], + usage: { prompt_tokens: 5, completion_tokens: 10 }, + }; + const result = backendToAnthropicResponse(raw, "deepseek-v4-flash-free"); + expect(result.content).toHaveLength(1); + expect(result.content[0].type).toBe("text"); + expect(result.stop_reason).toBe("end_turn"); + }); +}); + +describe("generateToolUseId", () => { + test("should generate unique IDs with toolu_ prefix", () => { + const id1 = generateToolUseId(); + const id2 = generateToolUseId(); + expect(id1).toMatch(/^toolu_/); + expect(id2).toMatch(/^toolu_/); + expect(id1).not.toBe(id2); + }); }); diff --git a/src/lib/anthropic-proxy.ts b/src/lib/anthropic-proxy.ts index f22a1ec..a830652 100644 --- a/src/lib/anthropic-proxy.ts +++ b/src/lib/anthropic-proxy.ts @@ -13,17 +13,40 @@ import type { ProxyPool, SessionProxyPool } from "./proxy-pool"; import { MODEL_ROUTES, type BackendConfig } from "./ai-proxy"; import { fetchWithRetry, fetchWithSessionRetry, SSELineBuffer, isDevMode, trackReader, releaseReader, wrapStreamWithCleanup, type FetchWithRetryResult } from "./fetch-utils"; +import { parseDSML, looksLikeDSML, isCompleteDSML } from "./dsml-parser"; // --- Types ------------------------------------------------------------------- -export interface AnthropicContentBlock { - type: "text"; - text: string; - cache_control?: { type: "ephemeral" }; +/** Anthropic tool definition (from request `tools` parameter). */ +export interface AnthropicToolDef { + name: string; + description?: string; + input_schema: Record; } +/** Tool_use content block for Anthropic response. */ +export interface ToolUseBlock { + type: "tool_use"; + id: string; + name: string; + input: unknown; +} + +/** tool_result content block (user role after tool execution). */ +interface ToolResultBlock { + type: "tool_result"; + tool_use_id: string; + content: string; +} + +/** Unified content block type: text, tool_use, or tool_result. */ +export type AnthropicContentBlock = + | { type: "text"; text: string; cache_control?: { type: "ephemeral" } } + | ToolUseBlock + | ToolResultBlock; + export interface AnthropicMessage { - role: "user" | "assistant"; + role: "user" | "assistant" | "tool"; content: string | AnthropicContentBlock[]; } @@ -43,6 +66,8 @@ export interface AnthropicRequest { top_k?: number; stop_sequences?: string[]; system?: string | AnthropicSystemBlock[]; + tools?: AnthropicToolDef[]; + tool_choice?: unknown; metadata?: Record; } @@ -50,9 +75,9 @@ interface AnthropicResponse { id: string; type: "message"; role: "assistant"; - content: Array<{ type: "text"; text: string }>; + content: AnthropicContentBlock[]; model: string; - stop_reason: "end_turn" | "max_tokens" | "stop_sequence" | null; + stop_reason: "end_turn" | "max_tokens" | "stop_sequence" | "tool_use" | null; stop_sequence: string | null; usage: { input_tokens: number; @@ -62,6 +87,11 @@ interface AnthropicResponse { }; } +/** Generate a unique tool_use ID. @internal Exported for testing. */ +export function generateToolUseId(): string { + return `toolu_${Date.now().toString(36)}_${crypto.randomUUID().slice(0, 8)}`; +} + // --- Model resolution ---------------------------------------------------------- /** Resolve a model name to a backend config (uses MODEL_ROUTES directly). */ @@ -89,6 +119,8 @@ interface BackendBody { top_k?: number; stream?: boolean; stop?: string | string[]; + tools?: AnthropicToolDef[]; + tool_choice?: unknown; } /** @@ -122,26 +154,75 @@ export function anthropicToBackend( // Convert Anthropic content blocks for OpenAI-compatible backend. // When a content block has cache_control, we keep it as a structured // content part so the backend (or downstream cache layer) can use it. - const messages: Array<{ role: string; content: string | Array> }> = anthReq.messages.map((m) => { + const messages: Array<{ role: string; content: string | Array>; tool_calls?: unknown[]; tool_call_id?: string }> = anthReq.messages.map((m) => { + // Tool-role messages (OpenAI format after tool_use was executed) + if (m.role === "tool") { + const msg = m as any; + return { + role: "tool", + content: typeof m.content === "string" ? m.content : "", + tool_call_id: msg.tool_use_id ?? "", + }; + } + if (typeof m.content === "string") { return { role: m.role, content: m.content }; } - // Check if any block has cache_control — if so, preserve as structured array - const hasCacheControl = m.content.some((c) => c.cache_control); - if (hasCacheControl) { - return { - role: m.role, - content: m.content.map((c) => { - const part: Record = { type: "text", text: c.text }; - if (c.cache_control) { - part.cache_control = c.cache_control; - } - return part; - }), - }; + + // Content is an array of blocks — may contain text, tool_use, tool_result + const textBlocks: string[] = []; + const toolCalls: Array<{ id: string; type: "function"; function: { name: string; arguments: string } }> = []; + let isToolResult = false; + let toolResultContent = ""; + let toolResultId = ""; + + for (const block of m.content) { + if (block.type === "text") { + textBlocks.push((block as any).text); + } else if (block.type === "tool_use") { + toolCalls.push({ + id: (block as any).id, + type: "function", + function: { + name: (block as any).name, + arguments: JSON.stringify((block as any).input ?? {}), + }, + }); + } else if (block.type === "tool_result") { + isToolResult = true; + toolResultContent = typeof (block as any).content === "string" + ? (block as any).content + : JSON.stringify((block as any).content); + toolResultId = (block as any).tool_use_id ?? ""; + } } - // No cache_control — flatten to plain text for simpler backend processing - return { role: m.role, content: m.content.map((c) => c.text).join("") }; + + // tool_result blocks come in user-role messages (Anthropic convention) + if (isToolResult) { + return { role: "tool", content: toolResultContent, tool_call_id: toolResultId }; + } + + const msg: any = { role: m.role }; + const hasCacheControl = m.content.some((c) => (c as any).cache_control); + + if (toolCalls.length > 0) { + // Assistant with tool calls: content is text, tool_calls separate + msg.content = textBlocks.join(""); + msg.tool_calls = toolCalls; + } else if (hasCacheControl) { + msg.content = m.content.map((c) => { + const part: Record = { type: "text", text: (c as any).text }; + if ((c as any).cache_control) { + part.cache_control = (c as any).cache_control; + } + return part; + }); + } else { + // No cache_control — flatten to plain text for simpler backend processing + msg.content = textBlocks.join(""); + } + + return msg; }); // Prepend system prompt as a system message if present. @@ -179,6 +260,13 @@ export function anthropicToBackend( stream: anthReq.stream, }; + if (anthReq.tools?.length) { + base.tools = anthReq.tools; + } + if (anthReq.tool_choice !== undefined) { + base.tool_choice = anthReq.tool_choice; + } + if (anthReq.stop_sequences?.length) { base.stop = anthReq.stop_sequences.length === 1 @@ -191,19 +279,22 @@ export function anthropicToBackend( if (anthropicVersion) { headers["anthropic-version"] = anthropicVersion; } + const adaptedReq: any = { + model: backendModel, + messages, + temperature: anthReq.temperature, + max_tokens: anthReq.max_tokens, + top_p: anthReq.top_p, + top_k: anthReq.top_k, + stream: anthReq.stream, + stop: anthReq.stop_sequences?.length === 1 + ? anthReq.stop_sequences[0] + : anthReq.stop_sequences, + }; + if (anthReq.tools?.length) adaptedReq.tools = anthReq.tools; + if (anthReq.tool_choice !== undefined) adaptedReq.tool_choice = anthReq.tool_choice; return { - body: config.adaptRequest({ - model: backendModel, - messages, - temperature: anthReq.temperature, - max_tokens: anthReq.max_tokens, - top_p: anthReq.top_p, - top_k: anthReq.top_k, - stream: anthReq.stream, - stop: anthReq.stop_sequences?.length === 1 - ? anthReq.stop_sequences[0] - : anthReq.stop_sequences, - }), + body: config.adaptRequest(adaptedReq), headers, }; } @@ -241,23 +332,60 @@ function extractUsage(raw: any): AnthropicResponse["usage"] { } /** - * Convert a backend JSON response body into Anthropic Messages format. - * Extracts token usage and cache metrics from the backend response. + * Convert a backend JSON response body into Anthropic Messages format, + * including DSML tool call detection. + * + * If the backend text response contains DSML markup (``), it is + * parsed and converted into Anthropic tool_use content blocks. The text + * portion before the DSML remains as a text block. Extracts token usage + * and cache metrics from the backend response. */ -function backendToAnthropicResponse( +export function backendToAnthropicResponse( raw: any, model: string, ): AnthropicResponse { const text = raw.choices?.[0]?.message?.content ?? raw.content ?? raw.text ?? ""; + const content: AnthropicContentBlock[] = []; + + // Check for DSML in the text + const parsedDSML = text ? parseDSML(text) : null; + + if (parsedDSML && parsedDSML.toolCalls.length > 0) { + // Add text before DSML if non-empty + if (parsedDSML.textBefore) { + content.push({ type: "text", text: parsedDSML.textBefore }); + } + + // Add a tool_use block for each parsed tool call + for (const tc of parsedDSML.toolCalls) { + content.push({ + type: "tool_use", + id: generateToolUseId(), + name: tc.name, + input: tc.args, + }); + } + + // Add text after DSML if non-empty + if (parsedDSML.textAfter) { + content.push({ type: "text", text: parsedDSML.textAfter }); + } + } else { + // No DSML — plain text response + content.push({ type: "text", text }); + } + return { id: raw.id ?? `msg_${Date.now()}`, type: "message", role: "assistant", - content: [{ type: "text", text }], + content, model, - stop_reason: raw.choices?.[0]?.finish_reason === "stop" ? "end_turn" : null, + stop_reason: parsedDSML + ? "tool_use" + : (raw.choices?.[0]?.finish_reason === "stop" ? "end_turn" : null), stop_sequence: raw.stop_sequence ?? null, usage: extractUsage(raw), }; @@ -483,16 +611,47 @@ function emitDoneEvents( encoder: TextEncoder, usage: AnthropicResponse["usage"], outputCounter: OutputCounter, + dsmlText?: string | null, ): void { if (usage.output_tokens === 0 && outputCounter.chars > 0) { usage.output_tokens = Math.max(1, Math.round(outputCounter.chars / 4)); } + // Check DSML buffer for tool calls + const dsmlResult = dsmlText ? parseDSML(dsmlText) : null; + const hasToolUse = dsmlResult !== null && dsmlResult.toolCalls.length > 0; + + // Close the current text content block controller.enqueue(encoder.encode('event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n')); + + // Emit tool_use blocks if DSML was found + if (hasToolUse) { + let blockIndex = 1; + for (const tc of dsmlResult!.toolCalls) { + const id = generateToolUseId(); + // content_block_start for tool_use + controller.enqueue(encoder.encode( + `event: content_block_start\ndata: ${JSON.stringify({ + type: "content_block_start", + index: blockIndex, + content_block: { type: "tool_use", id, name: tc.name, input: tc.args }, + })}\n\n`, + )); + // content_block_stop (full input available, no incremental delta needed) + controller.enqueue(encoder.encode( + `event: content_block_stop\ndata: ${JSON.stringify({ + type: "content_block_stop", + index: blockIndex, + })}\n\n`, + )); + blockIndex++; + } + } + controller.enqueue(encoder.encode( `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", - delta: { stop_reason: "end_turn", stop_sequence: null }, + delta: { stop_reason: hasToolUse ? "tool_use" : "end_turn", stop_sequence: null }, usage: { output_tokens: usage.output_tokens }, })}\n\n`, )); @@ -519,6 +678,8 @@ function emitErrorEvent( /** * Process one chunk from the upstream reader through the SSE line buffer * and emit adapted Anthropic SSE events for each complete line. + * When a DSML buffer is provided, text that looks like DSML is held back + * instead of being emitted as text deltas. * Returns true if the stream is done (reader returned done=true). */ async function processStreamChunk( @@ -531,6 +692,7 @@ async function processStreamChunk( config: BackendConfig, usage: AnthropicResponse["usage"], outputCounter: OutputCounter, + dsmlBuffer?: DSMLStreamBuffer, ): Promise { const { done, value } = await reader.read(); if (done) { @@ -538,7 +700,14 @@ async function processStreamChunk( const remaining = lineBuffer.flush(); if (remaining.length > 0) { const adapted = backendLineToAnthropicSSE(remaining, model, config, usage, outputCounter); - if (adapted) controller.enqueue(encoder.encode(adapted + "\n\n")); + if (adapted) { + const text = extractTextFromSSEEvent(adapted); + if (dsmlBuffer && text) { + dsmlBuffer.push(text); + } else if (adapted) { + controller.enqueue(encoder.encode(adapted + "\n\n")); + } + } } return true; } @@ -548,11 +717,64 @@ async function processStreamChunk( for (const line of lines) { const adapted = backendLineToAnthropicSSE(line, model, config, usage, outputCounter); - if (adapted) controller.enqueue(encoder.encode(adapted + "\n\n")); + if (adapted) { + const text = extractTextFromSSEEvent(adapted); + if (dsmlBuffer && text && (dsmlBuffer.isActive || looksLikeDSML(text))) { + dsmlBuffer.push(text); + } else { + controller.enqueue(encoder.encode(adapted + "\n\n")); + } + } } return false; } +/** Buffer for accumulating DSML content during streaming. */ +interface DSMLStreamBuffer { + text: string; + isActive: boolean; + push(chunk: string): void; + flush(): string | null; +} + +function createDSMLStreamBuffer(): DSMLStreamBuffer { + let buffer = ""; + let active = false; + + return { + get text() { return buffer; }, + get isActive() { return active; }, + + push(chunk: string) { + if (!active && looksLikeDSML(chunk)) { + active = true; + } + buffer += chunk; + }, + + flush() { + if (!buffer) return null; + const text = buffer; + buffer = ""; + active = false; + return isCompleteDSML(text) ? text : null; + }, + }; +} + +/** Extract plain text from a formatted content_block_delta SSE event. */ +function extractTextFromSSEEvent(event: string): string | null { + if (!event.startsWith("event: content_block_delta")) return null; + const dataMatch = event.match(/data:\s*(\{.*\})/); + if (!dataMatch) return null; + try { + const parsed = JSON.parse(dataMatch[1]); + return parsed.delta?.text ?? null; + } catch { + return null; + } +} + // --- Stream transformer -------------------------------------------------------- function transformAnthropicStream( @@ -568,6 +790,7 @@ function transformAnthropicStream( let phase: "init" | "block" | "done" = "init"; const outputCounter: OutputCounter = { chars: 0 }; const usage: AnthropicResponse["usage"] = { input_tokens: 0, output_tokens: 0 }; + const dsmlBuffer = createDSMLStreamBuffer(); let keepaliveTimer: ReturnType | null = null; const KEEPALIVE_INTERVAL_MS = 15_000; @@ -597,7 +820,7 @@ function transformAnthropicStream( let chunksProcessed = 0; while (phase === "block" && chunksProcessed < BATCH_SIZE) { - const isDone = await processStreamChunk(reader, lineBuffer, decoder, controller, encoder, model, config, usage, outputCounter); + const isDone = await processStreamChunk(reader, lineBuffer, decoder, controller, encoder, model, config, usage, outputCounter, dsmlBuffer); if (isDone) { stopKeepalive(); releaseReader(reader); @@ -613,7 +836,8 @@ function transformAnthropicStream( } if (phase === "done") { - emitDoneEvents(controller, encoder, usage, outputCounter); + const dsmlText = dsmlBuffer.flush(); + emitDoneEvents(controller, encoder, usage, outputCounter, dsmlText); } } catch (err) { stopKeepalive(); diff --git a/src/lib/dsml-parser.test.ts b/src/lib/dsml-parser.test.ts new file mode 100644 index 0000000..e5d4fe5 --- /dev/null +++ b/src/lib/dsml-parser.test.ts @@ -0,0 +1,278 @@ +import { test, expect, describe } from "bun:test"; +import { + parseDSML, + stripDSML, + looksLikeDSML, + isCompleteDSML, + createDSMLAccumulator, +} from "./dsml-parser"; + +describe("parseDSML", () => { + test("returns null for plain text without DSML", () => { + const result = parseDSML("Hello, world!"); + expect(result).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(parseDSML("")).toBeNull(); + }); + + test("parses single tool call with string args", () => { + const text = ` + +ls -la + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.toolCalls).toHaveLength(1); + expect(result!.toolCalls[0].name).toBe("Bash"); + expect(result!.toolCalls[0].args.command).toBe("ls -la"); + }); + + test("parses single tool call with JSON args", () => { + const text = ` + +/path/to/file.ts + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.toolCalls).toHaveLength(1); + expect(result!.toolCalls[0].name).toBe("Read"); + expect(result!.toolCalls[0].args.file_path).toBe("/path/to/file.ts"); + }); + + test("parses multiple tool calls", () => { + const text = ` + +ls + + +/tmp/test.txt + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.toolCalls).toHaveLength(2); + expect(result!.toolCalls[0].name).toBe("Bash"); + expect(result!.toolCalls[1].name).toBe("Read"); + }); + + test("extracts textBefore when there is thinking content before DSML", () => { + const text = `Let me check the file system. + + +ls -la + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.textBefore).toContain("Let me check the file system"); + expect(result!.toolCalls).toHaveLength(1); + expect(result!.toolCalls[0].name).toBe("Bash"); + }); + + test("extracts textAfter when there is content after DSML", () => { + const text = ` + +ls + + +Some follow-up text.`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.textAfter.trim()).toBe("Some follow-up text."); + expect(result!.toolCalls).toHaveLength(1); + }); + + test("handles multiline parameter values", () => { + const text = ` + +/path/to/file.ts +line 1 +line 2 +line 3 +new line 1 +new line 2 + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.toolCalls).toHaveLength(1); + expect(result!.toolCalls[0].name).toBe("Edit"); + expect(result!.toolCalls[0].args.old_string).toBe("line 1\nline 2\nline 3"); + expect(result!.toolCalls[0].args.new_string).toBe("new line 1\nnew line 2"); + }); + + test("handles text that looks like XML but is not DSML", () => { + const result = parseDSML("
hello
"); + expect(result).toBeNull(); + }); + + test("handles partial DSML (missing close tag)", () => { + const result = parseDSML(""); + expect(result).toBeNull(); + }); + + test("handles CDATA sections in parameter values", () => { + const text = ` + + + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.toolCalls).toHaveLength(1); + expect(result!.toolCalls[0].args.command).toBe('echo "hello world"'); + }); + + test("parses numeric parameter values in DSML", () => { + const text = ` + +1000 + +`; + const result = parseDSML(text); + expect(result).not.toBeNull(); + expect(result!.toolCalls[0].args.max_tokens).toBe(1000); + }); +}); + +describe("stripDSML", () => { + test("strips DSML markup from text", () => { + const text = `Let me think + + +ls + +`; + const stripped = stripDSML(text); + expect(stripped).not.toContain(""); + expect(stripped).not.toContain(" { + const text = "Hello world"; + expect(stripDSML(text)).toBe("Hello world"); + }); + + test("handles empty string", () => { + expect(stripDSML("")).toBe(""); + }); +}); + +describe("looksLikeDSML", () => { + test("detects opening tag", () => { + expect(looksLikeDSML("")).toBe(true); + }); + + test("detects tag", () => { + expect(looksLikeDSML('')).toBe(true); + }); + + test("returns false for plain text", () => { + expect(looksLikeDSML("Hello world")).toBe(false); + }); + + test("detects closing tags", () => { + expect(looksLikeDSML("")).toBe(true); + expect(looksLikeDSML("")).toBe(true); + }); + + test("detects tag", () => { + expect(looksLikeDSML('')).toBe(true); + }); +}); + +describe("isCompleteDSML", () => { + test("returns true for complete DSML block", () => { + const text = ` + +ls + +`; + expect(isCompleteDSML(text)).toBe(true); + }); + + test("returns false for incomplete DSML (no close)", () => { + expect(isCompleteDSML("")).toBe(false); + }); + + test("returns false for empty string", () => { + expect(isCompleteDSML("")).toBe(false); + }); +}); + +describe("DSMLAccumulator", () => { + test("accumulates partial DSML across chunks", () => { + const acc = createDSMLAccumulator(); + + // Chunk 1: partial + const r1 = acc.add("\n\n"); + expect(r1.result).toBeNull(); + expect(r1.consumed).toBe(0); + + // Chunk 2: still partial + const r2 = acc.add('ls -la\n'); + expect(r2.result).toBeNull(); + expect(r2.consumed).toBe(0); + + // Chunk 3: complete + const r3 = acc.add("\n"); + expect(r3.result).not.toBeNull(); + expect(r3.result!.toolCalls).toHaveLength(1); + expect(r3.result!.toolCalls[0].name).toBe("Bash"); + expect(r3.result!.toolCalls[0].args.command).toBe("ls -la"); + expect(r3.consumed).toBeGreaterThan(0); + }); + + test("flush returns null when no DSML accumulated", () => { + const acc = createDSMLAccumulator(); + acc.add("Hello world"); + expect(acc.flush()).toBeNull(); + }); + + test("flush returns parsed DSML when add left unparsed content", () => { + const acc = createDSMLAccumulator(); + // Simulate a scenario where DSML was partially buffered but not complete via add + acc.add(""); + + // Now complete it with a direct accumulation (simulating stream end) + const result = acc.flush(); + expect(result).toBeNull(); // not complete yet — missing close tags + }); + + test("add returns parsed DSML when complete in single chunk", () => { + const acc = createDSMLAccumulator(); + const result = acc.add("y").result; + expect(result).not.toBeNull(); + expect(result!.toolCalls).toHaveLength(1); + expect(result!.toolCalls[0].name).toBe("Test"); + }); + + test("handles non-DSML text then DSML", () => { + const acc = createDSMLAccumulator(); + + // Non-DSML chunks first + const r1 = acc.add("Hello "); + expect(r1.result).toBeNull(); + const r2 = acc.add("world"); + expect(r2.result).toBeNull(); + + // DSML starts + const r3 = acc.add("ls"); + expect(r3.result).not.toBeNull(); + expect(r3.result!.toolCalls).toHaveLength(1); + expect(r3.result!.textBefore).toBe("Hello world"); + }); + + test("buffer overflow flushes as non-DSML", () => { + const acc = createDSMLAccumulator(); + const bigString = "a".repeat(60_000); + const result = acc.add(bigString); + expect(result.result).toBeNull(); + expect(result.consumed).toBe(60_000); + }); +}); diff --git a/src/lib/dsml-parser.ts b/src/lib/dsml-parser.ts new file mode 100644 index 0000000..c87bb39 --- /dev/null +++ b/src/lib/dsml-parser.ts @@ -0,0 +1,266 @@ +/** + * DSML (DeepSeek Markup Language) parser. + * + * DeepSeek models return tool calls embedded in text content using + * DSML markup instead of structured JSON fields. This module detects + * and parses that markup into a format the proxy can convert into + * standard tool_use (Anthropic) / tool_calls (OpenAI) blocks. + * + * DSML format (DeepSeek output): + * ``` + * + * + * value1 + * value2 + * ... + * + * + * ``` + * + * The markup can appear: + * - As the entire response content + * - After thinking/reasoning text (e.g. `......`) + * - Mixed with regular text + * - Spanning multiple SSE chunks during streaming + */ + +// --- Types ------------------------------------------------------------------- + +export interface ParsedDSML { + /** Text that appeared before the first DSML tag (may include thinking). */ + textBefore: string; + /** Parsed tool call definitions. */ + toolCalls: ToolCallDef[]; + /** Text that appeared after the last DSML tag. */ + textAfter: string; +} + +export interface ToolCallDef { + /** The tool/function name, e.g. "Bash", "Read", "Edit". */ + name: string; + /** The parsed JSON arguments object. */ + args: Record; + /** + * Raw argument map (string->string before JSON parse). + * Only populated when `args` could not be fully parsed. + */ + rawArgs?: Record; +} + +// --- Constants --------------------------------------------------------------- + +/** Maximum bytes to buffer when detecting DSML in a stream before giving up. */ +export const MAX_DSML_BUFFER = 50_000; + +// --- Regex patterns ---------------------------------------------------------- + +// Match opening (case-insensitive, with optional whitespace) +const TOOL_CALLS_OPEN_RE = /\s*/i; +// Match closing (case-insensitive) +const TOOL_CALLS_CLOSE_RE = /<\/tool_calls>/i; +// Match with optional whitespace +const INVOKE_OPEN_RE = //i; +// Match (case-insensitive) +const INVOKE_CLOSE_RE = /<\/invoke>/i; +// Match value +const PARAM_RE = /\s*([\s\S]*?)\s*<\/parameter>/gi; +// Match CDATA sections +const CDATA_RE = //g; + +// --- Parser ------------------------------------------------------------------ + +/** + * Try to parse DSML markup from a text string. + * Returns null if no DSML markup is detected. + */ +export function parseDSML(text: string): ParsedDSML | null { + if (!text || typeof text !== "string") return null; + + const openMatch = text.match(TOOL_CALLS_OPEN_RE); + if (!openMatch) return null; + + const closeMatch = text.match(TOOL_CALLS_CLOSE_RE); + if (!closeMatch) return null; + + // Extract text before the first tag + const textBefore = text.slice(0, openMatch.index!); + + // Extract the content between and + const toolCallsStart = openMatch.index! + openMatch[0].length; + const toolCallsEnd = closeMatch.index!; + const toolCallsBody = text.slice(toolCallsStart, toolCallsEnd); + + // Text after the closing + const textAfter = text.slice(toolCallsEnd + closeMatch[0].length); + + // Parse individual blocks from the tool calls body + const toolCalls = parseInvokeBlocks(toolCallsBody); + + if (toolCalls.length === 0) return null; + + return { textBefore, toolCalls, textAfter }; +} + +/** + * Extract tool calls from text by parsing DSML, then strip the DSML markup + * leaving only non-DSML content. + */ +export function stripDSML(text: string): string { + if (!text) return text; + return text + .replace(TOOL_CALLS_OPEN_RE, "") + .replace(TOOL_CALLS_CLOSE_RE, "") + .replace(INVOKE_OPEN_RE, "") + .replace(INVOKE_CLOSE_RE, "") + .replace(PARAM_RE, "") + .replace(CDATA_RE, "$1") + .trim(); +} + +/** + * Check if a text string appears to be starting DSML markup. + * Useful for stream buffering decisions. + */ +export function looksLikeDSML(text: string): boolean { + if (!text) return false; + const trimmed = text.trim(); + return ( + trimmed.startsWith("... structure is present. + */ +export function isCompleteDSML(text: string): boolean { + return TOOL_CALLS_OPEN_RE.test(text) && TOOL_CALLS_CLOSE_RE.test(text); +} + +/** + * Stream-ready DSML detection: buffered accumulator. + * Accumulate chunks until DSML is complete or buffer max is reached. + * Returns { result: ParsedDSML | null, consumed: number } where consumed + * is how many bytes of the buffer were consumed by the DSML block. + */ +export interface DSMLAccumulator { + buffer: string; + flush(): ParsedDSML | null; + add(chunk: string): { result: ParsedDSML | null; consumed: number }; +} + +export function createDSMLAccumulator(): DSMLAccumulator { + let buffer = ""; + + return { + get buffer() { + return buffer; + }, + + add(chunk: string) { + buffer += chunk; + + // If buffer exceeds max without completing DSML, flush as non-DSML + if (buffer.length > MAX_DSML_BUFFER) { + const saved = buffer; + buffer = ""; + return { result: null, consumed: saved.length }; + } + + // Only try to parse if we have a complete DSML block + if (isCompleteDSML(buffer)) { + const result = parseDSML(buffer); + if (result) { + // Calculate consumed bytes: up to the end of + const closeMatch = buffer.match(TOOL_CALLS_CLOSE_RE); + const consumed = closeMatch + ? closeMatch.index! + closeMatch[0].length + : buffer.length; + buffer = buffer.slice(consumed); + return { result, consumed }; + } + } + + // Not yet complete or not DSML + return { result: null, consumed: 0 }; + }, + + flush() { + if (!buffer) return null; + const result = isCompleteDSML(buffer) ? parseDSML(buffer) : null; + buffer = ""; + return result; + }, + }; +} + +// --- Internal helpers -------------------------------------------------------- + +/** + * Parse blocks from the body of a section. + * Returns a list of tool call definitions. + */ +function parseInvokeBlocks(body: string): ToolCallDef[] { + const results: ToolCallDef[] = []; + let remaining = body; + + // First, unwrap any CDATA sections + remaining = remaining.replace(CDATA_RE, (_, content) => content); + + while (remaining.length > 0) { + const invokeMatch = remaining.match(INVOKE_OPEN_RE); + if (!invokeMatch) break; + + const name = invokeMatch[1]; + const invokeStart = invokeMatch.index! + invokeMatch[0].length; + const closeMatch = remaining.slice(invokeStart).match(INVOKE_CLOSE_RE); + + if (!closeMatch) break; // malformed — no closing + + const paramsBody = remaining.slice(invokeStart, invokeStart + closeMatch.index!); + + // Parse parameters + const args = parseParameters(paramsBody); + + results.push({ name, args }); + + // Advance past this invoke block + remaining = remaining.slice(invokeStart + closeMatch.index! + closeMatch[0].length); + } + + return results; +} + +/** + * Parse value blocks into key-value pairs. + * Values are attempted as JSON parse, falling back to string. + */ +function parseParameters(body: string): Record { + const args: Record = {}; + let match: RegExpExecArray | null; + + // Reset regex state + PARAM_RE.lastIndex = 0; + + while ((match = PARAM_RE.exec(body)) !== null) { + const key = match[1]; + let value: unknown = match[2].trim(); + + // Try to parse the value as JSON + if (value !== "") { + try { + value = JSON.parse(value as string); + } catch { + // Keep as string — not JSON + } + } + + args[key] = value; + } + + return args; +}