fix(stream): robust reasoning/content split + flush un-tagged output
The incremental </think> search missed tags split across tokens, inverting reasoning/content classification. Detect the first </think> on the full buffer and track the content boundary as a byte offset. If the model never closes </think>, flush the buffered text as content so clients always receive the response. Chat UI now renders reasoning_content too. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
b636496497
commit
7cec411cba
@@ -72,6 +72,13 @@
|
||||
display: flex; align-items: center; gap: 6px;
|
||||
}
|
||||
.msg .tool-call::before { content: '\1F527'; }
|
||||
.msg .reasoning {
|
||||
font-size: 12px; font-style: italic;
|
||||
color: var(--text2);
|
||||
white-space: pre-wrap; word-break: break-word;
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding-bottom: 8px; margin-bottom: 8px;
|
||||
}
|
||||
.msg.error {
|
||||
background: #2a1818; border-color: #4a2828; color: #f08080;
|
||||
}
|
||||
@@ -186,8 +193,33 @@ async function send() {
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let full = '';
|
||||
let reasoning = '';
|
||||
let toolCalls = null;
|
||||
|
||||
// Re-render the assistant bubble: reasoning (muted) above the answer.
|
||||
function render() {
|
||||
el.innerHTML = '';
|
||||
if (reasoning) {
|
||||
const r = document.createElement('div');
|
||||
r.className = 'reasoning';
|
||||
r.textContent = reasoning;
|
||||
el.appendChild(r);
|
||||
}
|
||||
if (full) {
|
||||
const c = document.createElement('div');
|
||||
c.textContent = full;
|
||||
el.appendChild(c);
|
||||
}
|
||||
if (toolCalls?.length) {
|
||||
for (const tc of toolCalls) {
|
||||
const t = document.createElement('div');
|
||||
t.className = 'tool-call';
|
||||
t.textContent = 'Calling tool: ' + (tc.function?.name || 'tool');
|
||||
el.appendChild(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
@@ -206,18 +238,20 @@ async function send() {
|
||||
const delta = chunk.choices?.[0]?.delta;
|
||||
const finish = chunk.choices?.[0]?.finish_reason;
|
||||
|
||||
if (delta?.reasoning_content) {
|
||||
reasoning += delta.reasoning_content;
|
||||
render();
|
||||
}
|
||||
if (delta?.content) {
|
||||
full += delta.content;
|
||||
el.textContent = full;
|
||||
render();
|
||||
}
|
||||
if (delta?.tool_calls) {
|
||||
toolCalls = delta.tool_calls;
|
||||
render();
|
||||
}
|
||||
if (finish === 'tool_calls' && toolCalls) {
|
||||
const t = document.createElement('div');
|
||||
t.className = 'tool-call';
|
||||
t.textContent = 'Calling tool: ' + toolCalls.map(tc => tc.function?.name).join(', ');
|
||||
el.appendChild(t);
|
||||
if (finish === 'tool_calls') {
|
||||
render();
|
||||
}
|
||||
} catch (e) { /* skip malformed chunk */ }
|
||||
}
|
||||
|
||||
@@ -176,7 +176,9 @@ async fn handle_streaming(
|
||||
let mut sampler = SendSampler(chat::build_sampler(¶ms));
|
||||
let mut text_buf = String::new();
|
||||
let mut sent_len: usize = 0;
|
||||
let mut think_done = false;
|
||||
// Byte offset in text_buf where the content phase begins (right after
|
||||
// the first `</think>`); None while still thinking.
|
||||
let mut content_start: Option<usize> = None;
|
||||
|
||||
let outcome = engine.generate(
|
||||
&input_tokens,
|
||||
@@ -187,11 +189,13 @@ async fn handle_streaming(
|
||||
&mut |_token, piece| {
|
||||
text_buf.push_str(piece);
|
||||
|
||||
// `was_thinking` is passed to split_stream_chunk so the chunk
|
||||
// containing the first </think> is treated as the boundary.
|
||||
let was_thinking = !think_done;
|
||||
if !think_done && text_buf.contains("</think>") {
|
||||
think_done = true;
|
||||
// Robust boundary detection on the *full* buffer — a `</think>`
|
||||
// tag may be split across tokens, which would defeat a search
|
||||
// over the incremental fragment only.
|
||||
if content_start.is_none() {
|
||||
if let Some(pos) = text_buf.find("</think>") {
|
||||
content_start = Some(pos + 8);
|
||||
}
|
||||
}
|
||||
|
||||
let new_text = &text_buf[sent_len..];
|
||||
@@ -199,7 +203,8 @@ async fn handle_streaming(
|
||||
return true;
|
||||
}
|
||||
|
||||
let (reasoning, content) = chat::split_stream_chunk(new_text, was_thinking);
|
||||
let (reasoning, content) =
|
||||
chat::split_stream_chunk(new_text, sent_len, content_start);
|
||||
|
||||
if let Some(reasoning) = reasoning {
|
||||
let chunk = SseChunk::delta(
|
||||
@@ -251,6 +256,28 @@ async fn handle_streaming(
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
};
|
||||
|
||||
// If the model never emitted `</think>`, everything was streamed
|
||||
// as reasoning_content. Flush it as content so the client always
|
||||
// receives the response text.
|
||||
if content_start.is_none() && !text_buf.is_empty() {
|
||||
let cleaned = chat::strip_markup(&text_buf);
|
||||
if !cleaned.is_empty() {
|
||||
let chunk = SseChunk::delta(
|
||||
chat_id.clone(),
|
||||
created,
|
||||
model_name.clone(),
|
||||
SseDelta {
|
||||
role: None,
|
||||
content: Some(cleaned),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
);
|
||||
let event = serde_json::to_string(&chunk).unwrap();
|
||||
let _ = tx.blocking_send(Ok(Event::default().data(event)));
|
||||
}
|
||||
}
|
||||
|
||||
// Single-shot tool-calls delta (this model emits whole blocks).
|
||||
let mut sent_tool_calls = false;
|
||||
if outcome.finish == FinishReason::ToolCalls {
|
||||
|
||||
Reference in New Issue
Block a user