fix(stream): robust reasoning/content split + flush un-tagged output

The incremental </think> search missed tags split across tokens, inverting
reasoning/content classification. Detect the first </think> on the full
buffer and track the content boundary as a byte offset. If the model never
closes </think>, flush the buffered text as content so clients always
receive the response. Chat UI now renders reasoning_content too.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
asepharyana
2026-08-03 11:03:26 +07:00
co-authored by Claude Opus 5
parent b636496497
commit 7cec411cba
4 changed files with 134 additions and 50 deletions
+40 -6
View File
@@ -72,6 +72,13 @@
display: flex; align-items: center; gap: 6px;
}
.msg .tool-call::before { content: '\1F527'; }
.msg .reasoning {
font-size: 12px; font-style: italic;
color: var(--text2);
white-space: pre-wrap; word-break: break-word;
border-bottom: 1px solid var(--border);
padding-bottom: 8px; margin-bottom: 8px;
}
.msg.error {
background: #2a1818; border-color: #4a2828; color: #f08080;
}
@@ -186,8 +193,33 @@ async function send() {
const decoder = new TextDecoder();
let buffer = '';
let full = '';
let reasoning = '';
let toolCalls = null;
// Re-render the assistant bubble: reasoning (muted) above the answer.
function render() {
el.innerHTML = '';
if (reasoning) {
const r = document.createElement('div');
r.className = 'reasoning';
r.textContent = reasoning;
el.appendChild(r);
}
if (full) {
const c = document.createElement('div');
c.textContent = full;
el.appendChild(c);
}
if (toolCalls?.length) {
for (const tc of toolCalls) {
const t = document.createElement('div');
t.className = 'tool-call';
t.textContent = 'Calling tool: ' + (tc.function?.name || 'tool');
el.appendChild(t);
}
}
}
while (true) {
const { done, value } = await reader.read();
if (done) break;
@@ -206,18 +238,20 @@ async function send() {
const delta = chunk.choices?.[0]?.delta;
const finish = chunk.choices?.[0]?.finish_reason;
if (delta?.reasoning_content) {
reasoning += delta.reasoning_content;
render();
}
if (delta?.content) {
full += delta.content;
el.textContent = full;
render();
}
if (delta?.tool_calls) {
toolCalls = delta.tool_calls;
render();
}
if (finish === 'tool_calls' && toolCalls) {
const t = document.createElement('div');
t.className = 'tool-call';
t.textContent = 'Calling tool: ' + toolCalls.map(tc => tc.function?.name).join(', ');
el.appendChild(t);
if (finish === 'tool_calls') {
render();
}
} catch (e) { /* skip malformed chunk */ }
}
+34 -7
View File
@@ -176,7 +176,9 @@ async fn handle_streaming(
let mut sampler = SendSampler(chat::build_sampler(&params));
let mut text_buf = String::new();
let mut sent_len: usize = 0;
let mut think_done = false;
// Byte offset in text_buf where the content phase begins (right after
// the first `</think>`); None while still thinking.
let mut content_start: Option<usize> = None;
let outcome = engine.generate(
&input_tokens,
@@ -187,11 +189,13 @@ async fn handle_streaming(
&mut |_token, piece| {
text_buf.push_str(piece);
// `was_thinking` is passed to split_stream_chunk so the chunk
// containing the first </think> is treated as the boundary.
let was_thinking = !think_done;
if !think_done && text_buf.contains("</think>") {
think_done = true;
// Robust boundary detection on the *full* buffer — a `</think>`
// tag may be split across tokens, which would defeat a search
// over the incremental fragment only.
if content_start.is_none() {
if let Some(pos) = text_buf.find("</think>") {
content_start = Some(pos + 8);
}
}
let new_text = &text_buf[sent_len..];
@@ -199,7 +203,8 @@ async fn handle_streaming(
return true;
}
let (reasoning, content) = chat::split_stream_chunk(new_text, was_thinking);
let (reasoning, content) =
chat::split_stream_chunk(new_text, sent_len, content_start);
if let Some(reasoning) = reasoning {
let chunk = SseChunk::delta(
@@ -251,6 +256,28 @@ async fn handle_streaming(
total_tokens: prompt_tokens + completion_tokens,
};
// If the model never emitted `</think>`, everything was streamed
// as reasoning_content. Flush it as content so the client always
// receives the response text.
if content_start.is_none() && !text_buf.is_empty() {
let cleaned = chat::strip_markup(&text_buf);
if !cleaned.is_empty() {
let chunk = SseChunk::delta(
chat_id.clone(),
created,
model_name.clone(),
SseDelta {
role: None,
content: Some(cleaned),
tool_calls: None,
reasoning_content: None,
},
);
let event = serde_json::to_string(&chunk).unwrap();
let _ = tx.blocking_send(Ok(Event::default().data(event)));
}
}
// Single-shot tool-calls delta (this model emits whole blocks).
let mut sent_tool_calls = false;
if outcome.finish == FinishReason::ToolCalls {