fix: skip leading EOS tokens in streaming + non-streaming

- Skip <|im_end|> generated as first token (prevents empty responses)
- Clean <think> tags as plain text in generated output
This commit is contained in:
Asep Haryana
2026-07-26 16:09:04 +07:00
parent 495b9ed126
commit 59c77108a5
2 changed files with 43 additions and 0 deletions
+10
View File
@@ -219,6 +219,16 @@ impl LlamaEngine {
let mut current = inner.sample(sampler);
// Skip leading EOS tokens (like <|im_end|> as first token)
while output.is_empty() && self.model.is_eog_token(current) {
let pos = input_tokens.len() as i32 + output.len() as i32;
if let Err(e) = inner.decode(current, pos) {
tracing::info!(" Decode error: {e}");
break;
}
current = inner.sample(sampler);
}
for _ in 0..max_tokens {
if self.model.is_eog_token(current) {
break;
+33
View File
@@ -186,6 +186,39 @@ async fn handle_streaming(
break;
}
// Skip leading EOS tokens (like <|im_end|> at start of generation)
if state.engine.is_eog(current) && text_buf.is_empty() {
// safety bound: don't skip more than 10
if count >= max_tokens || count > 10 {
let chunk = serde_json::to_string(&SseChunk {
id: chat_id.clone(),
object: "chat.completion.chunk".into(),
created,
model: model_name.clone(),
choices: vec![SseChoice {
index: 0,
delta: SseDelta {
role: None,
content: None,
tool_calls: None,
},
finish_reason: Some("stop".into()),
}],
})
.unwrap();
let _ = tx.send(Ok(Event::default().data(chunk))).await;
break;
}
count += 1;
let pos = input_tokens.len() as i32 + count as i32;
if let Err(e) = inner.decode(current, pos) {
info!(" Decode error: {e}");
break;
}
current = inner.sample(&mut sampler);
continue;
}
if state.engine.is_eog(current) {
let reason = if has_tools && text_buf.contains("<tool_call>") {
"tool_calls"