fix: skip leading EOS tokens in streaming + non-streaming
- Skip <|im_end|> generated as first token (prevents empty responses) - Clean <think> tags as plain text in generated output
This commit is contained in:
@@ -219,6 +219,16 @@ impl LlamaEngine {
|
|||||||
|
|
||||||
let mut current = inner.sample(sampler);
|
let mut current = inner.sample(sampler);
|
||||||
|
|
||||||
|
// Skip leading EOS tokens (like <|im_end|> as first token)
|
||||||
|
while output.is_empty() && self.model.is_eog_token(current) {
|
||||||
|
let pos = input_tokens.len() as i32 + output.len() as i32;
|
||||||
|
if let Err(e) = inner.decode(current, pos) {
|
||||||
|
tracing::info!(" Decode error: {e}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
current = inner.sample(sampler);
|
||||||
|
}
|
||||||
|
|
||||||
for _ in 0..max_tokens {
|
for _ in 0..max_tokens {
|
||||||
if self.model.is_eog_token(current) {
|
if self.model.is_eog_token(current) {
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -186,6 +186,39 @@ async fn handle_streaming(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip leading EOS tokens (like <|im_end|> at start of generation)
|
||||||
|
if state.engine.is_eog(current) && text_buf.is_empty() {
|
||||||
|
// safety bound: don't skip more than 10
|
||||||
|
if count >= max_tokens || count > 10 {
|
||||||
|
let chunk = serde_json::to_string(&SseChunk {
|
||||||
|
id: chat_id.clone(),
|
||||||
|
object: "chat.completion.chunk".into(),
|
||||||
|
created,
|
||||||
|
model: model_name.clone(),
|
||||||
|
choices: vec![SseChoice {
|
||||||
|
index: 0,
|
||||||
|
delta: SseDelta {
|
||||||
|
role: None,
|
||||||
|
content: None,
|
||||||
|
tool_calls: None,
|
||||||
|
},
|
||||||
|
finish_reason: Some("stop".into()),
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let _ = tx.send(Ok(Event::default().data(chunk))).await;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
count += 1;
|
||||||
|
let pos = input_tokens.len() as i32 + count as i32;
|
||||||
|
if let Err(e) = inner.decode(current, pos) {
|
||||||
|
info!(" Decode error: {e}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
current = inner.sample(&mut sampler);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if state.engine.is_eog(current) {
|
if state.engine.is_eog(current) {
|
||||||
let reason = if has_tools && text_buf.contains("<tool_call>") {
|
let reason = if has_tools && text_buf.contains("<tool_call>") {
|
||||||
"tool_calls"
|
"tool_calls"
|
||||||
|
|||||||
Reference in New Issue
Block a user