fix(plan): perbaiki fixture test shaping agar benar-benar men-drop pesan

Ditemukan implementer Task 5: fixture pesan pendek (~8 token nyata
lewat tiktoken untuk 20 pesan = ~160 token) tidak pernah melebihi
target 700 token (70% dari max_wire_tokens=1000), jadi force=true pun
tidak pernah men-drop satu pesan pun -- test placeholder-summary dan
keeps-most-recent lulus secara vakum tanpa benar-benar menguji jalur
drop. Diverifikasi ulang dengan context::tokens::count_tokens nyata:
fixture baru (20 pesan ~49 token = ~980 token total) melebihi target
700 dengan nyaman.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
asepharyana
2026-07-16 07:55:48 +07:00
co-authored by Claude Sonnet 5
parent e080d9fc6b
commit 3a6e32d8f3
@@ -1057,11 +1057,29 @@ mod tests {
assert_eq!(result.len(), messages.len());
}
/// Build a message whose real BPE token count is large enough that 20
/// of them (~49 tokens each, ~980 total — verified empirically with
/// `context::tokens::count_tokens`) comfortably exceed
/// `shape_messages`'s 70%-of-1000 = 700 token target, guaranteeing
/// several get dropped. A short fixture like `format!("message {i}")`
/// (~8 tokens each, ~160 total for 20) stays entirely under budget
/// with real BPE counting and would make these tests pass vacuously
/// (nothing ever gets dropped, so "must survive shaping" and "falls
/// back to placeholder" hold trivially without exercising the actual
/// drop logic) — this was a real bug caught during Task 5's first
/// implementation attempt.
fn padded_message(i: usize) -> String {
format!(
"message number {i} with some padding text {}",
"additional padding content to increase token count substantially ".repeat(5),
)
}
#[test]
fn shape_messages_always_preserves_the_first_system_message() {
let mut messages = vec![ChatMessage::system("system prompt")];
for i in 0..20 {
messages.push(ChatMessage::user(format!("message {i}")));
messages.push(ChatMessage::user(padded_message(i)));
}
let result = shape_messages(&messages, 100_000, 1000, true, None);
assert_eq!(result[0].content.as_deref(), Some("system prompt"));
@@ -1071,7 +1089,7 @@ mod tests {
fn shape_messages_without_a_client_falls_back_to_placeholder_summary() {
let mut messages = vec![ChatMessage::system("system prompt")];
for i in 0..20 {
messages.push(ChatMessage::user(format!("message number {i} with some padding text")));
messages.push(ChatMessage::user(padded_message(i)));
}
let result = shape_messages(&messages, 100_000, 1000, true, None);
let has_placeholder = result.iter().any(|m| {
@@ -1084,7 +1102,7 @@ mod tests {
fn shape_messages_keeps_most_recent_messages_over_older_ones() {
let mut messages = vec![ChatMessage::system("system prompt")];
for i in 0..20 {
messages.push(ChatMessage::user(format!("message number {i} with some padding text")));
messages.push(ChatMessage::user(padded_message(i)));
}
let result = shape_messages(&messages, 100_000, 1000, true, None);
let last_content = messages.last().unwrap().content.clone();