fix(context): batasi squash_log ke tool bash saja
Ditemukan reviewer whole-branch final: looks_log_shaped murni berbasis konten (>=3 baris berpola error/warn/fail), jadi hasil grep/search yang match ke kode error-handling ikut lolos ambang itu -- padahal squash_log punya cap keras 20 error + 10 warning tanpa budget byte, diam-diam membuang match yang sah di luar cap itu. Sekarang hanya tool bash (penghasil log sungguhan) yang boleh lewat squash_log; tool lain yang kebetulan konten-nya mirip log jatuh ke squash_generic yang lebih longgar (head/tail + budget byte). Tambah test regresi yang membedakan kedua jalur lewat retensi baris terakhir. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e075eb7acc
commit
7d99cd6618
@@ -167,7 +167,14 @@ shape:
|
|||||||
lines, up to 10 highest-scored warning lines, all summary lines, plus a ±2-line context
|
lines, up to 10 highest-scored warning lines, all summary lines, plus a ±2-line context
|
||||||
window around each kept line → single `[N lines omitted]` marker for drops (not
|
window around each kept line → single `[N lines omitted]` marker for drops (not
|
||||||
comment-shaped, per rtk's own finding on LLM confusion). Applied when the output isn't
|
comment-shaped, per rtk's own finding on LLM confusion). Applied when the output isn't
|
||||||
valid JSON and has ≥3 lines matching error/warn/stack-trace patterns.
|
valid JSON, the tool is `bash`, and the output has ≥3 lines matching error/warn/stack-trace
|
||||||
|
patterns. The tool restriction (added after the final whole-branch review) matters: a `grep`
|
||||||
|
result full of matches against error-handling code trips the same ≥3-line keyword threshold
|
||||||
|
as a real build log, but `squash_log`'s hard 20-error/10-warning cap has no byte budget and
|
||||||
|
would silently drop legitimate matches past it — the wrong compressor for search results.
|
||||||
|
Only `bash` (the actual log-producing tool) routes through `squash_log`; every other tool
|
||||||
|
whose output happens to look log-shaped falls through to the gentler, byte-budgeted
|
||||||
|
`squash_generic` instead.
|
||||||
- `squash_generic(&str, budget) -> String` — importance-ranked truncation: keeps the first 10
|
- `squash_generic(&str, budget) -> String` — importance-ranked truncation: keeps the first 10
|
||||||
and last 10 lines plus any line matching a small "looks important" heuristic (non-blank,
|
and last 10 lines plus any line matching a small "looks important" heuristic (non-blank,
|
||||||
not a byte-for-byte repeat of the immediately preceding line), single `[N lines omitted]`
|
not a byte-for-byte repeat of the immediately preceding line), single `[N lines omitted]`
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
//! ever enter conversation history, dispatching by content shape.
|
//! ever enter conversation history, dispatching by content shape.
|
||||||
//!
|
//!
|
||||||
//! Flow: `apply(tool_name, output)` -> `read` tool or under the size
|
//! Flow: `apply(tool_name, output)` -> `read` tool or under the size
|
||||||
//! floor? pass through unchanged : valid JSON? `squash_json` : looks
|
//! floor? pass through unchanged : valid JSON? `squash_json` : tool is
|
||||||
//! log-shaped? `squash_log` : `squash_generic`.
|
//! `bash` and looks log-shaped? `squash_log` : `squash_generic`.
|
||||||
//!
|
//!
|
||||||
//! Why: a single large `bash`/`grep` result can dominate a
|
//! Why: a single large `bash`/`grep` result can dominate a
|
||||||
//! conversation's token budget even on its first occurrence, long
|
//! conversation's token budget even on its first occurrence, long
|
||||||
@@ -30,6 +30,17 @@ const GENERIC_BUDGET_BYTES: usize = SQUASH_FLOOR_BYTES * 2;
|
|||||||
/// agent's view of real file content.
|
/// agent's view of real file content.
|
||||||
const NEVER_SQUASH: &[&str] = &["read"];
|
const NEVER_SQUASH: &[&str] = &["read"];
|
||||||
|
|
||||||
|
/// Tools whose output the log classifier is allowed to run on.
|
||||||
|
/// `looks_log_shaped` keys purely on content (>=3 error/warn/fail-shaped
|
||||||
|
/// lines), which a `grep`/`search` result full of matches against
|
||||||
|
/// error-handling code would trip just as easily as a real build log —
|
||||||
|
/// but `squash_log` caps at 20 error + 10 warning lines with no byte
|
||||||
|
/// budget, silently dropping legitimate matches past that cap. Only
|
||||||
|
/// `bash` (the actual log-producing tool) is allowed to route through
|
||||||
|
/// it; everything else that looks log-shaped falls through to the
|
||||||
|
/// gentler, byte-budgeted `squash_generic` instead.
|
||||||
|
const LOG_SHAPED_TOOLS: &[&str] = &["bash"];
|
||||||
|
|
||||||
/// Compress a tool's raw output before it's stored in conversation
|
/// Compress a tool's raw output before it's stored in conversation
|
||||||
/// history.
|
/// history.
|
||||||
///
|
///
|
||||||
@@ -43,7 +54,7 @@ pub fn apply(tool_name: &str, output: &str) -> String {
|
|||||||
if serde_json::from_str::<serde_json::Value>(output).is_ok() {
|
if serde_json::from_str::<serde_json::Value>(output).is_ok() {
|
||||||
return squash_json(output);
|
return squash_json(output);
|
||||||
}
|
}
|
||||||
if looks_log_shaped(output) {
|
if LOG_SHAPED_TOOLS.contains(&tool_name) && looks_log_shaped(output) {
|
||||||
return squash_log(output);
|
return squash_log(output);
|
||||||
}
|
}
|
||||||
squash_generic(output, GENERIC_BUDGET_BYTES)
|
squash_generic(output, GENERIC_BUDGET_BYTES)
|
||||||
@@ -379,6 +390,32 @@ mod tests {
|
|||||||
assert!(result.len() < text.len());
|
assert!(result.len() < text.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_bash_tool_with_log_shaped_content_is_not_log_compressed() {
|
||||||
|
// A grep result whose matched lines all mention "error" would
|
||||||
|
// trip `looks_log_shaped`'s >=3-line keyword threshold just like
|
||||||
|
// a real build log — but `squash_log` caps at 20 highest-scored
|
||||||
|
// error lines with no guaranteed tail retention, silently
|
||||||
|
// dropping legitimate matches past that cap. Only `bash` is
|
||||||
|
// treated as log-shaped; `grep` must fall through to
|
||||||
|
// `squash_generic`, which always keeps the first and last 10
|
||||||
|
// lines regardless of score. With every line tied at the same
|
||||||
|
// score, a `squash_log` route would keep indices 0-19 (stable
|
||||||
|
// sort preserves original order on ties) and drop index 49 —
|
||||||
|
// so asserting the tail survives is a route-distinguishing
|
||||||
|
// check, not just a content check.
|
||||||
|
let lines: Vec<String> = (0..50)
|
||||||
|
.map(|i| format!("src/file{i}.rs:{i}: error handling for case {i}"))
|
||||||
|
.collect();
|
||||||
|
let text = lines.join("\n");
|
||||||
|
assert!(text.len() > SQUASH_FLOOR_BYTES);
|
||||||
|
|
||||||
|
let result = apply("grep", &text);
|
||||||
|
|
||||||
|
assert!(result.contains("src/file0.rs:0: error handling for case 0"), "generic keeps head");
|
||||||
|
assert!(result.contains("src/file49.rs:49: error handling for case 49"), "generic keeps tail — squash_log would have dropped this");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn generic_large_text_is_truncated_with_omission_marker() {
|
fn generic_large_text_is_truncated_with_omission_marker() {
|
||||||
let lines: Vec<String> = (0..500).map(|i| format!("line number {i} of plain output")).collect();
|
let lines: Vec<String> = (0..500).map(|i| format!("line number {i} of plain output")).collect();
|
||||||
|
|||||||
Reference in New Issue
Block a user