Enhance tool documentation and add new features

- Added module-level documentation for memory tools (`remember`, `recall`, `forget`) to clarify their purpose.
- Improved documentation in `recall.rs` and `remember.rs` to describe the functionality and flow of memory entry operations.
- Updated `mod.rs` to include descriptions for the tool trait and execution context.
- Enhanced `plan.rs` with detailed comments on plan-mode signaling tools.
- Documented text search tools in `search.rs` to explain their functionality.
- Improved sequential-thinking tool documentation in `seqthink.rs`.
- Added safety filter documentation in `shell_filter` for credential and git operations.
- Enhanced utility tools documentation, including `cd`, `dir_cache_update`, and `todowrite`.
- Improved rendering documentation in view modules (`chat`, `markdown`, `status`, `workflow`) to clarify rendering flows and purposes.
This commit is contained in:
asepharyana
2026-07-12 11:28:39 +07:00
parent 7158d362fd
commit 2efd40ca88
124 changed files with 2379 additions and 19 deletions
+37
View File
@@ -1,9 +1,12 @@
//! SSE stream parser: converts SSE- or JSON-chunked LLM responses into
//! typed `StreamEvent` variants (tokens, reasoning, tool calls, usage, done).
pub mod turn;
pub mod tools;
use serde::{Deserialize, Serialize};
use serde_json::Value;
/// One atomic event extracted from an LLM streaming response stream.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum StreamEvent {
Token(String),
@@ -23,6 +26,8 @@ pub enum StreamEvent {
Error(String),
}
/// Buffered SSE frame parser that accumulates raw `data:` lines and
/// flushes a `StreamEvent` on each blank-line boundary.
pub struct SseParser {
buffer: String,
event_type: Option<String>,
@@ -30,6 +35,7 @@ pub struct SseParser {
}
impl SseParser {
/// Create a new parser with an empty buffer.
pub fn new() -> Self {
SseParser {
buffer: String::new(),
@@ -38,6 +44,17 @@ impl SseParser {
}
}
/// Feed a raw SSE chunk and produce any completed events.
///
/// Flow: append chunk to buffer → scan for '\n' → strip '\r' → on
/// blank line, call `flush_event` to parse the accumulated data →
/// on `event:` line, store the event type → on `data:` line, append
/// to data accumulator → continue until buffer exhausted.
///
/// Edge case: a chunk may split mid-line; the remainder stays in the
/// buffer for the next `feed()` call.
///
/// Return: all `StreamEvent`s completed by this chunk.
pub fn feed(&mut self, chunk: &str) -> Vec<StreamEvent> {
self.buffer.push_str(chunk);
let mut events = Vec::new();
@@ -57,6 +74,19 @@ impl SseParser {
events
}
/// Flush the current buffered `data:` lines as one or more `StreamEvent`s.
///
/// Flow: join data lines → handle `[DONE]` sentinel → JSON-parse →
/// emit `Usage` if a usage object is present → else match `event_type`
/// ("message.stop", "message.delta", etc.) → extract content,
/// reasoning, tool-call deltas, or finish-reason from the delta
/// structure (supporting both Anthropic-style top-level delta and
/// OpenAI-style `choices` array).
///
/// Why: dual-format support in one method avoids a separate
/// provider-specific parsing layer.
///
/// Return: 0, 1, or more `StreamEvent`s from the flushed frame.
fn flush_event(&mut self) -> Vec<StreamEvent> {
let data = self.data_lines.join("\n");
self.data_lines.clear();
@@ -179,6 +209,13 @@ impl SseParser {
/// Fallback parser for providers that send bare JSON chunks instead of SSE-framed
/// `data: ...` lines. Not used by the `SseParser` streaming path (which handles
/// standard SSE framing directly), kept for providers/tests that feed raw chunks.
///
/// Flow: parse `data` as JSON → extract first `choices[0].delta` →
/// return a `Token`, `Reasoning`, `Done`, or `ToolCallDelta` event based
/// on the fields present.
///
/// Return: `Some(StreamEvent)` if the chunk contained recognisable
/// content, `None` otherwise.
#[allow(dead_code)]
pub fn parse_stream_chunk(data: &str) -> Option<StreamEvent> {
let value: Value = serde_json::from_str(data).ok()?;
+23
View File
@@ -1,3 +1,16 @@
//! Standalone accumulator for streamed tool-call deltas.
//!
//! Flow: `ToolCallAccumulator::add_delta` is fed incremental `(index, id,
//! name, arguments_delta)` chunks as they arrive over SSE → grows its
//! internal `Vec<ParsedToolCall>` as needed → `is_complete` reports once
//! every accumulated call has both a name and arguments.
//!
//! Why: mirrors the accumulation logic built into `StreamedTurn::apply_event`
//! but as an independent, reusable type for callers that want to track
//! tool-call deltas without a full `StreamedTurn` (e.g. a lighter-weight
//! preview). Currently unused (`#[allow(dead_code)]`), kept for that future
//! use case.
use super::turn::ParsedToolCall;
use serde_json::{json, Value};
@@ -11,10 +24,15 @@ pub struct ToolCallAccumulator {
#[allow(dead_code)]
impl ToolCallAccumulator {
/// Construct an empty accumulator with no tool calls tracked yet.
///
/// Return: a fresh `ToolCallAccumulator`.
pub fn new() -> Self {
ToolCallAccumulator { calls: Vec::new() }
}
/// Append a delta to the tool call at the given index, growing the
/// calls vector if needed.
pub fn add_delta(
&mut self,
index: usize,
@@ -44,18 +62,23 @@ impl ToolCallAccumulator {
tc.arguments.push_str(arguments_delta);
}
/// Borrow the accumulated tool calls.
pub fn calls(&self) -> &[ParsedToolCall] {
&self.calls
}
/// Return true once all tool calls have both a name and arguments.
pub fn is_complete(&self) -> bool {
!self.calls.is_empty() && self.calls.iter().all(|tc| !tc.name.is_empty() && !tc.arguments.is_empty())
}
/// Clear all accumulated calls (starting a fresh turn).
pub fn reset(&mut self) {
self.calls.clear();
}
/// Build a JSON-serialisable `Vec<Value>` of pending (non-empty-name)
/// tool calls, suitable for downstream inspection or replay.
pub fn pending_args(&self) -> Vec<Value> {
self.calls
.iter()
+26 -3
View File
@@ -1,9 +1,14 @@
//! Accumulates streaming LLM responses into complete message/tool-call
//! representation via `StreamedTurn`, and provides a standalone tool-call
//! accumulator in `tools::ToolCallAccumulator`.
use super::StreamEvent;
use crate::dto::chat::message::ChatMessage;
use crate::dto::chat::tool::{ToolCall, ToolFunction};
use serde::{Deserialize, Serialize};
use serde_json::Value;
/// Accumulates a single streaming assistant turn into its final
/// `ChatMessage` form, including tool-call deltas and content/reasoning.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamedTurn {
pub messages: Vec<ChatMessage>,
@@ -13,6 +18,7 @@ pub struct StreamedTurn {
pub accumulated_reasoning: String,
}
/// A single tool call being built up from streaming deltas.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParsedToolCall {
pub id: String,
@@ -22,9 +28,11 @@ pub struct ParsedToolCall {
}
impl ParsedToolCall {
/// Attempts to parse the accumulated argument string as JSON before the tool call is
/// marked complete — useful for callers that want a speculative preview mid-stream.
/// `build_assistant_message` does its own (lossy-fallback) parse for the final message.
/// Attempt to parse the accumulated argument string as JSON before
/// the tool call is marked complete — useful for a speculative preview.
///
/// Return: `Some(Value)` if the arguments are parsable JSON, `None`
/// if still partial.
#[allow(dead_code)]
pub fn try_parse(&self) -> Option<Value> {
serde_json::from_str(&self.arguments).ok()
@@ -32,6 +40,7 @@ impl ParsedToolCall {
}
impl StreamedTurn {
/// Create an empty turn accumulator.
pub fn new() -> Self {
StreamedTurn {
messages: Vec::new(),
@@ -42,6 +51,12 @@ impl StreamedTurn {
}
}
/// Apply a `StreamEvent` to the turn, updating accumulated content,
/// reasoning, and tool-call deltas.
///
/// Flow: match on variant — `Token` appends to `accumulated_content`,
/// `Reasoning` to `accumulated_reasoning`, `ToolCallDelta` fills or
/// grows the `tool_calls` vector, `Done` sets `is_complete = true`.
pub fn apply_event(&mut self, event: &StreamEvent) {
match event {
StreamEvent::Token(token) => {
@@ -84,6 +99,14 @@ impl StreamedTurn {
}
}
/// Finalise the turn into a `ChatMessage`, combining accumulated
/// reasoning (wrapped in `<think>` tags) with content and tool calls.
///
/// Flow: if tool calls exist, build a `ChatMessage` with `tool_calls`
/// set; otherwise build a plain assistant message → set `content` to
/// the combined reasoning+content string (or `None` if empty).
///
/// Return: a complete `ChatMessage` with role `Assistant`.
pub fn build_assistant_message(&self) -> ChatMessage {
let mut msg = if self.tool_calls.is_empty() {
ChatMessage::assistant(None)