feat(lsp): implement LSP client and server management

- Added LspClient for handling communication with LSP servers, including methods for initialization, notifications, and requests.
- Introduced LspManager to manage multiple LSP server connections, allowing for connection, disconnection, and retrieval of server capabilities.
- Created tools for connecting to LSP servers, retrieving diagnostics, hover information, code completion, definitions, and references.
- Enhanced UI rendering to display token usage and settings in the overlay.
- Updated status bar to show current token usage and selected provider/model.

src/app/runtime/shortsend.rs

@@ -3,34 +3,27 @@
 //! LLM API.
 use crate::dto::chat::message::ChatMessage;
 
-const MAX_WIRE_TOKENS: usize = 2_000_000;
-const MIN_MESSAGES_BEFORE_SHAPE: usize = 20;
-const ENGAGE_HYSTERESIS: usize = 5;
-
 /// Decide whether the message list should be shaped (compacted) before
 /// sending to the LLM.
 ///
-/// Flow: skip shaping if fewer than `MIN_MESSAGES_BEFORE_SHAPE` messages
-/// → once past that threshold, use hysteresis (require 5 more messages
-/// before re-engaging if shaping is currently active) to avoid oscillation.
+/// Flow: trigger based on token estimate. If `token_estimate` exceeds
+/// `MAX_WIRE_TOKENS * 0.8`, we shape. We also apply hysteresis so it doesn't
+/// flutter.
 ///
 /// Return: `true` if shaping should be applied.
-pub fn should_shape(total_messages: usize, prev_shaped: bool) -> bool {
-    if total_messages < MIN_MESSAGES_BEFORE_SHAPE {
-        return false;
-    }
+pub fn should_shape(token_estimate: usize, max_wire_tokens: usize, prev_shaped: bool) -> bool {
     let threshold = if prev_shaped {
-        MIN_MESSAGES_BEFORE_SHAPE + ENGAGE_HYSTERESIS
+        (max_wire_tokens as f32 * 0.85) as usize
     } else {
-        MIN_MESSAGES_BEFORE_SHAPE
+        (max_wire_tokens as f32 * 0.90) as usize
     };
-    total_messages >= threshold
+    token_estimate >= threshold
 }
 
 /// Compact a long message list by dropping middle messages and inserting
 /// a summary placeholder.
 ///
-/// Flow: if the estimated token count is within budget, return messages
+/// Flow: if the estimated token count is within budget and not forced, return messages
 /// unchanged → otherwise keep the system message and the most recent
 /// messages (up to `MAX_WIRE_TOKENS / 200` of them) with a `[prior
 /// conversation compacted]` system message in between.
@@ -38,25 +31,75 @@ pub fn should_shape(total_messages: usize, prev_shaped: bool) -> bool {
 /// Why: keeps context-size overhead roughly constant regardless of
 /// session length.
 ///
-/// Return: a new Vec<ChatMessage> that preserves the first message and
-/// the tail.
-pub fn shape_messages(messages: &[ChatMessage], token_count: usize) -> Vec<ChatMessage> {
-    if token_count <= MAX_WIRE_TOKENS || messages.len() < 10 {
+pub fn shape_messages(
+    messages: &[ChatMessage],
+    token_count: usize,
+    max_wire_tokens: usize,
+    force: bool,
+    client: Option<&crate::service::provider::LlmClient>,
+) -> Vec<ChatMessage> {
+    if !force && (token_count <= max_wire_tokens || messages.len() < 5) {
         return messages.to_vec();
     }
-    let keep_recent = messages
-        .iter()
-        .rev()
-        .take(MAX_WIRE_TOKENS / 200)
-        .cloned()
-        .collect::<Vec<_>>();
-    let mut result = Vec::new();
-    if let Some(first) = messages.first() {
-        result.push(first.clone());
+    
+    let target_tokens = (max_wire_tokens as f32 * 0.70) as usize;
+    let mut current_tokens = 0;
+    let mut keep_recent = Vec::new();
+    let mut dropped_msgs = Vec::new();
+
+    // Always keep the very first message (System Prompt) which we don't count here
+    // as we just blindly preserve it later.
+    let mut msgs_to_eval = messages.to_vec();
+    let first = if !msgs_to_eval.is_empty() {
+        Some(msgs_to_eval.remove(0))
+    } else {
+        None
+    };
+
+    // Iterate backwards from the most recent to oldest
+    for m in msgs_to_eval.into_iter().rev() {
+        let text = m.content.as_deref().unwrap_or("");
+        let msg_tokens = text.len() / 4;
+        
+        if current_tokens + msg_tokens <= target_tokens {
+            current_tokens += msg_tokens;
+            keep_recent.push(m);
+        } else {
+            dropped_msgs.push(m); // These will end up in reverse chronological order
+        }
     }
-    result.push(ChatMessage::system(
-        "[prior conversation compacted]".to_string(),
-    ));
+    
+    // Reverse dropped_msgs so they are back in chronological order
+    dropped_msgs.reverse();
+    
+    let mut result = Vec::new();
+    if let Some(f) = first {
+        result.push(f);
+    }
+    
+    if !dropped_msgs.is_empty() {
+        let mut summary_text = "[prior conversation compacted]".to_string();
+        
+        if let Some(llm) = client {
+            let prompt = format!(
+                "Summarize the following dropped conversation history briefly. Focus on main goals, decisions made, and files modified, so the context is preserved for future turns. Keep it concise.\n\nHistory:\n{}",
+                dropped_msgs.iter()
+                    .map(|m| format!("[{}]: {}", if m.role == crate::dto::chat::message::Role::User { "User" } else { "Assistant" }, m.content.as_deref().unwrap_or("")))
+                    .collect::<Vec<_>>()
+                    .join("\n\n")
+            );
+            
+            let req_msgs = vec![ChatMessage::user(prompt)];
+            if let Ok(resp) = llm.chat_with_tools_non_streaming(&req_msgs, None) {
+                if let Some(content) = resp.0.content {
+                    summary_text = format!("[Summary of compacted prior conversation:\n{}\n]", content);
+                }
+            }
+        }
+        
+        result.push(ChatMessage::system(summary_text));
+    }
+    
     result.extend(keep_recent.into_iter().rev());
     result
 }