//! Multi-language code symbol index — functions, classes, variables, structs, //! enums, interfaces, traits, modules across all major programming languages. //! //! # Flow //! //! `rebuild(workspace)` → walk files by extension → dispatch to per-language //! extractor → merge into `SymbolIndex` → `search(query)` or `list()`. //! //! # Supported Languages //! //! | Language | Extensions | Symbols extracted | //! |-------------|-------------------|--------------------------------------------| //! | Rust | `.rs` | fn, struct, enum, trait, mod, impl, type, const, macro | //! | TypeScript | `.ts`, `.tsx` | function, class, interface, type, enum, const, variable | //! | JavaScript | `.js`, `.jsx`, `.mjs` | function, class, const, variable, module exports | //! | Python | `.py` | def, async def, class, module-level assignment | //! | Go | `.go` | func, type, struct, interface, const, var | //! | Generic | `.c`, `.h`, `.cpp`, `.hpp`, `.java`, `.rb`, `.rs` fallback | line-based heuristic | use crate::tools::{Tool, ToolCtx}; use anyhow::Result; use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::collections::HashMap; use std::sync::Mutex; use tracing::{debug, info, instrument}; // --------------------------------------------------------------------------- // Language & SymbolKind enums // --------------------------------------------------------------------------- /// Programming language of a code symbol. #[derive(Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)] pub enum Language { Rust, TypeScript, JavaScript, Python, Go, Other, } impl std::fmt::Display for Language { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Language::Rust => write!(f, "rust"), Language::TypeScript => write!(f, "typescript"), Language::JavaScript => write!(f, "javascript"), Language::Python => write!(f, "python"), Language::Go => write!(f, "go"), Language::Other => write!(f, "other"), } } } /// The kind of a code symbol. #[derive(Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)] pub enum SymbolKind { Function, Struct, Enum, Trait, /// module, namespace, package Module, /// impl block (Rust-specific) Impl, /// type alias Type, /// const value Constant, Macro, /// Class (TS/JS/Python/Go) Class, /// Interface (TS/Go) Interface, /// Variable (top-level let/var/assignment) Variable, Other, } impl std::fmt::Display for SymbolKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { SymbolKind::Function => write!(f, "fn"), SymbolKind::Struct => write!(f, "struct"), SymbolKind::Enum => write!(f, "enum"), SymbolKind::Trait => write!(f, "trait"), SymbolKind::Module => write!(f, "mod"), SymbolKind::Impl => write!(f, "impl"), SymbolKind::Type => write!(f, "type"), SymbolKind::Constant => write!(f, "const"), SymbolKind::Macro => write!(f, "macro"), SymbolKind::Class => write!(f, "class"), SymbolKind::Interface => write!(f, "interface"), SymbolKind::Variable => write!(f, "var"), SymbolKind::Other => write!(f, "symbol"), } } } /// A single code symbol entry in the index. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CodeSymbol { /// Symbol name (e.g. "run_agent", "AppStateRest", "main"). pub name: String, /// Kind of symbol. pub kind: SymbolKind, /// Language this symbol was extracted from. pub language: Language, /// File path relative to workspace root. pub file: String, /// Line number (1-indexed). pub line: usize, /// Parent symbol (e.g. struct name for impl methods). pub parent: Option, /// Doc comment or leading comment text, if any. pub doc_comment: Option, /// Short context (the declaration line). pub context: String, } /// The in-memory symbol index, shared via a global static. static SYMBOL_INDEX: Mutex> = Mutex::new(None); // --------------------------------------------------------------------------- // Language-specific regexes (lazily compiled) // --------------------------------------------------------------------------- struct LangRegexes { // Rust rust_fn: Regex, rust_struct: Regex, rust_enum: Regex, rust_trait: Regex, rust_mod: Regex, rust_impl: Regex, rust_type: Regex, rust_const: Regex, rust_macro: Regex, // TypeScript / JavaScript ts_fn: Regex, ts_class: Regex, ts_interface: Regex, ts_type: Regex, ts_enum: Regex, ts_var_export: Regex, // Python py_def: Regex, py_class: Regex, py_async_def: Regex, // Go go_func: Regex, go_type: Regex, go_struct: Regex, go_interface: Regex, go_const: Regex, go_var: Regex, } impl LangRegexes { fn new() -> Self { LangRegexes { // Rust rust_fn: Regex::new( r"(?m)^\s*(?:pub\s+)?(?:(?:unsafe\s+)?async\s+)?fn\s+(\w+)", ) .expect("rust fn regex"), rust_struct: Regex::new(r"(?m)^\s*(?:pub\s+)?struct\s+(\w+)") .expect("rust struct regex"), rust_enum: Regex::new(r"(?m)^\s*(?:pub\s+)?enum\s+(\w+)") .expect("rust enum regex"), rust_trait: Regex::new( r"(?m)^\s*(?:pub\s+)?(?:(?:unsafe\s+)?)?trait\s+(\w+)", ) .expect("rust trait regex"), rust_mod: Regex::new(r"(?m)^\s*(?:pub\s+)?mod\s+(\w+)") .expect("rust mod regex"), rust_impl: Regex::new( r"(?m)^\s*(?:pub\s+)?(?:unsafe\s+)?impl(?:\s*<[^>]*>)?\s+(?:for\s+)?(\w+)", ) .expect("rust impl regex"), rust_type: Regex::new(r"(?m)^\s*(?:pub\s+)?type\s+(\w+)") .expect("rust type regex"), rust_const: Regex::new(r"(?m)^\s*(?:pub\s+)?const\s+(\w+)") .expect("rust const regex"), rust_macro: Regex::new( r"(?m)^\s*(?:pub\s+)?macro_rules!\s*\(\s*(\w+)", ) .expect("rust macro regex"), // TypeScript/JavaScript ts_fn: Regex::new( r"(?m)^\s*(?:export\s+)?(?:(?:async\s+)?function\s+|(?:public|private|protected)\s+)?(\w+)\s*(?:\(|=\s*(?:async\s+)?\()", ) .expect("ts fn regex"), ts_class: Regex::new( r"(?m)^\s*(?:export\s+)?(?:abstract\s+)?class\s+(\w+)", ) .expect("ts class regex"), ts_interface: Regex::new( r"(?m)^\s*(?:export\s+)?interface\s+(\w+)", ) .expect("ts interface regex"), ts_type: Regex::new( r"(?m)^\s*(?:export\s+)?type\s+(\w+)\s*=", ) .expect("ts type regex"), ts_enum: Regex::new(r"(?m)^\s*(?:export\s+)?enum\s+(\w+)") .expect("ts enum regex"), ts_var_export: Regex::new( r"(?m)^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*(?::\s*\w+\s*)?=", ) .expect("ts var regex"), // Python py_def: Regex::new(r"(?m)^\s*def\s+(\w+)").expect("py def regex"), py_class: Regex::new(r"(?m)^\s*class\s+(\w+)") .expect("py class regex"), py_async_def: Regex::new(r"(?m)^\s*async\s+def\s+(\w+)") .expect("py async def regex"), // Go go_func: Regex::new( r"(?m)^\s*func\s+(?:\([^)]*\)\s+)?(\w+)", ) .expect("go func regex"), go_type: Regex::new(r"(?m)^\s*type\s+(\w+)") .expect("go type regex"), go_struct: Regex::new(r"(?m)^\s*type\s+(\w+)\s+struct") .expect("go struct regex"), go_interface: Regex::new( r"(?m)^\s*type\s+(\w+)\s+interface", ) .expect("go interface regex"), go_const: Regex::new(r"(?m)^\s*const\s+(\w+)") .expect("go const regex"), go_var: Regex::new(r"(?m)^\s*var\s+(\w+)") .expect("go var regex"), } } } static LANG_REGEXES: std::sync::OnceLock = std::sync::OnceLock::new(); fn regexes() -> &'static LangRegexes { LANG_REGEXES.get_or_init(LangRegexes::new) } // --------------------------------------------------------------------------- // SymbolIndex // --------------------------------------------------------------------------- /// A symbol index cache. #[derive(Debug, Clone)] pub struct SymbolIndex { symbols: Vec, workspace_path: Option, } impl SymbolIndex { pub fn new() -> Self { SymbolIndex { symbols: Vec::new(), workspace_path: None, } } pub fn is_empty(&self) -> bool { self.symbols.is_empty() } pub fn len(&self) -> usize { self.symbols.len() } /// Rebuild the index by walking the workspace and extracting symbols /// from all supported languages. pub fn rebuild(&mut self, workspace: &str) -> Result { let path = std::path::Path::new(workspace); if !path.exists() { anyhow::bail!("workspace path does not exist: {workspace}"); } let mut symbols = Vec::new(); let walker = ignore::Walk::new(path); // Pre-compile per-extension dispatch table. type ExtDispatch = HashMap<&'static str, fn(&str, &str) -> Vec>; let mut ext_dispatch = ExtDispatch::new(); ext_dispatch.insert("rs", extract_rust); ext_dispatch.insert("ts", extract_typescript); ext_dispatch.insert("tsx", extract_typescript); ext_dispatch.insert("mts", extract_typescript); ext_dispatch.insert("js", extract_javascript); ext_dispatch.insert("jsx", extract_javascript); ext_dispatch.insert("mjs", extract_javascript); ext_dispatch.insert("py", extract_python); ext_dispatch.insert("go", extract_go); for entry in walker.flatten() { let file_path = entry.path(); if !file_path.is_file() { continue; } let ext = file_path .extension() .and_then(|e| e.to_str()) .unwrap_or(""); let extractor = match ext_dispatch.get(ext) { Some(f) => f, _ => continue, // unsupported extension }; let rel_path = file_path .strip_prefix(path) .unwrap_or(file_path) .display() .to_string(); match std::fs::read_to_string(file_path) { Ok(content) => { let file_symbols = extractor(&content, &rel_path); symbols.extend(file_symbols); } Err(e) => { debug!(file = %rel_path, error = %e, "failed to read file for indexing"); } } } symbols.sort_by(|a, b| a.name.cmp(&b.name)); self.symbols = symbols; self.workspace_path = Some(workspace.to_string()); let count = self.symbols.len(); info!(symbol_count = count, workspace = %workspace, "symbol index rebuilt"); Ok(count) } /// Search indexed symbols by query. pub fn search(&self, query: &str, max_results: usize) -> Vec<&CodeSymbol> { if self.symbols.is_empty() { return Vec::new(); } let query_lower = query.to_lowercase(); let query_words: Vec = query_lower .split_whitespace() .map(|s| s.to_string()) .collect(); let mut scored: Vec<(i32, &CodeSymbol)> = self .symbols .iter() .filter_map(|sym| { let score = score_symbol(sym, &query_lower, &query_words); if score > 0 { Some((score, sym)) } else { None } }) .collect(); scored.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.name.cmp(&b.1.name))); scored .into_iter() .take(max_results) .map(|(_, sym)| sym) .collect() } /// Return all indexed symbols, optionally filtered by language and/or kind. pub fn list( &self, language_filter: Option, kind_filter: Option, file_filter: Option<&str>, max_results: usize, ) -> Vec<&CodeSymbol> { let iter: Vec<&CodeSymbol> = self .symbols .iter() .filter(|s| { language_filter.as_ref().is_none_or(|l| s.language == *l) && kind_filter.as_ref().is_none_or(|k| s.kind == *k) && file_filter.is_none_or(|f| s.file.contains(f)) }) .take(max_results) .collect(); iter } /// Count symbols by language. pub fn count_by_language(&self) -> Vec<(Language, usize)> { let mut counts: HashMap = HashMap::new(); for sym in &self.symbols { *counts.entry(sym.language.clone()).or_default() += 1; } let mut sorted: Vec<(Language, usize)> = counts.into_iter().collect(); sorted.sort_by_key(|b| std::cmp::Reverse(b.1)); sorted } /// Count symbols by kind. pub fn count_by_kind(&self) -> Vec<(SymbolKind, usize)> { let mut counts: HashMap = HashMap::new(); for sym in &self.symbols { *counts.entry(sym.kind.clone()).or_default() += 1; } let mut sorted: Vec<(SymbolKind, usize)> = counts.into_iter().collect(); sorted.sort_by_key(|b| std::cmp::Reverse(b.1)); sorted } } impl Default for SymbolIndex { fn default() -> Self { Self::new() } } /// Score a symbol against a search query. fn score_symbol(sym: &CodeSymbol, query_lower: &str, query_words: &[String]) -> i32 { let name_lower = sym.name.to_lowercase(); let mut score: i32 = 0; if name_lower == *query_lower { score += 1000; } if name_lower.starts_with(query_lower) { score += 500; } if name_lower.contains(query_lower) { score += 200; } for word in query_words { if name_lower.contains(word) { score += 50; } } if let Some(ref doc) = sym.doc_comment { let doc_lower = doc.to_lowercase(); if doc_lower.contains(query_lower) { score += 30; } for word in query_words { if doc_lower.contains(word) { score += 10; } } } let context_lower = sym.context.to_lowercase(); if context_lower.contains(query_lower) { score += 20; } score } // --------------------------------------------------------------------------- // Extract doc comments helper (Rust ///) // --------------------------------------------------------------------------- fn extract_doc_comments(lines: &[&str]) -> HashMap { let mut map = HashMap::new(); let mut i = 0; while i < lines.len() { let line = lines[i].trim(); if line.starts_with("///") { let mut doc = String::new(); while i < lines.len() { let l = lines[i].trim(); if l.starts_with("///") { if !doc.is_empty() { doc.push(' '); } doc.push_str(l.trim_start_matches("///").trim()); i += 1; } else { break; } } let target_line = find_next_declaration_line(lines, i); if let Some(tl) = target_line { map.insert(tl + 1, doc); } } else { i += 1; } } map } fn find_next_declaration_line(lines: &[&str], start: usize) -> Option { lines[start..] .iter() .position(|line| { let trimmed = line.trim(); !trimmed.is_empty() && !trimmed.starts_with("///") && !trimmed.starts_with("//!") && !trimmed.starts_with('#') }) .map(|pos| start + pos) } // --------------------------------------------------------------------------- // Rust extractor // --------------------------------------------------------------------------- fn extract_rust(content: &str, rel_path: &str) -> Vec { let r = regexes(); let mut symbols = Vec::new(); let lines: Vec<&str> = content.lines().collect(); let doc_comments = extract_doc_comments(&lines); for (i, line) in lines.iter().enumerate() { let line_num = i + 1; let trimmed = line.trim(); let entries: Vec<(Option<&str>, SymbolKind, &Regex)> = vec![ (None, SymbolKind::Function, &r.rust_fn), (None, SymbolKind::Struct, &r.rust_struct), (None, SymbolKind::Enum, &r.rust_enum), (None, SymbolKind::Trait, &r.rust_trait), (None, SymbolKind::Module, &r.rust_mod), (None, SymbolKind::Type, &r.rust_type), (None, SymbolKind::Constant, &r.rust_const), (None, SymbolKind::Macro, &r.rust_macro), ]; for (_parent, kind, re) in &entries { if let Some(caps) = re.captures(trimmed) { let name = caps .get(1) .expect("capture group 1 exists by regex") .as_str() .to_string(); let doc = doc_comments.get(&line_num).cloned(); symbols.push(CodeSymbol { name, kind: kind.clone(), language: Language::Rust, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc, context: trimmed.to_string(), }); } } // Parse impl blocks for methods if let Some(caps) = r.rust_impl.captures(trimmed) { let impl_for = caps .get(1) .expect("capture group 1 exists by regex") .as_str() .to_string(); let mut brace_depth: i32 = 0; let mut started = false; for (j, l) in lines[i..].iter().enumerate() { for ch in l.chars() { match ch { '{' => { brace_depth += 1; started = true; } '}' => { brace_depth -= 1; } _ => {} } } if started && brace_depth <= 0 && j > 1 { break; } if j > 0 { let inner_line = l.trim(); if let Some(mcaps) = r.rust_fn.captures(inner_line) { let method_name = mcaps .get(1) .expect("capture group 1 exists by regex") .as_str() .to_string(); let abs_line = i + j + 1; let doc = doc_comments.get(&abs_line).cloned(); symbols.push(CodeSymbol { name: format!("{impl_for}::{method_name}"), kind: SymbolKind::Function, language: Language::Rust, file: rel_path.to_string(), line: abs_line, parent: Some(impl_for.clone()), doc_comment: doc, context: inner_line.to_string(), }); } } } } } symbols } // --------------------------------------------------------------------------- // TypeScript extractor // --------------------------------------------------------------------------- fn extract_typescript(content: &str, rel_path: &str) -> Vec { let r = regexes(); let mut symbols = Vec::new(); let lines: Vec<&str> = content.lines().collect(); // Extract leading JSDoc/TSDoc comments let doc_comments = extract_ts_doc(&lines); for (i, line) in lines.iter().enumerate() { let line_num = i + 1; let trimmed = line.trim(); let doc = doc_comments.get(&line_num).cloned(); // Functions: `function name(` or `async function name(` or `name = function(` if let Some(caps) = r.ts_fn.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); // Skip arrow lambda captures that aren't function names if !name.starts_with('(') && name != "function" && name != "async" { symbols.push(CodeSymbol { name, kind: SymbolKind::Function, language: Language::TypeScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc.clone(), context: trimmed.to_string(), }); } continue; } // Classes if let Some(caps) = r.ts_class.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Class, language: Language::TypeScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc.clone(), context: trimmed.to_string(), }); continue; } // Interfaces if let Some(caps) = r.ts_interface.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Interface, language: Language::TypeScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc.clone(), context: trimmed.to_string(), }); continue; } // Type aliases if let Some(caps) = r.ts_type.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Type, language: Language::TypeScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc.clone(), context: trimmed.to_string(), }); continue; } // Enums if let Some(caps) = r.ts_enum.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Enum, language: Language::TypeScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc.clone(), context: trimmed.to_string(), }); continue; } // Const/let/var (module-level variables) if let Some(caps) = r.ts_var_export.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); // Only capture top-level (indentation 0) or exported let is_top_level = line.starts_with(|c: char| !c.is_whitespace()) || trimmed.starts_with("export"); if is_top_level { let kind = if trimmed.contains("const ") { SymbolKind::Constant } else { SymbolKind::Variable }; symbols.push(CodeSymbol { name, kind, language: Language::TypeScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: doc, context: trimmed.to_string(), }); } } } symbols } /// Extract leading `/** ... */` JSDoc or `///` comments. fn extract_ts_doc(lines: &[&str]) -> HashMap { let mut map = HashMap::new(); let mut i = 0; while i < lines.len() { let line = lines[i].trim(); if line.starts_with("/**") || line.starts_with("///") { let mut doc = String::new(); let mut in_block = line.starts_with("/**"); if in_block { // Single-line /** ... */ if line.ends_with("*/") && line.len() > 4 { let content = line.trim_start_matches("/**").trim_end_matches("*/").trim(); if !content.is_empty() { doc.push_str(content); } in_block = false; } while in_block && i < lines.len() { let l = lines[i].trim(); let l = l.trim_start_matches('*').trim(); if l.ends_with("*/") { doc.push_str(l.trim_end_matches("*/").trim()); break; } doc.push(' '); doc.push_str(l); i += 1; } } else { // /// style while i < lines.len() { let l = lines[i].trim(); if l.starts_with("///") { if !doc.is_empty() { doc.push(' '); } doc.push_str(l.trim_start_matches("///").trim()); i += 1; } else { break; } } } let target_line = find_next_declaration_line(lines, i); if let Some(tl) = target_line { map.insert(tl + 1, doc); } } else { i += 1; } } map } // --------------------------------------------------------------------------- // JavaScript extractor (subset of TypeScript, no TS-specific syntax) // --------------------------------------------------------------------------- fn extract_javascript(content: &str, rel_path: &str) -> Vec { let r = regexes(); let mut symbols = Vec::new(); let lines: Vec<&str> = content.lines().collect(); for (i, line) in lines.iter().enumerate() { let line_num = i + 1; let trimmed = line.trim(); // Functions if let Some(caps) = r.ts_fn.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); if !name.starts_with('(') && name != "function" && name != "async" { symbols.push(CodeSymbol { name, kind: SymbolKind::Function, language: Language::JavaScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); } continue; } // Classes if let Some(caps) = r.ts_class.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Class, language: Language::JavaScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); continue; } // Module-level const/let/var if let Some(caps) = r.ts_var_export.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); let is_top_level = line.starts_with(|c: char| !c.is_whitespace()); if is_top_level { let kind = if trimmed.contains("const ") { SymbolKind::Constant } else { SymbolKind::Variable }; symbols.push(CodeSymbol { name, kind, language: Language::JavaScript, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); } } } symbols } // --------------------------------------------------------------------------- // Python extractor // --------------------------------------------------------------------------- fn extract_python(content: &str, rel_path: &str) -> Vec { let r = regexes(); let mut symbols = Vec::new(); let lines: Vec<&str> = content.lines().collect(); // Track current class for method nesting // Reset when we see a non-indented line outside a class. let mut current_class: Option = None; for (i, line) in lines.iter().enumerate() { let line_num = i + 1; let trimmed = line.trim(); let indent = line.len() - trimmed.len(); // Reset class tracking when we leave its indentation level. if current_class.is_some() && indent == 0 && !trimmed.is_empty() { current_class = None; } // Class if let Some(caps) = r.py_class.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); current_class = Some(name.clone()); symbols.push(CodeSymbol { name, kind: SymbolKind::Class, language: Language::Python, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); continue; } // Async def if let Some(caps) = r.py_async_def.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); let full_name = current_class .as_ref() .map(|c| format!("{c}.{name}")) .unwrap_or_else(|| name.clone()); symbols.push(CodeSymbol { name: full_name, kind: SymbolKind::Function, language: Language::Python, file: rel_path.to_string(), line: line_num, parent: current_class.clone(), doc_comment: None, context: trimmed.to_string(), }); continue; } // Def if let Some(caps) = r.py_def.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); // Detect __init__ or other dunder methods let full_name = current_class .as_ref() .map(|c| format!("{c}.{name}")) .unwrap_or_else(|| name.clone()); symbols.push(CodeSymbol { name: full_name, kind: SymbolKind::Function, language: Language::Python, file: rel_path.to_string(), line: line_num, parent: current_class.clone(), doc_comment: None, context: trimmed.to_string(), }); continue; } // Module-level variable assignment: `NAME = value` (UPPER_CASE = constant) let is_top_level = line.starts_with(|c: char| !c.is_whitespace()) && !trimmed.starts_with('#') && !trimmed.starts_with("def ") && !trimmed.starts_with("class ") && !trimmed.starts_with("import ") && !trimmed.starts_with("from ") && !trimmed.starts_with("@") && !trimmed.starts_with("return") && !trimmed.starts_with("if ") && !trimmed.starts_with("elif ") && !trimmed.starts_with("else:") && !trimmed.starts_with("for ") && !trimmed.starts_with("while ") && !trimmed.starts_with("try:") && !trimmed.starts_with("except") && !trimmed.starts_with("with ") && !trimmed.starts_with("raise") && !trimmed.starts_with("pass") && !trimmed.starts_with("self.") && !trimmed.starts_with("cls.") && trimmed.contains(" = ") && !trimmed.contains("=="); if is_top_level { let name = trimmed.split('=').next().unwrap_or("").trim().to_string(); if !name.is_empty() && !name.starts_with('_') && !name.contains(' ') { let kind = if name.chars().all(|c| c.is_uppercase() || c == '_') { SymbolKind::Constant } else { SymbolKind::Variable }; symbols.push(CodeSymbol { name, kind, language: Language::Python, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); } } } symbols } // --------------------------------------------------------------------------- // Go extractor // --------------------------------------------------------------------------- fn extract_go(content: &str, rel_path: &str) -> Vec { let r = regexes(); let mut symbols = Vec::new(); let lines: Vec<&str> = content.lines().collect(); for (i, line) in lines.iter().enumerate() { let line_num = i + 1; let trimmed = line.trim(); // Struct: `type Name struct {` if let Some(caps) = r.go_struct.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Struct, language: Language::Go, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); continue; } // Interface: `type Name interface {` if let Some(caps) = r.go_interface.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Interface, language: Language::Go, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); continue; } // Type alias: `type Name Xxx` if let Some(caps) = r.go_type.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); // Skip if already captured as struct/interface if !trimmed.contains(" struct") && !trimmed.contains(" interface") { symbols.push(CodeSymbol { name, kind: SymbolKind::Type, language: Language::Go, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); } continue; } // Func if let Some(caps) = r.go_func.captures(trimmed) { let name = caps.get(1).unwrap().as_str().to_string(); symbols.push(CodeSymbol { name, kind: SymbolKind::Function, language: Language::Go, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); continue; } // Const if let Some(caps) = r.go_const.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Constant, language: Language::Go, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); continue; } // Var if let Some(caps) = r.go_var.captures(trimmed) { symbols.push(CodeSymbol { name: caps.get(1).unwrap().as_str().to_string(), kind: SymbolKind::Variable, language: Language::Go, file: rel_path.to_string(), line: line_num, parent: None, doc_comment: None, context: trimmed.to_string(), }); } } symbols } // --------------------------------------------------------------------------- // Format indexed symbols as a system-prompt-style listing // --------------------------------------------------------------------------- /// Format the entire symbol index as a compact, prompt-friendly listing. /// /// The output is a markdown table grouped by language then file: /// /// ```text /// ## Indexed Symbols (342 total) /// /// ### rust (210) /// apps/domain/src/core/message.rs /// fn user, fn assistant, fn system, fn tool, fn tool_result /// enum Role /// struct ChatMessage /// ... /// ``` pub fn format_symbol_listing(index: &SymbolIndex) -> String { let mut out = String::new(); let total = index.len(); out.push_str(&format!("## Indexed Symbols ({total} total)\n\n")); let by_lang = index.count_by_language(); if by_lang.is_empty() { out.push_str("_No symbols indexed. Rebuild the index first._\n"); return out; } // Group by language → file → symbol let mut by_lang_file: std::collections::BTreeMap< String, std::collections::BTreeMap>, > = std::collections::BTreeMap::new(); for sym in &index.symbols { let lang_str = sym.language.to_string(); by_lang_file .entry(lang_str) .or_default() .entry(sym.file.clone()) .or_default() .push(sym); } for (lang, files) in &by_lang_file { let count: usize = files.values().map(|v| v.len()).sum(); out.push_str(&format!("### {lang} ({count})\n")); for (file, syms) in files { out.push_str(&format!(" {file}\n")); // Group by kind for compact listing let mut by_kind: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for sym in syms { let kind_str = sym.kind.to_string(); by_kind .entry(kind_str) .or_default() .push(sym.name.as_str()); } for (kind, names) in &by_kind { out.push_str(&format!(" {kind}: {}\n", names.join(", "))); } } out.push('\n'); } out } // --------------------------------------------------------------------------- // Tool: SemanticSearch — search the symbol index // --------------------------------------------------------------------------- /// Search for code symbols across all indexed languages. /// /// Flow: ensure index is built → search by query → return formatted results. pub struct SemanticSearch; impl Tool for SemanticSearch { fn name(&self) -> &'static str { "semantic_search" } fn description(&self) -> &'static str { "Search for code symbols (functions, structs, classes, interfaces, variables) \ by name, concept, or meaning across Rust, TypeScript, JavaScript, Python, and Go" } fn parameters(&self) -> Value { json!({ "type": "object", "properties": { "query": { "type": "string", "description": "Search query — symbol name, concept, or meaning" }, "kind": { "type": "string", "enum": ["fn", "class", "struct", "enum", "interface", "trait", "const", "var", "mod", "all"], "description": "Filter by symbol kind (default: all)", "default": "all" }, "language": { "type": "string", "enum": ["rust", "typescript", "javascript", "python", "go", "all"], "description": "Filter by language (default: all)", "default": "all" }, "max_results": { "type": "integer", "description": "Maximum results (default 10, max 30)", "default": 10 }, "rebuild_index": { "type": "boolean", "description": "Force rebuild the symbol index before searching (default false)", "default": false } }, "required": ["query"] }) } #[instrument(skip(self, ctx, args))] fn run(&self, ctx: &ToolCtx, args: &Value) -> Result { let query = crate::tools::arg_str(args, "query")?; let kind_filter = args.get("kind").and_then(|v| v.as_str()).unwrap_or("all"); let lang_filter = args .get("language") .and_then(|v| v.as_str()) .unwrap_or("all"); let max_results = args .get("max_results") .and_then(|v| v.as_u64()) .unwrap_or(10) .min(30) as usize; let rebuild = args .get("rebuild_index") .and_then(|v| v.as_bool()) .unwrap_or(false); let workspace = ctx .workspaces .first() .map(|p| p.to_string_lossy().to_string()) .unwrap_or_else(|| ".".to_string()); info!( query = %query, kind = %kind_filter, lang = %lang_filter, max_results, rebuild, "semantic search" ); let mut guard = SYMBOL_INDEX .lock() .map_err(|e| anyhow::anyhow!("index lock failed: {e}"))?; let index = guard.get_or_insert_with(SymbolIndex::new); if rebuild || index.is_empty() { let count = index.rebuild(&workspace)?; debug!(symbol_count = count, "symbol index rebuilt"); } // Map kind filter to enum let target_kind = match kind_filter { "fn" => Some(SymbolKind::Function), "class" => Some(SymbolKind::Class), "struct" => Some(SymbolKind::Struct), "enum" => Some(SymbolKind::Enum), "interface" => Some(SymbolKind::Interface), "trait" => Some(SymbolKind::Trait), "const" => Some(SymbolKind::Constant), "var" => Some(SymbolKind::Variable), "mod" => Some(SymbolKind::Module), _ => None, }; let target_lang = match lang_filter { "rust" => Some(Language::Rust), "typescript" => Some(Language::TypeScript), "javascript" => Some(Language::JavaScript), "python" => Some(Language::Python), "go" => Some(Language::Go), _ => None, }; let results = index.search(&query, max_results * 2); let filtered: Vec<&&CodeSymbol> = results .iter() .filter(|s| target_kind.as_ref().is_none_or(|k| s.kind == *k)) .filter(|s| target_lang.as_ref().is_none_or(|l| s.language == *l)) .take(max_results) .collect(); if filtered.is_empty() { return Ok(format!( "No symbols found matching '{query}'.\n\ Try a different query, or use `rebuild_index: true` to rebuild the index first.\n\ Index has {} symbols across {} languages.", index.len(), index.count_by_language().len(), )); } let total = index.len(); info!(matched = filtered.len(), total_indexed = total, "semantic search completed"); let mut by_file: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for sym in &filtered { by_file.entry(sym.file.clone()).or_default().push(*sym); } let mut output = format!( "## Semantic Search Results\n\n**Query:** {query}\n**Index size:** {total} symbols\n**Matches:** {}\n\n", filtered.len() ); for (file, symbols) in &by_file { output.push_str(&format!("### `{file}`\n\n")); for sym in symbols { let lang_str = sym.language.to_string(); let kind_str = sym.kind.to_string(); let parent_str = sym .parent .as_ref() .map(|p| format!(" [{p}]")) .unwrap_or_default(); let doc_str = sym .doc_comment .as_ref() .map(|d| { let truncated: String = d.chars().take(100).collect(); format!(" — {truncated}") }) .unwrap_or_default(); output.push_str(&format!( "- `{kind_str}` **{}**{} `[{lang_str}]` at line {} `{}`{}{}\n", sym.name, parent_str, sym.line, sym.context.trim(), doc_str, if sym.context.trim().len() > 80 { "…" } else { "" } )); } output.push('\n'); } output.push_str(&format!( "---\n*{} symbols indexed across {} languages. Use `rebuild_index: true` to refresh.*\n", total, index.count_by_language().len(), )); Ok(output) } } // --------------------------------------------------------------------------- // Tool: RebuildIndex — explicitly rebuild the symbol index // --------------------------------------------------------------------------- /// Rebuild the code symbol index for all supported languages. pub struct RebuildIndex; impl Tool for RebuildIndex { fn name(&self) -> &'static str { "rebuild_index" } fn description(&self) -> &'static str { "Rebuild the code symbol index for semantic search (supports Rust, TypeScript, JavaScript, Python, Go)" } fn parameters(&self) -> Value { json!({ "type": "object", "properties": {} }) } #[instrument(skip(self, ctx, _args))] fn run(&self, ctx: &ToolCtx, _args: &Value) -> Result { let workspace = ctx .workspaces .first() .map(|p| p.to_string_lossy().to_string()) .unwrap_or_else(|| ".".to_string()); info!("rebuilding multi-language symbol index"); let mut guard = SYMBOL_INDEX .lock() .map_err(|e| anyhow::anyhow!("index lock failed: {e}"))?; let index = guard.get_or_insert_with(SymbolIndex::new); let count = index.rebuild(&workspace)?; let by_lang = index.count_by_language(); let mut out = format!( "Symbol index rebuilt successfully. {} symbols indexed.\n\n", count ); out.push_str("By language:\n"); for (lang, c) in &by_lang { out.push_str(&format!(" {lang}: {c}\n")); } Ok(out) } } // --------------------------------------------------------------------------- // Tool: ListSymbols — list all indexed symbols // --------------------------------------------------------------------------- /// List all indexed symbols grouped by language and file. /// /// This is designed to provide a compact symbol listing for system-prompt /// context so the AI agent knows what functions, variables, and types exist. pub struct ListSymbols; impl Tool for ListSymbols { fn name(&self) -> &'static str { "list_symbols" } fn description(&self) -> &'static str { "List all indexed code symbols (functions, classes, variables, structs, interfaces, \ types, constants) across Rust, TypeScript, JavaScript, Python, and Go. \ Optionally filter by language, kind, or file path." } fn parameters(&self) -> Value { json!({ "type": "object", "properties": { "language": { "type": "string", "enum": ["rust", "typescript", "javascript", "python", "go", "all"], "description": "Filter by language (default: all)", "default": "all" }, "kind": { "type": "string", "enum": ["fn", "class", "struct", "enum", "interface", "trait", "const", "var", "mod", "all"], "description": "Filter by symbol kind (default: all)", "default": "all" }, "file": { "type": "string", "description": "Filter by file path substring (e.g. 'auth/', 'domain/')" }, "max_results": { "type": "integer", "description": "Maximum symbols to list (default 50, max 200)", "default": 50 }, "rebuild_index": { "type": "boolean", "description": "Force rebuild before listing (default false)", "default": false } } }) } #[instrument(skip(self, ctx, args))] fn run(&self, ctx: &ToolCtx, args: &Value) -> Result { let lang_filter = args .get("language") .and_then(|v| v.as_str()) .unwrap_or("all"); let kind_filter = args.get("kind").and_then(|v| v.as_str()).unwrap_or("all"); let file_filter = args.get("file").and_then(|v| v.as_str()); let max_results = args .get("max_results") .and_then(|v| v.as_u64()) .unwrap_or(50) .min(200) as usize; let rebuild = args .get("rebuild_index") .and_then(|v| v.as_bool()) .unwrap_or(false); let workspace = ctx .workspaces .first() .map(|p| p.to_string_lossy().to_string()) .unwrap_or_else(|| ".".to_string()); let mut guard = SYMBOL_INDEX .lock() .map_err(|e| anyhow::anyhow!("index lock failed: {e}"))?; let index = guard.get_or_insert_with(SymbolIndex::new); if rebuild || index.is_empty() { let count = index.rebuild(&workspace)?; info!(symbol_count = count, "symbol index rebuilt for list"); } let target_lang = match lang_filter { "rust" => Some(Language::Rust), "typescript" => Some(Language::TypeScript), "javascript" => Some(Language::JavaScript), "python" => Some(Language::Python), "go" => Some(Language::Go), _ => None, }; let target_kind = match kind_filter { "fn" => Some(SymbolKind::Function), "class" => Some(SymbolKind::Class), "struct" => Some(SymbolKind::Struct), "enum" => Some(SymbolKind::Enum), "interface" => Some(SymbolKind::Interface), "trait" => Some(SymbolKind::Trait), "const" => Some(SymbolKind::Constant), "var" => Some(SymbolKind::Variable), "mod" => Some(SymbolKind::Module), _ => None, }; let symbols = index.list(target_lang, target_kind, file_filter, max_results); let total = index.len(); let by_lang = index.count_by_language(); if symbols.is_empty() { return Ok(format!( "No symbols match the filters. Index has {total} total symbols.\n\ Languages: {}", by_lang .iter() .map(|(l, c)| format!("{l}: {c}")) .collect::>() .join(", ") )); } let mut out = format!( "## Indexed Symbols\n\n**Total:** {total} | **Showing:** {} | **Filter:** lang={lang_filter}, kind={kind_filter}\n\n", symbols.len() ); // Group by language let mut by_lang_map: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for sym in &symbols { by_lang_map .entry(sym.language.to_string()) .or_default() .push(*sym); } for (lang, syms) in &by_lang_map { out.push_str(&format!("### {lang}\n\n")); let mut by_file: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for sym in syms { by_file.entry(sym.file.clone()).or_default().push(*sym); } for (file, file_syms) in &by_file { out.push_str(&format!("`{file}`:\n")); for sym in file_syms { out.push_str(&format!( " `{}` {} L{}\n", sym.kind, sym.name, sym.line, )); } } out.push('\n'); } out.push_str("---\n"); out.push_str(&format!( "By language: {}\n", by_lang .iter() .map(|(l, c)| format!("{l}: {c}")) .collect::>() .join(", ") )); out.push_str(&format!( "By kind: {}\n", index .count_by_kind() .iter() .map(|(k, c)| format!("{k}: {c}")) .collect::>() .join(", ") )); Ok(out) } } #[cfg(test)] mod tests { use super::*; #[test] fn test_extract_rust_functions() { let content = "pub fn hello() {}\nfn world() {}\n"; let symbols = extract_rust(content, "test.rs"); assert_eq!(symbols.len(), 2); assert_eq!(symbols[0].name, "hello"); assert_eq!(symbols[1].name, "world"); } #[test] fn test_extract_rust_struct() { let content = "pub struct MyStruct {}\nstruct Private;\n"; let symbols = extract_rust(content, "test.rs"); assert!(symbols.iter().any(|s| s.name == "MyStruct")); assert!(symbols.iter().any(|s| s.name == "Private")); } #[test] fn test_extract_typescript_function_and_class() { let content = "function hello() {}\nexport class User {}\ninterface Person {}\n"; let symbols = extract_typescript(content, "test.ts"); assert!(symbols.iter().any(|s| s.name == "hello" && s.kind == SymbolKind::Function)); assert!(symbols.iter().any(|s| s.name == "User" && s.kind == SymbolKind::Class)); assert!(symbols.iter().any(|s| s.name == "Person" && s.kind == SymbolKind::Interface)); } #[test] fn test_extract_typescript_const() { let content = "export const API_URL = 'http://example.com';\nconst MAX_RETRIES = 3;\n"; let symbols = extract_typescript(content, "test.ts"); assert!(symbols.iter().any(|s| s.name == "API_URL" && s.kind == SymbolKind::Constant)); assert!(symbols.iter().any(|s| s.name == "MAX_RETRIES" && s.kind == SymbolKind::Constant)); } #[test] fn test_extract_python_def_and_class() { let content = "class MyClass:\n def method(self):\n pass\n\ndef top_func():\n pass\n"; let symbols = extract_python(content, "test.py"); assert!(symbols.iter().any(|s| s.name == "MyClass" && s.kind == SymbolKind::Class)); assert!(symbols.iter().any(|s| s.name == "MyClass.method" && s.kind == SymbolKind::Function)); assert!(symbols.iter().any(|s| s.name == "top_func" && s.kind == SymbolKind::Function)); } #[test] fn test_extract_python_variable() { let content = "DATABASE_URL = 'postgres://localhost'\nconfig_path = '/etc/app'\n"; let symbols = extract_python(content, "test.py"); assert!(symbols.iter().any(|s| s.name == "DATABASE_URL" && s.kind == SymbolKind::Constant)); assert!(symbols.iter().any(|s| s.name == "config_path" && s.kind == SymbolKind::Variable)); } #[test] fn test_extract_go_func_and_struct() { let content = "func main() {}\nfunc (s *Server) Serve() {}\ntype Config struct {\n Name string\n}\n"; let symbols = extract_go(content, "test.go"); assert!(symbols.iter().any(|s| s.name == "main" && s.kind == SymbolKind::Function)); assert!(symbols.iter().any(|s| s.name == "Serve" && s.kind == SymbolKind::Function)); assert!(symbols.iter().any(|s| s.name == "Config" && s.kind == SymbolKind::Struct)); } #[test] fn test_extract_go_const_and_var() { let content = "const VERSION = \"1.0\"\nvar DefaultPort = 8080\n"; let symbols = extract_go(content, "test.go"); assert!(symbols.iter().any(|s| s.name == "VERSION" && s.kind == SymbolKind::Constant)); assert!(symbols.iter().any(|s| s.name == "DefaultPort" && s.kind == SymbolKind::Variable)); } #[test] fn test_index_rebuild_and_search() { let dir = std::env::temp_dir().join(format!("sstest_{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); std::fs::write(dir.join("test.rs"), "pub fn search_me() {}\n").unwrap(); std::fs::write(dir.join("test.ts"), "export const FOO = 42;\n").unwrap(); let mut index = SymbolIndex::new(); let count = index .rebuild(&dir.to_string_lossy()) .expect("rebuild should succeed"); assert!(count >= 2, "should index at least 2 symbols, got {count}"); let results = index.search("search_me", 10); assert!(!results.is_empty(), "should find search_me"); let results = index.search("FOO", 10); assert!(!results.is_empty(), "should find FOO"); let by_lang = index.count_by_language(); assert!( by_lang.iter().any(|(l, _)| *l == Language::Rust), "should have Rust symbols" ); assert!( by_lang.iter().any(|(l, _)| *l == Language::TypeScript), "should have TypeScript symbols" ); } #[test] fn test_empty_extraction() { let symbols = extract_rust("// just a comment\n", "empty.rs"); assert!(symbols.is_empty()); } #[test] fn test_search_empty_index() { let index = SymbolIndex::new(); assert!(index.search("anything", 10).is_empty()); } }