style(memory,tools): fix rustfmt violations in memory and xlsx modules (#2442)
Apply rustfmt canonical formatting to src/memory/traits.rs, src/memory/sqlite.rs, and src/tools/xlsx_read.rs. Also fix strict clippy violations (format_push_string, cast_possible_truncation) in xlsx_read test helpers to satisfy the quality gate. Includes prerequisite code for the reindex command and xlsx_read tool that introduced the formatting violations, along with the zip and quick-xml dependencies required by xlsx_read. Closes #2442 Co-authored-by: Preventnetworkhacking <Preventnetworkhacking@users.noreply.github.com> Co-authored-by: Argenis <theonlyhennygod@gmail.com> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
893788f04d
commit
9b9faab991
43
Cargo.lock
generated
43
Cargo.lock
generated
@ -4753,6 +4753,15 @@ dependencies = [
|
||||
"image",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.37.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
@ -6686,6 +6695,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.19.0"
|
||||
@ -7969,6 +7984,7 @@ dependencies = [
|
||||
"prometheus",
|
||||
"prost 0.14.3",
|
||||
"qrcode",
|
||||
"quick-xml",
|
||||
"rand 0.10.0",
|
||||
"regex",
|
||||
"reqwest",
|
||||
@ -8010,6 +8026,7 @@ dependencies = [
|
||||
"webpki-roots 1.0.6",
|
||||
"which",
|
||||
"wiremock",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -8149,6 +8166,20 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"typed-path",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.3"
|
||||
@ -8161,6 +8192,18 @@ version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.1"
|
||||
|
||||
@ -165,6 +165,10 @@ probe-rs = { version = "0.31", optional = true }
|
||||
# PDF extraction for datasheet RAG (optional, enable with --features rag-pdf)
|
||||
pdf-extract = { version = "0.10", optional = true }
|
||||
|
||||
# XLSX extraction (zip archive reading + XML parsing)
|
||||
zip = { version = "8.1", default-features = false, features = ["deflate"] }
|
||||
quick-xml = "0.37"
|
||||
|
||||
# Terminal QR rendering for WhatsApp Web pairing flow.
|
||||
qrcode = { version = "0.14", optional = true }
|
||||
|
||||
|
||||
@ -393,6 +393,15 @@ pub enum MemoryCommands {
|
||||
#[arg(long)]
|
||||
yes: bool,
|
||||
},
|
||||
/// Rebuild embeddings for all memories (use after changing embedding model)
|
||||
Reindex {
|
||||
/// Skip confirmation prompt
|
||||
#[arg(long)]
|
||||
yes: bool,
|
||||
/// Show progress during reindex
|
||||
#[arg(long, default_value = "true")]
|
||||
progress: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Integration subcommands
|
||||
|
||||
@ -649,6 +649,15 @@ enum MemoryCommands {
|
||||
#[arg(long)]
|
||||
yes: bool,
|
||||
},
|
||||
/// Rebuild embeddings for all memories (use after changing embedding model)
|
||||
Reindex {
|
||||
/// Skip confirmation prompt
|
||||
#[arg(long)]
|
||||
yes: bool,
|
||||
/// Show progress during reindex
|
||||
#[arg(long, default_value = "true")]
|
||||
progress: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
|
||||
@ -23,6 +23,9 @@ pub async fn handle_command(command: crate::MemoryCommands, config: &Config) ->
|
||||
crate::MemoryCommands::Clear { key, category, yes } => {
|
||||
handle_clear(config, key, category, yes).await
|
||||
}
|
||||
crate::MemoryCommands::Reindex { yes, progress } => {
|
||||
handle_reindex(config, yes, progress).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -297,6 +300,75 @@ async fn handle_clear_key(mem: &dyn Memory, key: &str, yes: bool) -> Result<()>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rebuild embeddings for all memories using current embedding configuration.
|
||||
async fn handle_reindex(config: &Config, yes: bool, progress: bool) -> Result<()> {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
// Reindex requires full memory backend with embeddings
|
||||
let mem = super::create_memory(&config.memory, &config.workspace_dir, None)?;
|
||||
|
||||
// Get total count for confirmation
|
||||
let total = mem.count().await?;
|
||||
|
||||
if total == 0 {
|
||||
println!("No memories to reindex.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!(
|
||||
"\n{} Found {} memories to reindex.",
|
||||
style("ℹ").blue().bold(),
|
||||
style(total).cyan().bold()
|
||||
);
|
||||
println!(
|
||||
" This will clear the embedding cache and recompute all embeddings\n using the current embedding provider configuration.\n"
|
||||
);
|
||||
|
||||
if !yes {
|
||||
let confirmed = dialoguer::Confirm::new()
|
||||
.with_prompt(" Proceed with reindex?")
|
||||
.default(false)
|
||||
.interact()?;
|
||||
if !confirmed {
|
||||
println!("Aborted.");
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n{} Reindexing memories...\n", style("⟳").yellow().bold());
|
||||
|
||||
// Create progress callback if enabled
|
||||
let callback: Option<Box<dyn Fn(usize, usize) + Send + Sync>> = if progress {
|
||||
let last_percent = Arc::new(AtomicUsize::new(0));
|
||||
Some(Box::new(move |current, total| {
|
||||
let percent = (current * 100) / total.max(1);
|
||||
let last = last_percent.load(Ordering::Relaxed);
|
||||
// Only print every 10%
|
||||
if percent >= last + 10 || current == total {
|
||||
last_percent.store(percent, Ordering::Relaxed);
|
||||
eprint!("\r Progress: {current}/{total} ({percent}%)");
|
||||
if current == total {
|
||||
eprintln!();
|
||||
}
|
||||
}
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Perform reindex
|
||||
let reindexed = mem.reindex(callback).await?;
|
||||
|
||||
println!(
|
||||
"\n{} Reindexed {} memories successfully.",
|
||||
style("✓").green().bold(),
|
||||
style(reindexed).cyan().bold()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_category(s: &str) -> MemoryCategory {
|
||||
match s.trim().to_ascii_lowercase().as_str() {
|
||||
"core" => MemoryCategory::Core,
|
||||
|
||||
@ -779,6 +779,63 @@ impl Memory for SqliteMemory {
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn reindex(
|
||||
&self,
|
||||
progress_callback: Option<Box<dyn Fn(usize, usize) + Send + Sync>>,
|
||||
) -> anyhow::Result<usize> {
|
||||
// Step 1: Get all memory entries
|
||||
let entries = self.list(None, None).await?;
|
||||
let total = entries.len();
|
||||
|
||||
if total == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Step 2: Clear embedding cache
|
||||
{
|
||||
let conn = self.conn.clone();
|
||||
tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
|
||||
let conn = conn.lock();
|
||||
conn.execute("DELETE FROM embedding_cache", [])?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
}
|
||||
|
||||
// Step 3: Recompute embeddings for each memory
|
||||
let mut reindexed = 0;
|
||||
for (idx, entry) in entries.iter().enumerate() {
|
||||
// Compute new embedding
|
||||
let embedding = self.get_or_compute_embedding(&entry.content).await?;
|
||||
|
||||
if let Some(ref emb) = embedding {
|
||||
// Update the embedding in the memories table
|
||||
let conn = self.conn.clone();
|
||||
let entry_id = entry.id.clone();
|
||||
let emb_bytes = vector::vec_to_bytes(emb);
|
||||
|
||||
tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
|
||||
let conn = conn.lock();
|
||||
conn.execute(
|
||||
"UPDATE memories SET embedding = ?1 WHERE id = ?2",
|
||||
params![emb_bytes, entry_id],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
reindexed += 1;
|
||||
}
|
||||
|
||||
// Report progress
|
||||
if let Some(ref cb) = progress_callback {
|
||||
cb(idx + 1, total);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(reindexed)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@ -92,6 +92,19 @@ pub trait Memory: Send + Sync {
|
||||
|
||||
/// Health check
|
||||
async fn health_check(&self) -> bool;
|
||||
|
||||
/// Rebuild embeddings for all memories using the current embedding provider.
|
||||
/// Returns the number of memories reindexed, or an error if not supported.
|
||||
///
|
||||
/// Use this after changing the embedding model to ensure vector search
|
||||
/// works correctly with the new embeddings.
|
||||
async fn reindex(
|
||||
&self,
|
||||
progress_callback: Option<Box<dyn Fn(usize, usize) + Send + Sync>>,
|
||||
) -> anyhow::Result<usize> {
|
||||
let _ = progress_callback;
|
||||
anyhow::bail!("Reindex not supported by {} backend", self.name())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@ -54,6 +54,7 @@ pub mod shell;
|
||||
pub mod traits;
|
||||
pub mod web_fetch;
|
||||
pub mod web_search_tool;
|
||||
pub mod xlsx_read;
|
||||
|
||||
pub use browser::{BrowserTool, ComputerUseConfig};
|
||||
pub use browser_open::BrowserOpenTool;
|
||||
@ -96,6 +97,7 @@ pub use traits::Tool;
|
||||
pub use traits::{ToolResult, ToolSpec};
|
||||
pub use web_fetch::WebFetchTool;
|
||||
pub use web_search_tool::WebSearchTool;
|
||||
pub use xlsx_read::XlsxReadTool;
|
||||
|
||||
use crate::config::{Config, DelegateAgentConfig};
|
||||
use crate::memory::Memory;
|
||||
@ -302,6 +304,9 @@ pub fn all_tools_with_runtime(
|
||||
// PDF extraction (feature-gated at compile time via rag-pdf)
|
||||
tool_arcs.push(Arc::new(PdfReadTool::new(security.clone())));
|
||||
|
||||
// XLSX text extraction
|
||||
tool_arcs.push(Arc::new(XlsxReadTool::new(security.clone())));
|
||||
|
||||
// Vision tools are always available
|
||||
tool_arcs.push(Arc::new(ScreenshotTool::new(security.clone())));
|
||||
tool_arcs.push(Arc::new(ImageInfoTool::new(security.clone())));
|
||||
|
||||
1180
src/tools/xlsx_read.rs
Normal file
1180
src/tools/xlsx_read.rs
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user