Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
431 changes: 307 additions & 124 deletions examples/batch_processing.rs

Large diffs are not rendered by default.

66 changes: 43 additions & 23 deletions examples/content_aggregation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,12 @@
//! cargo run --example content_aggregation
//! ```

use indextree::Arena;
use vectorless::document::NodeId;
use vectorless::retrieval::content::{
ContentAggregator, ContentAggregatorConfig, BudgetAllocator, AllocationStrategy,
StructureBuilder, OutputFormat, RelevanceScorer, ScoringStrategyConfig,
ContentChunk, ScoringContext,
AllocationStrategy, BudgetAllocator, ContentAggregator, ContentAggregatorConfig, ContentChunk,
OutputFormat, RelevanceScorer, ScoringContext, ScoringStrategyConfig, StructureBuilder,
};
use vectorless::document::NodeId;
use indextree::Arena;

fn make_node_id() -> NodeId {
let mut arena = Arena::new();
Expand Down Expand Up @@ -78,9 +77,12 @@ fn main() {
println!("\nScored chunks:");
for chunk in &chunks {
let relevance = scorer.score_chunk(chunk, &ctx);
println!(" - '{}' (depth {}): score {:.3}",
chunk.title, chunk.depth, relevance.score);
println!(" Components: keyword={:.2}, bm25={:.2}, depth_penalty={:.2}, density={:.2}",
println!(
" - '{}' (depth {}): score {:.3}",
chunk.title, chunk.depth, relevance.score
);
println!(
" Components: keyword={:.2}, bm25={:.2}, depth_penalty={:.2}, density={:.2}",
relevance.components.keyword_score,
relevance.components.bm25_score,
relevance.components.depth_penalty,
Expand All @@ -99,12 +101,14 @@ fn main() {

let strategies = vec![
("Greedy", AllocationStrategy::Greedy),
("Hierarchical (20%/level)", AllocationStrategy::Hierarchical { min_per_level: 0.2 }),
(
"Hierarchical (20%/level)",
AllocationStrategy::Hierarchical { min_per_level: 0.2 },
),
];

for (name, strategy) in strategies {
let allocator = BudgetAllocator::new(200)
.with_strategy(strategy);
let allocator = BudgetAllocator::new(200).with_strategy(strategy);

let result = allocator.allocate(scored.clone(), 2);

Expand All @@ -114,9 +118,15 @@ fn main() {
println!(" Avg score: {:.3}", result.stats.avg_score);

for content in &result.selected {
let trunc = if content.is_truncated() { " [truncated]" } else { "" };
println!(" - '{}' ({} tokens, score {:.2}){}",
content.title, content.tokens, content.score, trunc);
let trunc = if content.is_truncated() {
" [truncated]"
} else {
""
};
println!(
" - '{}' ({} tokens, score {:.2}){}",
content.title, content.tokens, content.score, trunc
);
}
}

Expand All @@ -129,16 +139,20 @@ fn main() {
("Flat", OutputFormat::Flat),
];

let allocator = BudgetAllocator::new(500)
.with_strategy(AllocationStrategy::Greedy);
let allocator = BudgetAllocator::new(500).with_strategy(AllocationStrategy::Greedy);
let result = allocator.allocate(scored.clone(), 2);

for (name, format) in formats {
let builder = StructureBuilder::new(format);
let tree = vectorless::document::DocumentTree::new("Test", "");
let structured = builder.build(result.selected.clone(), &tree);

println!("\n{} Output ({} chars, {} tokens):", name, structured.content.len(), structured.metadata.total_tokens);
println!(
"\n{} Output ({} chars, {} tokens):",
name,
structured.content.len(),
structured.metadata.total_tokens
);
let preview = if structured.content.len() > 300 {
format!("{}...", &structured.content[..300])
} else {
Expand All @@ -153,12 +167,18 @@ fn main() {

let configs = vec![
("Default (4000 tokens)", ContentAggregatorConfig::default()),
("Conservative (1000 tokens)", ContentAggregatorConfig::new()
.with_token_budget(1000)
.with_min_relevance(0.3)),
("High Precision (2000 tokens, 0.5 threshold)", ContentAggregatorConfig::new()
.with_token_budget(2000)
.with_min_relevance(0.5)),
(
"Conservative (1000 tokens)",
ContentAggregatorConfig::new()
.with_token_budget(1000)
.with_min_relevance(0.3),
),
(
"High Precision (2000 tokens, 0.5 threshold)",
ContentAggregatorConfig::new()
.with_token_budget(2000)
.with_min_relevance(0.5),
),
];

for (name, config) in configs {
Expand Down
105 changes: 61 additions & 44 deletions examples/events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
//! cargo run --example events
//! ```

use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};

use vectorless::client::{EngineBuilder, EventEmitter, IndexEvent, QueryEvent};

Expand All @@ -36,49 +36,55 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {

let events = EventEmitter::new()
// Index events
.on_index(move |e| {
match e {
IndexEvent::Started { path } => {
println!(" [INDEX] Started: {}", path);
}
IndexEvent::FormatDetected { format } => {
println!(" [INDEX] Format: {:?}", format);
}
IndexEvent::TreeBuilt { node_count } => {
println!(" [INDEX] Tree built: {} nodes", node_count);
}
IndexEvent::Complete { doc_id } => {
println!(" [INDEX] Complete: {}", &doc_id[..8]);
index_count_clone.fetch_add(1, Ordering::SeqCst);
}
IndexEvent::Error { message } => {
println!(" [INDEX] Error: {}", message);
}
_ => {}
.on_index(move |e| match e {
IndexEvent::Started { path } => {
println!(" [INDEX] Started: {}", path);
}
IndexEvent::FormatDetected { format } => {
println!(" [INDEX] Format: {:?}", format);
}
IndexEvent::TreeBuilt { node_count } => {
println!(" [INDEX] Tree built: {} nodes", node_count);
}
IndexEvent::Complete { doc_id } => {
println!(" [INDEX] Complete: {}", &doc_id[..8]);
index_count_clone.fetch_add(1, Ordering::SeqCst);
}
IndexEvent::Error { message } => {
println!(" [INDEX] Error: {}", message);
}
_ => {}
})
// Query events
.on_query(move |e| {
match e {
QueryEvent::Started { query } => {
println!(" [QUERY] Started: \"{}\"", query);
query_count_clone.fetch_add(1, Ordering::SeqCst);
}
QueryEvent::NodeVisited { title, score, .. } => {
println!(" [QUERY] Visited: \"{}\" (score: {:.2})", title, score);
nodes_visited_clone.fetch_add(1, Ordering::SeqCst);
}
QueryEvent::CandidateFound { node_id, score } => {
println!(" [QUERY] Candidate: {} (score: {:.2})", &node_id[..8], score);
}
QueryEvent::Complete { total_results, confidence } => {
println!(" [QUERY] Complete: {} results, confidence: {:.2}", total_results, confidence);
}
QueryEvent::Error { message } => {
println!(" [QUERY] Error: {}", message);
}
_ => {}
.on_query(move |e| match e {
QueryEvent::Started { query } => {
println!(" [QUERY] Started: \"{}\"", query);
query_count_clone.fetch_add(1, Ordering::SeqCst);
}
QueryEvent::NodeVisited { title, score, .. } => {
println!(" [QUERY] Visited: \"{}\" (score: {:.2})", title, score);
nodes_visited_clone.fetch_add(1, Ordering::SeqCst);
}
QueryEvent::CandidateFound { node_id, score } => {
println!(
" [QUERY] Candidate: {} (score: {:.2})",
&node_id[..8],
score
);
}
QueryEvent::Complete {
total_results,
confidence,
} => {
println!(
" [QUERY] Complete: {} results, confidence: {:.2}",
total_results, confidence
);
}
QueryEvent::Error { message } => {
println!(" [QUERY] Error: {}", message);
}
_ => {}
});

println!(" ✓ Event handlers configured\n");
Expand Down Expand Up @@ -122,7 +128,9 @@ The event system uses handlers that can be attached to the engine builder.
// 4. Query the document (events will fire)
println!("Step 4: Querying document (watch events)...\n");

let result = engine.query(&doc_id, "What features are available?").await?;
let result = engine
.query(&doc_id, "What features are available?")
.await?;
println!();

// 5. Show results
Expand All @@ -137,9 +145,18 @@ The event system uses handlers that can be attached to the engine builder.

// 6. Show statistics
println!("Step 6: Event statistics:");
println!(" - Index events fired: {}", index_count.load(Ordering::SeqCst));
println!(" - Query events fired: {}", query_count.load(Ordering::SeqCst));
println!(" - Nodes visited: {}", nodes_visited.load(Ordering::SeqCst));
println!(
" - Index events fired: {}",
index_count.load(Ordering::SeqCst)
);
println!(
" - Query events fired: {}",
query_count.load(Ordering::SeqCst)
);
println!(
" - Nodes visited: {}",
nodes_visited.load(Ordering::SeqCst)
);
println!();

// 7. Cleanup
Expand Down
5 changes: 1 addition & 4 deletions examples/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,7 @@ token_budget = 4000
println!(" - Queries: {}", stats.query_count.get());
println!(" - Cache hits: {}", stats.cache_hits.get());
println!(" - Cache misses: {}", stats.cache_misses.get());
println!(
" - Cache hit rate: {:.1}%",
stats.cache_hit_rate() * 100.0
);
println!(" - Cache hit rate: {:.1}%", stats.cache_hit_rate() * 100.0);
if let Some(avg_time) = stats.avg_query_time() {
println!(" - Avg query time: {:?}", avg_time);
}
Expand Down
9 changes: 7 additions & 2 deletions examples/storage_async.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ async fn main() -> vectorless::Result<()> {
println!("2. Adding documents...");
workspace.add(&create_doc("doc-1", "Document One")).await?;
workspace.add(&create_doc("doc-2", "Document Two")).await?;
workspace.add(&create_doc("doc-3", "Document Three")).await?;
workspace
.add(&create_doc("doc-3", "Document Three"))
.await?;
println!(" ✓ Added 3 documents\n");

// 3. Concurrent access example
Expand Down Expand Up @@ -83,7 +85,10 @@ async fn main() -> vectorless::Result<()> {
let len2 = ws2.len().await;
let len3 = ws3.len().await;

println!(" ws1.len() = {}, ws2.len() = {}, ws3.len() = {}", len1, len2, len3);
println!(
" ws1.len() = {}, ws2.len() = {}, ws3.len() = {}",
len1, len2, len3
);
println!(" ✓ All clones share the same state\n");

// Cleanup
Expand Down
13 changes: 11 additions & 2 deletions examples/storage_backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
use std::collections::HashMap;
use std::sync::{Arc, RwLock};

use vectorless::Result;
use vectorless::document::DocumentTree;
use vectorless::storage::{DocumentMeta, PersistedDocument, StorageBackend, Workspace};
use vectorless::Result;

/// A simple in-memory backend with logging.
///
Expand All @@ -42,7 +42,16 @@ impl StorageBackend for LoggingMemoryBackend {
fn get(&self, key: &str) -> Result<Option<Vec<u8>>> {
let data = self.data.read().unwrap();
let result = data.get(key).cloned();
println!(" [{}] GET '{}' -> {}", self.name, key, if result.is_some() { "found" } else { "not found" });
println!(
" [{}] GET '{}' -> {}",
self.name,
key,
if result.is_some() {
"found"
} else {
"not found"
}
);
Ok(result)
}

Expand Down
33 changes: 21 additions & 12 deletions examples/storage_compression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
//! cargo run --example storage_compression
//! ```

use vectorless::storage::{GzipCodec, IdentityCodec, Codec};
use vectorless::Result;
use vectorless::storage::{Codec, GzipCodec, IdentityCodec};

fn main() -> Result<()> {
println!("=== Compression Example ===\n");
Expand All @@ -35,8 +35,10 @@ fn main() -> Result<()> {
let identity_decoded = identity.decode(&identity_encoded)?;

println!(" Encoded size: {} bytes", identity_encoded.len());
println!(" Compression ratio: {:.1}%",
(identity_encoded.len() as f64 / original.len() as f64) * 100.0);
println!(
" Compression ratio: {:.1}%",
(identity_encoded.len() as f64 / original.len() as f64) * 100.0
);
assert_eq!(original.to_vec(), identity_decoded);
println!(" ✓ Roundtrip verified\n");

Expand All @@ -47,10 +49,12 @@ fn main() -> Result<()> {
let gzip = GzipCodec::new(level);
let compressed = gzip.encode(original)?;

println!(" Level {}: {} bytes ({:.1}% of original)",
level,
compressed.len(),
(compressed.len() as f64 / original.len() as f64) * 100.0);
println!(
" Level {}: {} bytes ({:.1}% of original)",
level,
compressed.len(),
(compressed.len() as f64 / original.len() as f64) * 100.0
);
}
println!();

Expand All @@ -62,8 +66,11 @@ fn main() -> Result<()> {
let decoded = gzip.decode(&encoded)?;

assert_eq!(original.to_vec(), decoded);
println!(" ✓ Encoded {} bytes -> {} bytes",
original.len(), encoded.len());
println!(
" ✓ Encoded {} bytes -> {} bytes",
original.len(),
encoded.len()
);
println!(" ✓ Decoded back to {} bytes", decoded.len());
println!(" ✓ Data integrity verified\n");

Expand All @@ -80,9 +87,11 @@ fn main() -> Result<()> {
println!("5. Summary:");
println!(" Original: {} bytes", original.len());
println!(" Identity: {} bytes (100.0%)", identity_encoded.len());
println!(" Gzip (lvl6): {} bytes ({:.1}%)",
encoded.len(),
(encoded.len() as f64 / original.len() as f64) * 100.0);
println!(
" Gzip (lvl6): {} bytes ({:.1}%)",
encoded.len(),
(encoded.len() as f64 / original.len() as f64) * 100.0
);
println!();

println!("✓ Compression example complete!");
Expand Down
Loading
Loading