Skip to content

Commit b06adf4

Browse files
JSKittyclaude
andcommitted
feat: NIP-77 negentropy for MLS group sync + parallel boot
Replace O(n) per-group relay fetches with NIP-77 set reconciliation for MLS groups. Exchanges fingerprints with relays to identify only missing events, then fetches the delta — near-instant when up to date. Boot sync runs DM and MLS negentropy concurrently via tokio::join!, reducing total sync from ~70s to ~1.2s. - Add sync_mls_groups_quick() with negentropy relay racing + background straggler gap-filling (matches DM pattern) - Add load_mls_negentropy_items() with SQL-level since filtering - Modify sync_group_since_cursor to accept prefetched events (skip relay fetch for batched multi-group sync) - Remove mls_processed_events cleanup (needed for negentropy state) - Widen DM quick phase to 7 days to match MLS window - Use negentropy for single-relay reconnection sync Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 2349fbe commit b06adf4

6 files changed

Lines changed: 386 additions & 73 deletions

File tree

src-tauri/src/commands/mls.rs

Lines changed: 276 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1141,7 +1141,7 @@ pub async fn sync_mls_groups_now(
11411141

11421142
if let Some(id) = group_id {
11431143
// Sync specific group since last cursor
1144-
mls.sync_group_since_cursor(&id)
1144+
mls.sync_group_since_cursor(&id, None)
11451145
.await
11461146
.map_err(|e| e.to_string())
11471147
} else {
@@ -1163,7 +1163,7 @@ pub async fn sync_mls_groups_now(
11631163
let mut total_new: u32 = 0;
11641164

11651165
for gid in group_ids {
1166-
match mls.sync_group_since_cursor(&gid).await {
1166+
match mls.sync_group_since_cursor(&gid, None).await {
11671167
Ok((processed, new_msgs)) => {
11681168
total_processed = total_processed.saturating_add(processed);
11691169
total_new = total_new.saturating_add(new_msgs);
@@ -1187,6 +1187,279 @@ pub async fn sync_mls_groups_now(
11871187
.map_err(|e| format!("Task join error: {}", e))?
11881188
}
11891189

1190+
/// Quick MLS group sync using NIP-77 negentropy set reconciliation.
1191+
/// Exchanges fingerprints with relays to identify only missing events, then fetches
1192+
/// the delta — near-instant when already up to date. Only syncs groups active within 7 days.
1193+
pub async fn sync_mls_groups_quick() -> Result<(u32, u32), String> {
1194+
use futures_util::StreamExt;
1195+
1196+
tokio::task::spawn_blocking(move || {
1197+
let rt = tokio::runtime::Handle::current();
1198+
rt.block_on(async move {
1199+
let mls = MlsService::new_persistent_static().map_err(|e| e.to_string())?;
1200+
1201+
// Load all non-evicted groups
1202+
let groups = db::load_mls_groups().await.unwrap_or_default();
1203+
let active_groups: Vec<_> = groups.into_iter()
1204+
.filter(|g| !g.evicted)
1205+
.collect();
1206+
1207+
if active_groups.is_empty() {
1208+
println!("[MLS] Quick sync: no groups to sync");
1209+
return Ok((0, 0));
1210+
}
1211+
1212+
// Load cursors to determine which groups are recently active
1213+
let cursors = mls.read_event_cursors().await.unwrap_or_default();
1214+
let now_secs = Timestamp::now().as_secs();
1215+
let seven_days_ago = now_secs.saturating_sub(7 * 24 * 3600);
1216+
1217+
// Filter to recently-active groups:
1218+
// - Groups with a cursor last_seen_at within 7 days (had recent messages)
1219+
// - Groups created within 7 days (newly joined, need initial sync)
1220+
let recent_groups: Vec<_> = active_groups.into_iter()
1221+
.filter(|g| {
1222+
let cursor_recent = cursors.get(&g.group_id)
1223+
.map(|c| c.last_seen_at >= seven_days_ago)
1224+
.unwrap_or(false);
1225+
let created_recent = g.created_at >= seven_days_ago;
1226+
cursor_recent || created_recent
1227+
})
1228+
.collect();
1229+
1230+
if recent_groups.is_empty() {
1231+
println!("[MLS] Quick sync: no recently-active groups (within 7d)");
1232+
return Ok((0, 0));
1233+
}
1234+
1235+
// Compute earliest cursor across all recent groups for the `since` filter
1236+
let min_since = recent_groups.iter()
1237+
.map(|g| {
1238+
cursors.get(&g.group_id)
1239+
.map(|c| c.last_seen_at)
1240+
.unwrap_or_else(|| {
1241+
if g.created_at > 0 { g.created_at } else { seven_days_ago }
1242+
})
1243+
})
1244+
.min()
1245+
.unwrap_or(seven_days_ago);
1246+
1247+
let group_ids: Vec<String> = recent_groups.iter()
1248+
.map(|g| g.group_id.clone())
1249+
.collect();
1250+
1251+
// Load known MLS event IDs for negentropy fingerprinting (SQL-filtered)
1252+
let neg_items = db::load_mls_negentropy_items(Some(min_since)).unwrap_or_default();
1253+
1254+
println!("[MLS] Quick sync (negentropy): {} groups, {} known items, since={}",
1255+
recent_groups.len(), neg_items.len(), min_since);
1256+
1257+
// Build filter for negentropy reconciliation
1258+
let filter = Filter::new()
1259+
.kind(Kind::MlsGroupMessage)
1260+
.since(Timestamp::from_secs(min_since))
1261+
.custom_tags(
1262+
SingleLetterTag::lowercase(Alphabet::H),
1263+
group_ids.iter().map(|s| s.as_str()),
1264+
);
1265+
1266+
// Negentropy dry-run: exchange fingerprints to find missing events
1267+
let sync_opts = nostr_sdk::SyncOptions::new()
1268+
.direction(nostr_sdk::SyncDirection::Down)
1269+
.initial_timeout(std::time::Duration::from_secs(10))
1270+
.dry_run();
1271+
1272+
let client = NOSTR_CLIENT.get().ok_or("Nostr client not initialized")?;
1273+
1274+
// Get Relay objects for trusted relays
1275+
let relay_map = client.relays().await;
1276+
let trusted_urls = active_trusted_relays().await;
1277+
let trusted_relays: Vec<(String, nostr_sdk::Relay)> = trusted_urls.iter()
1278+
.filter_map(|url| {
1279+
let normalized = url.trim_end_matches('/');
1280+
relay_map.iter()
1281+
.find(|(u, _)| u.as_str().trim_end_matches('/') == normalized)
1282+
.map(|(_, r)| (url.to_string(), r.clone()))
1283+
})
1284+
.collect();
1285+
drop(relay_map);
1286+
1287+
if trusted_relays.is_empty() {
1288+
println!("[MLS] Quick sync: no trusted relays available");
1289+
return Ok((0, 0));
1290+
}
1291+
1292+
// Race all trusted relays — first to reconcile drives sync
1293+
let mut relay_futs = futures_util::stream::FuturesUnordered::new();
1294+
for (url, relay) in &trusted_relays {
1295+
let url = url.clone();
1296+
let relay = relay.clone();
1297+
let f = filter.clone();
1298+
let items = neg_items.clone();
1299+
let opts = sync_opts.clone();
1300+
relay_futs.push(async move {
1301+
let result = tokio::time::timeout(
1302+
std::time::Duration::from_secs(10),
1303+
relay.sync_with_items(f, items, &opts),
1304+
).await;
1305+
(url, result)
1306+
});
1307+
}
1308+
1309+
// Drain until first successful reconciliation
1310+
let mut missing_ids: Vec<EventId> = Vec::new();
1311+
let mut primary_succeeded = false;
1312+
while let Some((url, result)) = relay_futs.next().await {
1313+
match result {
1314+
Ok(Ok(recon)) => {
1315+
missing_ids = recon.remote.into_iter().collect();
1316+
println!("[MLS] Quick sync: {} reconciled, {} missing events",
1317+
url, missing_ids.len());
1318+
primary_succeeded = true;
1319+
break;
1320+
}
1321+
Ok(Err(e)) => eprintln!("[MLS] Quick sync: {} negentropy failed: {}", url, e),
1322+
Err(_) => eprintln!("[MLS] Quick sync: {} negentropy timed out (10s)", url),
1323+
}
1324+
}
1325+
1326+
// Spawn background task for remaining relays — they fill gaps silently
1327+
if primary_succeeded && !relay_futs.is_empty() {
1328+
let primary_set: std::collections::HashSet<EventId> = missing_ids.iter().copied().collect();
1329+
let bg_client = client.clone();
1330+
let bg_group_ids: Vec<String> = recent_groups.iter().map(|g| g.group_id.clone()).collect();
1331+
tokio::spawn(async move {
1332+
let mut extra_ids: Vec<EventId> = Vec::new();
1333+
while let Some((url, result)) = relay_futs.next().await {
1334+
match result {
1335+
Ok(Ok(recon)) => {
1336+
let new: Vec<EventId> = recon.remote.into_iter()
1337+
.filter(|id| !primary_set.contains(id))
1338+
.collect();
1339+
if !new.is_empty() {
1340+
println!("[MLS][BG] {} reconciled: {} additional missing events", url, new.len());
1341+
extra_ids.extend(new);
1342+
} else {
1343+
println!("[MLS][BG] {} reconciled: 0 additional", url);
1344+
}
1345+
}
1346+
Ok(Err(e)) => eprintln!("[MLS][BG] {} negentropy failed: {}", url, e),
1347+
Err(_) => eprintln!("[MLS][BG] {} timed out (10s)", url),
1348+
}
1349+
}
1350+
1351+
// Fetch + process extra events found by background relays
1352+
if !extra_ids.is_empty() {
1353+
println!("[MLS][BG] Fetching {} additional events from background relays", extra_ids.len());
1354+
match bg_client.fetch_events_from(
1355+
active_trusted_relays().await,
1356+
Filter::new().ids(extra_ids).kind(Kind::MlsGroupMessage),
1357+
std::time::Duration::from_secs(15),
1358+
).await {
1359+
Ok(events) => {
1360+
// Group by h-tag and process per group
1361+
let mut by_group: std::collections::HashMap<String, Vec<nostr_sdk::Event>> =
1362+
std::collections::HashMap::new();
1363+
for event in events {
1364+
if let Some(h_tag) = event.tags.find(TagKind::SingleLetter(
1365+
SingleLetterTag::lowercase(Alphabet::H),
1366+
)) {
1367+
if let Some(gid) = h_tag.content() {
1368+
by_group.entry(gid.to_string()).or_default().push(event);
1369+
}
1370+
}
1371+
}
1372+
if let Ok(mls) = MlsService::new_persistent_static() {
1373+
for gid in &bg_group_ids {
1374+
if let Some(group_events) = by_group.remove(gid) {
1375+
match mls.sync_group_since_cursor(gid, Some(group_events)).await {
1376+
Ok((_, new)) if new > 0 => {
1377+
println!("[MLS][BG] {} new messages for group {}", new, &gid[..8.min(gid.len())]);
1378+
}
1379+
Err(e) => eprintln!("[MLS][BG] sync failed for {}: {}", &gid[..8.min(gid.len())], e),
1380+
_ => {}
1381+
}
1382+
}
1383+
}
1384+
}
1385+
println!("[MLS][BG] Background relay sync complete");
1386+
}
1387+
Err(e) => eprintln!("[MLS][BG] Fetch error: {}", e),
1388+
}
1389+
}
1390+
});
1391+
}
1392+
1393+
if missing_ids.is_empty() {
1394+
println!("[MLS] Quick sync: no missing events (already up to date)");
1395+
return Ok((0, 0));
1396+
}
1397+
1398+
// Fetch only the missing events
1399+
let events = client
1400+
.fetch_events_from(
1401+
active_trusted_relays().await,
1402+
Filter::new()
1403+
.ids(missing_ids)
1404+
.kind(Kind::MlsGroupMessage),
1405+
std::time::Duration::from_secs(15),
1406+
)
1407+
.await
1408+
.map_err(|e| format!("MLS negentropy fetch failed: {}", e))?;
1409+
1410+
println!("[MLS] Quick sync: fetched {} missing events for {} groups",
1411+
events.len(), recent_groups.len());
1412+
1413+
// Group events by h-tag value
1414+
let mut events_by_group: std::collections::HashMap<String, Vec<nostr_sdk::Event>> =
1415+
std::collections::HashMap::new();
1416+
for event in events {
1417+
if let Some(h_tag) = event.tags.find(TagKind::SingleLetter(
1418+
SingleLetterTag::lowercase(Alphabet::H),
1419+
)) {
1420+
if let Some(gid) = h_tag.content() {
1421+
events_by_group
1422+
.entry(gid.to_string())
1423+
.or_default()
1424+
.push(event);
1425+
}
1426+
}
1427+
}
1428+
1429+
// Process each group's events through the engine (with pre-fetched events)
1430+
let mut total_processed: u32 = 0;
1431+
let mut total_new: u32 = 0;
1432+
1433+
for group in &recent_groups {
1434+
let group_events = events_by_group.remove(&group.group_id).unwrap_or_default();
1435+
if group_events.is_empty() {
1436+
continue;
1437+
}
1438+
1439+
match mls.sync_group_since_cursor(&group.group_id, Some(group_events)).await {
1440+
Ok((processed, new_msgs)) => {
1441+
total_processed = total_processed.saturating_add(processed);
1442+
total_new = total_new.saturating_add(new_msgs);
1443+
}
1444+
Err(e) => {
1445+
eprintln!("[MLS] Quick sync failed for group {}: {}", group.group_id, e);
1446+
}
1447+
}
1448+
1449+
if let Err(e) = sync_mls_group_participants(group.group_id.clone()).await {
1450+
eprintln!("[MLS] Failed to sync participants for group {}: {}", group.group_id, e);
1451+
}
1452+
}
1453+
1454+
println!("[MLS] Quick sync complete (negentropy): {} processed, {} new messages",
1455+
total_processed, total_new);
1456+
Ok((total_processed, total_new))
1457+
})
1458+
})
1459+
.await
1460+
.map_err(|e| format!("Task join error: {}", e))?
1461+
}
1462+
11901463
/// Sync the participants array for an MLS group chat with the actual members from the engine
11911464
/// This ensures chat.participants is always up-to-date
11921465
/// (Internal helper - not a Tauri command)
@@ -1556,7 +1829,7 @@ pub async fn accept_mls_welcome(welcome_event_id_hex: String) -> Result<bool, St
15561829
// Immediately prefetch recent MLS messages for this group so the chat list shows previews
15571830
// and ordering without requiring the user to open the chat. This loads a recent slice
15581831
// (48h window by default in sync_group_since_cursor) rather than full history.
1559-
match mls.sync_group_since_cursor(&nostr_group_id).await {
1832+
match mls.sync_group_since_cursor(&nostr_group_id, None).await {
15601833
Ok((processed, new_msgs)) => {
15611834
println!("[MLS] Post-accept initial sync (epoch={}): processed={}, new={}", welcome_epoch, processed, new_msgs);
15621835
// Optional: let UI know initial sync finished for this group

src-tauri/src/commands/sync.rs

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,8 @@ pub async fn fetch_messages<R: Runtime>(
172172
}
173173
}
174174

175-
// Also sync MLS group messages after single-relay reconnection
176-
if let Err(e) = crate::commands::mls::sync_mls_groups_now(None).await {
175+
// Also sync MLS group messages after single-relay reconnection (negentropy)
176+
if let Err(e) = crate::commands::mls::sync_mls_groups_quick().await {
177177
eprintln!("[Single-Relay Sync] Failed to sync MLS groups: {}", e);
178178
}
179179

@@ -414,10 +414,15 @@ pub async fn fetch_messages<R: Runtime>(
414414
} // STATE lock released — no lock held during network operations
415415

416416
// ========================================================================
417-
// Negentropy (NIP-77) set reconciliation — single-pass sync
417+
// Negentropy (NIP-77) + MLS group sync — run concurrently
418418
// ========================================================================
419419

420420
let sync_start = std::time::Instant::now();
421+
422+
// Run DM quick phase and MLS group sync concurrently
423+
let (_dm_new_messages, _mls_result) = tokio::join!(
424+
// Task A: DM Quick Phase (negentropy reconciliation)
425+
async {
421426
let mut new_messages_count: u32 = 0;
422427

423428
// Load our known wrapper IDs + timestamps for reconciliation fingerprinting
@@ -426,9 +431,9 @@ pub async fn fetch_messages<R: Runtime>(
426431
println!("[Sync] Loaded {} negentropy items ({} with valid timestamps)",
427432
negentropy_items.len(), valid_ts_count);
428433

429-
// Quick phase: last 2 days — tiny item set for near-instant reconciliation.
434+
// Quick phase: last 7 days — small item set for near-instant reconciliation.
430435
// Shows recent offline messages within ~1s. Full archive sync runs in background after.
431-
let quick_since = Timestamp::now().as_secs().saturating_sub(2 * 24 * 3600);
436+
let quick_since = Timestamp::now().as_secs().saturating_sub(7 * 24 * 3600);
432437
let quick_items: Vec<(EventId, Timestamp)> = negentropy_items.iter()
433438
.filter(|(_, ts)| ts.as_secs() >= quick_since)
434439
.cloned()
@@ -437,7 +442,7 @@ pub async fn fetch_messages<R: Runtime>(
437442
.pubkey(my_public_key)
438443
.kind(Kind::GiftWrap)
439444
.since(Timestamp::from_secs(quick_since));
440-
println!("[Sync] Quick phase: {} items (last 2d), full: {}", quick_items.len(), negentropy_items.len());
445+
println!("[Sync] Quick phase: {} items (last 7d), full: {}", quick_items.len(), negentropy_items.len());
441446

442447
// Dry-run negentropy reconciliation — exchange fingerprints only
443448
// This identifies which events the relay has that we don't, without transferring data.
@@ -698,6 +703,17 @@ pub async fn fetch_messages<R: Runtime>(
698703
// Quick phase done — recent messages visible to user
699704
println!("[Sync] Quick phase: {:.2?}, {} new messages", sync_start.elapsed(), new_messages_count);
700705

706+
new_messages_count
707+
},
708+
// Task B: MLS Quick Sync (batched single-request fetch for recently-active groups)
709+
async {
710+
let mls_start = std::time::Instant::now();
711+
if let Err(e) = crate::commands::mls::sync_mls_groups_quick().await {
712+
eprintln!("[Sync] Parallel MLS group sync failed: {}", e);
713+
}
714+
println!("[Sync] MLS group sync: {:.2?}", mls_start.elapsed());
715+
}
716+
);
701717
// ========================================================================
702718
// Archive sync — full negentropy reconciliation (drives sync UI)
703719
// ========================================================================

0 commit comments

Comments
 (0)