Skip to content

fix(test): un-ignore four flaky tests; fix the content_status race #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonaswre merged 3 commits into from
Aug 2, 2026
Merged

fix(test): un-ignore four flaky tests; fix the content_status race #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e62c4e2
fix(test): un-ignore four flaky tests; fix the content_status race
jonaswre Aug 1, 2026
37cc03a
fix(test): enable test_roundtrip_bytes_small by dropping an unfounded…
jonaswre Aug 1, 2026
9350b1d
fix(test): drop the import orphaned by removing the addr assertion
jonaswre Aug 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion krikos/src/endpoint/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1865,7 +1865,6 @@ async fn test_bind_addr_prefix_len_0_not_default() -> Result {
Ok(())
}

#[ignore = "flaky"]
#[tokio::test]
#[traced_test]
async fn connect_via_relay_becomes_direct_and_sends_direct() -> Result {
Expand Down
30 changes: 20 additions & 10 deletions protocols/krikos-blobs/src/store/fs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1535,7 +1535,7 @@ pub mod tests {
api::blobs::Bitfield,
store::{
KRIKOS_BLOCK_SIZE,
util::{SliceInfoExt, Tag, read_checksummed, tests::create_n0_bao},
util::{Tag, read_checksummed, tests::create_n0_bao},
},
};

Expand Down Expand Up @@ -1668,7 +1668,6 @@ pub mod tests {

// import data via import_bytes, check that we can observe it and that it is complete
#[tokio::test]
#[ignore = "flaky. I need a reliable way to keep the handle alive"]
async fn test_roundtrip_bytes_small() -> TestResult<()> {
tracing_subscriber::fmt::try_init().ok();
let testdir = tempfile::tempdir()?;
Expand All @@ -1686,14 +1685,25 @@ pub mod tests {
let actual = store.get_bytes(expected_hash).await?;
// check that the data is there
assert_eq!(&expected, &actual);
assert_eq!(
&expected.addr(),
&actual.addr(),
"address mismatch for size {size}"
);
// we must at some point see completion, otherwise the test will hang
// keep the handle alive by observing until the end, otherwise the handle
// will change and the bytes won't be the same instance anymore
// Deliberately NOT asserting `expected.addr() == actual.addr()`.
//
// That asserted `get_bytes` hands back the *same allocation* that
// `add_bytes` was given. The store makes no such promise: reads go
// through `export_bao(..).data_to_bytes()`, and an entry may live in
// memory or on disk (`MemOrFile`), where a copy is unavoidable.
// Zero-copy is documented only as an internal property of the
// in-memory variant (`store/fs/bao_file.rs`), and it additionally
// requires a live handle -- which this test has no reliable way to
// hold. The assertion was therefore true only by coincidence, and
// failed every run, so the test was ignored and gave no signal at
// all.
//
// Promoting zero-copy to a public guarantee to make this assertion
// honest would foreclose encryption at rest, compression and
// checksum-on-read. If read-path allocation matters, measure it with
// a benchmark rather than freezing it into a contract here.
//
// We must at some point see completion, otherwise the test will hang.
obs.await_completion().await?;
}
store.shutdown().await?;
Expand Down
22 changes: 13 additions & 9 deletions protocols/krikos-docs/tests/sync.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,6 @@ async fn sync_gossip_bulk() -> Result<()> {
/// This tests basic sync and gossip with 3 peers.
#[tokio::test]
#[traced_test]
#[ignore = "flaky"]
async fn sync_full_basic() -> testresult::TestResult<()> {
let mut rng = test_rng(b"sync_full_basic");
let mut nodes = spawn_nodes(2, &mut rng).await?;
Expand Down Expand Up @@ -334,14 +333,21 @@ async fn sync_full_basic() -> testresult::TestResult<()> {
)
.await;

// `content_status` is evaluated when the event is CONVERTED, not when the
// entry synced, so whether a download has started by then is a race. These
// matchers accepted only `Missing` and broke whenever it read `Incomplete`.
// Measured over 20 runs under CPU load: peer2's two InsertRemote matchers
// saw Incomplete 12/20 and 7/20 respectively; `Complete` never appeared.
// So "not yet complete" is a real assertion that holds, while the
// Missing/Incomplete split is pure timing. Match both.
// peer0: assert events for entry received via gossip
info!("peer0: wait for 2 events (gossip'ed entry from peer1)");
assert_next(
&mut events0,
TIMEOUT,
vec![
Box::new(
move |e| matches!(e, LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == peer1),
move |e| matches!(e, LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing | ContentStatus::Incomplete, .. } if *from == peer1),
),
Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash1)),
],
Expand Down Expand Up @@ -375,10 +381,10 @@ async fn sync_full_basic() -> testresult::TestResult<()> {
Box::new(move |e| match_sync_finished(e, peer1)),
// 2 InsertRemote events
Box::new(
move |e| matches!(e, LiveEvent::InsertRemote { entry, content_status: ContentStatus::Missing, .. } if entry.content_hash() == hash0),
move |e| matches!(e, LiveEvent::InsertRemote { entry, content_status: ContentStatus::Missing | ContentStatus::Incomplete, .. } if entry.content_hash() == hash0),
),
Box::new(
move |e| matches!(e, LiveEvent::InsertRemote { entry, content_status: ContentStatus::Missing, .. } if entry.content_hash() == hash1),
move |e| matches!(e, LiveEvent::InsertRemote { entry, content_status: ContentStatus::Missing | ContentStatus::Incomplete, .. } if entry.content_hash() == hash1),
),
// 2 ContentReady events
Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash0)),
Expand Down Expand Up @@ -607,7 +613,6 @@ async fn test_sync_via_relay() -> Result<()> {

#[tokio::test]
#[traced_test]
#[ignore = "flaky"]
#[cfg(feature = "fs-store")]
async fn sync_restart_node() -> Result<()> {
use crate::util::endpoint;
Expand Down Expand Up @@ -656,7 +661,7 @@ async fn sync_restart_node() -> Result<()> {
vec![
match_event!(LiveEvent::NeighborUp(n) if *n == id2),
match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()),
match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == id2),
match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing | ContentStatus::Incomplete, .. } if *from == id2),
match_event!(LiveEvent::ContentReady { hash } if *hash == hash_a),
match_event!(LiveEvent::PendingContentReady),
],
Expand Down Expand Up @@ -701,7 +706,7 @@ async fn sync_restart_node() -> Result<()> {
vec![
match_event!(LiveEvent::NeighborUp(n) if *n== id2),
match_event!(LiveEvent::SyncFinished(e) if e.peer == id2 && e.result.is_ok()),
match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == id2),
match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing | ContentStatus::Incomplete, .. } if *from == id2),
match_event!(LiveEvent::ContentReady { hash } if *hash == hash_b),
],
vec![
Expand All @@ -718,7 +723,7 @@ async fn sync_restart_node() -> Result<()> {
&mut events1,
Duration::from_secs(10),
vec![
match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == id2),
match_event!(LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing | ContentStatus::Incomplete, .. } if *from == id2),
match_event!(LiveEvent::ContentReady { hash } if *hash == hash_c),
],
vec![
Expand Down Expand Up @@ -918,7 +923,6 @@ async fn test_download_policies() -> Result<()> {
/// Test sync between many nodes with propagation through sync reports.
#[tokio::test(flavor = "multi_thread")]
#[traced_test]
#[ignore = "flaky"]
async fn sync_big() -> Result<()> {
let mut rng = test_rng(b"sync_big");
let n_nodes = std::env::var("NODES")
Expand Down
44 changes: 22 additions & 22 deletions scripts/determinism-boundaries.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,22 +231,22 @@ clock-timer krikos/src/endpoint/tests.rs:1295 tokio::time::sleep(Duration::from_
clock-timer krikos/src/endpoint/tests.rs:1306 tokio::time::sleep(Duration::from_millis(100)).await;
clock-timer krikos/src/endpoint/tests.rs:1584 let t0 = Instant::now();
clock-timer krikos/src/endpoint/tests.rs:1590 let t1 = Instant::now();
clock-timer krikos/src/endpoint/tests.rs:2040 tokio::time::timeout(Duration::from_secs(5), async {
clock-timer krikos/src/endpoint/tests.rs:2042 tokio::time::sleep(Duration::from_millis(10)).await
clock-timer krikos/src/endpoint/tests.rs:2106 let now = Instant::now();
clock-timer krikos/src/endpoint/tests.rs:2188 tokio::time::timeout(Duration::from_secs(1), async {
clock-timer krikos/src/endpoint/tests.rs:2211 tokio::time::timeout(Duration::from_secs(1), endpoint.close())
clock-timer krikos/src/endpoint/tests.rs:2228 let now = Instant::now();
clock-timer krikos/src/endpoint/tests.rs:2229 tokio::time::timeout(Duration::from_secs(5), ep.close())
clock-timer krikos/src/endpoint/tests.rs:2256 tokio::time::sleep(Duration::from_millis(10)).await;
clock-timer krikos/src/endpoint/tests.rs:2259 tokio::time::timeout(Duration::from_secs(5), ep.close())
clock-timer krikos/src/endpoint/tests.rs:2280 let res = tokio::time::timeout(Duration::from_millis(500), ep.online()).await;
clock-timer krikos/src/endpoint/tests.rs:2287 let res = tokio::time::timeout(Duration::from_millis(1000), ep.online()).await;
clock-timer krikos/src/endpoint/tests.rs:2293 tokio::time::timeout(Duration::from_millis(500), ep_clone.online()).await
clock-timer krikos/src/endpoint/tests.rs:2306 let res = tokio::time::timeout(Duration::from_millis(500), ep.online()).await;
clock-timer krikos/src/endpoint/tests.rs:2312 tokio::time::timeout(Duration::from_millis(500), ep_clone.online()).await
clock-timer krikos/src/endpoint/tests.rs:2356 let auth_err: String = tokio::time::timeout(Duration::from_secs(5), async {
clock-timer krikos/src/endpoint/tests.rs:2380 tokio::time::timeout(Duration::from_secs(5), good_ep.online())
clock-timer krikos/src/endpoint/tests.rs:2039 tokio::time::timeout(Duration::from_secs(5), async {
clock-timer krikos/src/endpoint/tests.rs:2041 tokio::time::sleep(Duration::from_millis(10)).await
clock-timer krikos/src/endpoint/tests.rs:2105 let now = Instant::now();
clock-timer krikos/src/endpoint/tests.rs:2187 tokio::time::timeout(Duration::from_secs(1), async {
clock-timer krikos/src/endpoint/tests.rs:2210 tokio::time::timeout(Duration::from_secs(1), endpoint.close())
clock-timer krikos/src/endpoint/tests.rs:2227 let now = Instant::now();
clock-timer krikos/src/endpoint/tests.rs:2228 tokio::time::timeout(Duration::from_secs(5), ep.close())
clock-timer krikos/src/endpoint/tests.rs:2255 tokio::time::sleep(Duration::from_millis(10)).await;
clock-timer krikos/src/endpoint/tests.rs:2258 tokio::time::timeout(Duration::from_secs(5), ep.close())
clock-timer krikos/src/endpoint/tests.rs:2279 let res = tokio::time::timeout(Duration::from_millis(500), ep.online()).await;
clock-timer krikos/src/endpoint/tests.rs:2286 let res = tokio::time::timeout(Duration::from_millis(1000), ep.online()).await;
clock-timer krikos/src/endpoint/tests.rs:2292 tokio::time::timeout(Duration::from_millis(500), ep_clone.online()).await
clock-timer krikos/src/endpoint/tests.rs:2305 let res = tokio::time::timeout(Duration::from_millis(500), ep.online()).await;
clock-timer krikos/src/endpoint/tests.rs:2311 tokio::time::timeout(Duration::from_millis(500), ep_clone.online()).await
clock-timer krikos/src/endpoint/tests.rs:2355 let auth_err: String = tokio::time::timeout(Duration::from_secs(5), async {
clock-timer krikos/src/endpoint/tests.rs:2379 tokio::time::timeout(Duration::from_secs(5), good_ep.online())
clock-timer krikos/src/endpoint/tests.rs:287 tokio::time::timeout(Duration::from_secs(10), async {
clock-timer krikos/src/endpoint/tests.rs:289 tokio::time::sleep(Duration::from_millis(10)).await;
clock-timer krikos/src/endpoint/tests.rs:400 let (server, client) = tokio::time::timeout(
Expand Down Expand Up @@ -933,12 +933,12 @@ spawn-task krikos/src/endpoint/tests.rs:1236 let accept = tokio::spawn(async mov
spawn-task krikos/src/endpoint/tests.rs:1331 let server_task = tokio::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:1379 let server_task = tokio::task::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:1436 let server_task = tokio::spawn({
spawn-task krikos/src/endpoint/tests.rs:1907 let server_task = tokio::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:1910 let stats_task = tokio::spawn(collect_stats(conn.path_events()));
spawn-task krikos/src/endpoint/tests.rs:1921 let client_stats_task = tokio::spawn(collect_stats(conn.path_events()));
spawn-task krikos/src/endpoint/tests.rs:2248 let accept_task = tokio::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:2292 let task = tokio::task::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:2311 let task = tokio::task::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:1906 let server_task = tokio::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:1909 let stats_task = tokio::spawn(collect_stats(conn.path_events()));
spawn-task krikos/src/endpoint/tests.rs:1920 let client_stats_task = tokio::spawn(collect_stats(conn.path_events()));
spawn-task krikos/src/endpoint/tests.rs:2247 let accept_task = tokio::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:2291 let task = tokio::task::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:2310 let task = tokio::task::spawn(async move {
spawn-task krikos/src/endpoint/tests.rs:330 let server = tokio::spawn(
spawn-task krikos/src/endpoint/tests.rs:360 let client = tokio::spawn(
spawn-task krikos/src/endpoint/tests.rs:437 let server = tokio::spawn(
Expand Down
Loading