diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cbe2207691..fde50905ae1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,8 @@ env: rust_nightly: nightly-2025-10-12 # Pin a specific miri version rust_miri_nightly: nightly-2026-06-29 + rust_emscripten_nightly: nightly-2026-08-17 + emsdk_version: '6.0.8' rust_clippy: '1.88' # When updating this, also update: # - README.md @@ -509,7 +511,7 @@ jobs: # We don't care about the semver of unstable tokio features. features: ${{ env.TOKIO_STABLE_FEATURES }} - name: Check semver for rest of the workspace - if: ${{ !startsWith(github.event.pull_request.base.ref, 'tokio-1.') }} + if: ${{ !startsWith(github.event.pull_request.base.ref, 'tokio-1.') && !startsWith(github.ref_name, 'tokio-1.') }} uses: obi1kenobi/cargo-semver-checks-action@v2 with: rust-toolchain: ${{ env.rust_stable }} @@ -1180,9 +1182,9 @@ jobs: - name: Install cargo-nextest, wasmtime uses: taiki-e/install-action@v2 with: - # Wasmtime v47.0.0 or later should work, but we stick with a known, + # Wasmtime v48.0.0 or later should work, but we stick with a known, # specific version to avoid surprises: - tool: cargo-nextest,wasmtime-cli@47.0.2 + tool: cargo-nextest,wasmtime-cli@48.0.2 - uses: Swatinem/rust-cache@v2 @@ -1193,6 +1195,66 @@ jobs: RUSTFLAGS: --cfg tokio_unstable CARGO_TARGET_WASM32_WASIP2_RUNNER: wasmtime run -Sinherit-network + wasm32-unknown-emscripten: + name: test tokio for wasm32-unknown-emscripten + needs: basics + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Install Rust ${{ env.rust_stable }} + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.rust_stable }} + targets: wasm32-unknown-emscripten + + - name: Install Emscripten + uses: emscripten-core/setup-emsdk@v16 + with: + version: ${{ env.emsdk_version }} + + - uses: actions/setup-node@v7 + with: + node-version: 26 + + - uses: Swatinem/rust-cache@v2 + + - name: Install cargo-hack + uses: taiki-e/install-action@v2 + with: + tool: cargo-hack + + - name: Check tokio feature matrix for emscripten + run: cargo hack check -p tokio --each-feature --exclude-features full,net,process,signal,rt-multi-thread,io-uring,taskdump,schedule-latency --target wasm32-unknown-emscripten + working-directory: tokio + + - name: Test tokio for emscripten (JSPI) + run: cargo test -p tokio --target wasm32-unknown-emscripten --features "rt,time,sync,macros,fs,io-util,io-std,test-util" --tests + working-directory: tokio + env: + CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER: node + RUSTFLAGS: "-Dwarnings -Clink-args=-sALLOW_MEMORY_GROWTH=1 -Clink-args=-sEXIT_RUNTIME=1 -Clink-args=-sJSPI -Clink-args=-sSTACK_SIZE=1048576" + + - name: Test tokio for emscripten (non-JSPI) + run: cargo test -p tokio --target wasm32-unknown-emscripten --features "rt,time,sync,macros,fs,io-util,io-std,test-util" --test rt_emscripten_block_on --test sync_emscripten_blocking + working-directory: tokio + env: + CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER: node + RUSTFLAGS: "-Dwarnings -Clink-args=-sALLOW_MEMORY_GROWTH=1 -Clink-args=-sEXIT_RUNTIME=1 -Clink-args=-sSTACK_SIZE=1048576" + + - name: Install Rust ${{ env.rust_emscripten_nightly }} + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: ${{ env.rust_emscripten_nightly }} + targets: wasm32-unknown-emscripten + components: rust-src + + - name: Test tokio for emscripten (pthreads) + run: cargo +${{ env.rust_emscripten_nightly }} test -Zbuild-std=std,panic_abort -p tokio --target wasm32-unknown-emscripten --features "rt,rt-multi-thread,time,sync,macros,fs,io-util,io-std,test-util" --tests + working-directory: tokio + env: + CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER: node + RUSTFLAGS: "-Dwarnings -Ctarget-feature=+atomics,+bulk-memory,+mutable-globals -Clink-args=-pthread -Clink-args=-sPTHREAD_POOL_SIZE=8 -Clink-args=-sINITIAL_MEMORY=134217728 -Clink-args=-sEXIT_RUNTIME=1 -Clink-args=-sSTACK_SIZE=1048576" + check-external-types: name: check-external-types (${{ matrix.os }}) needs: basics @@ -1352,7 +1414,7 @@ jobs: - name: Test in FreeBSD uses: vmactions/freebsd-vm@v1 with: - release: '14.4' + release: '14.5' envs: "TOKIO_STABLE_FEATURES RUSTFLAGS" sync: rsync copyback: false @@ -1381,7 +1443,7 @@ jobs: RUSTFLAGS: --cfg docsrs --cfg tokio_unstable RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable -Dwarnings with: - release: '14.4' + release: '14.5' envs: "TOKIO_STABLE_FEATURES RUSTDOCFLAGS RUSTFLAGS" sync: rsync copyback: false @@ -1406,7 +1468,7 @@ jobs: - name: Test in FreeBSD uses: vmactions/freebsd-vm@v1 with: - release: '14.4' + release: '14.5' envs: "TOKIO_STABLE_FEATURES RUSTFLAGS" sync: rsync copyback: false diff --git a/.github/workflows/uring-kernel-version-test.yml b/.github/workflows/uring-kernel-version-test.yml index ada6c94a09b..85dcf912e77 100644 --- a/.github/workflows/uring-kernel-version-test.yml +++ b/.github/workflows/uring-kernel-version-test.yml @@ -81,6 +81,7 @@ jobs: mount -t sysfs sysfs /sys mkdir -p /tmp && mount -t tmpfs -o mode=1777 tmpfs /tmp for f in /bin/tests/*; do RUST_BACKTRACE=1 "$f" ; done + echo "ALL_TESTS_COMPLETED" EOF chmod +x initramfs/init @@ -94,11 +95,14 @@ jobs: set -euo pipefail set +e + # 3 GiB guest (the runner has 16 GB): the initramfs holds ~760 MB of + # unstripped test binaries, and `open_many_files` in `fs_uring` had + # 200-340 MB of anon RSS when the 1 GiB guest OOM-killed it. qemu-system-x86_64 \ -kernel linux-${{ env.KERNEL_VERSION }}/arch/x86/boot/bzImage \ -initrd initramfs.cpio.gz \ -append "console=ttyS0 rootfstype=ramfs panic=1" \ - -nographic -no-reboot -m 1024 -action panic=exit-failure 2>&1 | tee qemu-output.log + -nographic -no-reboot -m 3072 -action panic=exit-failure 2>&1 | tee qemu-output.log qemu_status=${PIPESTATUS[0]} tee_status=${PIPESTATUS[1]} set -e @@ -115,6 +119,14 @@ jobs: if grep -q "test result: FAILED" qemu-output.log; then echo "tests reported failures" exit 1 - else - echo "all tests passed" fi + + # A binary killed before it prints its summary (OOM, panic in init) + # never writes `test result: FAILED`, so also require the marker that + # init prints after the last binary. + if ! grep -q "ALL_TESTS_COMPLETED" qemu-output.log; then + echo "tests did not run to completion" + exit 1 + fi + + echo "all tests passed" diff --git a/README.md b/README.md index e7c1c81fe08..d16f19b17df 100644 --- a/README.md +++ b/README.md @@ -217,18 +217,18 @@ warrants a patch release with a fix for the bug, it will be backported and released as a new patch release for each LTS minor version. Our current LTS releases are: - * `1.47.x` - LTS release until September 2026. (MSRV 1.70) * `1.51.x` - LTS release until March 2027. (MSRV 1.71) + * `1.53.x` - LTS release until September 2027. (MSRV 1.71) Each LTS release will continue to receive backported fixes for at least a year. If you wish to use a fixed minor release in your project, we recommend that you use an LTS release. To use a fixed minor version, you can specify the version with a tilde. For -example, to specify that you wish to use the newest `1.47.x` patch release, you +example, to specify that you wish to use the newest `1.51.x` patch release, you can use the following dependency specification: ```text -tokio = { version = "~1.47", features = [...] } +tokio = { version = "~1.51", features = [...] } ``` ### Previous LTS releases @@ -242,6 +242,7 @@ tokio = { version = "~1.47", features = [...] } * `1.36.x` - LTS release until March 2025. * `1.38.x` - LTS release until July 2025. * `1.43.x` - LTS release until March 2026. + * `1.47.x` - LTS release until September 2026. ## License diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 6e6b58e0319..546f02b9be1 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -106,5 +106,10 @@ name = "remote_spawn" path = "remote_spawn.rs" harness = false +[[bench]] +name = "net_accept" +path = "net_accept.rs" +harness = false + [lints] workspace = true diff --git a/benches/net_accept.rs b/benches/net_accept.rs new file mode 100644 index 00000000000..c9b4c541667 --- /dev/null +++ b/benches/net_accept.rs @@ -0,0 +1,330 @@ +//! Benchmark the path from `accept` to the first read and write on the +//! accepted socket. The peer's request is already queued when the socket is +//! accepted, as it is for a client racing a busy server, so the first read +//! never needs to wait; this measures whether the runtime makes it wait for +//! the I/O driver anyway. These benches are a form of regression testing and +//! not a general purpose benchmark. +//! +//! Idle shapes, time per connection: +//! - `block_on`: accept + read + write inside `block_on`, one connection per +//! iteration. +//! - `serial_task`: one spawned task accepts and serves a batch of connections +//! in order, never yielding except on I/O. +//! - `task_per_conn`: an accept loop spawns one task per connection (the usual +//! server shape). +//! +//! `live/*`: a current-thread server runs on its own thread for the whole +//! bench and also drains `busy` other connections that a writer thread keeps +//! readable; the timed quantity is a blocking client's connect + request + +//! 2-byte reply. `inline_first_read` reads the request in the accept loop, +//! `task_per_conn` in a spawned task, `cap8` sets `max_io_events_per_tick(8)`. + +use std::io::Write; +use std::net::{SocketAddr, TcpStream as StdStream}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::runtime::{self, Runtime}; + +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; + +const REQUEST: &[u8] = &[1; 64]; +const BATCH: u64 = 64; + +/// A client that has connected and sent its request before the server accepts. +fn client(addr: SocketAddr) -> StdStream { + let mut c = StdStream::connect(addr).unwrap(); + c.set_nodelay(true).unwrap(); + c.write_all(REQUEST).unwrap(); + c +} + +async fn serve(mut s: TcpStream) { + let mut buf = [0u8; REQUEST.len()]; + s.read_exact(&mut buf).await.unwrap(); + s.write_all(b"ok").await.unwrap(); +} + +fn rt_current_thread() -> Runtime { + runtime::Builder::new_current_thread() + .enable_io() + .build() + .unwrap() +} + +fn rt_multi_thread(workers: usize) -> Runtime { + runtime::Builder::new_multi_thread() + .worker_threads(workers) + .enable_io() + .build() + .unwrap() +} + +fn block_on(c: &mut Criterion, name: &str, rt: Runtime) { + let listener = rt.block_on(TcpListener::bind("127.0.0.1:0")).unwrap(); + let addr = listener.local_addr().unwrap(); + + c.bench_function(name, |b| { + b.iter_batched( + || client(addr), + |_client| { + rt.block_on(async { + let (s, _) = listener.accept().await.unwrap(); + serve(s).await; + }) + }, + BatchSize::PerIteration, + ) + }); +} + +/// Runs `iters` connections in batches of `BATCH`: the clients for a batch are +/// connected and have written before the timed section, which runs `server` +/// for that batch as one spawned task. +fn spawned(c: &mut Criterion, name: &str, rt: Runtime, server: F) +where + F: Fn(Arc, u64) -> Fut + Copy + Send + 'static, + Fut: std::future::Future + Send + 'static, +{ + let listener = Arc::new(rt.block_on(TcpListener::bind("127.0.0.1:0")).unwrap()); + let addr = listener.local_addr().unwrap(); + + c.bench_function(name, |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + let mut left = iters; + while left > 0 { + let n = left.min(BATCH); + left -= n; + let clients: Vec = (0..n).map(|_| client(addr)).collect(); + let l = listener.clone(); + let start = Instant::now(); + rt.block_on(async move { tokio::spawn(server(l, n)).await.unwrap() }); + total += start.elapsed(); + drop(clients); + } + total + }) + }); +} + +async fn serial_task(listener: Arc, n: u64) { + for _ in 0..n { + let (s, _) = listener.accept().await.unwrap(); + serve(s).await; + } +} + +async fn task_per_conn(listener: Arc, n: u64) { + let mut handlers = Vec::with_capacity(n as usize); + for _ in 0..n { + let (s, _) = listener.accept().await.unwrap(); + handlers.push(tokio::spawn(serve(s))); + } + for h in handlers { + h.await.unwrap(); + } +} + +fn block_on_current_thread(c: &mut Criterion) { + block_on(c, "block_on/current_thread", rt_current_thread()); +} + +fn block_on_multi_thread_1(c: &mut Criterion) { + block_on(c, "block_on/multi_thread_1", rt_multi_thread(1)); +} + +fn serial_task_multi_thread_1(c: &mut Criterion) { + spawned( + c, + "serial_task/multi_thread_1", + rt_multi_thread(1), + serial_task, + ); +} + +fn serial_task_multi_thread_4(c: &mut Criterion) { + spawned( + c, + "serial_task/multi_thread_4", + rt_multi_thread(4), + serial_task, + ); +} + +fn task_per_conn_multi_thread_1(c: &mut Criterion) { + spawned( + c, + "task_per_conn/multi_thread_1", + rt_multi_thread(1), + task_per_conn, + ); +} + +fn task_per_conn_multi_thread_4(c: &mut Criterion) { + spawned( + c, + "task_per_conn/multi_thread_4", + rt_multi_thread(4), + task_per_conn, + ); +} + +criterion_group!( + net_accept, + block_on_current_thread, + block_on_multi_thread_1, + serial_task_multi_thread_1, + serial_task_multi_thread_4, + task_per_conn_multi_thread_1, + task_per_conn_multi_thread_4 +); + +// --------------------------------------------------------------------------- +// Live server: the runtime runs for the whole bench on its own thread and +// keeps draining the background connections, so both sides do the same +// background work; the client measures connect -> request -> reply. + +use std::io::Read; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; + +struct Live { + addr: SocketAddr, + stop: Arc, + drained: Arc, + writer: Option>, +} + +impl Drop for Live { + fn drop(&mut self) { + self.stop.store(true, Relaxed); + if let Some(h) = self.writer.take() { + let _ = h.join(); + } + // The server thread is left running; it holds no resources the next + // bench needs and exits with the process. + eprintln!( + " (background bytes drained by the server: {})", + self.drained.load(Relaxed) + ); + } +} + +fn live(busy: usize, inline: bool, events_per_tick: Option) -> Live { + let std_l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let std_bg = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + std_l.set_nonblocking(true).unwrap(); + std_bg.set_nonblocking(true).unwrap(); + let (addr, bg_addr) = (std_l.local_addr().unwrap(), std_bg.local_addr().unwrap()); + let accepted = Arc::new(AtomicU64::new(0)); + let drained = Arc::new(AtomicU64::new(0)); + let (accepted2, drained2) = (accepted.clone(), drained.clone()); + + std::thread::spawn(move || { + let mut b = runtime::Builder::new_current_thread(); + b.enable_io(); + if let Some(n) = events_per_tick { + b.max_io_events_per_tick(n); + } + let rt = b.build().unwrap(); + rt.block_on(async move { + let bg = TcpListener::from_std(std_bg).unwrap(); + tokio::spawn(async move { + loop { + let (mut s, _) = bg.accept().await.unwrap(); + accepted2.fetch_add(1, Relaxed); + let drained = drained2.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 256]; + loop { + match s.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => { + drained.fetch_add(n as u64, Relaxed); + } + } + } + }); + } + }); + let l = TcpListener::from_std(std_l).unwrap(); + loop { + let (s, _) = l.accept().await.unwrap(); + if inline { + serve(s).await; + } else { + tokio::spawn(serve(s)); + } + } + }); + }); + + let mut clients = Vec::with_capacity(busy); + for _ in 0..busy { + let c = StdStream::connect(bg_addr).unwrap(); + c.set_nonblocking(true).unwrap(); + c.set_nodelay(true).unwrap(); + clients.push(c); + } + while accepted.load(Relaxed) < busy as u64 { + std::thread::sleep(Duration::from_millis(1)); + } + let stop = Arc::new(AtomicBool::new(false)); + let stop2 = stop.clone(); + let writer = std::thread::spawn(move || { + let chunk = [2u8; 32]; + while !stop2.load(Relaxed) { + for c in &mut clients { + let _ = c.write(&chunk); + } + std::thread::sleep(Duration::from_micros(200)); + } + }); + Live { + addr, + stop, + drained, + writer: Some(writer), + } +} + +fn live_bench(c: &mut Criterion, name: &str, busy: usize, inline: bool, cap: Option) { + let srv = live(busy, inline, cap); + c.bench_function(name, |b| { + b.iter(|| { + let mut c = StdStream::connect(srv.addr).unwrap(); + c.set_nodelay(true).unwrap(); + c.write_all(REQUEST).unwrap(); + let mut reply = [0u8; 2]; + c.read_exact(&mut reply).unwrap(); + }) + }); +} + +fn live_task_per_conn_0(c: &mut Criterion) { + live_bench(c, "live/task_per_conn/busy_0", 0, false, None); +} +fn live_task_per_conn_1024(c: &mut Criterion) { + live_bench(c, "live/task_per_conn/busy_1024", 1024, false, None); +} +fn live_inline_0(c: &mut Criterion) { + live_bench(c, "live/inline_first_read/busy_0", 0, true, None); +} +fn live_inline_1024(c: &mut Criterion) { + live_bench(c, "live/inline_first_read/busy_1024", 1024, true, None); +} +fn live_task_per_conn_cap8_1024(c: &mut Criterion) { + live_bench(c, "live/task_per_conn_cap8/busy_1024", 1024, false, Some(8)); +} + +criterion_group!( + net_accept_live, + live_task_per_conn_0, + live_task_per_conn_1024, + live_inline_0, + live_inline_1024, + live_task_per_conn_cap8_1024 +); +criterion_main!(net_accept, net_accept_live); diff --git a/benches/sync_broadcast.rs b/benches/sync_broadcast.rs index 3e82c0e5d57..e2dd72f4913 100644 --- a/benches/sync_broadcast.rs +++ b/benches/sync_broadcast.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use tokio::sync::{broadcast, Notify}; use criterion::measurement::WallTime; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkGroup, Criterion}; +use criterion::{ + black_box, criterion_group, criterion_main, BenchmarkGroup, Criterion, Throughput, +}; fn rt() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() @@ -77,6 +79,38 @@ fn bench_contention(c: &mut Criterion) { group.finish(); } -criterion_group!(contention, bench_contention); +fn bench_try_recv(c: &mut Criterion) { + let mut group = c.benchmark_group("try_recv"); + const MESSAGES: usize = 256; + + for receiver_count in [1usize, 4, 16] { + group.throughput(Throughput::Elements((MESSAGES * receiver_count) as u64)); + group.bench_function(receiver_count.to_string(), |b| { + let (tx, first_rx) = broadcast::channel::(MESSAGES); + let mut receivers = Vec::with_capacity(receiver_count); + receivers.push(first_rx); + + for _ in 1..receiver_count { + receivers.push(tx.subscribe()); + } + + b.iter(|| { + for message in 0..MESSAGES { + tx.send(black_box(message)).unwrap(); + } + + for rx in &mut receivers { + for _ in 0..MESSAGES { + black_box(rx.try_recv().unwrap()); + } + } + }); + }); + } + + group.finish(); +} + +criterion_group!(contention, bench_contention, bench_try_recv); criterion_main!(contention); diff --git a/spellcheck.dic b/spellcheck.dic index 9d447933443..73aefddabcf 100644 --- a/spellcheck.dic +++ b/spellcheck.dic @@ -1,4 +1,4 @@ -327 +332 & + < @@ -106,6 +106,8 @@ dns DNS DoS dwOpenMode +Emscripten +Emscripten's endian enqueue enqueued @@ -161,6 +163,7 @@ IP IPv4 IPv6 iteratively +JSPI Kotlin's latencies Lauck @@ -213,6 +216,7 @@ plaintext poller POSIX proxied +pthreads qos RAII RCU @@ -323,6 +327,7 @@ Wakers wakeup wakeups WASI +Wasm watchOS workstealing ZST diff --git a/tokio-stream/src/empty.rs b/tokio-stream/src/empty.rs index d90dff502cb..46bc4dfd103 100644 --- a/tokio-stream/src/empty.rs +++ b/tokio-stream/src/empty.rs @@ -1,4 +1,5 @@ use crate::Stream; +use futures_core::FusedStream; use core::marker::PhantomData; use core::pin::Pin; @@ -56,3 +57,9 @@ impl Stream for Empty { (0, Some(0)) } } + +impl FusedStream for Empty { + fn is_terminated(&self) -> bool { + true + } +} diff --git a/tokio-stream/src/once.rs b/tokio-stream/src/once.rs index 7740c4db3d6..f81a6b8ed41 100644 --- a/tokio-stream/src/once.rs +++ b/tokio-stream/src/once.rs @@ -1,4 +1,5 @@ use crate::Stream; +use futures_core::FusedStream; use core::pin::Pin; use core::task::{Context, Poll}; @@ -60,3 +61,9 @@ impl Stream for Once { } } } + +impl FusedStream for Once { + fn is_terminated(&self) -> bool { + self.value.is_none() + } +} diff --git a/tokio-stream/src/pending.rs b/tokio-stream/src/pending.rs index 450031ef6ed..17d946cd6cb 100644 --- a/tokio-stream/src/pending.rs +++ b/tokio-stream/src/pending.rs @@ -49,6 +49,6 @@ impl Stream for Pending { } fn size_hint(&self) -> (usize, Option) { - (0, None) + (0, Some(0)) } } diff --git a/tokio-stream/src/stream_close.rs b/tokio-stream/src/stream_close.rs index a820c30d742..2ed5c11c34b 100644 --- a/tokio-stream/src/stream_close.rs +++ b/tokio-stream/src/stream_close.rs @@ -1,4 +1,5 @@ use crate::Stream; +use futures_core::FusedStream; use pin_project_lite::pin_project; use std::pin::Pin; use std::task::{Context, Poll}; @@ -91,3 +92,12 @@ where } } } + +impl FusedStream for StreamNotifyClose +where + S: Stream, +{ + fn is_terminated(&self) -> bool { + self.inner.is_none() + } +} diff --git a/tokio-stream/src/stream_ext/throttle.rs b/tokio-stream/src/stream_ext/throttle.rs index 4e71debf3ce..919d90cf148 100644 --- a/tokio-stream/src/stream_ext/throttle.rs +++ b/tokio-stream/src/stream_ext/throttle.rs @@ -1,7 +1,7 @@ //! Slow down a stream by enforcing a delay between items. use crate::Stream; -use tokio::time::{Duration, Instant, Sleep}; +use tokio::time::{sleep, Duration, Sleep}; use std::future::Future; use std::pin::Pin; @@ -14,7 +14,7 @@ where T: Stream, { Throttle { - delay: tokio::time::sleep_until(Instant::now() + duration), + delay: None, duration, has_delayed: true, stream, @@ -28,7 +28,7 @@ pin_project! { #[must_use = "streams do nothing unless polled"] pub struct Throttle { #[pin] - delay: Sleep, + delay: Option, duration: Duration, // Set to true when `delay` has returned ready, but `stream` hasn't. @@ -73,7 +73,9 @@ impl Stream for Throttle { let dur = *me.duration; if !*me.has_delayed && !is_zero(dur) { - ready!(me.delay.as_mut().poll(cx)); + if let Some(delay) = me.delay.as_mut().as_pin_mut() { + ready!(delay.poll(cx)); + } *me.has_delayed = true; } @@ -81,7 +83,7 @@ impl Stream for Throttle { if value.is_some() { if !is_zero(dur) { - me.delay.reset(Instant::now() + dur); + me.delay.set(Some(sleep(dur))); } *me.has_delayed = false; @@ -89,6 +91,10 @@ impl Stream for Throttle { Poll::Ready(value) } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } } fn is_zero(dur: Duration) -> bool { diff --git a/tokio-stream/src/stream_ext/timeout.rs b/tokio-stream/src/stream_ext/timeout.rs index d863af1dbdb..d968ff0c1fd 100644 --- a/tokio-stream/src/stream_ext/timeout.rs +++ b/tokio-stream/src/stream_ext/timeout.rs @@ -1,6 +1,6 @@ use crate::stream_ext::Fuse; use crate::Stream; -use tokio::time::{Instant, Sleep}; +use tokio::time::{sleep, Sleep}; use core::future::Future; use core::pin::Pin; @@ -29,8 +29,7 @@ pub struct Elapsed(()); impl Timeout { pub(super) fn new(stream: S, duration: Duration) -> Self { - let next = Instant::now() + duration; - let deadline = tokio::time::sleep_until(next); + let deadline = sleep(duration); Timeout { stream: Fuse::new(stream), @@ -45,13 +44,12 @@ impl Stream for Timeout { type Item = Result; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let me = self.project(); + let mut me = self.project(); match me.stream.poll_next(cx) { Poll::Ready(v) => { if v.is_some() { - let next = Instant::now() + *me.duration; - me.deadline.reset(next); + me.deadline.set(sleep(*me.duration)); *me.poll_deadline = true; } return Poll::Ready(v.map(Ok)); diff --git a/tokio-stream/tests/stream_fused.rs b/tokio-stream/tests/stream_fused.rs index 6a8de9c78a5..bdea69cddca 100644 --- a/tokio-stream/tests/stream_fused.rs +++ b/tokio-stream/tests/stream_fused.rs @@ -1,11 +1,53 @@ use futures_core::FusedStream; -use tokio_stream::StreamExt; +use tokio_stream::{Stream, StreamExt, StreamNotifyClose}; + +use std::pin::Pin; +use std::task::{Context, Poll}; + +struct EndOnce { + ended: bool, +} + +impl Stream for EndOnce { + type Item = (); + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.ended { + panic!("stream polled after returning None"); + } + + self.ended = true; + Poll::Ready(None) + } +} // Helper: a fused base stream built from a vec fn fused_iter(items: Vec) -> impl FusedStream { tokio_stream::iter(items).fuse() } +#[tokio::test] +async fn empty_is_terminated_immediately() { + let mut stream = tokio_stream::empty::(); + assert!(stream.is_terminated()); + assert_eq!(stream.next().await, None); + assert!(stream.is_terminated()); +} + +#[tokio::test] +async fn once_not_terminated_before_polled() { + let stream = tokio_stream::once(1); + assert!(!stream.is_terminated()); +} + +#[tokio::test] +async fn once_terminated_after_item_yielded() { + let mut stream = tokio_stream::once(1); + assert_eq!(stream.next().await, Some(1)); + assert!(stream.is_terminated()); + assert_eq!(stream.next().await, None); +} + // ── map ────────────────────────────────────────────────────────────────────── #[tokio::test] @@ -206,3 +248,23 @@ async fn merge_terminated_only_after_both_done() { assert!(stream.is_terminated()); assert_eq!(collected.len(), 2); } + +#[tokio::test] +async fn stream_notify_close_is_terminated_after_close_notification() { + let mut stream = StreamNotifyClose::new(tokio_stream::iter(vec![1])); + assert!(!stream.is_terminated()); + assert_eq!(stream.next().await, Some(Some(1))); + assert!(!stream.is_terminated()); + assert_eq!(stream.next().await, Some(None)); + assert!(stream.is_terminated()); + assert_eq!(stream.next().await, None); +} + +#[tokio::test] +async fn stream_notify_close_does_not_poll_inner_after_close_notification() { + let mut stream = StreamNotifyClose::new(EndOnce { ended: false }); + assert!(!stream.is_terminated()); + assert_eq!(stream.next().await, Some(None)); + assert!(stream.is_terminated()); + assert_eq!(stream.next().await, None); +} diff --git a/tokio-stream/tests/stream_pending.rs b/tokio-stream/tests/stream_pending.rs index 87b5d03bda2..16251bd3748 100644 --- a/tokio-stream/tests/stream_pending.rs +++ b/tokio-stream/tests/stream_pending.rs @@ -6,7 +6,7 @@ async fn basic_usage() { let mut stream = stream::pending::(); for _ in 0..2 { - assert_eq!(stream.size_hint(), (0, None)); + assert_eq!(stream.size_hint(), (0, Some(0))); let mut next = task::spawn(async { stream.next().await }); assert_pending!(next.poll()); diff --git a/tokio-stream/tests/stream_stream_map.rs b/tokio-stream/tests/stream_stream_map.rs index d600db1eaa0..1eb59d08011 100644 --- a/tokio-stream/tests/stream_stream_map.rs +++ b/tokio-stream/tests/stream_stream_map.rs @@ -219,7 +219,7 @@ fn size_hint_without_upper() { map.insert("a", pin_box(stream::iter(vec![1]))); map.insert("b", pin_box(stream::iter(vec![1, 2]))); - map.insert("c", pin_box(pending())); + map.insert("c", pin_box(futures::stream::poll_fn(|_| Poll::Pending))); let size_hint = map.size_hint(); assert_eq!(size_hint, (3, None)); diff --git a/tokio-stream/tests/stream_timeout.rs b/tokio-stream/tests/stream_timeout.rs index 19b4c3d533f..5e824aa5b59 100644 --- a/tokio-stream/tests/stream_timeout.rs +++ b/tokio-stream/tests/stream_timeout.rs @@ -107,3 +107,11 @@ async fn no_timeouts() { assert_ready_eq!(stream.poll_next(), Some(Ok(5))); assert_ready_eq!(stream.poll_next(), None); } + +#[tokio::test] +async fn duration_max_does_not_overflow() { + let stream = stream::iter([1]).timeout(Duration::MAX); + let mut stream = task::spawn(stream); + + assert_ready_eq!(stream.poll_next(), Some(Ok(1))); +} diff --git a/tokio-stream/tests/time_throttle.rs b/tokio-stream/tests/time_throttle.rs index e6c9917be3b..5e01688683c 100644 --- a/tokio-stream/tests/time_throttle.rs +++ b/tokio-stream/tests/time_throttle.rs @@ -2,11 +2,32 @@ #![cfg(all(feature = "time", feature = "sync", feature = "io-util"))] use tokio::time; -use tokio_stream::StreamExt; +use tokio_stream::{Stream, StreamExt}; use tokio_test::*; use std::time::Duration; +#[test] +fn throttle_can_be_created_outside_runtime() { + let _stream = futures::stream::iter([1, 2]).throttle(Duration::from_millis(1)); +} + +#[test] +fn zero_duration_does_not_require_time_driver() { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(async { + let stream = futures::stream::iter([1, 2]).throttle(Duration::from_millis(0)); + tokio::pin!(stream); + + assert_eq!(stream.next().await, Some(1)); + assert_eq!(stream.next().await, Some(2)); + assert_eq!(stream.next().await, None); + }); +} + #[tokio::test] async fn usage() { time::pause(); @@ -26,3 +47,17 @@ async fn usage() { assert_ready!(stream.poll_next()); } + +#[tokio::test] +async fn duration_max_does_not_overflow() { + let mut stream = task::spawn(futures::stream::iter([1]).throttle(Duration::MAX)); + + assert_ready_eq!(stream.poll_next(), Some(1)); +} + +#[tokio::test] +async fn size_hint() { + let stream = futures::stream::iter([1, 2, 3]).throttle(Duration::from_secs(1)); + + assert_eq!(stream.size_hint(), (3, Some(3))); +} diff --git a/tokio/Cargo.toml b/tokio/Cargo.toml index e260bbc5bd9..0cd6fa059ff 100644 --- a/tokio/Cargo.toml +++ b/tokio/Cargo.toml @@ -101,7 +101,7 @@ bytes = { version = "1.2.1", optional = true } mio = { version = "1.2.0", optional = true, default-features = false } parking_lot = { version = "0.12.0", optional = true } -[target.'cfg(any(not(target_family = "wasm"), all(target_os = "wasi", not(target_env = "p1"))))'.dependencies] +[target.'cfg(any(not(target_family = "wasm"), target_os = "emscripten", all(target_os = "wasi", not(target_env = "p1"))))'.dependencies] socket2 = { version = "0.6.3", optional = true, features = ["all"] } # Currently unstable. The API exposed by these features may be broken at any time. @@ -130,6 +130,8 @@ libc = { version = "0.2.168", optional = true } [target.'cfg(unix)'.dev-dependencies] libc = { version = "0.2.168" } + +[target.'cfg(all(unix, not(target_os = "emscripten")))'.dev-dependencies] nix = { version = "0.31.0", default-features = false, features = ["aio", "fs", "socket"] } [target.'cfg(windows)'.dependencies.windows-sys] @@ -146,22 +148,26 @@ features = [ [dev-dependencies] tokio-test = "0.4.0" tokio-stream = "0.1" -tokio-util = { version = "0.7", features = ["rt"] } futures = { version = "0.3.0", features = ["async-await"] } futures-test = "0.3.31" mockall = "0.13.0" async-stream = "0.3" futures-concurrency = "7.6.3" +[target.'cfg(not(target_os = "emscripten"))'.dev-dependencies] +tokio-util = { version = "0.7", features = ["rt"] } + [target.'cfg(not(target_family = "wasm"))'.dev-dependencies] socket2 = "0.6.0" -tempfile = "3.1.0" proptest = "1" +[target.'cfg(any(not(target_family = "wasm"), target_os = "emscripten"))'.dev-dependencies] +tempfile = "3.1.0" + [target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dev-dependencies] rand = "0.9" -[target.'cfg(all(target_family = "wasm", not(target_os = "wasi")))'.dev-dependencies] +[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dev-dependencies] wasm-bindgen-test = "0.3.0" [target.'cfg(target_os = "freebsd")'.dev-dependencies] diff --git a/tokio/README.md b/tokio/README.md index e7c1c81fe08..d16f19b17df 100644 --- a/tokio/README.md +++ b/tokio/README.md @@ -217,18 +217,18 @@ warrants a patch release with a fix for the bug, it will be backported and released as a new patch release for each LTS minor version. Our current LTS releases are: - * `1.47.x` - LTS release until September 2026. (MSRV 1.70) * `1.51.x` - LTS release until March 2027. (MSRV 1.71) + * `1.53.x` - LTS release until September 2027. (MSRV 1.71) Each LTS release will continue to receive backported fixes for at least a year. If you wish to use a fixed minor release in your project, we recommend that you use an LTS release. To use a fixed minor version, you can specify the version with a tilde. For -example, to specify that you wish to use the newest `1.47.x` patch release, you +example, to specify that you wish to use the newest `1.51.x` patch release, you can use the following dependency specification: ```text -tokio = { version = "~1.47", features = [...] } +tokio = { version = "~1.51", features = [...] } ``` ### Previous LTS releases @@ -242,6 +242,7 @@ tokio = { version = "~1.47", features = [...] } * `1.36.x` - LTS release until March 2025. * `1.38.x` - LTS release until July 2025. * `1.43.x` - LTS release until March 2026. + * `1.47.x` - LTS release until September 2026. ## License diff --git a/tokio/src/blocking.rs b/tokio/src/blocking.rs index f172399d5ef..f3c621dacbe 100644 --- a/tokio/src/blocking.rs +++ b/tokio/src/blocking.rs @@ -1,12 +1,46 @@ cfg_rt! { + #[cfg(not(target_os = "emscripten"))] pub(crate) use crate::runtime::spawn_blocking; cfg_fs! { + #[cfg(not(target_os = "emscripten"))] #[allow(unused_imports)] pub(crate) use crate::runtime::spawn_mandatory_blocking; } + #[cfg(not(target_os = "emscripten"))] pub(crate) use crate::task::JoinHandle; + + // Emscripten's filesystem is synchronous, so `fs` and `io-std` run inline + // here, pthread builds included. Public `task::spawn_blocking` is unaffected. + // + // The completed future is wrapped in `Coop` so that polling it consumes + // task budget exactly like the native `task::JoinHandle::poll` does. The + // `fs` and `io-std` consumers rely on that budget for their yield points: + // without it a loop of always-ready file reads never returns `Pending` + // and starves every other task on the single-threaded runtime. + #[cfg(target_os = "emscripten")] + pub(crate) type JoinHandle = + crate::task::coop::Coop>>; + + #[cfg(target_os = "emscripten")] + pub(crate) fn spawn_blocking(f: F) -> JoinHandle + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + crate::task::coop::cooperative(std::future::ready(Ok(f()))) + } + + #[cfg(all(target_os = "emscripten", feature = "fs"))] + #[allow(dead_code)] // unit tests replace this with the `fs::mocks` version + pub(crate) fn spawn_mandatory_blocking(f: F) -> Option> + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + Some(spawn_blocking(f)) + } } cfg_not_rt! { diff --git a/tokio/src/fs/file.rs b/tokio/src/fs/file.rs index d0275535b05..fe7f1810b5a 100644 --- a/tokio/src/fs/file.rs +++ b/tokio/src/fs/file.rs @@ -571,6 +571,10 @@ impl File { /// Although Tokio uses a sensible default value for this buffer size, this function would be /// useful for changing that default depending on the situation. /// + /// # Panics + /// + /// This function panics if `max_buf_size` is 0. + /// /// # Examples /// /// ```no_run @@ -590,7 +594,9 @@ impl File { /// # Ok(()) /// # } /// ``` + #[track_caller] pub fn set_max_buf_size(&mut self, max_buf_size: usize) { + assert!(max_buf_size > 0, "`max_buf_size` must be greater than 0"); self.max_buf_size = max_buf_size; } diff --git a/tokio/src/io/stdio_common.rs b/tokio/src/io/stdio_common.rs index 72fd97d917d..902bf619c90 100644 --- a/tokio/src/io/stdio_common.rs +++ b/tokio/src/io/stdio_common.rs @@ -108,6 +108,7 @@ where #[cfg(test)] #[cfg(not(loom))] +#[cfg(not(target_os = "emscripten"))] mod tests { use crate::io::blocking::DEFAULT_MAX_BUF_SIZE; use crate::io::AsyncWriteExt; diff --git a/tokio/src/io/util/async_write_ext.rs b/tokio/src/io/util/async_write_ext.rs index 913b824cc92..6496a80149f 100644 --- a/tokio/src/io/util/async_write_ext.rs +++ b/tokio/src/io/util/async_write_ext.rs @@ -286,8 +286,9 @@ cfg_io_util! { /// /// This method will continuously call [`write`] until /// [`buf.has_remaining()`](bytes::Buf::has_remaining) returns false. This method will not - /// return until the entire buffer has been successfully written or an error occurs. The - /// first error generated will be returned. + /// return until the entire buffer has been successfully written or an error occurs. + /// Errors of kind [`ErrorKind::Interrupted`] are ignored and the write is retried. The first + /// other error generated will be returned. /// /// The buffer is advanced after each chunk is successfully written. After failure, /// `src.chunk()` will return the chunk that failed to write. @@ -331,6 +332,7 @@ cfg_io_util! { /// ``` /// /// [`write`]: AsyncWriteExt::write + /// [`ErrorKind::Interrupted`]: std::io::ErrorKind::Interrupted fn write_all_buf<'a, B>(&'a mut self, src: &'a mut B) -> WriteAllBuf<'a, Self, B> where Self: Sized + Unpin, @@ -349,8 +351,9 @@ cfg_io_util! { /// /// This method will continuously call [`write`] until there is no more data /// to be written. This method will not return until the entire buffer - /// has been successfully written or such an error occurs. The first - /// error generated from this method will be returned. + /// has been successfully written or such an error occurs. Errors of kind + /// [`ErrorKind::Interrupted`] are ignored and the write is retried. The first + /// other error generated from this method will be returned. /// /// # Cancel safety /// @@ -362,7 +365,8 @@ cfg_io_util! { /// /// # Errors /// - /// This function will return the first error that [`write`] returns. + /// This function will ignore errors of kind [`ErrorKind::Interrupted`] and + /// will otherwise return the first error that [`write`] returns. /// /// # Examples /// @@ -384,6 +388,7 @@ cfg_io_util! { /// ``` /// /// [`write`]: AsyncWriteExt::write + /// [`ErrorKind::Interrupted`]: std::io::ErrorKind::Interrupted fn write_all<'a>(&'a mut self, src: &'a [u8]) -> WriteAll<'a, Self> where Self: Unpin, diff --git a/tokio/src/io/util/buf_writer.rs b/tokio/src/io/util/buf_writer.rs index ea9076ea693..5b9fc965727 100644 --- a/tokio/src/io/util/buf_writer.rs +++ b/tokio/src/io/util/buf_writer.rs @@ -70,6 +70,7 @@ impl BufWriter { break; } Ok(n) => *me.written += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, Err(e) => { ret = Err(e); break; diff --git a/tokio/src/io/util/copy_bidirectional.rs b/tokio/src/io/util/copy_bidirectional.rs index ea40a3b4c95..1b22b1a0b80 100644 --- a/tokio/src/io/util/copy_bidirectional.rs +++ b/tokio/src/io/util/copy_bidirectional.rs @@ -90,6 +90,10 @@ where /// /// This method is the same as the [`copy_bidirectional()`], except that it allows you to set the /// size of the internal buffers used when copying data. +/// +/// # Panics +/// +/// Panics if either buffer size is zero. #[cfg_attr(docsrs, doc(cfg(feature = "io-util")))] pub async fn copy_bidirectional_with_sizes( a: &mut A, @@ -101,6 +105,15 @@ where A: AsyncRead + AsyncWrite + Unpin + ?Sized, B: AsyncRead + AsyncWrite + Unpin + ?Sized, { + assert!( + a_to_b_buf_size > 0, + "`a_to_b_buf_size` must be greater than 0" + ); + assert!( + b_to_a_buf_size > 0, + "`b_to_a_buf_size` must be greater than 0" + ); + copy_bidirectional_impl( a, b, diff --git a/tokio/src/io/util/copy_buf.rs b/tokio/src/io/util/copy_buf.rs index 30b0837ff17..c18f7c8daa5 100644 --- a/tokio/src/io/util/copy_buf.rs +++ b/tokio/src/io/util/copy_buf.rs @@ -18,6 +18,7 @@ cfg_io_util! { reader: &'a mut R, writer: &'a mut W, amt: u64, + need_flush: bool, } /// Asynchronously copies the entire contents of a reader into a writer. @@ -75,6 +76,7 @@ cfg_io_util! { reader, writer, amt: 0, + need_flush: false, }.await } } @@ -87,9 +89,61 @@ where type Output = io::Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + #[cfg(any( + feature = "fs", + feature = "io-std", + feature = "net", + feature = "process", + feature = "rt", + feature = "signal", + feature = "sync", + feature = "time", + ))] + // Keep track of task budget + let coop = ready!(crate::task::coop::poll_proceed(cx)); loop { let me = &mut *self; - let buffer = ready!(Pin::new(&mut *me.reader).poll_fill_buf(cx))?; + let buffer = match Pin::new(&mut *me.reader).poll_fill_buf(cx) { + Poll::Ready(Ok(buffer)) => { + #[cfg(any( + feature = "fs", + feature = "io-std", + feature = "net", + feature = "process", + feature = "rt", + feature = "signal", + feature = "sync", + feature = "time", + ))] + coop.made_progress(); + buffer + } + Poll::Ready(Err(err)) => { + #[cfg(any( + feature = "fs", + feature = "io-std", + feature = "net", + feature = "process", + feature = "rt", + feature = "signal", + feature = "sync", + feature = "time", + ))] + coop.made_progress(); + return Poll::Ready(Err(err)); + } + Poll::Pending => { + // Try flushing when the reader has no progress to avoid deadlock + // when the reader depends on buffered writer. + if me.need_flush { + ready!(Pin::new(&mut *me.writer).poll_flush(cx))?; + me.need_flush = false; + } + + return Poll::Pending; + } + }; + if buffer.is_empty() { ready!(Pin::new(&mut self.writer).poll_flush(cx))?; return Poll::Ready(Ok(self.amt)); @@ -100,6 +154,7 @@ where return Poll::Ready(Err(std::io::ErrorKind::WriteZero.into())); } self.amt += i as u64; + self.need_flush = true; Pin::new(&mut *self.reader).consume(i); } } diff --git a/tokio/src/io/util/read_exact.rs b/tokio/src/io/util/read_exact.rs index e9e5afbf0b9..b79455e9560 100644 --- a/tokio/src/io/util/read_exact.rs +++ b/tokio/src/io/util/read_exact.rs @@ -57,7 +57,11 @@ where // if our buffer is empty, then we need to read some data to continue. let rem = me.buf.remaining(); if rem != 0 { - ready!(Pin::new(&mut *me.reader).poll_read(cx, me.buf))?; + match ready!(Pin::new(&mut *me.reader).poll_read(cx, me.buf)) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e).into(), + } if me.buf.remaining() == rem { return Err(eof()).into(); } diff --git a/tokio/src/io/util/read_to_end.rs b/tokio/src/io/util/read_to_end.rs index 05665026d29..7efd1dc9f3b 100644 --- a/tokio/src/io/util/read_to_end.rs +++ b/tokio/src/io/util/read_to_end.rs @@ -45,6 +45,7 @@ pub(super) fn read_to_end_internal( loop { let ret = ready!(poll_read_to_end(buf, reader.as_mut(), cx)); match ret { + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, Err(err) => return Poll::Ready(Err(err)), Ok(0) => return Poll::Ready(Ok(mem::replace(num_read, 0))), Ok(num) => { diff --git a/tokio/src/io/util/read_until.rs b/tokio/src/io/util/read_until.rs index fbe2609a64c..fae8cc9056c 100644 --- a/tokio/src/io/util/read_until.rs +++ b/tokio/src/io/util/read_until.rs @@ -53,7 +53,11 @@ pub(super) fn read_until_internal( ) -> Poll> { loop { let (done, used) = { - let available = ready!(reader.as_mut().poll_fill_buf(cx))?; + let available = match ready!(reader.as_mut().poll_fill_buf(cx)) { + Ok(available) => available, + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Poll::Ready(Err(e)), + }; if let Some(i) = memchr::memchr(delimiter, available) { buf.extend_from_slice(&available[..=i]); (true, i + 1) diff --git a/tokio/src/io/util/write_all.rs b/tokio/src/io/util/write_all.rs index 8330d89e1fc..260e77bd48e 100644 --- a/tokio/src/io/util/write_all.rs +++ b/tokio/src/io/util/write_all.rs @@ -40,7 +40,13 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let me = self.project(); while !me.buf.is_empty() { - let n = ready!(Pin::new(&mut *me.writer).poll_write(cx, me.buf))?; + let n = loop { + match ready!(Pin::new(&mut *me.writer).poll_write(cx, me.buf)) { + Ok(n) => break n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Poll::Ready(Err(e)), + } + }; { let (_, rest) = mem::take(&mut *me.buf).split_at(n); *me.buf = rest; diff --git a/tokio/src/io/util/write_all_buf.rs b/tokio/src/io/util/write_all_buf.rs index 02054913807..8443c77a49d 100644 --- a/tokio/src/io/util/write_all_buf.rs +++ b/tokio/src/io/util/write_all_buf.rs @@ -46,12 +46,20 @@ where let me = self.project(); while me.buf.has_remaining() { - let n = if me.writer.is_write_vectored() { - let mut slices = [IoSlice::new(&[]); MAX_VECTOR_ELEMENTS]; - let cnt = me.buf.chunks_vectored(&mut slices); - ready!(Pin::new(&mut *me.writer).poll_write_vectored(cx, &slices[..cnt]))? - } else { - ready!(Pin::new(&mut *me.writer).poll_write(cx, me.buf.chunk())?) + let n = loop { + let result = if me.writer.is_write_vectored() { + let mut slices = [IoSlice::new(&[]); MAX_VECTOR_ELEMENTS]; + let cnt = me.buf.chunks_vectored(&mut slices); + ready!(Pin::new(&mut *me.writer).poll_write_vectored(cx, &slices[..cnt])) + } else { + ready!(Pin::new(&mut *me.writer).poll_write(cx, me.buf.chunk())) + }; + + match result { + Ok(n) => break n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Poll::Ready(Err(e)), + } }; me.buf.advance(n); if n == 0 { diff --git a/tokio/src/lib.rs b/tokio/src/lib.rs index e62aab50659..06a96d1789f 100644 --- a/tokio/src/lib.rs +++ b/tokio/src/lib.rs @@ -424,8 +424,11 @@ //! //! ## `WASM` support //! -//! Tokio has some limited support for the `WASM` platform. Without the -//! `tokio_unstable` flag, the following features are supported: +//! Tokio has some limited support for Wasm platforms. +//! +//! Many of Tokio's feature flags are restricted on Wasm, and unsupported +//! combinations of feature flags will fail to build. However, all Wasm targets +//! can be built with the following features: //! //! * `sync` //! * `macros` @@ -434,26 +437,64 @@ //! * `time` //! //! Enabling any other feature (including `full`) will cause a compilation -//! failure. -//! -//! The `time` module will only work on `WASM` platforms that have support for -//! timers (e.g. wasm32-wasi). The timing functions will panic if used on a `WASM` -//! platform that does not support timers. -//! -//! Note also that if the runtime becomes indefinitely idle, it will panic -//! immediately instead of blocking forever. On platforms that don't support -//! time, this means that the runtime can never be idle in any way. -//! -//! ## Unstable `WASM` support -//! -//! Tokio also has unstable support for some additional `WASM` features. This -//! requires the use of the `tokio_unstable` flag. -//! -//! Using this flag enables the use of `tokio::net` on the wasm32-wasi target. -//! However, not all methods are available on the networking types as `WASI` -//! currently does not support the creation of new sockets from within `WASM`. -//! Because of this, sockets must currently be created via the `FromRawFd` -//! trait. +//! failure. Furthermore, some operations available under these feature flags +//! may panic if they are unsupported. For example: +//! +//! * Using timers will panic on Wasm targets that do not support blocking the +//! thread. +//! * If the runtime becomes indefinitely idle (e.g., the program triggers a +//! deadlock), then this will panic on most Wasm targets. +//! * Operations such as `spawn_blocking` that involve spawning threads will +//! panic on Wasm targets that do not support threads. +//! +//! All Wasm targets are still experimental, and breaking behavior changes can +//! occur in an effort to make Tokio use native Wasm operations. For instance, +//! the behavior of timers could be changed from panicking or blocking the +//! thread to starting a JavaScript timer. As another example, the +//! `spawn_blocking` method could be changed from starting a new thread to +//! creating a new cooperatively scheduled context of execution, if the Wasm +//! target supports such contexts. As a third example, the `#[tokio::main]` or +//! `#[tokio::test]` macros could be changed to work better with the Wasm +//! environment. +//! +//! ### `WASI` support +//! +//! The `wasm32-wasip1` and `wasm32-wasip2` targets support the above features. +//! Timers work correctly as blocking the thread is supported. +//! +//! Under the `tokio_unstable` flag, these targets support the use +//! of `tokio::net`. On `wasm32-wasip1`, not all methods are available on the +//! networking types as this target does not support the creation of new +//! sockets from within `WASM`. Because of this, sockets must currently be +//! created via the `FromRawFd` trait on `wasm32-wasip1`. The `wasm32-wasip2` +//! target does not have this limitation. +//! +//! ### Emscripten support +//! +//! The `wasm32-unknown-emscripten` target supports the single-threaded runtime +//! with the `rt`, `time`, `sync`, `macros`, `fs`, `io-util`, `io-std`, and +//! `test-util` features. The `rt-multi-thread` feature is additionally +//! supported when building with Emscripten pthreads (`-pthread`). The `net`, +//! `process`, and `signal` features are not supported. +//! +//! Emscripten's filesystem is synchronous, so `tokio::fs` and `io-std` run +//! their operations inline on the calling thread rather than on the blocking +//! pool, in pthreads builds too. +//! +//! When the build links [JSPI], a wait that would block, whether in +//! `block_on` or a blocking call such as `blocking_recv`, suspends on the +//! host event loop rather than blocking, resuming when a timer fires or when +//! a later call into the module wakes it. Without JSPI, such a wait panics; +//! the panic unwinds out of `block_on` and leaves the runtime usable. +//! +//! Suspension requires the current export to have been wrapped with +//! `WebAssembly.promising` (Emscripten's `ASYNCIFY_EXPORTS`). A wait from any +//! other activation, such as a plain host callback into the module, throws +//! `WebAssembly.SuspendError`. That is a foreign exception rather than a Rust +//! panic: it is not caught by `catch_unwind` and aborts at the first +//! `extern "C"` frame it reaches. +//! +//! [JSPI]: https://github.com/WebAssembly/js-promise-integration // Test that pointer width is compatible. This asserts that e.g. usize is at // least 32 bits, which a lot of components in Tokio currently assumes. @@ -467,6 +508,7 @@ compile_error! { #[cfg(all( not(tokio_unstable), target_family = "wasm", + not(target_os = "emscripten"), any( feature = "fs", feature = "io-std", @@ -478,6 +520,21 @@ compile_error! { ))] compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm."); +#[cfg(all( + target_os = "emscripten", + any(feature = "net", feature = "process", feature = "signal") +))] +compile_error!("Features net,process,signal are not supported on wasm32-unknown-emscripten."); + +#[cfg(all( + target_os = "emscripten", + feature = "rt-multi-thread", + not(target_feature = "atomics") +))] +compile_error!( + "The `rt-multi-thread` feature on wasm32-unknown-emscripten requires pthreads support (build with `-pthread`)." +); + #[cfg(all( tokio_unstable, feature = "taskdump", diff --git a/tokio/src/macros/cfg.rs b/tokio/src/macros/cfg.rs index 76165a50876..8c6ec4ea6da 100644 --- a/tokio/src/macros/cfg.rs +++ b/tokio/src/macros/cfg.rs @@ -398,6 +398,7 @@ macro_rules! cfg_process { #[cfg_attr(docsrs, doc(cfg(feature = "process")))] #[cfg(not(loom))] #[cfg(not(target_os = "wasi"))] + #[cfg(not(target_os = "emscripten"))] $item )* } @@ -405,16 +406,29 @@ macro_rules! cfg_process { macro_rules! cfg_process_driver { ($($item:item)*) => { - #[cfg(unix)] - #[cfg(not(loom))] - cfg_process! { $($item)* } + $( + #[cfg(all( + unix, + not(loom), + feature = "process", + not(target_os = "wasi"), + not(target_os = "emscripten"), + ))] + $item + )* } } macro_rules! cfg_not_process_driver { ($($item:item)*) => { $( - #[cfg(not(all(unix, not(loom), feature = "process")))] + #[cfg(not(all( + unix, + not(loom), + feature = "process", + not(target_os = "wasi"), + not(target_os = "emscripten"), + )))] $item )* } @@ -427,6 +441,7 @@ macro_rules! cfg_signal { #[cfg_attr(docsrs, doc(cfg(feature = "signal")))] #[cfg(not(loom))] #[cfg(not(target_os = "wasi"))] + #[cfg(not(target_os = "emscripten"))] $item )* } @@ -437,6 +452,7 @@ macro_rules! cfg_signal_internal { $( #[cfg(any(feature = "signal", all(unix, feature = "process")))] #[cfg(not(loom))] + #[cfg(not(target_os = "emscripten"))] $item )* } @@ -449,10 +465,15 @@ macro_rules! cfg_signal_internal_and_unix { } } -macro_rules! cfg_not_signal_internal { +macro_rules! cfg_not_signal_internal_and_unix { ($($item:item)*) => { $( - #[cfg(any(loom, not(unix), not(any(feature = "signal", all(unix, feature = "process")))))] + #[cfg(not(all( + unix, + any(feature = "signal", all(unix, feature = "process")), + not(loom), + not(target_os = "emscripten"), + )))] $item )* } @@ -712,15 +733,6 @@ macro_rules! cfg_not_wasip1 { } } -macro_rules! cfg_is_wasm_not_wasi { - ($($item:item)*) => { - $( - #[cfg(all(target_family = "wasm", not(target_os = "wasi")))] - $item - )* - } -} - /// Use this macro to provide two different implementations of the same API — one for stable /// builds and one for unstable builds. macro_rules! cfg_metrics_variant { diff --git a/tokio/src/net/tcp/listener.rs b/tokio/src/net/tcp/listener.rs index 61a54f11342..c7c788b38eb 100644 --- a/tokio/src/net/tcp/listener.rs +++ b/tokio/src/net/tcp/listener.rs @@ -166,7 +166,7 @@ impl TcpListener { .async_io(Interest::READABLE, || self.io.accept()) .await?; - let stream = TcpStream::new(mio)?; + let stream = TcpStream::new_accepted(mio)?; Ok((stream, addr)) } @@ -182,7 +182,7 @@ impl TcpListener { match self.io.accept() { Ok((io, addr)) => { - let io = TcpStream::new(io)?; + let io = TcpStream::new_accepted(io)?; return Poll::Ready(Ok((io, addr))); } Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { diff --git a/tokio/src/net/tcp/stream.rs b/tokio/src/net/tcp/stream.rs index cddc570df5d..17752bc84b5 100644 --- a/tokio/src/net/tcp/stream.rs +++ b/tokio/src/net/tcp/stream.rs @@ -165,6 +165,18 @@ impl TcpStream { Ok(TcpStream { io }) } + /// A stream returned by `accept`: assumed readable and writable, so that + /// its first read and write try the socket instead of waiting for the + /// driver's first event (see `Registration::assume_ready`). + pub(crate) fn new_accepted(connected: mio::net::TcpStream) -> io::Result { + let stream = TcpStream::new(connected)?; + stream + .io + .registration() + .assume_ready(Ready::READABLE | Ready::WRITABLE); + Ok(stream) + } + /// Creates new `TcpStream` from a `std::net::TcpStream`. /// /// This function is intended to be used to wrap a TCP stream from the diff --git a/tokio/src/net/unix/datagram/socket.rs b/tokio/src/net/unix/datagram/socket.rs index 1d6fbc3c852..9a7a45b82aa 100644 --- a/tokio/src/net/unix/datagram/socket.rs +++ b/tokio/src/net/unix/datagram/socket.rs @@ -434,6 +434,10 @@ impl UnixDatagram { let (a, b) = mio::net::UnixDatagram::pair()?; let a = UnixDatagram::new(a)?; let b = UnixDatagram::new(b)?; + // A fresh pair has empty buffers: writable now, readable only once + // the peer sends. + a.io.registration().assume_ready(Ready::WRITABLE); + b.io.registration().assume_ready(Ready::WRITABLE); Ok((a, b)) } diff --git a/tokio/src/net/unix/listener.rs b/tokio/src/net/unix/listener.rs index b39eedde31d..00cf9e38b02 100644 --- a/tokio/src/net/unix/listener.rs +++ b/tokio/src/net/unix/listener.rs @@ -214,7 +214,7 @@ impl UnixListener { .await?; let addr = SocketAddr(addr); - let stream = UnixStream::new(mio)?; + let stream = UnixStream::new_accepted(mio)?; Ok((stream, addr)) } @@ -227,7 +227,7 @@ impl UnixListener { pub fn poll_accept(&self, cx: &mut Context<'_>) -> Poll> { let (sock, addr) = ready!(self.io.registration().poll_read_io(cx, || self.io.accept()))?; let addr = SocketAddr(addr); - let sock = UnixStream::new(sock)?; + let sock = UnixStream::new_accepted(sock)?; Poll::Ready(Ok((sock, addr))) } } diff --git a/tokio/src/net/unix/stream.rs b/tokio/src/net/unix/stream.rs index 071ebad7f70..3be258a28c6 100644 --- a/tokio/src/net/unix/stream.rs +++ b/tokio/src/net/unix/stream.rs @@ -914,10 +914,24 @@ impl UnixStream { let (a, b) = mio::net::UnixStream::pair()?; let a = UnixStream::new(a)?; let b = UnixStream::new(b)?; + // A fresh pair has empty buffers: writable now, readable only once + // the peer writes. + a.io.registration().assume_ready(Ready::WRITABLE); + b.io.registration().assume_ready(Ready::WRITABLE); Ok((a, b)) } + /// See `TcpStream::new_accepted`. + pub(crate) fn new_accepted(stream: mio::net::UnixStream) -> io::Result { + let stream = UnixStream::new(stream)?; + stream + .io + .registration() + .assume_ready(Ready::READABLE | Ready::WRITABLE); + Ok(stream) + } + pub(crate) fn new(stream: mio::net::UnixStream) -> io::Result { let io = PollEvented::new(stream)?; Ok(UnixStream { io }) diff --git a/tokio/src/process/unix/orphan.rs b/tokio/src/process/unix/orphan.rs index 6aa0539fec4..93ba4d75aae 100644 --- a/tokio/src/process/unix/orphan.rs +++ b/tokio/src/process/unix/orphan.rs @@ -295,7 +295,7 @@ pub(crate) mod test { #[cfg_attr(miri, ignore)] // No `sigaction` on Miri #[test] fn does_not_register_signal_if_queue_empty() { - let (io_driver, io_handle) = IoDriver::new(1024).unwrap(); + let (io_driver, io_handle) = IoDriver::new(1024, None).unwrap(); let signal_driver = SignalDriver::new(io_driver, &io_handle).unwrap(); let handle = signal_driver.handle(); diff --git a/tokio/src/runtime/blocking/mod.rs b/tokio/src/runtime/blocking/mod.rs index b0e42264dd0..0e6fa204004 100644 --- a/tokio/src/runtime/blocking/mod.rs +++ b/tokio/src/runtime/blocking/mod.rs @@ -23,6 +23,10 @@ pub(crate) use task::BlockingTask; use crate::runtime::Builder; -pub(crate) fn create_blocking_pool(builder: &Builder, thread_cap: usize) -> BlockingPool { - BlockingPool::new(builder, thread_cap) +pub(crate) fn create_blocking_pool( + builder: &Builder, + thread_cap: usize, + scheduler_threads: usize, +) -> BlockingPool { + BlockingPool::new(builder, thread_cap, scheduler_threads) } diff --git a/tokio/src/runtime/blocking/pool.rs b/tokio/src/runtime/blocking/pool.rs index 5507bf136ed..6fea552cb88 100644 --- a/tokio/src/runtime/blocking/pool.rs +++ b/tokio/src/runtime/blocking/pool.rs @@ -7,7 +7,7 @@ use crate::runtime::blocking::sharded::ShardedImpl; use crate::runtime::blocking::{shutdown, BlockingTask}; use crate::runtime::builder::ThreadNameFn; use crate::runtime::task::{self, JoinHandle}; -use crate::runtime::{Builder, Callback, Handle, BOX_FUTURE_THRESHOLD}; +use crate::runtime::{AutoBox, Builder, Callback, Handle}; use crate::util::metric_atomics::MetricAtomicUsize; use crate::util::trace::{blocking_task, SpawnMeta}; @@ -93,6 +93,9 @@ struct Inner { // Maximum number of threads. thread_cap: usize, + // Number of runtime scheduler workers counted in `num_threads`. + scheduler_threads: usize, + // Customizable wait timeout. keep_alive: Duration, @@ -182,7 +185,7 @@ pub(crate) struct Task { #[derive(PartialEq, Eq)] pub(crate) enum Mandatory { - #[cfg_attr(not(feature = "fs"), allow(dead_code))] + #[cfg_attr(any(not(feature = "fs"), target_os = "emscripten"), allow(dead_code))] Mandatory, NonMandatory, } @@ -246,7 +249,8 @@ where cfg_fs! { #[cfg_attr(any( all(loom, not(test)), // the function is covered by loom tests - test + test, + target_os = "emscripten", // fs uses the inline shim ), allow(dead_code))] /// Runs the provided function on an executor dedicated to blocking /// operations. Tasks will be scheduled as mandatory, meaning they are @@ -265,7 +269,11 @@ cfg_fs! { // ===== impl BlockingPool ===== impl BlockingPool { - pub(crate) fn new(builder: &Builder, thread_cap: usize) -> BlockingPool { + pub(crate) fn new( + builder: &Builder, + thread_cap: usize, + scheduler_threads: usize, + ) -> BlockingPool { let (shutdown_tx, shutdown_rx) = shutdown::channel(); let keep_alive = builder.keep_alive.unwrap_or(KEEP_ALIVE); @@ -292,6 +300,7 @@ impl BlockingPool { after_start: builder.after_start.clone(), before_stop: builder.before_stop.clone(), thread_cap, + scheduler_threads, keep_alive, metrics: SpawnerMetrics::default(), }), @@ -355,7 +364,7 @@ impl Spawner { R: Send + 'static, { let fn_size = std::mem::size_of::(); - let (join_handle, spawn_result) = if fn_size > BOX_FUTURE_THRESHOLD { + let (join_handle, spawn_result) = if AutoBox::::SHOULD_BOX { self.spawn_blocking_inner( Box::new(func), Mandatory::NonMandatory, @@ -385,7 +394,8 @@ impl Spawner { #[track_caller] #[cfg_attr(any( all(loom, not(test)), // the function is covered by loom tests - test + test, + target_os = "emscripten", // fs uses the inline shim ), allow(dead_code))] pub(crate) fn spawn_mandatory_blocking(&self, rt: &Handle, func: F) -> Option> where @@ -393,7 +403,7 @@ impl Spawner { R: Send + 'static, { let fn_size = std::mem::size_of::(); - let (join_handle, spawn_result) = if fn_size > BOX_FUTURE_THRESHOLD { + let (join_handle, spawn_result) = if AutoBox::::SHOULD_BOX { self.spawn_blocking_inner( Box::new(func), Mandatory::Mandatory, @@ -444,6 +454,13 @@ impl Spawner { (handle, spawned) } + pub(crate) fn num_blocking_threads(&self) -> usize { + self.inner + .metrics + .num_threads() + .saturating_sub(self.inner.scheduler_threads) + } + fn spawn_task(&self, task: Task, rt: &Handle) -> Result<(), SpawnError> { // The `on_no_idle` closure runs under the same lock as the queue // push, exactly like the pre-refactor code that called @@ -472,7 +489,7 @@ impl Spawner { } Err(ref e) if is_temporary_os_thread_error(e) - && self.inner.metrics.num_threads() > 0 => + && self.num_blocking_threads() > 0 => { // OS temporarily failed to spawn a new thread. // The task will be picked up eventually by a currently @@ -515,10 +532,6 @@ impl Spawner { cfg_unstable_metrics! { impl Spawner { - pub(crate) fn num_threads(&self) -> usize { - self.inner.metrics.num_threads() - } - pub(crate) fn num_idle_threads(&self) -> usize { self.inner.metrics.num_idle_threads() } diff --git a/tokio/src/runtime/builder.rs b/tokio/src/runtime/builder.rs index 1143a5b8373..e62e3ec5f34 100644 --- a/tokio/src/runtime/builder.rs +++ b/tokio/src/runtime/builder.rs @@ -62,6 +62,7 @@ pub struct Builder { /// Whether or not to enable the I/O driver enable_io: bool, nevents: usize, + nevents_busy: Option, /// Whether or not to enable the time driver enable_time: bool, @@ -308,6 +309,7 @@ impl Builder { // I/O defaults to "off" enable_io: false, nevents: 1024, + nevents_busy: None, // Time defaults to "off" enable_time: false, @@ -1196,6 +1198,7 @@ impl Builder { enable_time: self.enable_time, start_paused: self.start_paused, nevents: self.nevents, + nevents_busy: self.nevents_busy, timer_flavor: self.timer_flavor, } } @@ -1759,7 +1762,7 @@ impl Builder { let (driver, driver_handle) = driver::Driver::new(cfg)?; // Blocking pool - let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads); + let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads, 0); let blocking_spawner = blocking_pool.spawner().clone(); // Generate a rng seed for this runtime. @@ -1825,48 +1828,100 @@ impl Builder { None } } -} -cfg_io_driver! { - impl Builder { - /// Enables the I/O driver. - /// - /// Doing this enables using net, process, signal, and some I/O types on - /// the runtime. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime; - /// - /// let rt = runtime::Builder::new_multi_thread() - /// .enable_io() - /// .build() - /// .unwrap(); - /// ``` - pub fn enable_io(&mut self) -> &mut Self { - self.enable_io = true; - self - } + /// Enables the I/O driver. + /// + /// Doing this enables using net, process, signal, and some I/O types on + /// the runtime. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_current_thread() + /// .enable_io() + /// .build() + /// .unwrap(); + /// ``` + pub fn enable_io(&mut self) -> &mut Self { + self.enable_io = true; + self + } - /// Enables the I/O driver and configures the max number of events to be - /// processed per tick. - /// - /// # Examples - /// - /// ``` - /// use tokio::runtime; - /// - /// let rt = runtime::Builder::new_current_thread() - /// .enable_io() - /// .max_io_events_per_tick(1024) - /// .build() - /// .unwrap(); - /// ``` - pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self { - self.nevents = capacity; - self - } + /// Sets the max number of I/O events processed per tick. + /// + /// To take a smaller batch on polls that do not wait, see + /// [`max_io_events_per_busy_tick`]. + /// + /// [`max_io_events_per_busy_tick`]: Builder::max_io_events_per_busy_tick + /// + /// # Panics + /// + /// Panics if `capacity` is zero. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_current_thread() + /// .enable_io() + /// .max_io_events_per_tick(1024) + /// .build() + /// .unwrap(); + /// ``` + #[track_caller] + pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self { + assert!(capacity > 0, "max_io_events_per_tick must be non-zero"); + self.nevents = capacity; + self + } + + /// Sets the max number of I/O events a worker processes when it polls + /// the driver while it still has tasks to run. + /// + /// A busy worker polls the driver every [`event_interval`] tasks, and + /// every task that poll wakes goes to its local queue. A large batch + /// overflows that queue, and under sustained overload the overflow + /// grows until requests time out. A small busy batch leaves the rest + /// in the kernel. An idle worker still takes up to + /// [`max_io_events_per_tick`] events. + /// + /// The runtime treats any poll that does not wait as busy. That + /// includes a park with a timer that has already expired, because the + /// worker runs that timer's task next. + /// + /// A multi-thread worker's local queue holds 256 tasks, so set + /// `max_io_events_per_tick` to at most 256 as well, with room for the + /// tasks those tasks wake. + /// + /// The default is to use the same value as [`max_io_events_per_tick`]. + /// + /// [`event_interval`]: Builder::event_interval + /// [`max_io_events_per_tick`]: Builder::max_io_events_per_tick + /// + /// # Panics + /// + /// Panics if `capacity` is zero. + /// + /// # Examples + /// + /// ``` + /// use tokio::runtime; + /// + /// let rt = runtime::Builder::new_current_thread() + /// .enable_io() + /// .max_io_events_per_tick(128) + /// .max_io_events_per_busy_tick(8) + /// .build() + /// .unwrap(); + /// ``` + #[track_caller] + pub fn max_io_events_per_busy_tick(&mut self, capacity: usize) -> &mut Self { + assert!(capacity > 0, "max_io_events_per_busy_tick must be non-zero"); + self.nevents_busy = Some(capacity); + self } } @@ -2133,7 +2188,7 @@ cfg_rt_multi_thread! { // Create the blocking pool let blocking_pool = - blocking::create_blocking_pool(self, self.max_blocking_threads + worker_threads); + blocking::create_blocking_pool(self, self.max_blocking_threads + worker_threads, worker_threads); let blocking_spawner = blocking_pool.spawner().clone(); // Generate a rng seed for this runtime. diff --git a/tokio/src/runtime/context.rs b/tokio/src/runtime/context.rs index 22c3e30630c..f1bded93308 100644 --- a/tokio/src/runtime/context.rs +++ b/tokio/src/runtime/context.rs @@ -165,7 +165,6 @@ cfg_rt! { CONTEXT.try_with(|ctx| ctx.current_task_id.get()).unwrap_or_default() } - #[cfg(tokio_unstable)] pub(crate) fn worker_index() -> Option { with_scheduler(|ctx| ctx.and_then(|c| c.worker_index())) } @@ -187,6 +186,22 @@ cfg_rt! { CONTEXT.with(|c| c.scheduler.set(v, f)) } + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + pub(crate) struct ClearSchedulerGuard(*const scheduler::Context); + + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + impl Drop for ClearSchedulerGuard { + fn drop(&mut self) { + CONTEXT.with(|c| c.scheduler.inner.set(self.0)); + } + } + + /// Unsets the scheduler context until the guard drops, on unwind too. + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + pub(crate) fn clear_scheduler() -> ClearSchedulerGuard { + CONTEXT.with(|c| ClearSchedulerGuard(c.scheduler.inner.replace(std::ptr::null()))) + } + #[track_caller] pub(super) fn with_scheduler(f: impl FnOnce(Option<&scheduler::Context>) -> R) -> R { let mut f = Some(f); diff --git a/tokio/src/runtime/driver.rs b/tokio/src/runtime/driver.rs index 92b2350db9d..e30c7da3474 100644 --- a/tokio/src/runtime/driver.rs +++ b/tokio/src/runtime/driver.rs @@ -40,12 +40,14 @@ pub(crate) struct Cfg { pub(crate) enable_pause_time: bool, pub(crate) start_paused: bool, pub(crate) nevents: usize, + pub(crate) nevents_busy: Option, pub(crate) timer_flavor: crate::runtime::TimerFlavor, } impl Driver { pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Handle)> { - let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io, cfg.nevents)?; + let (io_stack, io_handle, signal_handle) = + create_io_stack(cfg.enable_io, cfg.nevents, cfg.nevents_busy)?; let clock = create_clock(cfg.enable_pause_time, cfg.start_paused); @@ -100,7 +102,7 @@ impl Handle { pub(crate) fn signal(&self) -> &crate::runtime::signal::Handle { self.signal .as_ref() - .expect("there is no signal driver running, must be called from the context of Tokio runtime") + .expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.") } } @@ -126,6 +128,11 @@ impl Handle { pub(crate) fn clock(&self) -> &Clock { &self.clock } + + #[cfg(test)] + pub(crate) fn now(&self) -> u64 { + self.time().time_source().deadline_to_tick(self.clock.now()) + } } } @@ -146,12 +153,12 @@ cfg_io_driver! { Disabled(UnparkThread), } - fn create_io_stack(enabled: bool, nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> { + fn create_io_stack(enabled: bool, nevents: usize, nevents_busy: Option) -> io::Result<(IoStack, IoHandle, SignalHandle)> { #[cfg(loom)] assert!(!enabled); let ret = if enabled { - let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents)?; + let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents, nevents_busy)?; let (signal_driver, signal_handle) = create_signal_driver(io_driver, &io_handle)?; let process_driver = create_process_driver(signal_driver); @@ -212,7 +219,7 @@ cfg_not_io_driver! { #[derive(Debug)] pub(crate) struct IoStack(ParkThread); - fn create_io_stack(_enabled: bool, _nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> { + fn create_io_stack(_enabled: bool, _nevents: usize, _nevents_busy: Option) -> io::Result<(IoStack, IoHandle, SignalHandle)> { let park_thread = ParkThread::new(); let unpark_thread = park_thread.unpark(); Ok((IoStack(park_thread), unpark_thread, Default::default())) @@ -251,7 +258,7 @@ cfg_signal_internal_and_unix! { } } -cfg_not_signal_internal! { +cfg_not_signal_internal_and_unix! { pub(crate) type SignalHandle = (); cfg_io_driver! { diff --git a/tokio/src/runtime/handle.rs b/tokio/src/runtime/handle.rs index 27706bac093..578f14a3b9a 100644 --- a/tokio/src/runtime/handle.rs +++ b/tokio/src/runtime/handle.rs @@ -15,7 +15,7 @@ pub struct Handle { } use crate::runtime::task::JoinHandle; -use crate::runtime::BOX_FUTURE_THRESHOLD; +use crate::runtime::AutoBox; use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR}; use crate::util::trace::SpawnMeta; @@ -200,7 +200,7 @@ impl Handle { F::Output: Send + 'static, { let fut_size = mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.spawn_named(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { self.spawn_named(future, SpawnMeta::new_unnamed(fut_size)) @@ -341,7 +341,7 @@ impl Handle { #[track_caller] pub fn block_on(&self, future: F) -> F::Output { let fut_size = mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.block_on_inner(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { self.block_on_inner(future, SpawnMeta::new_unnamed(fut_size)) diff --git a/tokio/src/runtime/io/driver.rs b/tokio/src/runtime/io/driver.rs index cdd841894d0..b9456f6139f 100644 --- a/tokio/src/runtime/io/driver.rs +++ b/tokio/src/runtime/io/driver.rs @@ -29,6 +29,10 @@ pub(crate) struct Driver { /// Reuse the `mio::Events` value across calls to poll. events: mio::Events, + /// Buffer for polls that do not wait, if + /// `Builder::max_io_events_per_busy_tick` is set. + events_busy: Option, + /// The system event queue. poll: mio::Poll, } @@ -114,7 +118,7 @@ fn _assert_kinds() { impl Driver { /// Creates a new event loop, returning any error that happened during the /// creation. - pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> { + pub(crate) fn new(nevents: usize, nevents_busy: Option) -> io::Result<(Driver, Handle)> { let poll = mio::Poll::new()?; #[cfg(not(target_os = "wasi"))] let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?; @@ -123,6 +127,7 @@ impl Driver { let driver = Driver { signal_ready: false, events: mio::Events::with_capacity(nevents), + events_busy: nevents_busy.map(mio::Events::with_capacity), poll, }; @@ -181,7 +186,12 @@ impl Driver { handle.release_pending_registrations(); - let events = &mut self.events; + // A poll that does not wait takes the busy batch. Events it leaves + // behind stay queued in the kernel, so the next poll returns them. + let events = match (&mut self.events_busy, max_wait) { + (Some(busy), Some(wait)) if wait.is_zero() => busy, + _ => &mut self.events, + }; // Block waiting for an event to happen, peeling out how many events // happened. @@ -344,3 +354,34 @@ impl Direction { } } } + +#[cfg(all(test, unix, feature = "net", not(loom), not(miri)))] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn busy_turn_takes_busy_batch() { + let (mut driver, handle) = Driver::new(16, Some(2)).unwrap(); + let mut sources = Vec::new(); + for _ in 0..5 { + let (mut rx, mut tx) = mio::net::UnixStream::pair().unwrap(); + tx.write_all(b"x").unwrap(); + let reg = handle.add_source(&mut rx, Interest::READABLE).unwrap(); + sources.push((rx, tx, reg)); + } + + // A poll that does not wait takes the busy batch. + driver.turn(&handle, Some(Duration::ZERO)); + assert_eq!(driver.events_busy.as_ref().unwrap().iter().count(), 2); + + // The rest stays queued for the next poll, which takes the main batch. + driver.turn(&handle, Some(Duration::from_millis(100))); + assert_eq!(driver.events.iter().count(), 3); + + for (mut rx, _tx, reg) in sources { + handle.deregister_source(®, &mut rx).unwrap(); + } + handle.release_pending_registrations(); + } +} diff --git a/tokio/src/runtime/io/registration.rs b/tokio/src/runtime/io/registration.rs index bc5c54d4ade..f443dd6ef4c 100644 --- a/tokio/src/runtime/io/registration.rs +++ b/tokio/src/runtime/io/registration.rs @@ -104,6 +104,17 @@ impl Registration { self.shared.clear_readiness(event); } + /// Marks the resource ready without waiting for the driver's first event. + /// Used for sockets whose state is known when they are created: accepted + /// sockets are writable and usually already hold the peer's first bytes + /// (under load that first event can queue behind every established + /// connection's events), and both ends of a fresh `pair()` are writable. + /// A wrong guess costs one `WouldBlock`, which clears the readiness again. + #[cfg(feature = "net")] + pub(crate) fn assume_ready(&self, ready: crate::io::Ready) { + self.shared.assume_ready(ready); + } + // Uses the poll path, requiring the caller to ensure mutual exclusion for // correctness. Only the last task to call this function is notified. pub(crate) fn poll_read_ready(&self, cx: &mut Context<'_>) -> Poll> { diff --git a/tokio/src/runtime/io/scheduled_io.rs b/tokio/src/runtime/io/scheduled_io.rs index 6c567664b83..66dc6dae997 100644 --- a/tokio/src/runtime/io/scheduled_io.rs +++ b/tokio/src/runtime/io/scheduled_io.rs @@ -184,6 +184,17 @@ impl Default for ScheduledIo { } impl ScheduledIo { + /// Marks the resource ready in the given directions without an event from + /// the driver. Readiness may have false positives; an operation that finds + /// the resource not ready clears it as usual. Leaves the tick alone and + /// does nothing after shutdown. + pub(super) fn assume_ready(&self, ready: Ready) { + let _ = self.readiness.fetch_update(AcqRel, Acquire, |curr| { + (SHUTDOWN.unpack(curr) == 0) + .then(|| READINESS.pack(READINESS.unpack(curr) | ready.as_usize(), curr)) + }); + } + pub(crate) fn token(&self) -> mio::Token { mio::Token(super::EXPOSE_IO.expose_provenance(self)) } @@ -219,7 +230,11 @@ impl ScheduledIo { Tick::Set => tick.wrapping_add(1) % MAX_TICK, }; let ready = Ready::from_usize(READINESS.unpack(curr)); - Some(TICK.pack(new_tick, f(ready).as_usize())) + // Keep the shutdown bit, so that a clear after shutdown (a + // `WouldBlock` observed once the driver is gone) does not turn the + // next wait into a wait for an event that will never come. + let next = TICK.pack(new_tick, f(ready).as_usize()); + Some(SHUTDOWN.pack(SHUTDOWN.unpack(curr), next)) }); } diff --git a/tokio/src/runtime/jspi.rs b/tokio/src/runtime/jspi.rs new file mode 100644 index 00000000000..2c1979f763c --- /dev/null +++ b/tokio/src/runtime/jspi.rs @@ -0,0 +1,113 @@ +//! Minimal JSPI primitives for `wasm32-unknown-emscripten`. +//! +//! [`park`] suspends the calling activation on a promise held in a per-parker +//! slot on the host side, settled by a host timer at the deadline or by +//! [`unpark`] from a later activation (a host callback entering tokio). Both +//! the runtime driver and the thread parker behind `blocking_recv` and +//! friends park through here, so neither needs the `rt` feature. The runtime +//! stays entered while parked, so a `block_on` from another activation on the +//! thread during the park panics as a nested runtime. + +use std::sync::OnceLock; +use std::time::Duration; + +// Emscripten EM_JS convention: the `__em_js__` data export carries the +// JS body, and an `__asyncjs__` name gets `WebAssembly.Suspending` treatment +// under `-sJSPI`. `#[used]` is what exports it: on this target LLVM marks +// `llvm.used` symbols exported (the `EMSCRIPTEN_KEEPALIVE` mechanism), while +// rustc keeps `#[no_mangle]` statics out of the linker's export list. +// +// `ms < 0` is a park with no deadline. A zero-duration park is the +// scheduler's maintenance yield, and wants the cheapest resumption that +// still lets the host loop reach its timer phase. `setTimeout(0)` is clamped +// to a millisecond, while an immediate resumes after the current poll phase +// and schedules its successor into the next iteration, which begins by +// running expired timers. A microtask-flavoured queue (`queueMicrotask`, +// `process.nextTick`) would not do: those drain before the loop advances at +// all, so host timers could never fire and a self-waking task would starve +// them. Hosts without an immediate keep the clamped timeout. Timeouts above +// the host's 32-bit millisecond limit would be clamped to one, so they are +// capped; a spurious resume at the cap re-parks. +// +// A slot is only removed by its own `wake`. A park whose suspension failed +// (see `SuspendError` below) leaves its timer pending, and the slot id (the +// parker's address) may be reused before that fires; the stale timer must not +// remove the successor's entry. +#[allow(non_upper_case_globals)] +#[no_mangle] +#[used] +static __em_js____asyncjs__tokio_jspi_park: [u8; 555] = *b"(id, ms)<::>{ \ + return Asyncify.handleAsync(async () => { \ + const parks = Module.tokioParks || (Module.tokioParks = new Map()); \ + await new Promise((resolve) => { \ + const immediate = ms === 0 && typeof setImmediate == 'function'; \ + const done = () => { if (parks.get(id) === wake) parks.delete(id); resolve(); }; \ + const timer = ms < 0 ? undefined : immediate ? setImmediate(done) : setTimeout(done, Math.min(ms, 0x7fffffff)); \ + const wake = () => { if (timer !== undefined) (immediate ? clearImmediate : clearTimeout)(timer); done(); }; \ + parks.set(id, wake); \ + }); \ + }); \ + }\0"; + +#[allow(non_upper_case_globals)] +#[no_mangle] +#[used] +static __em_js__tokio_jspi_unpark: [u8; 91] = *b"(id)<::>{ \ + const wake = Module.tokioParks && Module.tokioParks.get(id); \ + if (wake) wake(); \ + }\0"; + +extern "C" { + /// Reports the `ASYNCIFY` build mode: 0 = none, 1 = legacy `Asyncify`, + /// 2 = JSPI. Only mode 2 supports Tokio's JSPI imports. + fn emscripten_has_asyncify() -> i32; +} + +#[link(wasm_import_module = "env")] +extern "C-unwind" { + // Suspending import: parks on the slot for `id`. Unit return. + // `Asyncify.handleAsync` keeps the runtime alive across the suspension. + // + // Suspension needs a `WebAssembly.promising` activation on the stack. + // From any other activation (a plain host callback into the module) the + // engine throws `WebAssembly.SuspendError` out of this import instead. It + // is a foreign exception to Rust: drops run as it unwinds, `catch_unwind` + // does not catch it, and it aborts at the first `extern "C"` frame. + #[link_name = "__asyncjs__tokio_jspi_park"] + fn tokio_jspi_park_import(id: usize, ms: f64); + + // Settles the slot for `id`, if parked. The resumption is a microtask, + // so it runs once the calling activation has returned to the host. + #[link_name = "tokio_jspi_unpark"] + fn tokio_jspi_unpark_import(id: usize); +} + +/// Whether JSPI suspension is available: linked with `-sJSPI`. +pub(crate) fn jspi_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + // SAFETY: an Emscripten libc query with no arguments and no side effects. + *ENABLED.get_or_init(|| unsafe { emscripten_has_asyncify() == 2 }) +} + +/// Suspend the owning activation until [`unpark`] is called for `id`, or +/// `dur` elapses on a host timer if given. +pub(crate) fn park(id: usize, dur: Option) { + let ms = dur.map_or(-1.0, |dur| dur.as_secs_f64() * 1000.0); + // A host activation entering tokio during the suspension shares this + // thread's locals but is not on the runtime: with the scheduler context + // left set, its wakes would take the on-runtime shortcut and never + // unpark. + #[cfg(feature = "rt")] + let _scheduler = crate::runtime::context::clear_scheduler(); + // SAFETY: the import takes plain scalars and returns nothing. Under + // `-sJSPI` it suspends this activation; the caller has checked + // `jspi_enabled`. + unsafe { tokio_jspi_park_import(id, ms) } +} + +/// Resume the activation parked under `id`, if any. +pub(crate) fn unpark(id: usize) { + // SAFETY: the import takes a plain scalar and returns nothing, and does + // not suspend. + unsafe { tokio_jspi_unpark_import(id) } +} diff --git a/tokio/src/runtime/local_runtime/runtime.rs b/tokio/src/runtime/local_runtime/runtime.rs index 6782f425a36..9b13ce2dab0 100644 --- a/tokio/src/runtime/local_runtime/runtime.rs +++ b/tokio/src/runtime/local_runtime/runtime.rs @@ -2,7 +2,7 @@ use crate::runtime::blocking::BlockingPool; use crate::runtime::scheduler::CurrentThread; -use crate::runtime::{context, Builder, EnterGuard, Handle, BOX_FUTURE_THRESHOLD}; +use crate::runtime::{context, AutoBox, Builder, EnterGuard, Handle}; use crate::task::JoinHandle; use crate::util::trace::SpawnMeta; @@ -158,7 +158,7 @@ impl LocalRuntime { // safety: spawn_local can only be called from `LocalRuntime`, which this is unsafe { - if std::mem::size_of::() > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.handle.spawn_local_named(Box::pin(future), meta) } else { self.handle.spawn_local_named(future, meta) @@ -225,7 +225,7 @@ impl LocalRuntime { let fut_size = mem::size_of::(); let meta = SpawnMeta::new_unnamed(fut_size); - if std::mem::size_of::() > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.block_on_inner(Box::pin(future), meta) } else { self.block_on_inner(future, meta) diff --git a/tokio/src/runtime/mod.rs b/tokio/src/runtime/mod.rs index 05bb5fa6ee1..66dc253ad40 100644 --- a/tokio/src/runtime/mod.rs +++ b/tokio/src/runtime/mod.rs @@ -418,6 +418,9 @@ pub(crate) mod context; pub(crate) mod park; +#[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] +pub(crate) mod jspi; + pub(crate) mod driver; pub(crate) mod scheduler; @@ -444,8 +447,6 @@ cfg_time! { #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))] pub(crate) mod time_alt; - use crate::time::Instant; - use std::task::{Context, Poll}; use std::pin::Pin; @@ -460,7 +461,7 @@ cfg_time! { impl Timer { #[cfg_attr(not(all(tokio_unstable, feature = "rt-multi-thread")), allow(unused_variables))] #[track_caller] - pub(crate) fn new(handle: scheduler::Handle, deadline: Instant) -> Self { + pub(crate) fn new(handle: scheduler::Handle, deadline: u64) -> Self { match handle.timer_flavor() { TimerFlavor::Traditional => { Timer::Traditional(time::TimerEntry::new(handle)) @@ -472,7 +473,7 @@ cfg_time! { } } - pub(crate) fn init(self: Pin<&mut Self>, deadline: Instant) { + pub(crate) fn init(self: Pin<&mut Self>, deadline: u64) { // Safety: we never move the inner entries. let this = unsafe { self.get_unchecked_mut() }; match this { @@ -494,7 +495,7 @@ cfg_time! { } #[cfg_attr(not(all(tokio_unstable, feature = "rt-multi-thread")), allow(unused_variables))] - pub(crate) fn reset(self: Pin<&mut Self>, handle: scheduler::Handle, deadline: Instant) { + pub(crate) fn reset(self: Pin<&mut Self>, handle: scheduler::Handle, deadline: u64) { // Safety: we never move the inner entries. let this = unsafe { self.get_unchecked_mut() }; match this { @@ -550,6 +551,8 @@ cfg_rt! { } cfg_fs! { + // Emscripten uses the inline shim in `crate::blocking`. + #[cfg_attr(target_os = "emscripten", allow(unused_imports))] pub(crate) use blocking::spawn_mandatory_blocking; } @@ -558,43 +561,45 @@ cfg_rt! { cfg_unstable! { pub use self::builder::UnhandledPanic; pub use crate::util::rand::RngSeed; + } - /// Returns the index of the current worker thread, if called from a - /// runtime worker thread. - /// - /// The returned value is a 0-based index matching the worker indices - /// used by [`RuntimeMetrics`] methods such as - /// [`worker_total_busy_duration`](RuntimeMetrics::worker_total_busy_duration). - /// - /// Returns `None` when called from outside a runtime worker thread - /// (for example, from a blocking thread or a non-Tokio thread). On the - /// multi-thread runtime, the thread that calls [`Runtime::block_on`] is - /// not a worker thread, so this also returns `None` there. - /// - /// For the current-thread runtime and [`LocalRuntime`], this always - /// returns `Some(0)` (including inside `block_on`, since the calling - /// thread *is* the worker thread). - /// - /// Note that the result may change across `.await` points, as the - /// task may be moved to a different worker thread by the scheduler. - /// - /// # Examples - /// - /// ``` - /// # #[cfg(not(target_family = "wasm"))] - /// # { - /// #[tokio::main(flavor = "multi_thread", worker_threads = 4)] - /// async fn main() { - /// let index = tokio::spawn(async { - /// tokio::runtime::worker_index() - /// }).await.unwrap(); - /// println!("Task ran on worker {:?}", index); - /// } - /// # } - /// ``` - pub fn worker_index() -> Option { - context::worker_index() - } + /// Returns the index of the current worker thread, if called from a + /// runtime worker thread. + /// + /// The returned value is a 0-based index matching the worker indices + /// used by [`RuntimeMetrics`] methods such as + /// [`worker_total_busy_duration`](RuntimeMetrics::worker_total_busy_duration). + /// + /// Returns `None` when called from outside a runtime worker thread + /// (for example, from a blocking thread or a non-Tokio thread). On the + /// multi-thread runtime, the thread that calls [`Runtime::block_on`] is + /// not a worker thread, so this also returns `None` there. + /// + /// For the current-thread runtime, this returns `Some(0)` when called from + /// the thread that currently owns the runtime driver. If multiple threads + /// call [`Runtime::block_on`] concurrently, calls on threads that do not + /// own the driver return `None`. A [`LocalRuntime`] can only be driven from + /// its owning thread, so calls inside `block_on` always return `Some(0)`. + /// + /// Note that the result may change across `.await` points, as the + /// task may be moved to a different worker thread by the scheduler. + /// + /// # Examples + /// + /// ``` + /// # #[cfg(not(target_family = "wasm"))] + /// # { + /// #[tokio::main(flavor = "multi_thread", worker_threads = 4)] + /// async fn main() { + /// let index = tokio::spawn(async { + /// tokio::runtime::worker_index() + /// }).await.unwrap(); + /// println!("Task ran on worker {:?}", index); + /// } + /// # } + /// ``` + pub fn worker_index() -> Option { + context::worker_index() } cfg_taskdump! { @@ -631,6 +636,24 @@ cfg_rt! { 16384 }; + /// Decides whether a future or closure of type `T` is boxed before it is + /// turned into a task, based on [`BOX_FUTURE_THRESHOLD`]. + /// + /// The decision is an associated constant rather than a runtime + /// comparison of `std::mem::size_of::()` so that only the taken branch + /// is instantiated. With a runtime `if`, both branches are instantiated + /// for every `T` (one task harness for `T`, one for `Pin>`), + /// doubling the generated code for every spawned future in a crate. A + /// branch on a constant that is known once `T` is known is pruned by the + /// monomorphization collector, so only the harness that is actually used + /// is generated. + pub(crate) struct AutoBox(std::marker::PhantomData); + + impl AutoBox { + /// `true` if a value of type `T` is larger than [`BOX_FUTURE_THRESHOLD`]. + pub(crate) const SHOULD_BOX: bool = std::mem::size_of::() > BOX_FUTURE_THRESHOLD; + } + mod thread_id; pub(crate) use thread_id::ThreadId; diff --git a/tokio/src/runtime/park.rs b/tokio/src/runtime/park.rs index cf502c40408..a8d73731dfa 100644 --- a/tokio/src/runtime/park.rs +++ b/tokio/src/runtime/park.rs @@ -28,6 +28,7 @@ const EMPTY: usize = 0; const PARKED: usize = 1; const NOTIFIED: usize = 2; +#[cfg(not(all(target_os = "emscripten", not(target_feature = "atomics"))))] tokio_thread_local! { static CURRENT_PARKER: ParkThread = ParkThread::new(); } @@ -76,6 +77,7 @@ impl ParkThread { // ==== impl Inner ==== impl Inner { + #[cfg(not(all(target_os = "emscripten", not(target_feature = "atomics"))))] fn park(&self) { // If we were previously notified then we consume this notification and // return quickly. @@ -123,7 +125,72 @@ impl Inner { } } + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + fn park(&self) { + self.park_jspi(None); + } + + /// Parks the activation until unparked or `dur` elapses, if given. + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + fn park_jspi(&self, dur: Option) { + // If we were previously notified then we consume this notification and + // return quickly. + if self + .state + .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst) + .is_ok() + { + return; + } + + // Without JSPI a real wait is impossible: suspending would trap and + // busy-waiting would starve the host loop the wake depends on. A + // zero-duration park returns immediately as on native. + if !crate::runtime::jspi::jspi_enabled() { + if dur == Some(Duration::ZERO) { + return; + } + panic!( + "cannot block on wasm32-unknown-emscripten: this wait would \ + suspend on the host event loop, which needs the build to \ + link `-sJSPI`" + ); + } + + // Suspend until a host timer fires or a later activation (a host + // callback entering tokio) unparks us. A zero-duration park still + // takes a host turn so timers and microtasks run mid-drive (the + // scheduler's maintenance yield). No threads: nothing runs between + // here and the suspension, so an unpark can only land while parked, + // where it settles the promise. + let old = self.state.swap(PARKED, SeqCst); + debug_assert_eq!(old, EMPTY, "inconsistent park state"); + + // Consume any notification delivered during the park so the token + // does not leak into the next one. A `SuspendError` unwinding out of + // the import must also leave the parker `EMPTY` for its next park. + struct Unpark<'a>(&'a Inner); + impl Drop for Unpark<'_> { + fn drop(&mut self) { + match self.0.state.swap(EMPTY, SeqCst) { + NOTIFIED => {} // got a notification, hurray! + PARKED => {} // timer deadline, alas + n => debug_assert!(false, "inconsistent park state: {n}"), + } + } + } + let _unpark = Unpark(self); + + crate::runtime::jspi::park(self.id(), dur); + } + + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + fn id(&self) -> usize { + self as *const Inner as usize + } + /// Parks the current thread for at most `dur`. + #[cfg(not(all(target_os = "emscripten", not(target_feature = "atomics"))))] fn park_timeout(&self, dur: Duration) { // Like `park` above we have a fast path for an already-notified thread, // and afterwards we start coordinating for a sleep. Return quickly. @@ -174,6 +241,11 @@ impl Inner { } } + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + fn park_timeout(&self, dur: Duration) { + self.park_jspi(Some(dur)); + } + fn unpark(&self) { // To ensure the unparked thread will observe any writes we made before // this call, we must perform a release operation that `park` can @@ -187,6 +259,10 @@ impl Inner { _ => panic!("inconsistent state in unpark"), } + // The parked activation is suspended in the host; settle its promise. + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + crate::runtime::jspi::unpark(self.id()); + // There is a period between when the parked thread sets `state` to // `PARKED` (or last checked `state` in the case of a spurious wake // up) and when it actually waits on `cvar`. If we were to notify @@ -231,6 +307,12 @@ use std::task::{RawWaker, RawWakerVTable, Waker}; /// Blocks the current thread using a condition variable. #[derive(Debug)] pub(crate) struct CachedParkThread { + // While a suspended stack is parked, host callbacks can still run tokio + // code on this thread, and a shared thread-local parker would hand this + // stack's notification token to that code. Each blocking call gets its + // own parker instead. + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + park: ParkThread, _anchor: PhantomData>, } @@ -241,6 +323,8 @@ impl CachedParkThread { /// the thread that the caller intends to park. pub(crate) fn new() -> CachedParkThread { CachedParkThread { + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + park: ParkThread::new(), _anchor: PhantomData, } } @@ -268,7 +352,12 @@ impl CachedParkThread { where F: FnOnce(&ParkThread) -> R, { - CURRENT_PARKER.try_with(|inner| f(inner)) + #[cfg(not(all(target_os = "emscripten", not(target_feature = "atomics"))))] + return CURRENT_PARKER.try_with(|inner| f(inner)); + + // See the comment on the `park` field. + #[cfg(all(target_os = "emscripten", not(target_feature = "atomics")))] + return Ok(f(&self.park)); } pub(crate) fn block_on(&mut self, f: F) -> Result { diff --git a/tokio/src/runtime/runtime.rs b/tokio/src/runtime/runtime.rs index 0a1a9b35d19..52c06e53a34 100644 --- a/tokio/src/runtime/runtime.rs +++ b/tokio/src/runtime/runtime.rs @@ -1,4 +1,4 @@ -use super::BOX_FUTURE_THRESHOLD; +use super::AutoBox; use crate::runtime::blocking::BlockingPool; use crate::runtime::scheduler::CurrentThread; use crate::runtime::{context, EnterGuard, Handle}; @@ -247,7 +247,7 @@ impl Runtime { F::Output: Send + 'static, { let fut_size = mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.handle .spawn_named(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { @@ -342,7 +342,7 @@ impl Runtime { #[track_caller] pub fn block_on(&self, future: F) -> F::Output { let fut_size = mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.block_on_inner(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { self.block_on_inner(future, SpawnMeta::new_unnamed(fut_size)) diff --git a/tokio/src/runtime/scheduler/current_thread/mod.rs b/tokio/src/runtime/scheduler/current_thread/mod.rs index 4766ba47bb3..b083e025043 100644 --- a/tokio/src/runtime/scheduler/current_thread/mod.rs +++ b/tokio/src/runtime/scheduler/current_thread/mod.rs @@ -662,7 +662,7 @@ cfg_unstable_metrics! { } pub(crate) fn num_blocking_threads(&self) -> usize { - self.blocking_spawner.num_threads() + self.blocking_spawner.num_blocking_threads() } pub(crate) fn num_idle_blocking_threads(&self) -> usize { diff --git a/tokio/src/runtime/scheduler/inject.rs b/tokio/src/runtime/scheduler/inject.rs index f23118cb1c9..8ccb6074415 100644 --- a/tokio/src/runtime/scheduler/inject.rs +++ b/tokio/src/runtime/scheduler/inject.rs @@ -14,6 +14,7 @@ pub(crate) use synced::Synced; cfg_rt_multi_thread! { mod rt_multi_thread; + pub(crate) use rt_multi_thread::InjectQueue; } mod metrics; @@ -36,7 +37,7 @@ impl Inject { } // Kind of annoying to have to include the cfg here - #[cfg(all(tokio_unstable, feature = "taskdump"))] + #[cfg(any(all(tokio_unstable, feature = "taskdump"), feature = "rt-multi-thread"))] pub(crate) fn is_closed(&self) -> bool { let synced = self.synced.lock(); self.shared.is_closed(&synced) diff --git a/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs b/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs index 44f7b635941..ffca6b406f1 100644 --- a/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs +++ b/tokio/src/runtime/scheduler/inject/rt_multi_thread.rs @@ -1,34 +1,102 @@ -use super::{Shared, Synced}; +use super::{Inject, Pop}; -use crate::runtime::scheduler::Lock; use crate::runtime::task; use std::sync::atomic::Ordering::Release; -impl<'a> Lock for &'a mut Synced { - type Handle = &'a mut Synced; +/// The multi-thread scheduler's inject queue. Each variant owns its queue and +/// lock topology. +pub(crate) enum InjectQueue { + /// A single queue behind a single mutex. + Locked(Inject), +} - fn lock(self) -> Self::Handle { - self +impl InjectQueue { + pub(crate) fn new() -> InjectQueue { + InjectQueue::Locked(Inject::new()) } -} -impl AsMut for Synced { - fn as_mut(&mut self) -> &mut Synced { - self + pub(crate) fn is_empty(&self) -> bool { + match self { + InjectQueue::Locked(q) => q.is_empty(), + } } -} -impl Shared { - /// Pushes several values into the queue. + pub(crate) fn len(&self) -> usize { + match self { + InjectQueue::Locked(q) => q.len(), + } + } + + pub(crate) fn is_closed(&self) -> bool { + match self { + InjectQueue::Locked(q) => q.is_closed(), + } + } + + /// Closes the queue, returns `true` if the queue was open when the + /// transition was made. + pub(crate) fn close(&self) -> bool { + match self { + InjectQueue::Locked(q) => q.close(), + } + } + + /// Pushes a value into the queue. /// - /// # Safety + /// This does nothing if the queue is closed. + pub(crate) fn push(&self, task: task::Notified) { + match self { + InjectQueue::Locked(q) => q.push(task), + } + } + + pub(crate) fn pop(&self) -> Option> { + match self { + InjectQueue::Locked(q) => q.pop(), + } + } + + /// Pushes several values into the queue. /// - /// Must be called with the same `Synced` instance returned by `Inject::new` + /// This does nothing if the queue is closed. + pub(crate) fn push_batch(&self, iter: I) + where + I: Iterator>, + { + match self { + InjectQueue::Locked(q) => q.push_batch(iter), + } + } + + /// Pops up to `n` values from the queue, passing an iterator over them to + /// `f`. Any values `f` does not consume are removed from the queue and + /// dropped. + pub(crate) fn pop_n(&self, n: usize, f: impl FnOnce(Pop<'_, T>) -> R) -> R { + match self { + InjectQueue::Locked(q) => q.pop_n(n, f), + } + } + + /// Pops every task from the queue into `dst`, atomically with respect to + /// concurrent pushes. + #[cfg(all(tokio_unstable, feature = "taskdump"))] + pub(crate) fn drain_into(&self, dst: &mut Vec>) { + match self { + InjectQueue::Locked(q) => q.drain_into(dst), + } + } +} + +impl Inject { + pub(crate) fn is_empty(&self) -> bool { + self.shared.is_empty() + } + + /// Pushes several values into the queue. #[inline] - pub(crate) unsafe fn push_batch(&self, shared: L, mut iter: I) + pub(crate) fn push_batch(&self, mut iter: I) where - L: Lock, I: Iterator>, { let first = match iter.next() { @@ -56,10 +124,9 @@ impl Shared { // Now that the tasks are linked together, insert them into the // linked list. // - // Safety: exactly the same safety requirements as `push_batch` method. - unsafe { - self.push_batch_inner(shared, first, prev, counter); - } + // safety: the batch was linked just above from `Notified`s this + // function took ownership of, satisfying both obligations. + unsafe { self.push_batch_inner(first, prev, counter) }; } /// Inserts several tasks that have been linked together into the queue. @@ -69,28 +136,32 @@ impl Shared { /// /// # Safety /// - /// Must be called with the same `Synced` instance returned by `Inject::new` + /// The caller must own the `Notified` for each of the `num` tasks, and + /// the tasks must be linked from `batch_head` to `batch_tail` through + /// their `queue_next` fields, with `batch_tail`'s `queue_next` unset. #[inline] - unsafe fn push_batch_inner( + unsafe fn push_batch_inner( &self, - shared: L, batch_head: task::RawTask, batch_tail: task::RawTask, num: usize, - ) where - L: Lock, - { + ) { debug_assert!(unsafe { batch_tail.get_queue_next().is_none() }); - let mut synced = shared.lock(); + let mut synced = self.synced.lock(); - if synced.as_mut().is_closed { + if synced.is_closed { + // Drop the lock before dropping the tasks: dropping a task can + // run arbitrary user `Drop` code, which may reentrantly acquire + // this lock by scheduling a task. drop(synced); let mut curr = Some(batch_head); while let Some(task) = curr { - // Safety: exactly the same safety requirements as `push_batch_inner`. + // safety: per this function's contract, the caller owns each + // task's `Notified` and linked the batch through `queue_next`; + // reconstituting the `Notified` here takes that ownership. curr = unsafe { task.get_queue_next() }; let _ = unsafe { task::Notified::::from_raw(task) }; @@ -99,8 +170,6 @@ impl Shared { return; } - let synced = synced.as_mut(); - if let Some(tail) = synced.tail { unsafe { tail.set_queue_next(Some(batch_head)); @@ -115,8 +184,29 @@ impl Shared { // // safety: All updates to the len atomic are guarded by the mutex. As // such, a non-atomic load followed by a store is safe. - let len = unsafe { self.len.unsync_load() }; + let len = unsafe { self.shared.len.unsync_load() }; - self.len.store(len + num, Release); + self.shared.len.store(len + num, Release); + } + + /// Pops up to `n` values from the queue, passing an iterator over them to + /// `f`. The queue lock is held while `f` runs, so any values `f` does not + /// consume are removed from the queue and dropped before the lock is + /// released. + pub(crate) fn pop_n(&self, n: usize, f: impl FnOnce(Pop<'_, T>) -> R) -> R { + let mut synced = self.synced.lock(); + // safety: passing correct `Synced` + f(unsafe { self.shared.pop_n(&mut synced, n) }) + } + + /// Pops every task from the queue into `dst`, holding the queue lock for + /// the entire drain so it is atomic with respect to concurrent pushes. + #[cfg(all(tokio_unstable, feature = "taskdump"))] + pub(crate) fn drain_into(&self, dst: &mut Vec>) { + let mut synced = self.synced.lock(); + // safety: passing correct `Synced` + while let Some(task) = unsafe { self.shared.pop(&mut synced) } { + dst.push(task); + } } } diff --git a/tokio/src/runtime/scheduler/lock.rs b/tokio/src/runtime/scheduler/lock.rs deleted file mode 100644 index 0901c2b37ca..00000000000 --- a/tokio/src/runtime/scheduler/lock.rs +++ /dev/null @@ -1,6 +0,0 @@ -/// A lock (mutex) yielding generic data. -pub(crate) trait Lock { - type Handle: AsMut; - - fn lock(self) -> Self::Handle; -} diff --git a/tokio/src/runtime/scheduler/mod.rs b/tokio/src/runtime/scheduler/mod.rs index 6175165cb2d..f8d2162bddf 100644 --- a/tokio/src/runtime/scheduler/mod.rs +++ b/tokio/src/runtime/scheduler/mod.rs @@ -17,9 +17,6 @@ cfg_rt_multi_thread! { mod block_in_place; pub(crate) use block_in_place::block_in_place; - mod lock; - use lock::Lock; - pub(crate) mod multi_thread; pub(crate) use multi_thread::MultiThread; } @@ -281,7 +278,6 @@ cfg_rt! { match_flavor!(self, Context(context) => context.defer(waker)); } - #[cfg(tokio_unstable)] pub(crate) fn worker_index(&self) -> Option { match self { Context::CurrentThread(_) => Some(0), diff --git a/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs b/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs index 985495c7561..c1f452f8184 100644 --- a/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs +++ b/tokio/src/runtime/scheduler/multi_thread/handle/metrics.rs @@ -30,10 +30,7 @@ impl Handle { } pub(crate) fn num_blocking_threads(&self) -> usize { - // workers are currently spawned using spawn_blocking - self.blocking_spawner - .num_threads() - .saturating_sub(self.num_workers()) + self.blocking_spawner.num_blocking_threads() } pub(crate) fn num_idle_blocking_threads(&self) -> usize { diff --git a/tokio/src/runtime/scheduler/multi_thread/idle.rs b/tokio/src/runtime/scheduler/multi_thread/idle.rs index 834bc2b66fc..e0b034c85a9 100644 --- a/tokio/src/runtime/scheduler/multi_thread/idle.rs +++ b/tokio/src/runtime/scheduler/multi_thread/idle.rs @@ -92,6 +92,10 @@ impl Idle { // Acquire the lock let mut lock = shared.synced.lock(); + // A worker that is already tracked as a sleeper would be counted twice, + // which corrupts `num_unparked` and the sleeper list. + debug_assert!(!lock.idle.sleepers.contains(&worker)); + // Decrement the number of unparked threads let ret = State::dec_num_unparked(&self.state, is_searching); @@ -151,6 +155,9 @@ impl Idle { } fn notify_should_wakeup(&self) -> bool { + // This must be a `SeqCst` RMW rather than a load: it is what makes + // the caller's preceding inject-queue push visible to a parking + // worker's subsequent unlocked queue-emptiness check. let state = State(self.state.fetch_add(0, SeqCst)); state.num_searching() == 0 && state.num_unparked() < self.num_workers } @@ -194,6 +201,7 @@ impl State { } let prev = State(cell.fetch_sub(dec, SeqCst)); + debug_assert!(prev.num_unparked() > 0, "{prev:?}"); is_searching && prev.num_searching() == 1 } diff --git a/tokio/src/runtime/scheduler/multi_thread/mod.rs b/tokio/src/runtime/scheduler/multi_thread/mod.rs index c04a3ef55a6..5bc7f100b7e 100644 --- a/tokio/src/runtime/scheduler/multi_thread/mod.rs +++ b/tokio/src/runtime/scheduler/multi_thread/mod.rs @@ -26,8 +26,6 @@ pub(crate) use worker::{Context, Launch, Shared}; cfg_taskdump! { mod trace; use trace::TraceStatus; - - pub(crate) use worker::Synced; } cfg_not_taskdump! { diff --git a/tokio/src/runtime/scheduler/multi_thread/worker.rs b/tokio/src/runtime/scheduler/multi_thread/worker.rs index 97a285c6116..ac16dd36b3c 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker.rs @@ -61,7 +61,7 @@ use crate::runtime; use crate::runtime::scheduler::multi_thread::{ idle, park, queue, Counters, Handle, Idle, Overflow, Parker, Stats, TraceStatus, Unparker, }; -use crate::runtime::scheduler::{inject, Defer, Lock}; +use crate::runtime::scheduler::{inject::InjectQueue, Defer}; use crate::runtime::task::OwnedTasks; use crate::runtime::{ blocking, driver, scheduler, task, Config, SchedulerMetrics, TimerFlavor, WorkerMetrics, @@ -72,6 +72,7 @@ use crate::util::atomic_cell::AtomicCell; use crate::util::rand::{FastRand, RngSeedGenerator}; use std::cell::RefCell; +use std::ops::ControlFlow; use std::task::Waker; use std::thread; use std::time::{Duration, Instant}; @@ -174,7 +175,7 @@ pub(crate) struct Shared { /// Global task queue used for: /// 1. Submit work to the scheduler while **not** currently on a worker thread. /// 2. Submit work to the scheduler when a worker run queue is saturated - pub(super) inject: inject::Shared>, + pub(super) inject: InjectQueue>, /// Coordinates idle workers idle: Idle, @@ -220,13 +221,16 @@ pub(crate) struct Synced { /// Synchronized state for `Idle`. pub(super) idle: idle::Synced, - /// Synchronized state for `Inject`. - pub(crate) inject: inject::Synced, - #[cfg(all(tokio_unstable, feature = "time"))] /// Timers pending to be registered. /// This is used to register a timer but the [`Core`] /// is not available in the current thread. + /// + /// This must stay under the same mutex as `idle`: a parking worker drains + /// it (via `try_lock`) only after publishing its parked state under this + /// lock, and `notify_if_work_pending` does not check for pending timers, + /// so sharing the lock with the parking transition is what prevents a + /// timer push from being stranded while every worker sleeps. inject_timers: Vec, } @@ -255,11 +259,6 @@ pub(crate) struct Context { /// Starts the workers pub(crate) struct Launch(Vec>); -/// Running a task may consume the core. If the core is still available when -/// running the task completes, it is returned. Otherwise, the worker will need -/// to stop processing. -type RunResult = Result, ()>; - /// A notified task handle type Notified = task::Notified>; @@ -316,7 +315,6 @@ pub(super) fn create( } let (idle, idle_synced) = Idle::new(size); - let (inject, inject_synced) = inject::Shared::new(); let schedule_latency_start = config.track_task_schedule_latency.then(Instant::now); let remotes_len = remotes.len(); @@ -325,12 +323,11 @@ pub(super) fn create( task_hooks: TaskHooks::from_config(&config), shared: Shared { remotes: remotes.into_boxed_slice(), - inject, + inject: InjectQueue::new(), idle, owned: OwnedTasks::new(size), synced: Mutex::new(Synced { idle: idle_synced, - inject: inject_synced, #[cfg(all(tokio_unstable, feature = "time"))] inject_timers: Vec::new(), }), @@ -559,9 +556,7 @@ fn run(worker: Arc) { context::set_scheduler(&cx, || { let cx = cx.expect_multi_thread(); - // This should always be an error. It only returns a `Result` to support - // using `?` to short circuit. - assert!(cx.run(core).is_err()); + cx.run(core); // Check if there are any deferred tasks to notify. This can happen when // the worker core is lost due to `block_in_place()` being called from @@ -572,7 +567,7 @@ fn run(worker: Arc) { } impl Context { - fn run(&self, mut core: Box) -> RunResult { + fn run(&self, mut core: Box) { // Reset `lifo_enabled` here in case the core was previously stolen from // a task that had the LIFO slot disabled. self.reset_lifo_enabled(&mut core); @@ -596,7 +591,10 @@ impl Context { // First, check work available to the current worker. if let Some(task) = core.next_task(&self.worker) { - core = self.run_task(task, core)?; + core = match self.run_task(task, core) { + ControlFlow::Continue(core) => core, + ControlFlow::Break(()) => return, + }; continue; } @@ -608,7 +606,10 @@ impl Context { if let Some(task) = core.steal_work(&self.worker) { // Found work, switch back to processing core.stats.start_processing_scheduled_tasks(); - core = self.run_task(task, core)?; + core = match self.run_task(task, core) { + ControlFlow::Continue(core) => core, + ControlFlow::Break(()) => return, + }; } else { // Wait for work core = if !self.defer.is_empty() { @@ -638,10 +639,12 @@ impl Context { core.pre_shutdown(&self.worker); // Signal shutdown self.worker.handle.shutdown_core(core); - Err(()) } - fn run_task(&self, task: Notified, mut core: Box) -> RunResult { + /// Running a task may consume the core. If the core is still available when + /// running the task completes, it is returned. Otherwise, the worker will need + /// to stop processing. + fn run_task(&self, task: Notified, mut core: Box) -> ControlFlow<(), Box> { let task = self.worker.handle.shared.owned.assert_owner(task); // Make sure the worker is not in the **searching** state. This enables @@ -716,7 +719,7 @@ impl Context { // In this case, we cannot call `reset_lifo_enabled()` // because the core was stolen. The stealer will handle // that at the top of `Context::run` - return Err(()); + return ControlFlow::Break(()); } }; @@ -726,7 +729,7 @@ impl Context { None => { self.reset_lifo_enabled(&mut core); core.stats.end_poll(); - return Ok(core); + return ControlFlow::Continue(core); } }; @@ -743,7 +746,7 @@ impl Context { // If we hit this point, the LIFO slot should be enabled. // There is no need to reset it. debug_assert!(core.lifo_enabled); - return Ok(core); + return ControlFlow::Continue(core); } // Track that we are about to run a task from the LIFO slot. @@ -1072,7 +1075,6 @@ impl Context { }) } - #[cfg(tokio_unstable)] pub(crate) fn worker_index(&self) -> usize { self.worker.index } @@ -1141,17 +1143,15 @@ impl Core { // and not pushed onto the local queue. let n = usize::max(1, n); - let mut synced = worker.handle.shared.synced.lock(); - // safety: passing in the correct `inject::Synced`. - let mut tasks = unsafe { worker.inject().pop_n(&mut synced.inject, n) }; - - // Pop the first task to return immediately - let ret = tasks.next(); + worker.inject().pop_n(n, |mut tasks| { + // Pop the first task to return immediately + let ret = tasks.next(); - // Push the rest of the on the run queue - self.run_queue.push_back(tasks); + // Push the rest of the on the run queue + self.run_queue.push_back(tasks); - ret + ret + }) } } @@ -1228,7 +1228,7 @@ impl Core { /// /// Returns true if the transition happened, false if there is work to do first. fn transition_to_parked(&mut self, worker: &Worker) -> bool { - // Workers should not park if they have work to do + // Workers should not park if they have work to do or are about to be traced if self.has_tasks() || self.is_traced { return false; } @@ -1256,8 +1256,9 @@ impl Core { /// Returns `true` if the transition happened. fn transition_from_parked(&mut self, worker: &Worker) -> bool { // If a task is in the lifo slot/run queue, then we must unpark regardless of - // being notified - if self.has_tasks() { + // being notified. Same when woken to be traced: a dump unparks worker + // threads without going through `Idle`, so the worker does it here. + if self.has_tasks() || self.is_traced { // When a worker wakes, it should only transition to the "searching" // state when the wake originates from another worker *or* a new task // is pushed. We do *not* want the worker to transition to "searching" @@ -1291,8 +1292,7 @@ impl Core { if !self.is_shutdown { // Check if the scheduler has been shutdown - let synced = worker.handle.shared.synced.lock(); - self.is_shutdown = worker.inject().is_closed(&synced.inject); + self.is_shutdown = worker.inject().is_closed(); } if !self.is_traced { @@ -1344,7 +1344,7 @@ impl Core { impl Worker { /// Returns a reference to the scheduler's injection queue. - fn inject(&self) -> &inject::Shared> { + fn inject(&self) -> &InjectQueue> { &self.handle.shared.inject } } @@ -1417,23 +1417,13 @@ impl Handle { } fn next_remote_task(&self) -> Option { - if self.shared.inject.is_empty() { - return None; - } - - let mut synced = self.shared.synced.lock(); - // safety: passing in correct `idle::Synced` - unsafe { self.shared.inject.pop(&mut synced.inject) } + self.shared.inject.pop() } fn push_remote_task(&self, task: Notified) { self.shared.scheduler_metrics.inc_remote_schedule_count(); - let mut synced = self.shared.synced.lock(); - // safety: passing in correct `idle::Synced` - unsafe { - self.shared.inject.push(&mut synced.inject, task); - } + self.shared.inject.push(task); } #[cfg(all(tokio_unstable, feature = "time"))] @@ -1458,11 +1448,7 @@ impl Handle { } pub(super) fn close(&self) { - if self - .shared - .inject - .close(&mut self.shared.synced.lock().inject) - { + if self.shared.inject.close() { self.notify_all(); } } @@ -1558,29 +1544,7 @@ impl Overflow> for Handle { where I: Iterator>>, { - unsafe { - self.shared.inject.push_batch(self, iter); - } - } -} - -pub(crate) struct InjectGuard<'a> { - lock: crate::loom::sync::MutexGuard<'a, Synced>, -} - -impl<'a> AsMut for InjectGuard<'a> { - fn as_mut(&mut self) -> &mut inject::Synced { - &mut self.lock.inject - } -} - -impl<'a> Lock for &'a Handle { - type Handle = InjectGuard<'a>; - - fn lock(self) -> Self::Handle { - InjectGuard { - lock: self.shared.synced.lock(), - } + self.shared.inject.push_batch(iter); } } diff --git a/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs b/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs index 312673034d3..bd057629323 100644 --- a/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs +++ b/tokio/src/runtime/scheduler/multi_thread/worker/taskdump.rs @@ -35,12 +35,9 @@ impl Handle { let owned = &self.shared.owned; let mut local = self.shared.steal_all(); - let synced = &self.shared.synced; let injection = &self.shared.inject; - // safety: `trace_multi_thread` is invoked with the same `synced` that `injection` - // was created with. - let traces = unsafe { trace_multi_thread(owned, &mut local, synced, injection) } + let traces = trace_multi_thread(owned, &mut local, injection) .into_iter() .map(|(id, trace)| dump::Task::new(id, trace)) .collect(); diff --git a/tokio/src/runtime/task/trace/mod.rs b/tokio/src/runtime/task/trace/mod.rs index de97a093897..c889eb63b84 100644 --- a/tokio/src/runtime/task/trace/mod.rs +++ b/tokio/src/runtime/task/trace/mod.rs @@ -386,21 +386,14 @@ pub(in crate::runtime) fn trace_current_thread( } cfg_rt_multi_thread! { - use crate::loom::sync::Mutex; + use crate::runtime::scheduler::inject::InjectQueue; use crate::runtime::scheduler::multi_thread; - use crate::runtime::scheduler::multi_thread::Synced; - use crate::runtime::scheduler::inject::Shared; - /// Trace and poll all tasks of the `current_thread` runtime. - /// - /// ## Safety - /// - /// Must be called with the same `synced` that `injection` was created with. - pub(in crate::runtime) unsafe fn trace_multi_thread( + /// Trace and poll all tasks of the `multi_thread` runtime. + pub(in crate::runtime) fn trace_multi_thread( owned: &OwnedTasks>, local: &mut multi_thread::queue::Local>, - synced: &Mutex, - injection: &Shared>, + injection: &InjectQueue>, ) -> Vec<(Id, Trace)> { let mut dequeued = Vec::new(); @@ -410,13 +403,7 @@ cfg_rt_multi_thread! { } // clear the injection queue - let mut synced = synced.lock(); - // Safety: exactly the same safety requirements as `trace_multi_thread` function. - while let Some(notified) = unsafe { injection.pop(&mut synced.inject) } { - dequeued.push(notified); - } - - drop(synced); + injection.drain_into(&mut dequeued); // precondition: we have drained the tasks from the local and injection // queues. diff --git a/tokio/src/runtime/tests/inject.rs b/tokio/src/runtime/tests/inject.rs index ccead5e024a..2195c7e7bbc 100644 --- a/tokio/src/runtime/tests/inject.rs +++ b/tokio/src/runtime/tests/inject.rs @@ -1,54 +1,44 @@ -use crate::runtime::scheduler::inject; +use crate::runtime::scheduler::Inject; #[test] fn push_and_pop() { const N: usize = 2; - let (inject, mut synced) = inject::Shared::new(); + let inject = Inject::new(); for i in 0..N { assert_eq!(inject.len(), i); let (task, _) = super::unowned(async {}); - unsafe { inject.push(&mut synced, task) }; + inject.push(task); } for i in 0..N { assert_eq!(inject.len(), N - i); - assert!(unsafe { inject.pop(&mut synced) }.is_some()); + assert!(inject.pop().is_some()); } println!("--------------"); - assert!(unsafe { inject.pop(&mut synced) }.is_none()); + assert!(inject.pop().is_none()); } #[test] fn push_batch_and_pop() { - let (inject, mut inject_synced) = inject::Shared::new(); + let inject = Inject::new(); - unsafe { - inject.push_batch( - &mut inject_synced, - (0..10).map(|_| super::unowned(async {}).0), - ); + inject.push_batch((0..10).map(|_| super::unowned(async {}).0)); - assert_eq!(5, inject.pop_n(&mut inject_synced, 5).count()); - assert_eq!(5, inject.pop_n(&mut inject_synced, 5).count()); - assert_eq!(0, inject.pop_n(&mut inject_synced, 5).count()); - } + assert_eq!(5, inject.pop_n(5, |tasks| tasks.count())); + assert_eq!(5, inject.pop_n(5, |tasks| tasks.count())); + assert_eq!(0, inject.pop_n(5, |tasks| tasks.count())); } #[test] fn pop_n_drains_on_drop() { - let (inject, mut inject_synced) = inject::Shared::new(); + let inject = Inject::new(); - unsafe { - inject.push_batch( - &mut inject_synced, - (0..10).map(|_| super::unowned(async {}).0), - ); - let _ = inject.pop_n(&mut inject_synced, 10); + inject.push_batch((0..10).map(|_| super::unowned(async {}).0)); + inject.pop_n(10, |_| ()); - assert_eq!(inject.len(), 0); - } + assert_eq!(inject.len(), 0); } diff --git a/tokio/src/runtime/time/entry.rs b/tokio/src/runtime/time/entry.rs index 5cc3ff60623..7ac816bb343 100644 --- a/tokio/src/runtime/time/entry.rs +++ b/tokio/src/runtime/time/entry.rs @@ -60,7 +60,6 @@ use crate::loom::sync::atomic::Ordering; use crate::runtime::scheduler; use crate::sync::AtomicWaker; -use crate::time::Instant; use crate::util::linked_list; use pin_project_lite::pin_project; @@ -478,12 +477,10 @@ impl TimerEntry { } } - pub(crate) fn init(self: Pin<&mut Self>, deadline: Instant) { - let tick = self.driver().time_source().deadline_to_tick(deadline); - + pub(crate) fn init(self: Pin<&mut Self>, deadline: u64) { unsafe { self.driver() - .reregister(&self.driver.driver().io, tick, (&self.inner).into()); + .reregister(&self.driver.driver().io, deadline, (&self.inner).into()); } } @@ -519,16 +516,14 @@ impl TimerEntry { unsafe { self.driver().clear_entry(NonNull::from(&self.inner)) }; } - pub(crate) fn reset(self: Pin<&mut Self>, deadline: Instant) { - let tick = self.driver().time_source().deadline_to_tick(deadline); - - if self.inner.extend_expiration(tick).is_ok() { + pub(crate) fn reset(self: Pin<&mut Self>, deadline: u64) { + if self.inner.extend_expiration(deadline).is_ok() { return; } unsafe { self.driver() - .reregister(&self.driver.driver().io, tick, (&self.inner).into()); + .reregister(&self.driver.driver().io, deadline, (&self.inner).into()); } } diff --git a/tokio/src/runtime/time/tests/mod.rs b/tokio/src/runtime/time/tests/mod.rs index 84c765af69e..713bef76d0d 100644 --- a/tokio/src/runtime/time/tests/mod.rs +++ b/tokio/src/runtime/time/tests/mod.rs @@ -1,6 +1,6 @@ -#![cfg(not(target_os = "wasi"))] +#![cfg(all(not(target_os = "wasi"), not(target_os = "emscripten")))] -use std::{task::Context, time::Duration}; +use std::task::Context; #[cfg(not(loom))] use futures::task::noop_waker_ref; @@ -46,13 +46,11 @@ fn single_timer() { let rt = rt(false); let handle = rt.handle(); - let handle_ = handle.clone(); + let inner = handle.inner.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(handle_.inner.clone()); + let entry = TimerEntry::new(inner.clone()); pin!(entry); - entry - .as_mut() - .init(handle_.inner.driver().clock().now() + Duration::from_secs(1)); + entry.as_mut().init(inner.driver().now() + 1000); block_on(std::future::poll_fn(|cx| entry.as_mut().poll_elapsed(cx))).unwrap(); }); @@ -75,13 +73,11 @@ fn drop_timer() { let rt = rt(false); let handle = rt.handle(); - let handle_ = handle.clone(); + let inner = handle.inner.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(handle_.inner.clone()); + let entry = TimerEntry::new(inner.clone()); pin!(entry); - entry - .as_mut() - .init(handle_.inner.driver().clock().now() + Duration::from_secs(1)); + entry.as_mut().init(inner.driver().now() + 1000); let _ = entry .as_mut() @@ -109,13 +105,11 @@ fn change_waker() { let rt = rt(false); let handle = rt.handle(); - let handle_ = handle.clone(); + let inner = handle.inner.clone(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(handle_.inner.clone()); + let entry = TimerEntry::new(inner.clone()); pin!(entry); - entry - .as_mut() - .init(handle_.inner.driver().clock().now() + Duration::from_secs(1)); + entry.as_mut().init(inner.driver().now() + 1000); let _ = entry .as_mut() @@ -144,20 +138,20 @@ fn reset_future() { let rt = rt(false); let handle = rt.handle(); - let handle_ = handle.clone(); + let inner = handle.clone().inner; let finished_early_ = finished_early.clone(); - let start = handle.inner.driver().clock().now(); + let start = handle.inner.driver().now(); let jh = thread::spawn(move || { - let entry = TimerEntry::new(handle_.inner.clone()); + let entry = TimerEntry::new(inner.clone()); pin!(entry); - entry.as_mut().init(start + Duration::from_secs(1)); + entry.as_mut().init(start + 1000); let _ = entry .as_mut() .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref())); - entry.as_mut().reset(start + Duration::from_secs(2)); + entry.as_mut().reset(start + 2000); // shouldn't complete before 2s block_on(std::future::poll_fn(|cx| entry.as_mut().poll_elapsed(cx))).unwrap(); @@ -169,19 +163,11 @@ fn reset_future() { let handle = handle.inner.driver().time(); - handle.process_at_time( - handle - .time_source() - .instant_to_tick(start + Duration::from_millis(1500)), - ); + handle.process_at_time(start + 1500); assert!(!finished_early.load(Ordering::Relaxed)); - handle.process_at_time( - handle - .time_source() - .instant_to_tick(start + Duration::from_millis(2500)), - ); + handle.process_at_time(start + 2500); jh.join().unwrap(); @@ -208,9 +194,7 @@ fn poll_process_levels() { for i in 0..normal_or_miri(1024, 64) { let mut entry = Box::pin(TimerEntry::new(handle.inner.clone())); - entry - .as_mut() - .init(handle.inner.driver().clock().now() + Duration::from_millis(i)); + entry.as_mut().init(handle.inner.driver().now() + i); let _ = entry .as_mut() @@ -243,8 +227,7 @@ fn poll_process_levels_targeted() { let e1 = TimerEntry::new(handle.inner.clone()); pin!(e1); - e1.as_mut() - .init(handle.inner.driver().clock().now() + Duration::from_millis(193)); + e1.as_mut().init(handle.inner.driver().now() + 193); let handle = handle.inner.driver().time(); diff --git a/tokio/src/runtime/time/wheel/level.rs b/tokio/src/runtime/time/wheel/level.rs index 4c0db573b44..4d44656f77f 100644 --- a/tokio/src/runtime/time/wheel/level.rs +++ b/tokio/src/runtime/time/wheel/level.rs @@ -111,8 +111,15 @@ impl Level { return None; } - // Get the slot for now using Maths - let now_slot = (now / slot_range(self.level)) as usize; + // Add the +1 offset for the `now_slot` to ignore the slot that `now` fits in, + // since it's the farthest timer that could appear from `now`. + // This is mostly relevant for the top level because it acts as a + // pseudo-ring buffer: timers that would logically go past the top level are + // fudged into it by `level_for` and the `MAX_DURATION` cap, so the slot holding + // `now` can be occupied by an entry that is a whole rotation away. + // For the lower levels `level_for` always places an entry in a slot other + // than the one holding `now`, so `now_slot` is always empty there. + let now_slot = ((now / slot_range(self.level)) % LEVEL_MULT as u64) as usize + 1; let occupied = self.occupied.rotate_right(now_slot as u32); let zeros = occupied.trailing_zeros() as usize; let slot = (zeros + now_slot) % LEVEL_MULT; @@ -131,7 +138,12 @@ impl Level { pub(crate) unsafe fn remove_entry(&mut self, item: NonNull) { let slot = slot_for(unsafe { item.as_ref().registered_when() }, self.level); - unsafe { self.slot[slot].remove(item) }; + unsafe { + assert!( + self.slot[slot].remove(item).is_some(), + "Attempt to remove item not present in the timing wheel" + ) + }; if self.slot[slot].is_empty() { // The bit is currently set debug_assert!(self.occupied & occupied_bit(slot) != 0); @@ -190,4 +202,113 @@ mod test { } } } + + fn level_with(level: usize, occupied: u64) -> Level { + let mut level = Level::new(level); + level.occupied = occupied; + level + } + + #[test] + fn next_occupied_slot_on_an_empty_level() { + assert_eq!(Level::new(0).next_occupied_slot(0), None); + assert_eq!(Level::new(5).next_occupied_slot(1 << 36), None); + } + + #[test] + fn next_occupied_slot_of_a_single_slot() { + // slot 10 of level 0, i.e. tick 10 of every 64-tick window + let level = level_with(0, 1 << 10); + + assert_eq!(level.next_occupied_slot(0), Some(10)); + assert_eq!(level.next_occupied_slot(9), Some(10)); + // `now` inside the slot itself, and past it + assert_eq!(level.next_occupied_slot(10), Some(10)); + assert_eq!(level.next_occupied_slot(11), Some(10)); + // `now` past the window: the slot is taken modulo 64 + assert_eq!(level.next_occupied_slot(64 + 3), Some(10)); + } + + #[test] + fn next_occupied_slot_picks_the_nearest_slot_forward() { + let level = level_with(0, (1 << 10) | (1 << 40)); + + assert_eq!(level.next_occupied_slot(0), Some(10)); + assert_eq!(level.next_occupied_slot(20), Some(40)); + // nothing left ahead in this window, so the scan wraps to slot 10 + assert_eq!(level.next_occupied_slot(41), Some(10)); + } + + #[test] + fn next_occupied_slot_skips_the_slot_holding_now() { + // The occurrence of slot 0 in this rotation has already started, so it + // can only be processed a full `level_range` later. Slot 1 is still + // ahead of `now` in this rotation and therefore expires first. + let level = level_with(5, 0b11); + + assert_eq!(level.next_occupied_slot(0), Some(1)); + assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(1)); + + // The same holds for any later slot, not just the adjacent one. + let level = level_with(5, 1 | (1 << 40)); + + assert_eq!(level.next_occupied_slot(0), Some(40)); + } + + #[test] + fn next_occupied_slot_of_the_slot_holding_now_when_it_is_the_only_one() { + // Nothing is ahead of `now`, so slot 0 is the earliest expiration even + // though it is reached only in the next rotation. + let level = level_with(5, 1); + + assert_eq!(level.next_occupied_slot(0), Some(0)); + assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(0)); + } + + #[test] + fn next_occupied_slot_of_the_last_slot() { + let level = level_with(5, 1 << 63); + + assert_eq!(level.next_occupied_slot(62 << 30), Some(63)); + assert_eq!(level.next_occupied_slot(63 << 30), Some(63)); + } + + #[test] + fn next_expiration_reports_the_start_of_the_slot() { + // slot 3 of level 1: slots are 64 ticks wide, so it starts at tick 192 + let expiration = level_with(1, 1 << 3).next_expiration(100).unwrap(); + + assert_eq!(expiration.level, 1); + assert_eq!(expiration.slot, 3); + assert_eq!(expiration.deadline, 192); + } + + #[test] + fn next_expiration_below_the_top_level() { + let level = level_with(4, 1 << 1); + + assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 24); + assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 24); + } + + #[test] + fn next_expiration_at_the_top_level() { + let level = level_with(5, 1 << 1); + + assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 30); + assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 30); + } + + #[test] + fn next_expiration_wraps_a_slot_at_or_behind_now() { + // Slot 0 of the top level starts at tick 0, so its next occurrence is a + // full rotation of the level away. + let level = level_with(5, 1 << 0); + + assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 36); + assert_eq!( + level.next_expiration((1 << 30) + 10).unwrap().deadline, + 1 << 36 + ); + } } diff --git a/tokio/src/runtime/time/wheel/mod.rs b/tokio/src/runtime/time/wheel/mod.rs index 38640a65d76..2c1413a1d8c 100644 --- a/tokio/src/runtime/time/wheel/mod.rs +++ b/tokio/src/runtime/time/wheel/mod.rs @@ -290,6 +290,8 @@ fn level_for(elapsed: u64, when: u64) -> usize { #[cfg(all(test, not(loom)))] mod test { + use std::pin::Pin; + use super::*; #[test] @@ -327,4 +329,178 @@ mod test { } } } + + #[must_use] + fn insert_entry(wheel: &mut Wheel, when: u64) -> Pin> { + let entry = Box::pin(TimerShared::new()); + + unsafe { entry.set_expiration(when) }; + + unsafe { wheel.insert(entry.as_ref().handle()).unwrap() }; + + entry + } + + #[test] + fn test_next_expiration_to_level_4() { + let wheel = &mut Wheel::new(); + + // that should occupy slot 1 of the level 4 of the wheel + let when = (1 << 24) + 10; + + let _entry = insert_entry(wheel, when); + + let expiration = wheel.next_expiration_time(); + // next expiration should be calculated as the start of the level 4 range + assert_eq!(expiration, Some(1 << 24)); + + // set the elapsed to the start of the previous expiration + wheel.poll(1 << 24); + + let expiration = wheel.next_expiration_time(); + // that should be equal of the LEVEL_WHEN which is 10 ms after previous expiration + assert_eq!(expiration, Some(when)); + + wheel.poll(when); + + assert!(wheel.next_expiration().is_none()); + } + + #[test] + fn test_next_expiration_to_level_5() { + let wheel = &mut Wheel::new(); + + // that will occupy slot 1 of the level 5 of the wheel + let when = (1 << 30) + 10; + let _entry = insert_entry(wheel, when); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(1 << 30)); + + // set the elapsed to the start of the previous expiration + wheel.poll(1 << 30); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some((1 << 30) + 10)); + + wheel.poll(when); + + assert!(wheel.next_expiration().is_none()); + } + + #[test] + fn test_next_expiration_after_level_5() { + let wheel = &mut Wheel::new(); + + // that will occupy slot 0 of the level 5 of the wheel + let when = (1 << 36) + 5; + let _entry = insert_entry(wheel, when); + + let expiration = wheel.next_expiration_time(); + + // that should come after the wheel + assert_eq!(expiration, Some(1 << 36)); + + // set the elapsed to the start of the previous expiration + wheel.poll(1 << 36); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(when)); + + wheel.poll(when); + + assert!(wheel.next_expiration_time().is_none()); + } + + #[test] + fn test_next_expiration_after_level_5_twice() { + let wheel = &mut Wheel::new(); + + // that will occupy slot 0 of the level 5 of the wheel + let when = (1 << 37) + 5; + let _entry = insert_entry(wheel, when); + + let expiration = wheel.next_expiration_time(); + + // that should come after the wheel + assert_eq!(expiration, Some(1 << 36)); + + // set the elapsed to the start of the previous expiration + wheel.poll(1 << 36); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(1 << 37)); + + wheel.poll(1 << 37); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(when)); + + wheel.poll(when); + + assert!(wheel.next_expiration_time().is_none()); + } + + #[test] + fn test_next_expiration_to_level_5_and_after_level_5() { + let wheel = &mut Wheel::new(); + + // that will occupy slot 0 of the level 5 of the wheel + let when = (1 << 36) + 1; + let _entry = insert_entry(wheel, when); + + // that will occupy slot 1 of the level 5 of the wheel + let when = (1 << 30) + 10; + let _entry = insert_entry(wheel, when); + + let expiration = wheel.next_expiration_time(); + + // this should point to the expiration of the slot 1 entry + // and not the slot 0 that is higher than 2^36 + assert_eq!(expiration, Some(1 << 30)); + + // set the elapsed to the start of the previous expiration + wheel.poll(1 << 30); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some((1 << 30) + 10)); + + wheel.poll(when); + + let expiration = wheel.next_expiration_time(); + + // The next expiration show refer the next loop of top level + assert_eq!(expiration, Some(1 << 36)); + } + + #[test] + fn test_next_expiration_at_slot_5_of_the_top_level() { + let wheel = &mut Wheel::new(); + + // move the wheel into slot 5 of the top level + wheel.poll(5 << 30); + + // that will occupy slot 5 of the level 5 of the wheel, the same slot the + // elapsed time sits in, so it is reachable only one rotation of the + // level later + let when = (1 << 36) + (5 << 30) + 1; + let _entry = insert_entry(wheel, when); + + // that will occupy slot 20 of the level 5 of the wheel, still ahead in + // the current rotation + let when = (20 << 30) + 10; + let _entry = insert_entry(wheel, when); + + let expiration = wheel.next_expiration_time(); + + // this should point to the start of the slot 20 and not to the slot 5 + // that comes around only after the whole level rotates + assert_eq!(expiration, Some(20 << 30)); + } } diff --git a/tokio/src/runtime/time_alt/timer.rs b/tokio/src/runtime/time_alt/timer.rs index 1798deeb73a..da0fe576b17 100644 --- a/tokio/src/runtime/time_alt/timer.rs +++ b/tokio/src/runtime/time_alt/timer.rs @@ -1,6 +1,5 @@ use super::{EntryHandle, TempLocalContext}; use crate::runtime::scheduler; -use crate::time::Instant; use std::pin::Pin; use std::task::{Context, Poll}; @@ -27,11 +26,10 @@ impl Drop for Timer { impl Timer { #[track_caller] - pub(crate) fn new(handle: scheduler::Handle, deadline: Instant) -> Self { - let tick = deadline_to_tick(&handle, deadline); + pub(crate) fn new(handle: scheduler::Handle, deadline: u64) -> Self { let entry = with_current_temp_local_context(&handle, |ctx| match ctx { Some(TempLocalContext::Running { registration_queue }) => { - let entry = EntryHandle::new(tick); + let entry = EntryHandle::new(deadline); unsafe { registration_queue.push_front(entry.clone()) } entry } @@ -39,7 +37,7 @@ impl Timer { Some(TempLocalContext::Shutdown) => panic!("{RUNTIME_SHUTTING_DOWN_ERROR}"), _ => { - let entry = EntryHandle::new(tick); + let entry = EntryHandle::new(deadline); push_from_remote(&handle, entry.clone()); entry } @@ -122,8 +120,3 @@ fn push_from_remote(sched_hdl: &scheduler::Handle, entry_hdl: EntryHandle) { sched_hdl.push_remote_timer(entry_hdl); } } - -fn deadline_to_tick(sched_hdl: &scheduler::Handle, deadline: Instant) -> u64 { - let time_hdl = sched_hdl.driver().time(); - time_hdl.time_source().deadline_to_tick(deadline) -} diff --git a/tokio/src/runtime/time_alt/wheel/level.rs b/tokio/src/runtime/time_alt/wheel/level.rs index 507998f80cf..b7f8da75e93 100644 --- a/tokio/src/runtime/time_alt/wheel/level.rs +++ b/tokio/src/runtime/time_alt/wheel/level.rs @@ -112,8 +112,15 @@ impl Level { return None; } - // Get the slot for now using Maths - let now_slot = (now / slot_range(self.level)) as usize; + // Add the +1 offset for the `now_slot` to ignore the slot that `now` fits in, + // since it's the farthest timer that could appear from `now`. + // This is mostly relevant for the top level because it acts as a + // pseudo-ring buffer: timers that would logically go past the top level are + // fudged into it by `level_for` and the `MAX_DURATION` cap, so the slot holding + // `now` can be occupied by an entry that is a whole rotation away. + // For the lower levels `level_for` always places an entry in a slot other + // than the one holding `now`, so `now_slot` is always empty there. + let now_slot = ((now / slot_range(self.level)) % LEVEL_MULT as u64) as usize + 1; let occupied = self.occupied.rotate_right(now_slot as u32); let zeros = occupied.trailing_zeros() as usize; let slot = (zeros + now_slot) % LEVEL_MULT; @@ -134,7 +141,12 @@ impl Level { pub(crate) unsafe fn remove_entry(&mut self, hdl: EntryHandle) { let slot = slot_for(hdl.deadline(), self.level); - unsafe { self.slot[slot].remove(NonNull::from(&hdl)) }; + unsafe { + assert!( + self.slot[slot].remove(NonNull::from(&hdl)).is_some(), + "Attempt to remove item not present in the timing wheel" + ) + }; if self.slot[slot].is_empty() { // The bit is currently set debug_assert!(self.occupied & occupied_bit(slot) != 0); @@ -193,4 +205,113 @@ mod test { } } } + + fn level_with(level: usize, occupied: u64) -> Level { + let mut level = Level::new(level); + level.occupied = occupied; + level + } + + #[test] + fn next_occupied_slot_on_an_empty_level() { + assert_eq!(Level::new(0).next_occupied_slot(0), None); + assert_eq!(Level::new(5).next_occupied_slot(1 << 36), None); + } + + #[test] + fn next_occupied_slot_of_a_single_slot() { + // slot 10 of level 0, i.e. tick 10 of every 64-tick window + let level = level_with(0, 1 << 10); + + assert_eq!(level.next_occupied_slot(0), Some(10)); + assert_eq!(level.next_occupied_slot(9), Some(10)); + // `now` inside the slot itself, and past it + assert_eq!(level.next_occupied_slot(10), Some(10)); + assert_eq!(level.next_occupied_slot(11), Some(10)); + // `now` past the window: the slot is taken modulo 64 + assert_eq!(level.next_occupied_slot(64 + 3), Some(10)); + } + + #[test] + fn next_occupied_slot_picks_the_nearest_slot_forward() { + let level = level_with(0, (1 << 10) | (1 << 40)); + + assert_eq!(level.next_occupied_slot(0), Some(10)); + assert_eq!(level.next_occupied_slot(20), Some(40)); + // nothing left ahead in this window, so the scan wraps to slot 10 + assert_eq!(level.next_occupied_slot(41), Some(10)); + } + + #[test] + fn next_occupied_slot_skips_the_slot_holding_now() { + // The occurrence of slot 0 in this rotation has already started, so it + // can only be processed a full `level_range` later. Slot 1 is still + // ahead of `now` in this rotation and therefore expires first. + let level = level_with(5, 0b11); + + assert_eq!(level.next_occupied_slot(0), Some(1)); + assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(1)); + + // The same holds for any later slot, not just the adjacent one. + let level = level_with(5, 1 | (1 << 40)); + + assert_eq!(level.next_occupied_slot(0), Some(40)); + } + + #[test] + fn next_occupied_slot_of_the_slot_holding_now_when_it_is_the_only_one() { + // Nothing is ahead of `now`, so slot 0 is the earliest expiration even + // though it is reached only in the next rotation. + let level = level_with(5, 1); + + assert_eq!(level.next_occupied_slot(0), Some(0)); + assert_eq!(level.next_occupied_slot((1 << 30) - 1), Some(0)); + } + + #[test] + fn next_occupied_slot_of_the_last_slot() { + let level = level_with(5, 1 << 63); + + assert_eq!(level.next_occupied_slot(62 << 30), Some(63)); + assert_eq!(level.next_occupied_slot(63 << 30), Some(63)); + } + + #[test] + fn next_expiration_reports_the_start_of_the_slot() { + // slot 3 of level 1: slots are 64 ticks wide, so it starts at tick 192 + let expiration = level_with(1, 1 << 3).next_expiration(100).unwrap(); + + assert_eq!(expiration.level, 1); + assert_eq!(expiration.slot, 3); + assert_eq!(expiration.deadline, 192); + } + + #[test] + fn next_expiration_below_the_top_level() { + let level = level_with(4, 1 << 1); + + assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 24); + assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 24); + } + + #[test] + fn next_expiration_at_the_top_level() { + let level = level_with(5, 1 << 1); + + assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 30); + assert_eq!(level.next_expiration(1000).unwrap().deadline, 1 << 30); + } + + #[test] + fn next_expiration_wraps_a_slot_at_or_behind_now() { + // Slot 0 of the top level starts at tick 0, so its next occurrence is a + // full rotation of the level away. + let level = level_with(5, 1 << 0); + + assert_eq!(level.next_expiration(0).unwrap().deadline, 1 << 36); + assert_eq!( + level.next_expiration((1 << 30) + 10).unwrap().deadline, + 1 << 36 + ); + } } diff --git a/tokio/src/runtime/time_alt/wheel/mod.rs b/tokio/src/runtime/time_alt/wheel/mod.rs index 1bb67d816d4..56aa038f22f 100644 --- a/tokio/src/runtime/time_alt/wheel/mod.rs +++ b/tokio/src/runtime/time_alt/wheel/mod.rs @@ -234,6 +234,7 @@ fn level_for(elapsed: u64, when: u64) -> usize { #[cfg(all(test, not(loom)))] mod test { + use super::super::cancellation_queue; use super::*; #[test] @@ -271,4 +272,180 @@ mod test { } } } + + #[must_use] + fn insert_entry(wheel: &mut Wheel, when: u64) -> EntryHandle { + let (cancel_tx, _cancel_rx) = cancellation_queue::new(); + let hdl = EntryHandle::new(when); + unsafe { wheel.insert(hdl.clone(), cancel_tx) }; + hdl + } + + fn poll(wheel: &mut Wheel, now: u64) { + let mut wq = WakeQueue::new(); + wheel.take_expired(now, &mut wq); + } + + #[test] + fn test_next_expiration_to_level_4() { + let mut wheel = Wheel::new(); + + // that should occupy slot 1 of the level 4 of the wheel + let when = (1 << 24) + 10; + + let _entry = insert_entry(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + // next expiration should be calculated as the start of the level 4 range + assert_eq!(expiration, Some(1 << 24)); + + // set the elapsed to the start of the previous expiration + poll(&mut wheel, 1 << 24); + + let expiration = wheel.next_expiration_time(); + // that should be equal of the LEVEL_WHEN which is 10 ms after previous expiration + assert_eq!(expiration, Some(when)); + + poll(&mut wheel, when); + + assert!(wheel.next_expiration_time().is_none()); + } + + #[test] + fn test_next_expiration_to_level_5() { + let mut wheel = Wheel::new(); + + // that will occupy slot 1 of the level 5 of the wheel + let when = (1 << 30) + 10; + let _entry = insert_entry(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(1 << 30)); + + // set the elapsed to the start of the previous expiration + poll(&mut wheel, 1 << 30); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some((1 << 30) + 10)); + + poll(&mut wheel, when); + + assert!(wheel.next_expiration_time().is_none()); + } + + #[test] + fn test_next_expiration_after_level_5() { + let mut wheel = Wheel::new(); + + // that will occupy slot 0 of the level 5 of the wheel + let when = (1 << 36) + 5; + let _entry = insert_entry(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + + // that should come after the wheel + assert_eq!(expiration, Some(1 << 36)); + + // set the elapsed to the start of the previous expiration + poll(&mut wheel, 1 << 36); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(when)); + + poll(&mut wheel, when); + + assert!(wheel.next_expiration_time().is_none()); + } + + #[test] + fn test_next_expiration_after_level_5_twice() { + let mut wheel = Wheel::new(); + + // that will occupy slot 0 of the level 5 of the wheel + let when = (1 << 37) + 5; + let _entry = insert_entry(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + + // that should come after the wheel + assert_eq!(expiration, Some(1 << 36)); + + // set the elapsed to the start of the previous expiration + poll(&mut wheel, 1 << 36); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(1 << 37)); + + poll(&mut wheel, 1 << 37); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some(when)); + + poll(&mut wheel, when); + + assert!(wheel.next_expiration_time().is_none()); + } + + #[test] + fn test_next_expiration_to_level_5_and_after_level_5() { + let mut wheel = Wheel::new(); + + // that will occupy slot 0 of the level 5 of the wheel + let when = (1 << 36) + 1; + let _entry = insert_entry(&mut wheel, when); + + // that will occupy slot 1 of the level 5 of the wheel + let when = (1 << 30) + 10; + let _entry = insert_entry(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + + // this should point to the expiration of the slot 1 entry + // and not the slot 0 that is higher than 2^36 + assert_eq!(expiration, Some(1 << 30)); + + // set the elapsed to the start of the previous expiration + poll(&mut wheel, 1 << 30); + + let expiration = wheel.next_expiration_time(); + + assert_eq!(expiration, Some((1 << 30) + 10)); + + poll(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + + // The next expiration show refer the next loop of top level + assert_eq!(expiration, Some(1 << 36)); + } + + #[test] + fn test_next_expiration_at_slot_5_of_the_top_level() { + let mut wheel = Wheel::new(); + + // move the wheel into slot 5 of the top level + poll(&mut wheel, 5 << 30); + + // that will occupy slot 5 of the level 5 of the wheel, the same slot the + // elapsed time sits in, so it is reachable only one rotation of the + // level later + let when = (1 << 36) + (5 << 30) + 1; + let _entry = insert_entry(&mut wheel, when); + + // that will occupy slot 20 of the level 5 of the wheel, still ahead in + // the current rotation + let when = (20 << 30) + 10; + let _entry = insert_entry(&mut wheel, when); + + let expiration = wheel.next_expiration_time(); + + // this should point to the start of the slot 20 and not to the slot 5 + // that comes around only after the whole level rotates + assert_eq!(expiration, Some(20 << 30)); + } } diff --git a/tokio/src/sync/broadcast.rs b/tokio/src/sync/broadcast.rs index f88972ba62d..89f8a212461 100644 --- a/tokio/src/sync/broadcast.rs +++ b/tokio/src/sync/broadcast.rs @@ -142,7 +142,7 @@ use std::future::Future; use std::marker::PhantomPinned; use std::pin::Pin; use std::ptr::NonNull; -use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst}; +use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release}; use std::task::{ready, Context, Poll, Waker}; /// Sending-half of the [`broadcast`] channel. @@ -408,16 +408,15 @@ struct Slot { /// /// When this goes to zero, the value is released. /// - /// An atomic is used as it is mutated concurrently with the slot read lock - /// acquired. - rem: AtomicUsize, + /// Access is protected by the slot mutex. + rem: usize, /// Uniquely identifies the `send` stored in the slot. pos: u64, /// The value being broadcast. /// - /// The value is set by `send` when the write lock is held. When a reader + /// The value is set by `send` while the slot mutex is held. When a reader /// drops, `rem` is decremented. When it hits zero, the value is dropped. val: Option, } @@ -532,7 +531,7 @@ const MAX_RECEIVERS: usize = usize::MAX >> 2; /// /// # Panics /// -/// This will panic if `capacity` is equal to `0`. +/// This will panic if `capacity` is equal to `0` or exceeds `usize::MAX / 2`. /// /// This pre-allocates space for `capacity` messages. Allocation failure may result in a panic or /// [an allocation error](std::alloc::handle_alloc_error). @@ -584,7 +583,7 @@ impl Sender { let buffer = (0..capacity).map(|i| { Mutex::new(Slot { - rem: AtomicUsize::new(0), + rem: 0, pos: (i as u64).wrapping_sub(capacity as u64), val: None, }) @@ -680,7 +679,7 @@ impl Sender { slot.pos = pos; // Set remaining receivers - slot.rem.with_mut(|v| *v = rem); + slot.rem = rem; // Write the value slot.val = Some(value); @@ -782,7 +781,7 @@ impl Sender { while low < high { let mid = low + (high - low) / 2; let idx = base_idx.wrapping_add(mid) & self.shared.mask; - if self.shared.buffer[idx].lock().rem.load(SeqCst) == 0 { + if self.shared.buffer[idx].lock().rem == 0 { low = mid + 1; } else { high = mid; @@ -824,7 +823,7 @@ impl Sender { let tail = self.shared.tail.lock(); let idx = (tail.pos.wrapping_sub(1) & self.shared.mask as u64) as usize; - self.shared.buffer[idx].lock().rem.load(SeqCst) == 0 + self.shared.buffer[idx].lock().rem == 0 } /// Returns the number of active receivers. @@ -1750,7 +1749,8 @@ impl<'a, T> RecvGuard<'a, T> { impl<'a, T> Drop for RecvGuard<'a, T> { fn drop(&mut self) { // Decrement the remaining counter - if 1 == self.slot.rem.fetch_sub(1, SeqCst) { + self.slot.rem -= 1; + if self.slot.rem == 0 { self.slot.val = None; } } diff --git a/tokio/src/sync/semaphore.rs b/tokio/src/sync/semaphore.rs index 537d0b3c8d7..f5740001fa4 100644 --- a/tokio/src/sync/semaphore.rs +++ b/tokio/src/sync/semaphore.rs @@ -1270,6 +1270,11 @@ impl<'a> SemaphorePermit<'a> { }) } + /// Returns the [`Semaphore`] from which this permit was acquired. + pub fn semaphore(&self) -> &Semaphore { + self.sem + } + /// Returns the number of permits held by `self`. pub fn num_permits(&self) -> usize { self.permits diff --git a/tokio/src/sync/tests/atomic_waker.rs b/tokio/src/sync/tests/atomic_waker.rs index b182574f325..8c5e1a113d0 100644 --- a/tokio/src/sync/tests/atomic_waker.rs +++ b/tokio/src/sync/tests/atomic_waker.rs @@ -15,7 +15,7 @@ impl AssertSync for AtomicWaker {} impl AssertSend for Waker {} impl AssertSync for Waker {} -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; #[test] diff --git a/tokio/src/sync/tests/notify.rs b/tokio/src/sync/tests/notify.rs index 540aa22444a..bfbc6f486c8 100644 --- a/tokio/src/sync/tests/notify.rs +++ b/tokio/src/sync/tests/notify.rs @@ -3,7 +3,7 @@ use std::future::Future; use std::sync::Arc; use std::task::{Context, RawWaker, RawWakerVTable, Waker}; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; #[test] diff --git a/tokio/src/sync/tests/semaphore_batch.rs b/tokio/src/sync/tests/semaphore_batch.rs index fb5e8fdd6f7..84f062d8f23 100644 --- a/tokio/src/sync/tests/semaphore_batch.rs +++ b/tokio/src/sync/tests/semaphore_batch.rs @@ -3,7 +3,7 @@ use tokio_test::*; const MAX_PERMITS: usize = crate::sync::Semaphore::MAX_PERMITS; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; #[test] diff --git a/tokio/src/sync/watch.rs b/tokio/src/sync/watch.rs index 96e1dad64f0..c122976d30c 100644 --- a/tokio/src/sync/watch.rs +++ b/tokio/src/sync/watch.rs @@ -747,6 +747,24 @@ impl Receiver { Ok(self.version != new_version) } + /// Checks if the channel has been closed. This happens when all [`Sender`]s + /// have been dropped. + /// + /// # Examples + /// + /// ``` + /// use tokio::sync::watch; + /// + /// let (tx, rx) = watch::channel(()); + /// assert!(!rx.is_closed()); + /// + /// drop(tx); + /// assert!(rx.is_closed()); + /// ``` + pub fn is_closed(&self) -> bool { + self.shared.state.load().is_closed() + } + /// Marks the state as changed. /// /// After invoking this method [`has_changed()`](Self::has_changed) diff --git a/tokio/src/task/builder.rs b/tokio/src/task/builder.rs index c253e853e97..a2357639860 100644 --- a/tokio/src/task/builder.rs +++ b/tokio/src/task/builder.rs @@ -1,6 +1,6 @@ #![allow(unreachable_pub)] use crate::{ - runtime::{Handle, BOX_FUTURE_THRESHOLD}, + runtime::{AutoBox, Handle}, task::{JoinHandle, LocalSet}, util::trace::SpawnMeta, }; @@ -90,7 +90,7 @@ impl<'a> Builder<'a> { Fut::Output: Send + 'static, { let fut_size = mem::size_of::(); - Ok(if fut_size > BOX_FUTURE_THRESHOLD { + Ok(if AutoBox::::SHOULD_BOX { super::spawn::spawn_inner(Box::pin(future), SpawnMeta::new(self.name, fut_size)) } else { super::spawn::spawn_inner(future, SpawnMeta::new(self.name, fut_size)) @@ -111,7 +111,7 @@ impl<'a> Builder<'a> { Fut::Output: Send + 'static, { let fut_size = mem::size_of::(); - Ok(if fut_size > BOX_FUTURE_THRESHOLD { + Ok(if AutoBox::::SHOULD_BOX { handle.spawn_named(Box::pin(future), SpawnMeta::new(self.name, fut_size)) } else { handle.spawn_named(future, SpawnMeta::new(self.name, fut_size)) @@ -142,7 +142,7 @@ impl<'a> Builder<'a> { Fut::Output: 'static, { let fut_size = mem::size_of::(); - Ok(if fut_size > BOX_FUTURE_THRESHOLD { + Ok(if AutoBox::::SHOULD_BOX { super::local::spawn_local_inner(Box::pin(future), SpawnMeta::new(self.name, fut_size)) } else { super::local::spawn_local_inner(future, SpawnMeta::new(self.name, fut_size)) @@ -167,7 +167,7 @@ impl<'a> Builder<'a> { Fut::Output: 'static, { let fut_size = mem::size_of::(); - Ok(if fut_size > BOX_FUTURE_THRESHOLD { + Ok(if AutoBox::::SHOULD_BOX { local_set.spawn_named(Box::pin(future), SpawnMeta::new(self.name, fut_size)) } else { local_set.spawn_named(future, SpawnMeta::new(self.name, fut_size)) @@ -213,7 +213,7 @@ impl<'a> Builder<'a> { { use crate::runtime::Mandatory; let fn_size = mem::size_of::(); - let (join_handle, spawn_result) = if fn_size > BOX_FUTURE_THRESHOLD { + let (join_handle, spawn_result) = if AutoBox::::SHOULD_BOX { handle.inner.blocking_spawner().spawn_blocking_inner( Box::new(function), Mandatory::NonMandatory, diff --git a/tokio/src/task/coop/mod.rs b/tokio/src/task/coop/mod.rs index 4a520e6f527..4041f6bdb7e 100644 --- a/tokio/src/task/coop/mod.rs +++ b/tokio/src/task/coop/mod.rs @@ -434,6 +434,7 @@ cfg_coop! { pin_project! { /// Future wrapper to ensure cooperative scheduling created by [`cooperative`]. #[must_use = "futures do nothing unless polled"] + #[derive(Debug)] pub struct Coop { #[pin] pub(crate) fut: F, @@ -496,7 +497,7 @@ cfg_coop! { mod test { use super::*; - #[cfg(all(target_family = "wasm", not(target_os = "wasi")))] + #[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; fn get() -> Budget { diff --git a/tokio/src/task/local.rs b/tokio/src/task/local.rs index 4d467d0fbf5..1e3ccf01c2e 100644 --- a/tokio/src/task/local.rs +++ b/tokio/src/task/local.rs @@ -5,7 +5,7 @@ use crate::runtime; use crate::runtime::task::{ self, JoinHandle, LocalOwnedTasks, SpawnLocation, Task, TaskHarnessScheduleHooks, }; -use crate::runtime::{context, ThreadId, BOX_FUTURE_THRESHOLD}; +use crate::runtime::{context, AutoBox, ThreadId}; use crate::sync::AtomicWaker; use crate::util::trace::SpawnMeta; use crate::util::RcCell; @@ -397,7 +397,7 @@ cfg_rt! { F::Output: 'static, { let fut_size = std::mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { spawn_local_inner(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { spawn_local_inner(future, SpawnMeta::new_unnamed(fut_size)) @@ -594,7 +594,7 @@ impl LocalSet { F::Output: 'static, { let fut_size = mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { self.spawn_named(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { self.spawn_named(future, SpawnMeta::new_unnamed(fut_size)) @@ -1276,7 +1276,7 @@ impl LocalState { // ensure they are on the same thread that owns the `LocalSet`. unsafe impl Send for LocalState {} -#[cfg(all(test, not(loom)))] +#[cfg(all(test, not(loom), not(target_os = "emscripten")))] mod tests { use super::*; diff --git a/tokio/src/task/spawn.rs b/tokio/src/task/spawn.rs index 149665a4d1b..ccc396ad7e3 100644 --- a/tokio/src/task/spawn.rs +++ b/tokio/src/task/spawn.rs @@ -1,4 +1,4 @@ -use crate::runtime::BOX_FUTURE_THRESHOLD; +use crate::runtime::AutoBox; use crate::task::JoinHandle; use crate::util::trace::SpawnMeta; @@ -177,7 +177,7 @@ cfg_rt! { F::Output: Send + 'static, { let fut_size = std::mem::size_of::(); - if fut_size > BOX_FUTURE_THRESHOLD { + if AutoBox::::SHOULD_BOX { spawn_inner(Box::pin(future), SpawnMeta::new_unnamed(fut_size)) } else { spawn_inner(future, SpawnMeta::new_unnamed(fut_size)) diff --git a/tokio/src/time/instant.rs b/tokio/src/time/instant.rs index faf62ee59e3..d3f46ab39b8 100644 --- a/tokio/src/time/instant.rs +++ b/tokio/src/time/instant.rs @@ -54,14 +54,6 @@ impl Instant { Instant { std } } - pub(crate) fn far_future() -> Instant { - // Roughly 30 years from now. - // API does not provide a way to obtain max `Instant` - // or convert specific date in the future to instant. - // 1000 years overflows on macOS, 100 years overflows on FreeBSD. - Self::now() + Duration::from_secs(86400 * 365 * 30) - } - /// Convert the value into a `std::time::Instant`. pub fn into_std(self) -> std::time::Instant { self.std diff --git a/tokio/src/time/interval.rs b/tokio/src/time/interval.rs index ff889a00f96..df13a4317a1 100644 --- a/tokio/src/time/interval.rs +++ b/tokio/src/time/interval.rs @@ -1,4 +1,4 @@ -use crate::time::{sleep_until, Duration, Instant, Sleep}; +use crate::time::{safe_delay, sleep_until, Duration, Instant, Sleep}; use crate::util::trace; use std::future::{poll_fn, Future}; @@ -133,7 +133,7 @@ fn internal_interval_at( Interval { delay: Box::pin(sleep_until(start)), - period, + period: safe_delay(period), missed_tick_behavior: MissedTickBehavior::default(), #[cfg(all(tokio_unstable, feature = "tracing"))] resource_span, @@ -337,21 +337,21 @@ impl MissedTickBehavior { Self::Burst => timeout + period, Self::Delay => now + period, Self::Skip => { - now + period - - Duration::from_nanos( - ((now - timeout).as_nanos() % period.as_nanos()) - .try_into() - // This operation is practically guaranteed not to - // fail, as in order for it to fail, `period` would - // have to be longer than `now - timeout`, and both - // would have to be longer than 584 years. - // - // If it did fail, there's not a good way to pass - // the error along to the user, so we just panic. - .expect( - "too much time has elapsed since the interval was supposed to tick", - ), - ) + let offset = Duration::from_nanos( + ((now - timeout).as_nanos() % period.as_nanos()) + .try_into() + // This operation is practically guaranteed not to + // fail, as in order for it to fail, `period` would + // have to be longer than `now - timeout`, and both + // would have to be longer than 584 years. + // + // If it did fail, there's not a good way to pass + // the error along to the user, so we just panic. + .expect( + "too much time has elapsed since the interval was supposed to tick", + ), + ); + now + (period - offset) } } } @@ -474,9 +474,7 @@ impl Interval { self.missed_tick_behavior .next_timeout(timeout, now, self.period) } else { - timeout - .checked_add(self.period) - .unwrap_or_else(Instant::far_future) + timeout + self.period }; // When we arrive here, the internal delay returned `Poll::Ready`. @@ -582,7 +580,8 @@ impl Interval { /// # } /// ``` pub fn reset_after(&mut self, after: Duration) { - self.delay.as_mut().reset(Instant::now() + after); + let deadline = Instant::now() + safe_delay(after); + self.delay.as_mut().reset(deadline); } /// Resets the interval to a [`crate::time::Instant`] deadline. diff --git a/tokio/src/time/mod.rs b/tokio/src/time/mod.rs index 8627a7838aa..67f805a941d 100644 --- a/tokio/src/time/mod.rs +++ b/tokio/src/time/mod.rs @@ -84,6 +84,12 @@ //! [`interval`]: crate::time::interval() //! [`sleep`]: sleep() +fn safe_delay(duration: Duration) -> Duration { + // Roughly 30 years from now. + // 1000 years overflows on macOS, 100 years overflows on FreeBSD. + duration.min(Duration::from_secs(86400 * 365 * 30)) +} + mod clock; pub(crate) use self::clock::Clock; cfg_test_util! { diff --git a/tokio/src/time/sleep.rs b/tokio/src/time/sleep.rs index c12bbdf9bcf..57ae4bb588b 100644 --- a/tokio/src/time/sleep.rs +++ b/tokio/src/time/sleep.rs @@ -1,5 +1,5 @@ use crate::runtime::{scheduler, Timer}; -use crate::time::{error::Error, Duration, Instant}; +use crate::time::{error::Error, safe_delay, Duration, Instant}; use crate::util::trace; use pin_project_lite::pin_project; @@ -121,12 +121,8 @@ pub fn sleep_until(deadline: Instant) -> Sleep { #[cfg_attr(docsrs, doc(alias = "wait"))] #[track_caller] pub fn sleep(duration: Duration) -> Sleep { - let location = trace::caller_location(); - - match Instant::now().checked_add(duration) { - Some(deadline) => Sleep::new_timeout(deadline, location), - None => Sleep::new_timeout(Instant::far_future(), location), - } + let deadline = Instant::now() + safe_delay(duration); + Sleep::new_timeout(deadline, trace::caller_location()) } pin_project! { @@ -296,10 +292,6 @@ impl Sleep { } } - pub(crate) fn far_future(location: Option<&'static Location<'static>>) -> Sleep { - Self::new_timeout(Instant::far_future(), location) - } - /// Returns the instant at which the future will complete. pub fn deadline(&self) -> Instant { self.deadline @@ -346,6 +338,8 @@ impl Sleep { *this.deadline = deadline; let handle = this.driver; + let time_source = handle.driver().time().time_source(); + let deadline = time_source.deadline_to_tick(deadline); #[cfg(all(tokio_unstable, feature = "tracing"))] { @@ -358,12 +352,10 @@ impl Sleep { tracing::trace_span!("runtime.resource.async_op.poll"); let clock = handle.driver().clock(); - let time_source = handle.driver().time().time_source(); let now = time_source.now(clock); - let tick = time_source.deadline_to_tick(deadline); tracing::trace!( target: "runtime::resource::state_update", - duration = tick.saturating_sub(now), + duration = deadline.saturating_sub(now), duration.unit = "ms", duration.op = "override", ); @@ -413,25 +405,25 @@ impl Sleep { Some(timer) => timer, None => { let handle = this.driver; + let time_source = handle.driver().time().time_source(); + let deadline = time_source.deadline_to_tick(*this.deadline); #[cfg(all(tokio_unstable, feature = "tracing"))] { let clock = handle.driver().clock(); - let time_source = handle.driver().time().time_source(); let now = time_source.now(clock); - let tick = time_source.deadline_to_tick(*this.deadline); tracing::trace!( target: "runtime::resource::state_update", - duration = tick.saturating_sub(now), + duration = deadline.saturating_sub(now), duration.unit = "ms", duration.op = "override", ); } - let timer = Timer::new(handle.clone(), *this.deadline); + let timer = Timer::new(handle.clone(), deadline); this.timer.set(Some(timer)); let mut timer = this.timer.as_pin_mut().unwrap(); - timer.as_mut().init(*this.deadline); + timer.as_mut().init(deadline); timer } }; diff --git a/tokio/src/time/timeout.rs b/tokio/src/time/timeout.rs index 48862cea2fe..1ffa8099f54 100644 --- a/tokio/src/time/timeout.rs +++ b/tokio/src/time/timeout.rs @@ -6,7 +6,7 @@ use crate::{ task::coop, - time::{error::Elapsed, sleep_until, Duration, Instant, Sleep}, + time::{error::Elapsed, Duration, Instant, Sleep}, util::trace, }; @@ -87,14 +87,15 @@ pub fn timeout(duration: Duration, future: F) -> Timeout where F: IntoFuture, { - let location = trace::caller_location(); - - let deadline = Instant::now().checked_add(duration); - let delay = match deadline { - Some(deadline) => Sleep::new_timeout(deadline, location), - None => Sleep::far_future(location), - }; - Timeout::new_with_delay(future.into_future(), delay) + // Closures don't preserve `#[track_caller]`. + #[allow(clippy::manual_map)] + Timeout { + value: future.into_future(), + delay: match Instant::now().checked_add(duration) { + Some(deadline) => Some(Sleep::new_timeout(deadline, trace::caller_location())), + None => None, + }, + } } /// Requires a `Future` to complete before the specified instant in time. @@ -165,8 +166,10 @@ pub fn timeout_at(deadline: Instant, future: F) -> Timeout where F: IntoFuture, { - let delay = sleep_until(deadline); - Timeout::new_with_delay(future.into_future(), delay) + Timeout { + value: future.into_future(), + delay: Some(Sleep::new_timeout(deadline, trace::caller_location())), + } } pin_project! { @@ -177,15 +180,11 @@ pin_project! { #[pin] value: T, #[pin] - delay: Sleep, + delay: Option, } } impl Timeout { - pub(crate) fn new_with_delay(value: T, delay: Sleep) -> Timeout { - Timeout { value, delay } - } - /// Gets a reference to the underlying value in this timeout. pub fn get_ref(&self) -> &T { &self.value @@ -218,7 +217,10 @@ where return Poll::Ready(Ok(v)); } - poll_delay(had_budget_before, me.delay, cx).map(Err) + match me.delay.as_pin_mut() { + Some(delay) => poll_delay(had_budget_before, delay, cx).map(Err), + None => Poll::Pending, + } } } diff --git a/tokio/src/util/idle_notified_set.rs b/tokio/src/util/idle_notified_set.rs index 4c5177c5630..3d9e7dd2b8c 100644 --- a/tokio/src/util/idle_notified_set.rs +++ b/tokio/src/util/idle_notified_set.rs @@ -492,7 +492,7 @@ unsafe impl linked_list::Link for ListEntry { } } -#[cfg(all(test, not(loom)))] +#[cfg(all(test, not(loom), not(target_os = "emscripten")))] mod tests { use crate::runtime::Builder; use crate::task::JoinSet; diff --git a/tokio/tests/dump.rs b/tokio/tests/dump.rs index e9d095652d3..ec8b7ff8ba6 100644 --- a/tokio/tests/dump.rs +++ b/tokio/tests/dump.rs @@ -202,3 +202,109 @@ fn notified_during_tracing() { ); }); } + +/// Regression tests for the scheduler's idle bookkeeping across a dump. +/// +/// Requesting a dump wakes every worker thread directly, without going through +/// the multi-threaded scheduler's idle bookkeeping. A worker that was parked +/// when the dump began must therefore perform the logical unpark itself, or the +/// bookkeeping is corrupted the next time it parks: it is recorded as a sleeper +/// twice and the count of unparked workers underflows. After that, the runtime +/// believes every worker is busy and never notifies one of newly spawned work. +mod dump_of_parked_workers { + use super::*; + + use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; + use std::sync::{mpsc, Arc}; + use std::time::{Duration, Instant}; + + const TIMEOUT: Duration = Duration::from_secs(10); + + /// Observes the worker threads entering and leaving the park path. + #[derive(Default)] + struct ParkState { + /// Number of workers currently in the park path. + parked: AtomicUsize, + /// Number of times a worker has entered the park path. + entries: AtomicUsize, + } + + impl ParkState { + fn wait_until(&self, mut condition: impl FnMut(&Self) -> bool, message: &str) { + let deadline = Instant::now() + TIMEOUT; + while !condition(self) { + assert!(Instant::now() < deadline, "{message}"); + std::thread::yield_now(); + } + } + + fn wait_for_all_workers_parked(&self, worker_threads: usize) { + self.wait_until( + |state| state.parked.load(SeqCst) == worker_threads, + "not every worker reached the park path", + ); + } + } + + fn assert_dump_preserves_idle_bookkeeping(worker_threads: usize) { + let park_state = Arc::new(ParkState::default()); + + let rt = runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(worker_threads) + .on_thread_park({ + let park_state = park_state.clone(); + move || { + park_state.entries.fetch_add(1, SeqCst); + park_state.parked.fetch_add(1, SeqCst); + } + }) + .on_thread_unpark({ + let park_state = park_state.clone(); + move || { + park_state.parked.fetch_sub(1, SeqCst); + } + }) + .build() + .unwrap(); + + // Dump only once every worker is idle. That is the state in which the + // dump wakes worker threads that the scheduler still counts as parked. + park_state.wait_for_all_workers_parked(worker_threads); + let entries_before_dump = park_state.entries.load(SeqCst); + + rt.block_on(rt.handle().dump()); + + // Every worker leaves the park path to be traced, then parks again. + park_state.wait_until( + |state| state.entries.load(SeqCst) >= entries_before_dump + worker_threads, + "not every worker was woken for the dump", + ); + park_state.wait_for_all_workers_parked(worker_threads); + + let (tx, rx) = mpsc::channel(); + rt.spawn(async move { + let _ = tx.send(()); + }); + + assert!( + rx.recv_timeout(TIMEOUT).is_ok(), + "the runtime did not run a task spawned after a dump" + ); + } + + #[test] + fn one_worker() { + assert_dump_preserves_idle_bookkeeping(1); + } + + #[test] + fn two_workers() { + assert_dump_preserves_idle_bookkeeping(2); + } + + #[test] + fn many_workers() { + assert_dump_preserves_idle_bookkeeping(8); + } +} diff --git a/tokio/tests/fs.rs b/tokio/tests/fs.rs index f5fb193f4f9..30d8690a52a 100644 --- a/tokio/tests/fs.rs +++ b/tokio/tests/fs.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support file operations +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi does not support file operations + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "fs" + ) +))] use tokio::fs; use tokio_test::assert_ok; @@ -16,6 +24,10 @@ async fn path_read_write() { } #[tokio::test] +#[cfg_attr( + target_os = "emscripten", + ignore = "emscripten libc does not implement dup()/try_clone()" +)] async fn try_clone_should_preserve_max_buf_size() { let buf_size = 128; let temp = tempdir(); diff --git a/tokio/tests/fs_canonicalize_dir.rs b/tokio/tests/fs_canonicalize_dir.rs index e7f6c68ea91..9f4d6d94a30 100644 --- a/tokio/tests/fs_canonicalize_dir.rs +++ b/tokio/tests/fs_canonicalize_dir.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tokio::fs; diff --git a/tokio/tests/fs_copy.rs b/tokio/tests/fs_copy.rs index fac64dccddc..ddf4c2bc586 100644 --- a/tokio/tests/fs_copy.rs +++ b/tokio/tests/fs_copy.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/tokio/tests/fs_dir.rs b/tokio/tests/fs_dir.rs index 3f28c1a07b3..f11ef03643d 100644 --- a/tokio/tests/fs_dir.rs +++ b/tokio/tests/fs_dir.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tokio::fs; use tokio_test::{assert_err, assert_ok}; diff --git a/tokio/tests/fs_file.rs b/tokio/tests/fs_file.rs index e5bc0bd87ff..ac29f3cb33b 100644 --- a/tokio/tests/fs_file.rs +++ b/tokio/tests/fs_file.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use futures::future::FutureExt; use std::io::prelude::*; @@ -124,6 +136,10 @@ async fn coop() { } #[tokio::test] +#[cfg_attr( + target_os = "emscripten", + ignore = "emscripten libc does not implement dup()/try_clone()" +)] async fn write_to_clone() { let tempfile = tempfile(); @@ -221,6 +237,14 @@ async fn set_max_buf_size_write() { assert_eq!(file.write(HELLO).await.unwrap(), 1); } +#[tokio::test] +#[should_panic(expected = "`max_buf_size` must be greater than 0")] +async fn set_max_buf_size_panics_on_zero() { + let tempfile = tempfile(); + let mut file = File::open(tempfile.path()).await.unwrap(); + file.set_max_buf_size(0); +} + #[tokio::test] #[cfg_attr(miri, ignore)] #[cfg(unix)] diff --git a/tokio/tests/fs_link.rs b/tokio/tests/fs_link.rs index ef143678bcf..7bb85417546 100644 --- a/tokio/tests/fs_link.rs +++ b/tokio/tests/fs_link.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tokio::fs; @@ -8,6 +20,7 @@ use tempfile::tempdir; #[tokio::test] #[cfg_attr(miri, ignore)] // No `linkat` in miri. +#[cfg_attr(target_os = "emscripten", ignore = "MEMFS rejects link() with EMLINK")] async fn test_hard_link() { let dir = tempdir().unwrap(); let src = dir.path().join("src.txt"); @@ -63,6 +76,7 @@ async fn test_symlink() { #[tokio::test] #[cfg_attr(miri, ignore)] // No `linkat` in miri. +#[cfg_attr(target_os = "emscripten", ignore = "MEMFS rejects link() with EMLINK")] async fn test_hard_link_error_source_not_found() { let dir = tempdir().unwrap(); let src = dir.path().join("nonexistent.txt"); @@ -72,6 +86,7 @@ async fn test_hard_link_error_source_not_found() { assert_eq!(err.kind(), std::io::ErrorKind::NotFound); } +// On emscripten MEMFS reports EEXIST before its unconditional link() EMLINK. #[tokio::test] #[cfg_attr(miri, ignore)] // No `linkat` in miri. async fn test_hard_link_error_destination_already_exists() { @@ -92,6 +107,7 @@ async fn test_hard_link_error_destination_already_exists() { #[tokio::test] #[cfg_attr(miri, ignore)] // No `linkat` in miri. +#[cfg_attr(target_os = "emscripten", ignore = "MEMFS rejects link() with EMLINK")] async fn test_hard_link_error_source_is_directory() { let dir = tempdir().unwrap(); let src_dir = dir.path().join("src_directory"); diff --git a/tokio/tests/fs_open_options.rs b/tokio/tests/fs_open_options.rs index 58982d679df..957cdd33379 100644 --- a/tokio/tests/fs_open_options.rs +++ b/tokio/tests/fs_open_options.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use std::io::Write; use tempfile::NamedTempFile; diff --git a/tokio/tests/fs_remove_dir_all.rs b/tokio/tests/fs_remove_dir_all.rs index 5c71bdfda63..996b6ee5e94 100644 --- a/tokio/tests/fs_remove_dir_all.rs +++ b/tokio/tests/fs_remove_dir_all.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/tokio/tests/fs_remove_file.rs b/tokio/tests/fs_remove_file.rs index ea477213988..9d584532e10 100644 --- a/tokio/tests/fs_remove_file.rs +++ b/tokio/tests/fs_remove_file.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/tokio/tests/fs_rename.rs b/tokio/tests/fs_rename.rs index 91bb39ee359..01041c191fa 100644 --- a/tokio/tests/fs_rename.rs +++ b/tokio/tests/fs_rename.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/tokio/tests/fs_try_exists.rs b/tokio/tests/fs_try_exists.rs index 5e698cf1886..9e11bb96307 100644 --- a/tokio/tests/fs_try_exists.rs +++ b/tokio/tests/fs_try_exists.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/tokio/tests/fs_uring_read.rs b/tokio/tests/fs_uring_read.rs index 4ce6d454ab6..1869c513696 100644 --- a/tokio/tests/fs_uring_read.rs +++ b/tokio/tests/fs_uring_read.rs @@ -21,6 +21,12 @@ use tokio::runtime::{Builder, Runtime}; use tokio_test::assert_pending; use tokio_util::task::TaskTracker; +use crate::support::io_uring::uring_fs_op_in_use; + +mod support { + pub(crate) mod io_uring; +} + fn multi_rt(n: usize) -> Box Runtime> { Box::new(move || { Builder::new_multi_thread() @@ -48,6 +54,10 @@ fn rt_combinations() -> Vec Runtime>> { #[test] fn shutdown_runtime_while_performing_io_uring_ops() { + if !uring_fs_op_in_use(io_uring::opcode::Read::CODE) { + return; + } + fn run(rt: Runtime) { let (done_tx, done_rx) = mpsc::channel(); let (_tmp, path) = create_tmp_files(1); @@ -133,6 +143,10 @@ async fn read_small_large_files() { #[tokio::test] async fn cancel_op_future() { + if !uring_fs_op_in_use(io_uring::opcode::Read::CODE) { + return; + } + let (_tmp_file, path): (Vec, Vec) = create_tmp_files(1); let path = path[0].clone(); @@ -143,7 +157,6 @@ async fn cancel_op_future() { tokio::pin!(fut); poll_fn(move |_| { - // If io_uring is enabled (and not falling back to the thread pool), // the first poll should return Pending. assert_pending!(fut.as_mut().poll(&mut Context::from_waker(Waker::noop()))); tx.send(true).unwrap(); diff --git a/tokio/tests/fs_uring_statx.rs b/tokio/tests/fs_uring_statx.rs index 3ddc5866e85..8673953ac4f 100644 --- a/tokio/tests/fs_uring_statx.rs +++ b/tokio/tests/fs_uring_statx.rs @@ -24,7 +24,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_test::assert_pending; use tokio_util::task::TaskTracker; -use crate::support::io_uring::io_uring_supported; +use crate::support::io_uring::{io_uring_supported, uring_fs_op_in_use}; mod support { pub(crate) mod io_uring; @@ -57,7 +57,7 @@ fn rt_combinations() -> Vec Runtime>> { #[test] fn shutdown_runtime_while_performing_io_uring_ops() { - if !io_uring_supported() { + if !uring_fs_op_in_use(io_uring::opcode::Statx::CODE) { return; } @@ -158,6 +158,12 @@ async fn stat_permission_denied() { return; } + // A 0o244 directory does not stop root, so the EACCES this test expects + // never happens when running as root (e.g. in the CI kernel test guest). + if unsafe { libc::geteuid() } == 0 { + return; + } + let dir = tempdir().unwrap(); let permission_denied_directory_path = dir.path().join("baz"); create_dir(&permission_denied_directory_path).await.unwrap(); @@ -233,7 +239,7 @@ async fn stat_path_name_too_long() { #[tokio::test] async fn cancel_op_future() { - if !io_uring_supported() { + if !uring_fs_op_in_use(io_uring::opcode::Statx::CODE) { return; } diff --git a/tokio/tests/fs_uring_statx_fd_leak_test.rs b/tokio/tests/fs_uring_statx_fd_leak_test.rs index fb28e97ee77..6fe6865d4af 100644 --- a/tokio/tests/fs_uring_statx_fd_leak_test.rs +++ b/tokio/tests/fs_uring_statx_fd_leak_test.rs @@ -3,7 +3,11 @@ feature = "io-uring", feature = "rt", feature = "fs", - target_os = "linux" + target_os = "linux", + // `tokio::fs::read` only takes the io_uring path on these targets (see the + // FIXME in `src/fs/read.rs`); on musl it falls back to `spawn_blocking` and + // the poll-count assumptions below do not hold. + any(target_env = "gnu", target_os = "android") ))] mod support { diff --git a/tokio/tests/fs_write.rs b/tokio/tests/fs_write.rs index a125e040875..dae2d7ba8a4 100644 --- a/tokio/tests/fs_write.rs +++ b/tokio/tests/fs_write.rs @@ -1,5 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // WASI does not support all fs operations +#![cfg(all( + any( + feature = "full", + all( + target_os = "emscripten", + feature = "fs", + feature = "macros", + feature = "rt", + feature = "io-util" + ) + ), + not(target_os = "wasi") +))] // WASI does not support all fs operations use tempfile::tempdir; use tokio::fs; diff --git a/tokio/tests/io_async_read.rs b/tokio/tests/io_async_read.rs index aaeadfa4c11..8fe8ca0139c 100644 --- a/tokio/tests/io_async_read.rs +++ b/tokio/tests/io_async_read.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::AsyncRead; diff --git a/tokio/tests/io_buf_reader.rs b/tokio/tests/io_buf_reader.rs index 0d3f6bafc20..2e69b1ce2d7 100644 --- a/tokio/tests/io_buf_reader.rs +++ b/tokio/tests/io_buf_reader.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] // https://github.com/rust-lang/futures-rs/blob/1803948ff091b4eabf7f3bf39e16bbbdefca5cc8/futures/tests/io_buf_reader.rs diff --git a/tokio/tests/io_buf_writer.rs b/tokio/tests/io_buf_writer.rs index d3acf62c784..68958160207 100644 --- a/tokio/tests/io_buf_writer.rs +++ b/tokio/tests/io_buf_writer.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] // https://github.com/rust-lang/futures-rs/blob/1803948ff091b4eabf7f3bf39e16bbbdefca5cc8/futures/tests/io_buf_writer.rs @@ -118,6 +126,21 @@ async fn buf_writer_inner_flushes() { assert_eq!(w, [0, 1]); } +#[tokio::test] +async fn buf_writer_flush_retries_interrupted() { + let inner = { + let mut builder = tokio_test::io::Builder::new(); + builder + .write_error(io::Error::from(io::ErrorKind::Interrupted)) + .write(b"hello"); + builder.build() + }; + let mut writer = BufWriter::with_capacity(6, inner); + + assert_eq!(writer.write(b"hello").await.unwrap(), 5); + writer.flush().await.unwrap(); +} + #[tokio::test] async fn buf_writer_seek() { let mut w = BufWriter::with_capacity(3, Cursor::new(Vec::new())); diff --git a/tokio/tests/io_chain.rs b/tokio/tests/io_chain.rs index 70398295be9..23574fb0005 100644 --- a/tokio/tests/io_chain.rs +++ b/tokio/tests/io_chain.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::AsyncReadExt; use tokio_test::assert_ok; diff --git a/tokio/tests/io_copy.rs b/tokio/tests/io_copy.rs index 3bde8e7fa69..ef1707fd6dc 100644 --- a/tokio/tests/io_copy.rs +++ b/tokio/tests/io_copy.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use bytes::BytesMut; use tokio::io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}; @@ -36,39 +44,39 @@ async fn copy() { assert_eq!(wr, b"hello world"); } -#[tokio::test] -async fn proxy() { - struct BufferedWd { - buf: BytesMut, - writer: io::DuplexStream, - } - - impl AsyncWrite for BufferedWd { - fn poll_write( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.get_mut().buf.extend_from_slice(buf); - Poll::Ready(Ok(buf.len())) - } +struct BufferedWd { + buf: BytesMut, + writer: io::DuplexStream, +} - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); +impl AsyncWrite for BufferedWd { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.get_mut().buf.extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } - while !this.buf.is_empty() { - let n = ready!(Pin::new(&mut this.writer).poll_write(cx, &this.buf))?; - let _ = this.buf.split_to(n); - } + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); - Pin::new(&mut this.writer).poll_flush(cx) + while !this.buf.is_empty() { + let n = ready!(Pin::new(&mut this.writer).poll_write(cx, &this.buf))?; + let _ = this.buf.split_to(n); } - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.writer).poll_shutdown(cx) - } + Pin::new(&mut this.writer).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.writer).poll_shutdown(cx) } +} +#[tokio::test] +async fn proxy() { let (rd, wd) = io::duplex(1024); let mut rd = rd.take(1024); let mut wd = BufferedWd { @@ -85,6 +93,24 @@ async fn proxy() { assert_eq!(n, 1024); } +#[tokio::test] +async fn proxy_buf() { + let (rd, wd) = io::duplex(1024); + let mut rd = io::BufReader::new(rd).take(1024); + let mut wd = BufferedWd { + buf: BytesMut::new(), + writer: wd, + }; + + // write start bytes + assert_ok!(wd.write_all(&[0x42; 512]).await); + assert_ok!(wd.flush().await); + + let n = assert_ok!(io::copy_buf(&mut rd, &mut wd).await); + + assert_eq!(n, 1024); +} + #[tokio::test] async fn copy_is_cooperative() { tokio::select! { @@ -99,3 +125,18 @@ async fn copy_is_cooperative() { _ = tokio::task::yield_now() => {} } } + +#[tokio::test] +async fn copy_buf_is_cooperative() { + tokio::select! { + biased; + _ = async { + loop { + let mut reader: &[u8] = b"hello"; + let mut writer: Vec = vec![]; + let _ = io::copy_buf(&mut reader, &mut writer).await; + } + } => {}, + _ = tokio::task::yield_now() => {} + } +} diff --git a/tokio/tests/io_copy_bidirectional.rs b/tokio/tests/io_copy_bidirectional.rs index 3cdce32d0ce..91f507b16ac 100644 --- a/tokio/tests/io_copy_bidirectional.rs +++ b/tokio/tests/io_copy_bidirectional.rs @@ -2,7 +2,9 @@ #![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support bind() use std::time::Duration; -use tokio::io::{self, copy_bidirectional, AsyncReadExt, AsyncWriteExt}; +use tokio::io::{ + self, copy_bidirectional, copy_bidirectional_with_sizes, AsyncReadExt, AsyncWriteExt, +}; use tokio::net::TcpStream; use tokio::task::JoinHandle; @@ -139,6 +141,24 @@ async fn immediate_exit_on_read_error() { assert!(copy_bidirectional(&mut a, &mut b).await.is_err()); } +#[test] +#[should_panic(expected = "`a_to_b_buf_size` must be greater than 0")] +fn copy_bidirectional_with_sizes_panics_on_zero_a_to_b_buffer() { + let mut a = tokio_test::io::Builder::new().build(); + let mut b = tokio_test::io::Builder::new().build(); + + tokio_test::block_on(copy_bidirectional_with_sizes(&mut a, &mut b, 0, 1)).unwrap(); +} + +#[test] +#[should_panic(expected = "`b_to_a_buf_size` must be greater than 0")] +fn copy_bidirectional_with_sizes_panics_on_zero_b_to_a_buffer() { + let mut a = tokio_test::io::Builder::new().build(); + let mut b = tokio_test::io::Builder::new().build(); + + tokio_test::block_on(copy_bidirectional_with_sizes(&mut a, &mut b, 1, 0)).unwrap(); +} + #[tokio::test] async fn copy_bidirectional_is_cooperative() { tokio::select! { diff --git a/tokio/tests/io_driver_drop.rs b/tokio/tests/io_driver_drop.rs index f3845590e56..419fa16268c 100644 --- a/tokio/tests/io_driver_drop.rs +++ b/tokio/tests/io_driver_drop.rs @@ -5,6 +5,11 @@ use tokio::net::TcpListener; use tokio::runtime; use tokio_test::{assert_err, assert_pending, assert_ready, task}; +use futures::task::{waker_ref, ArcWake}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::task::Context; + #[test] fn tcp_doesnt_block() { let rt = rt(); @@ -52,6 +57,55 @@ fn drop_wakes() { assert_ready!(task.poll()); } +struct RegistrationWaker { + // Keep the registration alive through the waker to recreate the cycle from #3481. + _listener: Arc, + dropped: Arc, +} + +impl ArcWake for RegistrationWaker { + fn wake_by_ref(_: &Arc) {} +} + +impl Drop for RegistrationWaker { + fn drop(&mut self) { + self.dropped.store(true, Ordering::Relaxed); + } +} + +#[test] +fn shutdown_drops_waker_holding_registration() { + let rt = rt(); + + let listener = { + let _enter = rt.enter(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + + listener.set_nonblocking(true).unwrap(); + + Arc::new(TcpListener::from_std(listener).unwrap()) + }; + + let dropped = Arc::new(AtomicBool::new(false)); + let task = Arc::new(RegistrationWaker { + _listener: listener.clone(), + dropped: dropped.clone(), + }); + { + let waker = waker_ref(&task); + let mut cx = Context::from_waker(&waker); + + assert_pending!(listener.poll_accept(&mut cx)); + } + + drop(listener); + drop(task); + assert!(!dropped.load(Ordering::Relaxed)); + drop(rt); + + assert!(dropped.load(Ordering::Relaxed)); +} + fn rt() -> runtime::Runtime { runtime::Builder::new_current_thread() .enable_all() diff --git a/tokio/tests/io_emscripten.rs b/tokio/tests/io_emscripten.rs new file mode 100644 index 00000000000..f0bb6a4f45b --- /dev/null +++ b/tokio/tests/io_emscripten.rs @@ -0,0 +1,65 @@ +//! Standard I/O tests for emscripten. +//! +//! `tokio::io::{stdout, stderr}` round through emscripten's libc to the JS +//! `print`/`printErr` hooks; these tests mostly check that writes don't fail — +//! observable output verification is left to manual `--nocapture` runs. +//! +//! `tokio::io::stdin` reads fd 0 synchronously in emscripten, so this only +//! completes when stdin is non-interactive (EOF), as under CI where +//! the runner's stdin is `/dev/null`. The contract worth pinning is "a stdin +//! read returns rather than deadlocking", not a specific errno. + +#![cfg(all( + target_os = "emscripten", + feature = "io-std", + feature = "io-util", + feature = "rt", + feature = "macros" +))] + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[tokio::test] +async fn stdout_write_completes() { + let mut out = tokio::io::stdout(); + out.write_all(b"hello from stdout\n").await.unwrap(); + out.flush().await.unwrap(); +} + +#[tokio::test] +async fn stderr_write_completes() { + let mut err = tokio::io::stderr(); + err.write_all(b"hello from stderr\n").await.unwrap(); + err.flush().await.unwrap(); +} + +#[tokio::test] +async fn stdout_large_multichunk_write_completes() { + // Exercises the BufWriter chunking inside `Stdout` (writes larger than + // the internal buffer force multiple underlying writes). + let mut out = tokio::io::stdout(); + let data = vec![b'x'; 64 * 1024]; + out.write_all(&data).await.unwrap(); + out.flush().await.unwrap(); +} + +#[tokio::test] +async fn stdout_interleaved_writes_complete() { + let mut out = tokio::io::stdout(); + for i in 0..16 { + out.write_all(format!("line {i}\n").as_bytes()) + .await + .unwrap(); + } + out.flush().await.unwrap(); +} + +#[tokio::test] +async fn stdin_read_does_not_hang() { + // The runner detaches stdin onto the null device, so a read must return + // promptly (Ok(0) EOF or an I/O error) rather than blocking the host + // loop. The runner's watchdog fails the test if this ever deadlocks. + let mut stdin = tokio::io::stdin(); + let mut buf = [0u8; 32]; + let _ = stdin.read(&mut buf).await; +} diff --git a/tokio/tests/io_fill_buf.rs b/tokio/tests/io_fill_buf.rs index 534417c855d..7cf44cdcc1c 100644 --- a/tokio/tests/io_fill_buf.rs +++ b/tokio/tests/io_fill_buf.rs @@ -1,5 +1,14 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support file operations +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util", + feature = "fs" + ) +))] use tempfile::NamedTempFile; use tokio::fs::File; diff --git a/tokio/tests/io_join.rs b/tokio/tests/io_join.rs index 9b9f1e5ae5b..7f753986330 100644 --- a/tokio/tests/io_join.rs +++ b/tokio/tests/io_join.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{join, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, Join, ReadBuf}; diff --git a/tokio/tests/io_lines.rs b/tokio/tests/io_lines.rs index 56fd27bea7a..20a441a2af1 100644 --- a/tokio/tests/io_lines.rs +++ b/tokio/tests/io_lines.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use std::io::{Error, ErrorKind}; use std::string::FromUtf8Error; diff --git a/tokio/tests/io_mem_stream.rs b/tokio/tests/io_mem_stream.rs index 9c4304203c0..b5965697c89 100644 --- a/tokio/tests/io_mem_stream.rs +++ b/tokio/tests/io_mem_stream.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use futures::FutureExt; use std::io::IoSlice; diff --git a/tokio/tests/io_panic.rs b/tokio/tests/io_panic.rs index 6189ecf9a46..d64f14f5232 100644 --- a/tokio/tests/io_panic.rs +++ b/tokio/tests/io_panic.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi does not support panic recovery + all(target_os = "emscripten", feature = "io-util") +))] #![cfg(panic = "unwind")] use std::task::{Context, Poll}; @@ -42,7 +45,7 @@ impl AsyncWrite for RW { } } -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "emscripten")))] mod unix { use std::os::unix::prelude::{AsRawFd, RawFd}; @@ -168,7 +171,7 @@ fn new_unsplit_zero_capacity_panic_caller() -> Result<(), Box> { } #[test] -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "emscripten")))] fn async_fd_new_panic_caller() -> Result<(), Box> { use tokio::io::unix::AsyncFd; use tokio::runtime::Builder; @@ -190,7 +193,7 @@ fn async_fd_new_panic_caller() -> Result<(), Box> { } #[test] -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "emscripten")))] fn async_fd_with_interest_panic_caller() -> Result<(), Box> { use tokio::io::unix::AsyncFd; use tokio::io::Interest; @@ -213,7 +216,7 @@ fn async_fd_with_interest_panic_caller() -> Result<(), Box> { } #[test] -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "emscripten")))] fn async_fd_try_new_panic_caller() -> Result<(), Box> { use tokio::io::unix::AsyncFd; use tokio::runtime::Builder; @@ -235,7 +238,7 @@ fn async_fd_try_new_panic_caller() -> Result<(), Box> { } #[test] -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "emscripten")))] fn async_fd_try_with_interest_panic_caller() -> Result<(), Box> { use tokio::io::unix::AsyncFd; use tokio::io::Interest; diff --git a/tokio/tests/io_read.rs b/tokio/tests/io_read.rs index 6bea0ac865e..f15d884586d 100644 --- a/tokio/tests/io_read.rs +++ b/tokio/tests/io_read.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi does not support panic recovery + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; diff --git a/tokio/tests/io_read_buf.rs b/tokio/tests/io_read_buf.rs index 49a4f86f8ad..33db330608a 100644 --- a/tokio/tests/io_read_buf.rs +++ b/tokio/tests/io_read_buf.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; diff --git a/tokio/tests/io_read_exact.rs b/tokio/tests/io_read_exact.rs index d0e659bd339..2cf9ad16796 100644 --- a/tokio/tests/io_read_exact.rs +++ b/tokio/tests/io_read_exact.rs @@ -1,7 +1,18 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] -use tokio::io::AsyncReadExt; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_test::assert_ok; #[tokio::test] @@ -13,3 +24,37 @@ async fn read_exact() { assert_eq!(n, 8); assert_eq!(buf[..], b"hello wo"[..]); } + +struct InterruptThenRead { + interrupted: bool, + data: &'static [u8], +} + +impl AsyncRead for InterruptThenRead { + fn poll_read( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if !self.interrupted { + self.interrupted = true; + return Poll::Ready(Err(io::Error::from(io::ErrorKind::Interrupted))); + } + let n = std::cmp::min(self.data.len(), buf.remaining()); + buf.put_slice(&self.data[..n]); + self.data = &self.data[n..]; + Poll::Ready(Ok(())) + } +} + +#[tokio::test] +async fn read_exact_retries_interrupted() { + let mut reader = InterruptThenRead { + interrupted: false, + data: b"hello", + }; + let mut buf = [0u8; 5]; + let n = reader.read_exact(&mut buf).await.unwrap(); + assert_eq!(n, 5); + assert_eq!(&buf, b"hello"); +} diff --git a/tokio/tests/io_read_line.rs b/tokio/tests/io_read_line.rs index bbc984b4505..ef3436cd2c4 100644 --- a/tokio/tests/io_read_line.rs +++ b/tokio/tests/io_read_line.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use std::io::ErrorKind; use std::str::Utf8Error; diff --git a/tokio/tests/io_read_to_end.rs b/tokio/tests/io_read_to_end.rs index 6573f8b68b2..f42ee77dc07 100644 --- a/tokio/tests/io_read_to_end.rs +++ b/tokio/tests/io_read_to_end.rs @@ -1,6 +1,15 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] +use std::io; use std::pin::Pin; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; @@ -17,6 +26,20 @@ async fn read_to_end() { assert_eq!(buf[..], b"hello world"[..]); } +#[tokio::test] +async fn read_to_end_retries_interrupted() { + let mut mock = Builder::new() + .read(b"hello") + .read_error(io::Error::from(io::ErrorKind::Interrupted)) + .read(b" world") + .build(); + let mut buf = Vec::new(); + + let n = mock.read_to_end(&mut buf).await.unwrap(); + assert_eq!(n, 11); + assert_eq!(buf, b"hello world"); +} + #[derive(Copy, Clone, Debug)] enum State { Initializing, diff --git a/tokio/tests/io_read_to_string.rs b/tokio/tests/io_read_to_string.rs index 040f7269293..67f61f46730 100644 --- a/tokio/tests/io_read_to_string.rs +++ b/tokio/tests/io_read_to_string.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use std::io; use std::str::Utf8Error; @@ -17,6 +25,20 @@ async fn read_to_string() { assert_eq!(buf[..], "hello world"[..]); } +#[tokio::test] +async fn read_to_string_retries_interrupted() { + let mut mock = Builder::new() + .read(b"hello") + .read_error(io::Error::from(io::ErrorKind::Interrupted)) + .read(b" world") + .build(); + let mut buf = String::new(); + + let n = mock.read_to_string(&mut buf).await.unwrap(); + assert_eq!(n, 11); + assert_eq!(buf, "hello world"); +} + #[tokio::test] async fn to_string_does_not_truncate_on_utf8_error() { let data = vec![0xff, 0xff, 0xff]; diff --git a/tokio/tests/io_read_until.rs b/tokio/tests/io_read_until.rs index 61800a0d9c1..3a438a7a1a2 100644 --- a/tokio/tests/io_read_until.rs +++ b/tokio/tests/io_read_until.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use std::io::ErrorKind; use tokio::io::{AsyncBufReadExt, BufReader, Error}; @@ -23,6 +31,20 @@ async fn read_until() { assert_eq!(buf, []); } +#[tokio::test] +async fn read_until_retries_interrupted() { + let mock = Builder::new() + .read_error(Error::from(ErrorKind::Interrupted)) + .read(b"hello world") + .build(); + let mut read = BufReader::new(mock); + let mut buf = vec![]; + + let n = read.read_until(b' ', &mut buf).await.unwrap(); + assert_eq!(n, 6); + assert_eq!(buf, b"hello "); +} + #[tokio::test] async fn read_until_not_all_ready() { let mock = Builder::new() diff --git a/tokio/tests/io_repeat.rs b/tokio/tests/io_repeat.rs index 5a48817b1a0..f9ad4db4a28 100644 --- a/tokio/tests/io_repeat.rs +++ b/tokio/tests/io_repeat.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(miri)))] +#![cfg(any( + all(feature = "full", not(miri)), + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::AsyncReadExt; diff --git a/tokio/tests/io_sink.rs b/tokio/tests/io_sink.rs index fb085c51561..749a7b260b3 100644 --- a/tokio/tests/io_sink.rs +++ b/tokio/tests/io_sink.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::AsyncWriteExt; diff --git a/tokio/tests/io_split.rs b/tokio/tests/io_split.rs index 983982ccaf9..45891484bc7 100644 --- a/tokio/tests/io_split.rs +++ b/tokio/tests/io_split.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi does not support panic recovery + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{ split, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf, ReadHalf, WriteHalf, diff --git a/tokio/tests/io_take.rs b/tokio/tests/io_take.rs index 1ae5f6908f5..dd735146adc 100644 --- a/tokio/tests/io_take.rs +++ b/tokio/tests/io_take.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi does not support panic recovery + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use std::pin::Pin; use std::task::{Context, Poll}; diff --git a/tokio/tests/io_util_empty.rs b/tokio/tests/io_util_empty.rs index 7a4b8c6a575..72fac1b140f 100644 --- a/tokio/tests/io_util_empty.rs +++ b/tokio/tests/io_util_empty.rs @@ -1,4 +1,12 @@ -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncSeekExt}; #[tokio::test] diff --git a/tokio/tests/io_write.rs b/tokio/tests/io_write.rs index 96cebc3313b..21bdcbe6e3b 100644 --- a/tokio/tests/io_write.rs +++ b/tokio/tests/io_write.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio_test::assert_ok; diff --git a/tokio/tests/io_write_all.rs b/tokio/tests/io_write_all.rs index 7ca02228a3c..877cb823d3b 100644 --- a/tokio/tests/io_write_all.rs +++ b/tokio/tests/io_write_all.rs @@ -1,8 +1,17 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio_test::assert_ok; +use tokio_test::io::Builder; use bytes::BytesMut; use std::cmp; @@ -49,3 +58,14 @@ async fn write_all() { assert_eq!(wr.buf, b"hello world"[..]); assert_eq!(wr.cnt, 3); } + +#[tokio::test] +async fn write_all_retries_interrupted() { + let mut mock = Builder::new() + .write(b"he") + .write_error(io::Error::from(io::ErrorKind::Interrupted)) + .write(b"llo") + .build(); + + mock.write_all(b"hello").await.unwrap(); +} diff --git a/tokio/tests/io_write_all_buf.rs b/tokio/tests/io_write_all_buf.rs index 52ad5965c09..edee44334b5 100644 --- a/tokio/tests/io_write_all_buf.rs +++ b/tokio/tests/io_write_all_buf.rs @@ -1,7 +1,16 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio_test::io::Builder; use tokio_test::{assert_err, assert_ok}; use bytes::{Buf, Bytes, BytesMut}; @@ -99,6 +108,7 @@ async fn write_buf_err() { async fn write_all_buf_vectored() { struct Wr { buf: BytesMut, + interrupted: bool, } impl AsyncWrite for Wr { fn poll_write( @@ -121,6 +131,11 @@ async fn write_all_buf_vectored() { _cx: &mut Context<'_>, bufs: &[io::IoSlice<'_>], ) -> Poll> { + if !self.interrupted { + self.interrupted = true; + return Poll::Ready(Err(io::Error::from(io::ErrorKind::Interrupted))); + } + for buf in bufs { self.buf.extend_from_slice(buf); } @@ -135,6 +150,7 @@ async fn write_all_buf_vectored() { let mut wr = Wr { buf: BytesMut::with_capacity(64), + interrupted: false, }; let mut buf = Bytes::from_static(b"hello") .chain(Bytes::from_static(b" ")) @@ -143,3 +159,16 @@ async fn write_all_buf_vectored() { wr.write_all_buf(&mut buf).await.unwrap(); assert_eq!(&wr.buf[..], b"hello world"); } + +#[tokio::test] +async fn write_all_buf_retries_interrupted() { + let mut mock = Builder::new() + .write(b"he") + .write_error(io::Error::from(io::ErrorKind::Interrupted)) + .write(b"llo") + .build(); + let mut buf = Bytes::from_static(b"hello"); + + mock.write_all_buf(&mut buf).await.unwrap(); + assert!(!buf.has_remaining()); +} diff --git a/tokio/tests/io_write_buf.rs b/tokio/tests/io_write_buf.rs index 8bd09ad62c9..dd74c73132f 100644 --- a/tokio/tests/io_write_buf.rs +++ b/tokio/tests/io_write_buf.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio_test::assert_ok; diff --git a/tokio/tests/io_write_int.rs b/tokio/tests/io_write_int.rs index 48a583d8c3f..c9c58fa8892 100644 --- a/tokio/tests/io_write_int.rs +++ b/tokio/tests/io_write_int.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "io-util" + ) +))] use tokio::io::{AsyncWrite, AsyncWriteExt}; diff --git a/tokio/tests/join_handle_panic.rs b/tokio/tests/join_handle_panic.rs index 248d5702f68..de505c36f1b 100644 --- a/tokio/tests/join_handle_panic.rs +++ b/tokio/tests/join_handle_panic.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi doesn't support panic recovery + all(target_os = "emscripten", feature = "rt", feature = "macros", feature = "time") +))] #![cfg(panic = "unwind")] struct PanicsOnDrop; diff --git a/tokio/tests/macros_join.rs b/tokio/tests/macros_join.rs index 4c6db26d8ae..8b304be8da4 100644 --- a/tokio/tests/macros_join.rs +++ b/tokio/tests/macros_join.rs @@ -2,13 +2,13 @@ #![allow(clippy::disallowed_names)] use std::sync::Arc; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] #[cfg(target_pointer_width = "64")] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use tokio::sync::{oneshot, Semaphore}; diff --git a/tokio/tests/macros_pin.rs b/tokio/tests/macros_pin.rs index 2de68ad031f..19c29d56fdb 100644 --- a/tokio/tests/macros_pin.rs +++ b/tokio/tests/macros_pin.rs @@ -1,9 +1,9 @@ #![cfg(feature = "macros")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; async fn one() {} diff --git a/tokio/tests/macros_select.rs b/tokio/tests/macros_select.rs index 3b403e4ce27..662645c0740 100644 --- a/tokio/tests/macros_select.rs +++ b/tokio/tests/macros_select.rs @@ -1,10 +1,10 @@ #![cfg(feature = "macros")] #![allow(clippy::disallowed_names)] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use tokio::sync::oneshot; diff --git a/tokio/tests/macros_try_join.rs b/tokio/tests/macros_try_join.rs index 03172ca2a2d..25f811c8e22 100644 --- a/tokio/tests/macros_try_join.rs +++ b/tokio/tests/macros_try_join.rs @@ -6,10 +6,10 @@ use std::{convert::Infallible, sync::Arc}; use tokio::sync::{oneshot, Semaphore}; use tokio_test::{assert_pending, assert_ready, task}; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; #[maybe_tokio_test] diff --git a/tokio/tests/rt_basic.rs b/tokio/tests/rt_basic.rs index 3fab2649a12..3adf7d48c3e 100644 --- a/tokio/tests/rt_basic.rs +++ b/tokio/tests/rt_basic.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all(target_os = "emscripten", feature = "rt", feature = "macros") +))] use tokio::runtime::{self, Runtime}; use tokio::sync::oneshot; @@ -249,7 +252,10 @@ fn spawn_two() { } } -#[cfg_attr(target_os = "wasi", ignore = "WASI: std::thread::spawn not supported")] +#[cfg_attr( + target_family = "wasm", + ignore = "wasm targets do not support std::thread::spawn" +)] #[test] fn spawn_remote() { let rt = rt(); @@ -346,7 +352,10 @@ mod unstable { } #[test] - #[cfg_attr(target_os = "wasi", ignore = "Wasi does not support panic recovery")] + #[cfg_attr( + target_family = "wasm", + ignore = "wasm targets do not support std::thread::spawn" + )] fn spawns_do_nothing() { use std::sync::Arc; @@ -375,7 +384,10 @@ mod unstable { } #[test] - #[cfg_attr(target_os = "wasi", ignore = "Wasi does not support panic recovery")] + #[cfg_attr( + target_family = "wasm", + ignore = "wasm targets do not support std::thread::spawn" + )] fn shutdown_all_concurrent_block_on() { const N: usize = 2; use std::sync::{mpsc, Arc}; diff --git a/tokio/tests/rt_blocking_thread_exhaust.rs b/tokio/tests/rt_blocking_thread_exhaust.rs new file mode 100644 index 00000000000..8c27ffabd7b --- /dev/null +++ b/tokio/tests/rt_blocking_thread_exhaust.rs @@ -0,0 +1,95 @@ +#![warn(rust_2018_idioms)] +#![cfg(all( + target_os = "linux", + feature = "full", + not(miri), + not(tokio_no_tuning_tests), + panic = "unwind", +))] + +use std::panic::{self, AssertUnwindSafe}; +use std::sync::mpsc; +use std::time::Duration; + +use tokio::runtime::Builder; +use tokio::time::timeout; + +#[test] +fn spawn_blocking_with_only_scheduler_workers() { + // Regression test for https://github.com/tokio-rs/tokio/issues/8406. + // + // On a multi-threaded runtime, the only pool threads present before the + // first `spawn_blocking` are the scheduler workers themselves. If the OS + // then refuses to create a new thread, the pool used to swallow the error + // assuming a busy thread would pick up the task, but scheduler workers + // never drain the blocking queue, so the task was orphaned forever. + // + // Pin the per-user process thread limit to the current thread count so + // that creating a new pool thread fails; `spawn_blocking` must panic + // with "OS can't spawn worker thread" instead of hanging. + + let (started_tx, started_rx) = mpsc::sync_channel(4); + let rt = Builder::new_multi_thread() + .worker_threads(4) + .on_thread_start(move || { + started_tx.send(()).unwrap(); + }) + .enable_all() + .build() + .unwrap(); + + for _ in 0..4 { + started_rx.recv().unwrap(); + } + + let threads = std::fs::read_dir("/proc/self/task").unwrap().count(); + let lim = libc::rlimit { + rlim_cur: threads as libc::rlim_t, + rlim_max: threads as libc::rlim_t, + }; + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NPROC, &lim) }, 0); + + // If the limit is not enforced for this process (e.g. when running as root + // or in a container), skip rather than fail. + match std::thread::Builder::new() + .name("probe".into()) + .spawn(|| {}) + { + Ok(h) => { + let _ = h.join(); + eprintln!("skip: setrlimit did not prevent thread creation"); + return; + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {} + Err(e) => panic!("unexpected thread-spawn error: {e}"), + } + + let res = panic::catch_unwind(AssertUnwindSafe(|| { + rt.block_on(async { + timeout( + Duration::from_secs(5), + tokio::task::spawn_blocking(|| 42u32), + ) + .await + }) + })); + + // The fixed code panics synchronously when no real pool thread can be + // created; the bug let the task hang, which the timeout turns into + // `Ok(Err(_))`. + let panic_err = match res { + Err(panic_err) => panic_err, + Ok(Err(_)) => panic!("spawn_blocking timed out"), + Ok(Ok(_)) => panic!("spawn_blocking unexpectedly succeeded"), + }; + + let msg = panic_err + .downcast_ref::<&str>() + .copied() + .or_else(|| panic_err.downcast_ref::().map(|s| s.as_str())) + .unwrap_or(""); + assert!( + msg.contains("OS can't spawn worker thread"), + "unexpected panic: {msg}" + ); +} diff --git a/tokio/tests/rt_busy_tick.rs b/tokio/tests/rt_busy_tick.rs new file mode 100644 index 00000000000..fdcf81f0f7d --- /dev/null +++ b/tokio/tests/rt_busy_tick.rs @@ -0,0 +1,86 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] + +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +#[test] +#[should_panic(expected = "max_io_events_per_busy_tick must be non-zero")] +fn zero_busy_tick_panics() { + tokio::runtime::Builder::new_multi_thread().max_io_events_per_busy_tick(0); +} + +#[test] +#[should_panic(expected = "max_io_events_per_tick must be non-zero")] +fn zero_io_events_per_tick_panics() { + tokio::runtime::Builder::new_multi_thread().max_io_events_per_tick(0); +} + +// Liveness check: the runtime always has runnable tasks, so its I/O polls do +// not wait and each takes one event. Echo traffic must still complete. +fn busy_runtime_gets_every_event(rt: tokio::runtime::Runtime) { + for _ in 0..8 { + rt.spawn(async { + loop { + tokio::task::yield_now().await; + } + }); + } + rt.block_on(async { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (mut s, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let mut buf = [0u8; 4]; + while s.read_exact(&mut buf).await.is_ok() { + s.write_all(&buf).await.unwrap(); + } + }); + } + }); + let clients: Vec<_> = (0..16) + .map(|_| { + tokio::spawn(async move { + let mut s = TcpStream::connect(addr).await.unwrap(); + for _ in 0..16 { + s.write_all(b"ping").await.unwrap(); + s.read_exact(&mut [0u8; 4]).await.unwrap(); + } + }) + }) + .collect(); + let all = async { + for c in clients { + c.await.unwrap(); + } + }; + tokio::time::timeout(Duration::from_secs(20), all) + .await + .expect("echo round trips did not finish"); + }); +} + +#[test] +fn multi_thread_busy_tick() { + // One worker, so the spinning tasks keep it from ever parking. + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .max_io_events_per_busy_tick(1) + .enable_all() + .build() + .unwrap(); + busy_runtime_gets_every_event(rt); +} + +#[test] +fn current_thread_busy_tick() { + let rt = tokio::runtime::Builder::new_current_thread() + .max_io_events_per_busy_tick(1) + .enable_all() + .build() + .unwrap(); + busy_runtime_gets_every_event(rt); +} diff --git a/tokio/tests/rt_emscripten_block_on.rs b/tokio/tests/rt_emscripten_block_on.rs new file mode 100644 index 00000000000..9c91d837dfd --- /dev/null +++ b/tokio/tests/rt_emscripten_block_on.rs @@ -0,0 +1,143 @@ +//! `Runtime::block_on` drives the scheduler synchronously to a fixed point: +//! immediate futures return their value. A wait that would block suspends on +//! the host loop when the build linked `-sJSPI` (see `rt_emscripten_jspi`) +//! and panics when it did not. Both CI lanes run this file. + +#![cfg(all( + target_os = "emscripten", + not(target_feature = "atomics"), + feature = "rt", + feature = "time", + feature = "sync", + feature = "macros" +))] + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::time::Duration; + +use tokio::runtime::Builder; + +extern "C" { + /// Emscripten's `ASYNCIFY` build mode; 2 is JSPI. + fn emscripten_has_asyncify() -> i32; +} + +fn jspi_linked() -> bool { + // SAFETY: an Emscripten libc query with no arguments and no side effects. + unsafe { emscripten_has_asyncify() == 2 } +} + +fn rt() -> tokio::runtime::Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +/// Assert `f` panics with the targeted would-suspend message. +fn assert_panics_cannot_block_on(f: impl FnOnce()) { + if cfg!(not(panic = "unwind")) { + return; + } + let err = catch_unwind(AssertUnwindSafe(f)).expect_err("expected a would-suspend panic"); + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + msg.contains("cannot block"), + "unexpected panic message: {msg}" + ); +} + +#[test] +fn block_on_returns_immediate_value() { + let out = rt().block_on(async { 1 + 2 }); + assert_eq!(out, 3); +} + +#[test] +fn block_on_drives_ready_spawned_tasks() { + // Spawned tasks that complete synchronously must be driven to + // completion within the same fixed-point pump. + let out = rt().block_on(async { + let a = tokio::spawn(async { 20 }); + let b = tokio::spawn(async { 22 }); + a.await.unwrap() + b.await.unwrap() + }); + assert_eq!(out, 42); +} + +#[test] +fn block_on_drives_many_ready_spawned_tasks() { + // Well past `event_interval`, so the scheduler's maintenance park runs + // mid-drive: a host turn under JSPI and a no-op without. + let out = rt().block_on(async { + let handles: Vec<_> = (0..1000u32) + .map(|i| tokio::spawn(async move { i })) + .collect(); + let mut sum = 0; + for h in handles { + sum += h.await.unwrap(); + } + sum + }); + assert_eq!(out, 499_500); +} + +#[test] +fn timer_wait_needs_jspi() { + let sleep = || { + rt().block_on(async { + tokio::time::sleep(Duration::from_millis(10)).await; + }); + }; + + if jspi_linked() { + sleep(); + } else { + assert_panics_cannot_block_on(sleep); + } +} + +#[test] +fn wait_without_a_deadline_needs_jspi() { + // A oneshot sent from a host callback: the only wake is an unpark from a + // later wasm activation, which needs the suspended activation to be + // resumable. + let recv = || { + let (tx, rx) = tokio::sync::oneshot::channel::(); + // Without JSPI the receiver is gone by the time this fires. + host_callback(10, move || { + let _ = tx.send(11); + }); + rt().block_on(async { rx.await.unwrap() }) + }; + + if jspi_linked() { + assert_eq!(recv(), 11); + } else { + assert_panics_cannot_block_on(|| { + recv(); + }); + } +} + +extern "C" { + fn emscripten_async_call( + func: extern "C" fn(*mut std::ffi::c_void), + arg: *mut std::ffi::c_void, + millis: i32, + ); +} + +/// Run `f` from a fresh wasm activation after a host timeout. +fn host_callback(millis: i32, f: impl FnOnce() + 'static) { + extern "C" fn trampoline(arg: *mut std::ffi::c_void) { + // SAFETY: `arg` is the `Box>` leaked below, and + // Emscripten invokes the callback exactly once. + let f = unsafe { Box::from_raw(arg as *mut Box) }; + f(); + } + let f: Box> = Box::new(Box::new(f)); + // SAFETY: an Emscripten API scheduling `trampoline(arg)` on the host loop. + unsafe { emscripten_async_call(trampoline, Box::into_raw(f) as *mut _, millis) } +} diff --git a/tokio/tests/rt_emscripten_jspi.rs b/tokio/tests/rt_emscripten_jspi.rs new file mode 100644 index 00000000000..ffbccd1a770 --- /dev/null +++ b/tokio/tests/rt_emscripten_jspi.rs @@ -0,0 +1,256 @@ +//! JSPI suspension contracts. With `-sJSPI` a would-block wait suspends the +//! activation until a host timer fires or a later activation unparks it; +//! without it the wait panics (see `rt_emscripten_block_on`), so every test +//! here returns early unless the build linked JSPI. +//! +//! NOTE: This is the only Emscripten test file with real timer tests. + +#![cfg(all( + target_os = "emscripten", + not(target_feature = "atomics"), + feature = "rt", + feature = "time", + feature = "sync", + feature = "macros" +))] + +use std::sync::Arc; +use std::time::Duration; + +use tokio::runtime::Builder; +use tokio::sync::Notify; +use tokio::time::{sleep, Instant}; + +fn rt() -> tokio::runtime::Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +extern "C" { + /// Emscripten's `ASYNCIFY` build mode; 2 is JSPI. + fn emscripten_has_asyncify() -> i32; +} + +fn jspi_linked() -> bool { + // SAFETY: an Emscripten libc query with no arguments and no side effects. + unsafe { emscripten_has_asyncify() == 2 } +} + +macro_rules! require_jspi { + () => { + if !jspi_linked() { + return; + } + }; +} + +fn is_nested_runtime_panic(e: &Box) -> bool { + e.downcast_ref::<&str>() + .map(|m| m.contains("Cannot start a runtime from within a runtime")) + .unwrap_or(false) +} + +#[test] +fn nested_block_on_still_panics() { + require_jspi!(); + if cfg!(not(panic = "unwind")) { + return; + } + let outer = rt(); + let res = outer.block_on(async { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let res = std::panic::catch_unwind(|| rt().block_on(async { 1 })); + std::panic::set_hook(hook); + res + }); + assert!(is_nested_runtime_panic(&res.unwrap_err())); +} + +#[test] +fn block_on_yield_now_takes_a_host_turn() { + require_jspi!(); + let out = rt().block_on(async { + tokio::task::yield_now().await; + 7 + }); + assert_eq!(out, 7); +} + +#[tokio::test] +async fn root_sleep_parks_and_resumes() { + require_jspi!(); + let start = tokio::time::Instant::now(); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!( + start.elapsed() >= Duration::from_millis(15), + "the park must actually wait out the timer deadline" + ); +} + +#[tokio::test] +async fn root_spawned_tasks_with_timers() { + require_jspi!(); + let out = async { + let a = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(5)).await; + 20 + }); + let b = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(10)).await; + 22 + }); + a.await.unwrap() + b.await.unwrap() + } + .await; + assert_eq!(out, 42); +} + +#[tokio::test] +async fn sequential_parks_inside_one_root() { + require_jspi!(); + // Each park must suspend and resume independently; leaf bookkeeping + // must balance across them. + for i in 0..3u32 { + let start = tokio::time::Instant::now(); + tokio::time::sleep(Duration::from_millis(2)).await; + assert!(start.elapsed() >= Duration::from_millis(1), "park {i}"); + } +} + +#[tokio::test] +async fn root_park_resumes_on_timer_driven_wake() { + require_jspi!(); + // The spawned task's timer bounds the driver park; on resume it sends + // and wakes the root future. + let (tx, rx) = tokio::sync::oneshot::channel::(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(5)).await; + tx.send(11).unwrap(); + }); + assert_eq!(rx.await.unwrap(), 11); +} + +extern "C" { + fn emscripten_async_call( + func: extern "C" fn(*mut std::ffi::c_void), + arg: *mut std::ffi::c_void, + millis: i32, + ); +} + +/// Run `f` from a fresh wasm activation after a host timeout. +fn host_callback(millis: i32, f: impl FnOnce() + 'static) { + extern "C" fn trampoline(arg: *mut std::ffi::c_void) { + // SAFETY: `arg` is the `Box>` leaked below, and + // Emscripten invokes the callback exactly once. + let f = unsafe { Box::from_raw(arg as *mut Box) }; + f(); + } + let f: Box> = Box::new(Box::new(f)); + // SAFETY: an Emscripten API scheduling `trampoline(arg)` on the host loop. + unsafe { emscripten_async_call(trampoline, Box::into_raw(f) as *mut _, millis) } +} + +// A host callback is a fresh activation entering tokio while the root +// activation is parked with no deadline; its send must resume the park. +#[test] +fn host_activation_wakes_park_without_deadline() { + require_jspi!(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(1); + host_callback(10, move || tx.try_send(11).unwrap()); + let out = rt().block_on(async { rx.recv().await.unwrap() }); + assert_eq!(out, 11); +} + +// The park is bounded by a far timer; the host callback's send must resume +// it at once rather than at that deadline. +#[test] +fn host_activation_wakes_timed_park_early() { + require_jspi!(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(1); + host_callback(10, move || tx.try_send(11).unwrap()); + let start = Instant::now(); + let out = rt().block_on(async { + tokio::time::timeout(Duration::from_secs(10), rx.recv()) + .await + .unwrap() + .unwrap() + }); + assert_eq!(out, 11); + assert!(start.elapsed() < Duration::from_secs(5)); +} + +// A spawned task woken from a host activation, with the root awaiting it. +#[test] +fn host_activation_wakes_spawned_task() { + require_jspi!(); + let notify = Arc::new(Notify::new()); + let n = notify.clone(); + host_callback(10, move || n.notify_one()); + let out = rt().block_on(async { + tokio::spawn(async move { + notify.notified().await; + 5 + }) + .await + .unwrap() + }); + assert_eq!(out, 5); +} + +// A self-rewaking task must not starve the real host timer: the +// event-interval park yields a 0ms host turn so the timer still fires. +#[tokio::test] +async fn greedy_task_does_not_starve_host_timer() { + require_jspi!(); + tokio::spawn(async { + loop { + tokio::task::yield_now().await; + } + }); + sleep(Duration::from_millis(5)).await; +} + +// When a nearer timer fires, the next park must re-arm for a still-pending +// farther timer rather than dropping it. +#[tokio::test] +async fn farther_timer_survives_nearer_timer_firing() { + require_jspi!(); + let start = Instant::now(); + + let notify = Arc::new(Notify::new()); + let n = notify.clone(); + let near = tokio::spawn(async move { + sleep(Duration::from_millis(5)).await; + n.notify_one(); + }); + let waiter = tokio::spawn(async move { + notify.notified().await; + }); + + sleep(Duration::from_millis(25)).await; + assert!( + start.elapsed() >= Duration::from_millis(25), + "farther timer did not hold its deadline" + ); + + near.await.unwrap(); + waiter.await.unwrap(); +} + +// A host activation can spawn onto the parked runtime: `tokio::spawn` sees +// the entered runtime's handle, and the spawn unparks the root to run it. +#[test] +fn host_activation_spawns_onto_parked_runtime() { + require_jspi!(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(2); + let tx2 = tx.clone(); + let runtime = rt(); + let handle = runtime.handle().clone(); + host_callback(10, move || { + tokio::spawn(async move { tx.send(1).await.unwrap() }); + handle.spawn(async move { tx2.send(2).await.unwrap() }); + }); + let out = runtime.block_on(async { rx.recv().await.unwrap() + rx.recv().await.unwrap() }); + assert_eq!(out, 3); +} diff --git a/tokio/tests/rt_handle.rs b/tokio/tests/rt_handle.rs index 8feb7207d0b..db1ee6f7124 100644 --- a/tokio/tests/rt_handle.rs +++ b/tokio/tests/rt_handle.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all(target_os = "emscripten", feature = "rt", feature = "macros") +))] use std::sync::Arc; use tokio::runtime::Runtime; diff --git a/tokio/tests/rt_multi_thread_emscripten.rs b/tokio/tests/rt_multi_thread_emscripten.rs new file mode 100644 index 00000000000..49e41731c5e --- /dev/null +++ b/tokio/tests/rt_multi_thread_emscripten.rs @@ -0,0 +1,72 @@ +#![warn(rust_2018_idioms)] +#![cfg(all( + target_os = "emscripten", + feature = "rt-multi-thread", + feature = "macros" +))] + +//! Multi-thread runtime tests for `wasm32-unknown-emscripten` built with +//! pthreads (`-pthread`). + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier}; +use std::time::Duration; + +#[test] +fn block_on_multi_thread() { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_time() + .build() + .unwrap(); + + let out = rt.block_on(async { + let jh = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(10)).await; + "hello" + }); + jh.await.unwrap() + }); + assert_eq!(out, "hello"); +} + +#[test] +fn spawn_blocking_runs_in_parallel() { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .build() + .unwrap(); + + // Both closures must be running concurrently for the barrier to release. + let barrier = Arc::new(Barrier::new(2)); + rt.block_on(async { + let a = tokio::task::spawn_blocking({ + let barrier = barrier.clone(); + move || { + barrier.wait(); + } + }); + let b = tokio::task::spawn_blocking(move || { + barrier.wait(); + }); + a.await.unwrap(); + b.await.unwrap(); + }); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn macro_multi_thread() { + static COUNT: AtomicUsize = AtomicUsize::new(0); + + let mut handles = Vec::new(); + for _ in 0..8 { + handles.push(tokio::spawn(async { + tokio::task::yield_now().await; + COUNT.fetch_add(1, Ordering::Relaxed); + })); + } + for handle in handles { + handle.await.unwrap(); + } + assert_eq!(COUNT.load(Ordering::Relaxed), 8); +} diff --git a/tokio/tests/rt_panic.rs b/tokio/tests/rt_panic.rs index 1bf05580ab6..f9c122ae26b 100644 --- a/tokio/tests/rt_panic.rs +++ b/tokio/tests/rt_panic.rs @@ -1,11 +1,16 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all(target_os = "emscripten", feature = "rt", feature = "macros") +))] #![cfg(not(target_os = "wasi"))] // Wasi doesn't support panic recovery #![cfg(panic = "unwind")] use futures::future; use std::error::Error; -use tokio::runtime::{Builder, Handle, Runtime}; +#[cfg(feature = "rt-multi-thread")] +use tokio::runtime::Builder; +use tokio::runtime::{Handle, Runtime}; mod support { pub mod panic; @@ -47,6 +52,7 @@ fn into_panic_panic_caller() -> Result<(), Box> { } #[test] +#[cfg(feature = "rt-multi-thread")] fn builder_worker_threads_panic_caller() -> Result<(), Box> { let panic_location_file = test_panic(|| { let _ = Builder::new_multi_thread().worker_threads(0).build(); @@ -59,6 +65,7 @@ fn builder_worker_threads_panic_caller() -> Result<(), Box> { } #[test] +#[cfg(feature = "rt-multi-thread")] fn builder_max_blocking_threads_panic_caller() -> Result<(), Box> { let panic_location_file = test_panic(|| { let _ = Builder::new_multi_thread().max_blocking_threads(0).build(); @@ -71,6 +78,7 @@ fn builder_max_blocking_threads_panic_caller() -> Result<(), Box> { } #[test] +#[cfg(feature = "rt-multi-thread")] fn builder_global_queue_interval_panic_caller() -> Result<(), Box> { let panic_location_file = test_panic(|| { let _ = Builder::new_multi_thread().global_queue_interval(0).build(); @@ -83,6 +91,7 @@ fn builder_global_queue_interval_panic_caller() -> Result<(), Box> { } #[test] +#[cfg(feature = "rt-multi-thread")] fn builder_event_interval_interval_panic_caller() -> Result<(), Box> { let panic_location_file = test_panic(|| { let _ = Builder::new_multi_thread().event_interval(0).build(); @@ -95,6 +104,7 @@ fn builder_event_interval_interval_panic_caller() -> Result<(), Box> } #[test] +#[cfg(feature = "rt-multi-thread")] fn builder_name_panic_caller() -> Result<(), Box> { let panic_location_file = test_panic(|| { let _ = Builder::new_multi_thread().name(" ").build(); diff --git a/tokio/tests/rt_shutdown_err.rs b/tokio/tests/rt_shutdown_err.rs index b92d8eb24f1..cf03debed25 100644 --- a/tokio/tests/rt_shutdown_err.rs +++ b/tokio/tests/rt_shutdown_err.rs @@ -1,8 +1,9 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any(feature = "full", all(target_os = "emscripten", feature = "rt")))] #![cfg(not(miri))] // No socket in miri. use std::io; +#[cfg(not(target_family = "wasm"))] use tokio::net::TcpListener; use tokio::runtime::Builder; @@ -10,6 +11,7 @@ fn rt() -> tokio::runtime::Runtime { Builder::new_current_thread().enable_all().build().unwrap() } +#[cfg(not(target_family = "wasm"))] // needs net (TcpListener) #[test] fn test_is_rt_shutdown_err() { let rt1 = rt(); diff --git a/tokio/tests/rt_time_start_paused.rs b/tokio/tests/rt_time_start_paused.rs index 1765d625e19..6ec020f7944 100644 --- a/tokio/tests/rt_time_start_paused.rs +++ b/tokio/tests/rt_time_start_paused.rs @@ -1,4 +1,12 @@ -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util" + ) +))] use tokio::time::{Duration, Instant}; diff --git a/tokio/tests/rt_worker_index.rs b/tokio/tests/rt_worker_index.rs index de88bdfd4da..f8e6055c3de 100644 --- a/tokio/tests/rt_worker_index.rs +++ b/tokio/tests/rt_worker_index.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", tokio_unstable, not(target_os = "wasi"),))] +#![cfg(all(feature = "full", not(target_os = "wasi"),))] + +use std::sync::Arc; +use std::thread; use tokio::runtime::{self, Runtime}; @@ -15,6 +18,46 @@ fn worker_index_current_thread() { }); } +#[test] +fn worker_index_handle_block_on_current_thread() { + let rt = runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let index = rt.handle().block_on(async { runtime::worker_index() }); + assert_eq!(index, None); +} + +#[test] +fn worker_index_current_thread_concurrent_block_on() { + let rt = Arc::new( + runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(), + ); + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + + let owner_rt = rt.clone(); + let owner = thread::spawn(move || { + owner_rt.block_on(async { + entered_tx.send(()).unwrap(); + release_rx.await.unwrap(); + runtime::worker_index() + }) + }); + + entered_rx.recv().unwrap(); + + let other_rt = rt.clone(); + let other = thread::spawn(move || other_rt.block_on(async { runtime::worker_index() })); + + assert_eq!(other.join().unwrap(), None); + release_tx.send(()).unwrap(); + assert_eq!(owner.join().unwrap(), Some(0)); +} + #[test] fn worker_index_local_runtime() { let rt = runtime::LocalRuntime::new().unwrap(); @@ -29,7 +72,7 @@ fn worker_index_outside_runtime() { assert_eq!(runtime::worker_index(), None); } -#[cfg(target_has_atomic = "64")] +#[cfg(all(tokio_unstable, target_has_atomic = "64"))] #[test] fn worker_index_matches_metrics_worker_thread_id() { let rt = runtime::Builder::new_multi_thread() @@ -72,12 +115,23 @@ fn worker_index_from_spawn_blocking() { } #[test] -fn worker_index_block_on_multi_thread() { - let rt = Runtime::new().unwrap(); - // block_on runs on the calling thread, not a worker thread - let index = rt.block_on(async { runtime::worker_index() }); - assert_eq!( - index, None, - "block_on thread is not a worker thread on multi-thread runtime" - ); +fn worker_index_multi_thread() { + let rt = runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let index = runtime::worker_index(); + assert_eq!(index, None); + }); +} + +#[tokio::test(flavor = "current_thread")] +async fn worker_index_tokio_test_current_thread() { + assert_eq!(runtime::worker_index(), Some(0)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn worker_index_tokio_test_multi_thread() { + assert_eq!(runtime::worker_index(), None); } diff --git a/tokio/tests/support/io_uring.rs b/tokio/tests/support/io_uring.rs index bdc57bf2adf..02cf46015be 100644 --- a/tokio/tests/support/io_uring.rs +++ b/tokio/tests/support/io_uring.rs @@ -17,17 +17,42 @@ use io_uring::IoUring; // to check if the fallback mechanism works, this comes with the limitation that we are not // able to run some checks (e.g., asserting a poll returns pending). This utility function // is useful when we want to run a test only in Linux targets where io_uring is supported. +#[allow(dead_code)] pub fn io_uring_supported() -> bool { match IoUring::new(256) { Ok(_) => true, - // The Kernel does not support io-uring - Err(e) if e.raw_os_error() == Some(libc::ENOSYS) => false, - Err(_) => unreachable!( - "The target should either support io_uring or return ENOSYS if not supported" + // ENOSYS: kernel does not support io_uring. + // EPERM: io_uring disabled via sysctl kernel.io_uring_disabled (#7691). + Err(e) + if e.raw_os_error() == Some(libc::ENOSYS) || e.raw_os_error() == Some(libc::EPERM) => + { + false + } + Err(e) => unreachable!( + "IoUring::new failed with an unexpected error (expected ENOSYS or EPERM): {e}" ), } } +/// Whether tokio uses io_uring, rather than the `spawn_blocking` fallback, for +/// the `fs` functions built on `opcode` (`Statx` for `try_exists`, `Read` for +/// `read`). Mirrors the gate in `tokio/src/fs`: the target must have +/// `libc::statx` (see the FIXME there about musl) and the kernel must support +/// the opcode. Tests that assert on the io_uring path (e.g. that the first +/// poll is `Pending`) return early when this is false, since the blocking +/// fallback may already be done by the first poll. +#[allow(dead_code)] +pub fn uring_fs_op_in_use(opcode: u8) -> bool { + if !cfg!(any(target_env = "gnu", target_os = "android")) { + return false; + } + let Ok(ring) = IoUring::new(2) else { + return false; + }; + let mut probe = io_uring::Probe::new(); + ring.submitter().register_probe(&mut probe).is_ok() && probe.is_supported(opcode) +} + #[allow(dead_code)] pub async fn assert_fds_are_not_leaking(count_before: usize, opened_files: usize, timeout: u64) { let fd_check_start = Instant::now(); diff --git a/tokio/tests/sync_barrier.rs b/tokio/tests/sync_barrier.rs index ac5977f24d8..a8f8e9790bc 100644 --- a/tokio/tests/sync_barrier.rs +++ b/tokio/tests/sync_barrier.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::Barrier; diff --git a/tokio/tests/sync_broadcast.rs b/tokio/tests/sync_broadcast.rs index 2bfe235e511..94bfde6bbc2 100644 --- a/tokio/tests/sync_broadcast.rs +++ b/tokio/tests/sync_broadcast.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::broadcast; @@ -563,7 +563,7 @@ fn sender_len() { } #[test] -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] fn sender_len_random() { let (tx, mut rx1) = broadcast::channel(16); let mut rx2 = tx.subscribe(); diff --git a/tokio/tests/sync_broadcast_weak.rs b/tokio/tests/sync_broadcast_weak.rs index 1e7fd6f2d67..ccca130e52e 100644 --- a/tokio/tests/sync_broadcast_weak.rs +++ b/tokio/tests/sync_broadcast_weak.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::broadcast::{self, channel}; diff --git a/tokio/tests/sync_emscripten_blocking.rs b/tokio/tests/sync_emscripten_blocking.rs new file mode 100644 index 00000000000..bf7e96e8f0b --- /dev/null +++ b/tokio/tests/sync_emscripten_blocking.rs @@ -0,0 +1,118 @@ +//! `blocking_recv` outside a runtime parks the thread parker directly, which +//! suspends under `-sJSPI` just as the runtime driver does, and panics +//! without it. Needs only the `sync` feature, so every Emscripten CI lane +//! runs this file, including a `sync`-only build. + +#![cfg(all( + target_os = "emscripten", + not(target_feature = "atomics"), + feature = "sync" +))] + +use std::panic::{catch_unwind, AssertUnwindSafe}; + +extern "C" { + /// Emscripten's `ASYNCIFY` build mode; 2 is JSPI. + fn emscripten_has_asyncify() -> i32; + + fn emscripten_async_call( + func: extern "C" fn(*mut std::ffi::c_void), + arg: *mut std::ffi::c_void, + millis: i32, + ); +} + +fn jspi_linked() -> bool { + // SAFETY: an Emscripten libc query with no arguments and no side effects. + unsafe { emscripten_has_asyncify() == 2 } +} + +/// Run `f` from a fresh wasm activation on the next host loop turn. +fn host_callback(f: impl FnOnce() + 'static) { + extern "C" fn trampoline(arg: *mut std::ffi::c_void) { + // SAFETY: `arg` is the `Box>` leaked below, and + // Emscripten invokes the callback exactly once. + let f = unsafe { Box::from_raw(arg as *mut Box) }; + f(); + } + let f: Box> = Box::new(Box::new(f)); + // SAFETY: an Emscripten API scheduling `trampoline(arg)` on the host loop. + unsafe { emscripten_async_call(trampoline, Box::into_raw(f) as *mut _, 0) } +} + +/// Assert `f` panics with the targeted would-suspend message. +fn assert_panics_cannot_block_on(f: impl FnOnce()) { + if cfg!(not(panic = "unwind")) { + return; + } + let err = catch_unwind(AssertUnwindSafe(f)).expect_err("expected a would-suspend panic"); + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + msg.contains("cannot block"), + "unexpected panic message: {msg}" + ); +} + +#[test] +fn mpsc_blocking_recv_wakes_from_host_activation() { + let recv = || { + let (tx, mut rx) = tokio::sync::mpsc::channel::(1); + // Without JSPI the receiver is gone by the time this fires. + host_callback(move || { + let _ = tx.try_send(11); + }); + rx.blocking_recv() + }; + + if jspi_linked() { + assert_eq!(recv(), Some(11)); + } else { + assert_panics_cannot_block_on(|| { + recv(); + }); + } +} + +#[test] +fn oneshot_blocking_recv_wakes_from_host_activation() { + let recv = || { + let (tx, rx) = tokio::sync::oneshot::channel::(); + host_callback(move || { + let _ = tx.send(11); + }); + rx.blocking_recv() + }; + + if jspi_linked() { + assert_eq!(recv(), Ok(11)); + } else { + assert_panics_cannot_block_on(|| { + let _ = recv(); + }); + } +} + +#[test] +fn blocking_recv_ready_value_needs_no_suspension() { + let (tx, mut rx) = tokio::sync::mpsc::channel::(1); + tx.try_send(3).unwrap(); + assert_eq!(rx.blocking_recv(), Some(3)); +} + +#[test] +fn sequential_blocking_recvs_reuse_the_parker() { + if !jspi_linked() { + return; + } + // Each call parks and resumes independently; the parker's notification + // token must not leak from one into the next. + for i in 0..3u32 { + let (tx, mut rx) = tokio::sync::mpsc::channel::(1); + host_callback(move || tx.try_send(i).unwrap()); + assert_eq!(rx.blocking_recv(), Some(i)); + } +} diff --git a/tokio/tests/sync_errors.rs b/tokio/tests/sync_errors.rs index 4e43c8f311e..2bc9b878111 100644 --- a/tokio/tests/sync_errors.rs +++ b/tokio/tests/sync_errors.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; fn is_error() {} diff --git a/tokio/tests/sync_mpsc.rs b/tokio/tests/sync_mpsc.rs index 398ab633927..207bef40d7f 100644 --- a/tokio/tests/sync_mpsc.rs +++ b/tokio/tests/sync_mpsc.rs @@ -2,12 +2,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use std::fmt; diff --git a/tokio/tests/sync_mpsc_weak.rs b/tokio/tests/sync_mpsc_weak.rs index fba0fe4e33a..f09e79dab9d 100644 --- a/tokio/tests/sync_mpsc_weak.rs +++ b/tokio/tests/sync_mpsc_weak.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::atomic::AtomicUsize; diff --git a/tokio/tests/sync_mutex.rs b/tokio/tests/sync_mutex.rs index 8d74addad75..37ecf7a2e20 100644 --- a/tokio/tests/sync_mutex.rs +++ b/tokio/tests/sync_mutex.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use tokio::sync::Mutex; diff --git a/tokio/tests/sync_mutex_owned.rs b/tokio/tests/sync_mutex_owned.rs index 28b2afbf32b..95f7ba82e4b 100644 --- a/tokio/tests/sync_mutex_owned.rs +++ b/tokio/tests/sync_mutex_owned.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use tokio::sync::Mutex; diff --git a/tokio/tests/sync_notify.rs b/tokio/tests/sync_notify.rs index ee7a9ecf0ad..3eb92716080 100644 --- a/tokio/tests/sync_notify.rs +++ b/tokio/tests/sync_notify.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::Notify; diff --git a/tokio/tests/sync_notify_owned.rs b/tokio/tests/sync_notify_owned.rs index 06a0f6ade57..cef574a68aa 100644 --- a/tokio/tests/sync_notify_owned.rs +++ b/tokio/tests/sync_notify_owned.rs @@ -1,7 +1,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; diff --git a/tokio/tests/sync_once_cell.rs b/tokio/tests/sync_once_cell.rs index a05438f24c8..acc2a1f427c 100644 --- a/tokio/tests/sync_once_cell.rs +++ b/tokio/tests/sync_once_cell.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util" + ) +))] use std::mem; use std::sync::atomic::{AtomicU32, Ordering}; diff --git a/tokio/tests/sync_oneshot.rs b/tokio/tests/sync_oneshot.rs index 08206ea1d33..8aaddbdb39a 100644 --- a/tokio/tests/sync_oneshot.rs +++ b/tokio/tests/sync_oneshot.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use tokio::sync::oneshot; diff --git a/tokio/tests/sync_panic.rs b/tokio/tests/sync_panic.rs index c781c846bf5..f2f79e0f308 100644 --- a/tokio/tests/sync_panic.rs +++ b/tokio/tests/sync_panic.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), + all(target_os = "emscripten", feature = "rt", feature = "sync") +))] #![cfg(panic = "unwind")] use std::{error::Error, sync::Arc}; diff --git a/tokio/tests/sync_rwlock.rs b/tokio/tests/sync_rwlock.rs index 2dc7b0a62ac..c51f9616cc6 100644 --- a/tokio/tests/sync_rwlock.rs +++ b/tokio/tests/sync_rwlock.rs @@ -1,12 +1,12 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test; -#[cfg(not(all(target_family = "wasm", not(target_os = "wasi"))))] +#[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::test as maybe_tokio_test; use std::task::Poll; diff --git a/tokio/tests/sync_semaphore.rs b/tokio/tests/sync_semaphore.rs index 113dee0646a..18abd0a718b 100644 --- a/tokio/tests/sync_semaphore.rs +++ b/tokio/tests/sync_semaphore.rs @@ -1,6 +1,6 @@ #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; @@ -168,6 +168,14 @@ fn split() { assert_eq!(sem.available_permits(), 5); } +#[test] +fn permit_semaphore() { + let sem = Semaphore::new(1); + let permit = sem.try_acquire().unwrap(); + + assert!(std::ptr::eq(permit.semaphore(), &sem)); +} + #[tokio::test] #[cfg(feature = "full")] async fn stress_test() { diff --git a/tokio/tests/sync_semaphore_owned.rs b/tokio/tests/sync_semaphore_owned.rs index 1bae1e9e0d1..031d297c572 100644 --- a/tokio/tests/sync_semaphore_owned.rs +++ b/tokio/tests/sync_semaphore_owned.rs @@ -1,6 +1,6 @@ #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use std::sync::Arc; diff --git a/tokio/tests/sync_set_once.rs b/tokio/tests/sync_set_once.rs index 5b6d88a9f51..99c0e6255cf 100644 --- a/tokio/tests/sync_set_once.rs +++ b/tokio/tests/sync_set_once.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "sync", + feature = "macros" + ) +))] use std::sync::{ atomic::{AtomicU32, Ordering}, diff --git a/tokio/tests/sync_watch.rs b/tokio/tests/sync_watch.rs index 48e4106841f..dbe754f5833 100644 --- a/tokio/tests/sync_watch.rs +++ b/tokio/tests/sync_watch.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms)] #![cfg(feature = "sync")] -#[cfg(all(target_family = "wasm", not(target_os = "wasi")))] +#[cfg(all(target_family = "wasm", target_os = "unknown"))] use wasm_bindgen_test::wasm_bindgen_test as test; use tokio::sync::watch; @@ -493,6 +493,19 @@ fn has_changed_errors_on_closed_channel_with_seen_value() { .expect_err("`has_changed` returns an error if and only if channel is closed."); } +#[test] +fn receiver_is_closed_after_all_senders_are_dropped() { + let (tx, rx) = watch::channel("A"); + let tx2 = tx.clone(); + assert!(!rx.is_closed()); + + drop(tx); + assert!(!rx.is_closed()); + + drop(tx2); + assert!(rx.is_closed()); +} + #[tokio::test] async fn wait_for_errors_on_closed_channel_true_predicate() { let (tx, mut rx) = watch::channel("A"); diff --git a/tokio/tests/task_abort.rs b/tokio/tests/task_abort.rs index 8de366454e0..101daad58fc 100644 --- a/tokio/tests/task_abort.rs +++ b/tokio/tests/task_abort.rs @@ -1,16 +1,22 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi doesn't support panic recovery + all(target_os = "emscripten", feature = "rt", feature = "time") +))] +#[cfg(not(target_family = "wasm"))] use std::sync::Arc; +#[cfg(not(target_family = "wasm"))] use std::thread::sleep; +#[cfg(not(target_family = "wasm"))] use tokio::time::Duration; use tokio::runtime::Builder; -#[cfg(panic = "unwind")] +#[cfg(all(panic = "unwind", not(target_family = "wasm")))] struct PanicOnDrop; -#[cfg(panic = "unwind")] +#[cfg(all(panic = "unwind", not(target_family = "wasm")))] impl Drop for PanicOnDrop { fn drop(&mut self) { panic!("Well what did you expect would happen..."); @@ -19,6 +25,7 @@ impl Drop for PanicOnDrop { /// Checks that a suspended task can be aborted without panicking as reported in /// issue #3157: . +#[cfg(not(target_family = "wasm"))] #[test] fn test_abort_without_panic_3157() { let rt = Builder::new_multi_thread() @@ -41,6 +48,7 @@ fn test_abort_without_panic_3157() { /// Checks that a suspended task can be aborted inside of a current_thread /// executor without panicking as reported in issue #3662: /// . +#[cfg(not(target_family = "wasm"))] #[test] fn test_abort_without_panic_3662() { use std::sync::atomic::{AtomicBool, Ordering}; @@ -104,6 +112,7 @@ fn test_abort_without_panic_3662() { /// Checks that a suspended LocalSet task can be aborted from a remote thread /// without panicking and without running the tasks destructor on the wrong thread. /// +#[cfg(not(target_family = "wasm"))] #[test] fn remote_abort_local_set_3929() { struct DropCheck { @@ -147,6 +156,7 @@ fn remote_abort_local_set_3929() { /// Checks that a suspended task can be aborted even if the `JoinHandle` is immediately dropped. /// issue #3964: . +#[cfg(not(target_family = "wasm"))] #[test] fn test_abort_wakes_task_3964() { let rt = Builder::new_current_thread().enable_time().build().unwrap(); @@ -177,8 +187,8 @@ fn test_abort_wakes_task_3964() { /// Checks that aborting a task whose destructor panics does not allow the /// panic to escape the task. +#[cfg(all(panic = "unwind", not(target_family = "wasm")))] #[test] -#[cfg(panic = "unwind")] fn test_abort_task_that_panics_on_drop_contained() { let rt = Builder::new_current_thread().enable_time().build().unwrap(); @@ -201,8 +211,8 @@ fn test_abort_task_that_panics_on_drop_contained() { } /// Checks that aborting a task whose destructor panics has the expected result. +#[cfg(all(panic = "unwind", not(target_family = "wasm")))] #[test] -#[cfg(panic = "unwind")] fn test_abort_task_that_panics_on_drop_returned() { let rt = Builder::new_current_thread().enable_time().build().unwrap(); diff --git a/tokio/tests/task_emscripten.rs b/tokio/tests/task_emscripten.rs new file mode 100644 index 00000000000..5f11b24fdb9 --- /dev/null +++ b/tokio/tests/task_emscripten.rs @@ -0,0 +1,17 @@ +#![cfg(all( + target_os = "emscripten", + not(target_feature = "atomics"), + feature = "rt", + feature = "macros" +))] + +/// There is no threadpool on a single-threaded JS worker: the public +/// `spawn_blocking` is unsupported on non-pthread emscripten, as on the other +/// single-threaded wasm targets. `tokio::fs` and +/// `tokio::io::{stdin, stdout, stderr}` do not rely on it there — their +/// syscalls complete synchronously. +#[tokio::test] +#[should_panic = "OS can't spawn worker thread"] +async fn spawn_blocking_is_unsupported() { + let _ = tokio::task::spawn_blocking(|| 42).await; +} diff --git a/tokio/tests/task_id.rs b/tokio/tests/task_id.rs index 0cbf80d5ace..1c54773af07 100644 --- a/tokio/tests/task_id.rs +++ b/tokio/tests/task_id.rs @@ -1,10 +1,19 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "sync", + feature = "macros" + ) +))] use std::error::Error; use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; +#[cfg(not(target_family = "wasm"))] use tokio::runtime::Runtime; use tokio::sync::oneshot; use tokio::task::{self, Id, LocalSet}; @@ -21,7 +30,7 @@ async fn task_id_spawn() { .unwrap(); } -#[cfg(not(target_os = "wasi"))] +#[cfg(not(any(target_os = "wasi", target_os = "emscripten")))] #[tokio::test(flavor = "current_thread")] async fn task_id_spawn_blocking() { task::spawn_blocking(|| println!("task id: {}", task::id())) @@ -38,7 +47,7 @@ async fn task_id_collision_current_thread() { assert_ne!(id1.unwrap(), id2.unwrap()); } -#[cfg(not(target_os = "wasi"))] +#[cfg(not(target_family = "wasm"))] #[tokio::test(flavor = "multi_thread")] async fn task_id_collision_multi_thread() { let handle1 = tokio::spawn(async { task::id() }); @@ -59,7 +68,7 @@ async fn task_ids_match_current_thread() { handle.await.unwrap(); } -#[cfg(not(target_os = "wasi"))] +#[cfg(not(target_family = "wasm"))] #[tokio::test(flavor = "multi_thread")] async fn task_ids_match_multi_thread() { let (tx, rx) = oneshot::channel(); @@ -72,7 +81,8 @@ async fn task_ids_match_multi_thread() { } #[cfg(not(target_os = "wasi"))] -#[tokio::test(flavor = "multi_thread")] +#[cfg_attr(target_os = "emscripten", tokio::test(flavor = "current_thread"))] +#[cfg_attr(not(target_os = "emscripten"), tokio::test(flavor = "multi_thread"))] async fn task_id_future_destructor_completion() { struct MyFuture { tx: Option>, @@ -100,7 +110,8 @@ async fn task_id_future_destructor_completion() { } #[cfg(not(target_os = "wasi"))] -#[tokio::test(flavor = "multi_thread")] +#[cfg_attr(target_os = "emscripten", tokio::test(flavor = "current_thread"))] +#[cfg_attr(not(target_os = "emscripten"), tokio::test(flavor = "multi_thread"))] async fn task_id_future_destructor_abort() { struct MyFuture { tx: Option>, @@ -205,7 +216,7 @@ fn task_try_id_outside_task() { assert_eq!(None, task::try_id()); } -#[cfg(not(target_os = "wasi"))] +#[cfg(not(target_family = "wasm"))] #[test] fn task_try_id_inside_block_on() { let rt = Runtime::new().unwrap(); @@ -248,7 +259,7 @@ async fn task_id_nested_spawn_local() { .await; } -#[cfg(not(target_os = "wasi"))] +#[cfg(not(target_family = "wasm"))] #[tokio::test(flavor = "multi_thread")] async fn task_id_block_in_place_block_on_spawn() { use tokio::runtime::Builder; @@ -283,6 +294,7 @@ fn task_id_outside_task_panic_caller() -> Result<(), Box> { Ok(()) } +#[cfg(not(target_family = "wasm"))] #[test] #[cfg_attr(not(panic = "unwind"), ignore)] fn task_id_inside_block_on_panic_caller() -> Result<(), Box> { diff --git a/tokio/tests/task_join_set.rs b/tokio/tests/task_join_set.rs index 38534d1b734..6d6f2b3159c 100644 --- a/tokio/tests/task_join_set.rs +++ b/tokio/tests/task_join_set.rs @@ -1,5 +1,14 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "sync", + feature = "macros", + feature = "time" + ) +))] use futures::future::{pending, FutureExt}; use std::panic; @@ -433,6 +442,7 @@ mod spawn_local { set.spawn_local(async {}); } + #[cfg(not(target_family = "wasm"))] #[tokio::test(flavor = "multi_thread")] #[should_panic( expected = "`spawn_local` called from outside of a `task::LocalSet` or `runtime::LocalRuntime`" diff --git a/tokio/tests/task_local.rs b/tokio/tests/task_local.rs index be9de724163..0341c7df57f 100644 --- a/tokio/tests/task_local.rs +++ b/tokio/tests/task_local.rs @@ -1,11 +1,15 @@ -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support threads +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi doesn't support threads + all(target_os = "emscripten", feature = "rt", feature = "macros") +))] use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; use tokio::sync::oneshot; -#[tokio::test(flavor = "multi_thread")] +#[cfg_attr(not(target_family = "wasm"), tokio::test(flavor = "multi_thread"))] +#[cfg_attr(target_family = "wasm", tokio::test)] async fn local() { tokio::task_local! { static REQ_ID: u32; diff --git a/tokio/tests/task_panic.rs b/tokio/tests/task_panic.rs index 8b4de2ada54..2221f634334 100644 --- a/tokio/tests/task_panic.rs +++ b/tokio/tests/task_panic.rs @@ -1,11 +1,16 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), + all(target_os = "emscripten", feature = "rt", feature = "macros") +))] #![cfg(panic = "unwind")] use futures::future; use std::error::Error; use tokio::runtime::Builder; -use tokio::task::{self, block_in_place}; +use tokio::task; +#[cfg(feature = "rt-multi-thread")] +use tokio::task::block_in_place; mod support { pub mod panic; @@ -13,6 +18,7 @@ mod support { use support::panic::test_panic; #[test] +#[cfg(feature = "rt-multi-thread")] fn block_in_place_panic_caller() -> Result<(), Box> { let panic_location_file = test_panic(|| { let rt = Builder::new_current_thread().enable_all().build().unwrap(); diff --git a/tokio/tests/task_yield_now.rs b/tokio/tests/task_yield_now.rs index e6fe5d2009a..343b9418125 100644 --- a/tokio/tests/task_yield_now.rs +++ b/tokio/tests/task_yield_now.rs @@ -1,4 +1,11 @@ -#![cfg(all(feature = "full", not(target_os = "wasi"), tokio_unstable))] +#![cfg(all( + any( + feature = "full", + all(target_os = "emscripten", feature = "rt", feature = "macros") + ), + not(target_os = "wasi"), + tokio_unstable +))] use tokio::task; use tokio_test::task::spawn; @@ -15,6 +22,7 @@ fn yield_now_outside_of_runtime() { assert!(task.poll().is_ready()); } +#[cfg(not(target_family = "wasm"))] #[tokio::test(flavor = "multi_thread")] async fn yield_now_external_executor_and_block_in_place() { let j = tokio::spawn(async { diff --git a/tokio/tests/tcp_accept_ready.rs b/tokio/tests/tcp_accept_ready.rs new file mode 100644 index 00000000000..d939d15586a --- /dev/null +++ b/tokio/tests/tcp_accept_ready.rs @@ -0,0 +1,210 @@ +#![warn(rust_2018_idioms)] +#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] + +//! An accepted socket starts out assumed readable and writable, so its first +//! read and write try the syscall instead of waiting for the driver's first +//! event. + +use futures::FutureExt; +use std::future::Future; +use std::io::{IoSlice, Write}; +use std::pin::pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncReadExt, AsyncWriteExt, Interest}; +use tokio::net::{TcpListener, TcpStream}; + +/// Waits until `n` bytes are queued on `s`, using a raw `MSG_PEEK` that does not touch +/// tokio's readiness, so a test never depends on loopback delivery timing. +fn wait_until_queued(s: &TcpStream, n: usize) { + if n == 0 { + return; + } + let sock = socket2::SockRef::from(s); + let mut buf = [std::mem::MaybeUninit::::uninit(); 64]; + assert!(n <= buf.len()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !matches!(sock.peek(&mut buf), Ok(m) if m >= n) { + assert!( + std::time::Instant::now() < deadline, + "peer's bytes never arrived" + ); + std::thread::yield_now(); + } +} + +async fn accepted_with(data: &[u8]) -> (TcpStream, std::net::TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut client = std::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + client.write_all(data).unwrap(); + let (s, _) = listener.accept().await.unwrap(); + wait_until_queued(&s, data.len()); + (s, client) +} + +#[tokio::test] +async fn accepted_stream_reads_without_waiting_for_an_event() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + // The peer connects and sends before we accept, as a client racing an + // overloaded server does. + let mut client = std::net::TcpStream::connect(addr).unwrap(); + client.write_all(b"hello").unwrap(); + + let (stream, _) = listener.accept().await.unwrap(); + wait_until_queued(&stream, 5); + + // `try_read` never waits for the driver; if the socket did not start out + // readable it would return `WouldBlock` here despite the queued data. + let mut buf = [0u8; 16]; + let n = stream + .try_read(&mut buf) + .expect("data was queued before the first read"); + assert_eq!(&buf[..n], b"hello"); + // The first write tries the socket too. + assert_eq!(stream.try_write(b"world").unwrap(), 5); +} + +#[tokio::test] +async fn accepted_stream_with_no_data_still_waits() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut client = std::net::TcpStream::connect(addr).unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + + // Nothing queued at accept time: the socket is assumed readable (a false + // positive) until a read finds nothing, and it is writable. + assert!(stream.readable().now_or_never().is_some()); + let mut buf = [0u8; 16]; + let err = stream.try_read(&mut buf).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); + assert!(stream.readable().now_or_never().is_none()); + assert_eq!(stream.try_write(b"early").unwrap(), 5); + // The normal event path still delivers data that arrives later. + client.write_all(b"later").unwrap(); + stream.readable().await.unwrap(); + let n = stream.try_read(&mut buf).unwrap(); + assert_eq!(&buf[..n], b"later"); +} + +// The `AsyncRead`/`AsyncWrite` path (`PollEvented`), which HTTP stacks use. +#[tokio::test] +async fn first_poll_read_and_write_are_ready() { + let (mut s, _c) = accepted_with(b"hello").await; + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + let mut buf = [0u8; 16]; + assert!(matches!( + pin!(s.read(&mut buf)).poll(&mut cx), + Poll::Ready(Ok(5)) + )); + assert!(matches!( + pin!(s.write(b"x")).poll(&mut cx), + Poll::Ready(Ok(1)) + )); +} + +// A read that fills the buffer keeps the socket readable, so draining +// continues without an event. +#[tokio::test] +async fn full_first_read_keeps_draining() { + let (mut s, _c) = accepted_with(b"helloworld").await; + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + let mut buf = [0u8; 5]; + assert!(matches!( + pin!(s.read(&mut buf)).poll(&mut cx), + Poll::Ready(Ok(5)) + )); + assert!(matches!( + pin!(s.read(&mut buf)).poll(&mut cx), + Poll::Ready(Ok(5)) + )); +} + +// After the runtime is gone: later reads still report the driver gone rather +// than hang. +#[test] +fn later_reads_after_runtime_drop_report_shutdown() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (mut s, _c) = rt.block_on(accepted_with(b"hello")); + drop(rt); + assert_eq!(s.try_read(&mut [0u8; 5]).unwrap(), 5); + // This read finds nothing and clears readiness after shutdown; the + // shutdown bit must survive that or the read below would hang. + let _ = s.try_read(&mut [0u8; 5]); + let rt2 = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt2.block_on(async { + let r = tokio::time::timeout(std::time::Duration::from_secs(1), s.read(&mut [0u8; 5])) + .await + .expect("read hung instead of reporting shutdown"); + assert!(tokio::runtime::is_rt_shutdown_err(&r.unwrap_err())); + }); +} + +#[cfg(unix)] +#[tokio::test] +async fn unix_accepted_stream_reads_without_an_event() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sock"); + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + let mut client = std::os::unix::net::UnixStream::connect(&path).unwrap(); + client.write_all(b"hello").unwrap(); + let (s, _) = listener.accept().await.unwrap(); + // Same-host UDS write is synchronous into the peer's buffer; peek to be sure. + let sock = socket2::SockRef::from(&s); + let mut peek = [std::mem::MaybeUninit::::uninit(); 8]; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !matches!(sock.peek(&mut peek), Ok(m) if m >= 5) { + assert!(std::time::Instant::now() < deadline); + std::thread::yield_now(); + } + let mut buf = [0u8; 16]; + assert_eq!(s.try_read(&mut buf).unwrap(), 5); +} + +// `peek` goes through `async_io`. +#[tokio::test] +async fn peek_first_poll_ready() { + let (s, _c) = accepted_with(b"hello").await; + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + let mut buf = [0u8; 16]; + let r = pin!(s.peek(&mut buf)).poll(&mut cx); + assert!(matches!(r, Poll::Ready(Ok(5))), "{r:?}"); +} + +// `write_vectored` goes through `poll_write_io` / `poll_io`. +#[tokio::test] +async fn write_vectored_first_poll_ready() { + let (mut s, _c) = accepted_with(b"").await; + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + let bufs = [IoSlice::new(b"ab"), IoSlice::new(b"cd")]; + let r = pin!(s.write_vectored(&bufs)).poll(&mut cx); + assert!(matches!(r, Poll::Ready(Ok(4))), "{r:?}"); +} + +// An `async_io` whose closure finds the assumed readiness wrong clears it, +// waits for the driver, and calls the closure again once data arrives. +#[tokio::test] +async fn async_io_wouldblock_falls_back_to_wait() { + let (s, mut c) = accepted_with(b"").await; + let calls = std::cell::Cell::new(0); + let fut = s.async_io(Interest::READABLE, || { + calls.set(calls.get() + 1); + let mut b = [std::mem::MaybeUninit::::uninit(); 8]; + socket2::SockRef::from(&s).recv(&mut b) + }); + let mut fut = pin!(fut); + assert!(fut.as_mut().now_or_never().is_none()); + c.write_all(b"xyz").unwrap(); + let n = tokio::time::timeout(std::time::Duration::from_secs(5), fut) + .await + .unwrap() + .unwrap(); + assert_eq!(n, 3); + assert!(calls.get() >= 2, "calls={}", calls.get()); +} diff --git a/tokio/tests/tcp_stream.rs b/tokio/tests/tcp_stream.rs index 5817074ca82..c5f1a0564e2 100644 --- a/tokio/tests/tcp_stream.rs +++ b/tokio/tests/tcp_stream.rs @@ -55,6 +55,13 @@ async fn try_read_write() { let (server, _) = listener.accept().await.unwrap(); let mut written = DATA.to_vec(); + // An accepted socket starts out assumed readable; a read that finds + // nothing clears that. + assert_eq!( + server.try_read(&mut [0; 1]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + // Track the server receiving data let mut readable = task::spawn(server.readable()); assert_pending!(readable.poll()); @@ -231,7 +238,10 @@ macro_rules! assert_not_writable_by_polling { async fn poll_read_ready() { let (mut client, mut server) = create_pair().await; - // Initial state - not readable. + // Initial state - an accepted socket is assumed readable until a read + // finds nothing. + assert_readable_by_polling!(server); + read_until_pending(&mut server); assert_not_readable_by_polling!(server); // There is data in the buffer - readable. @@ -317,6 +327,13 @@ async fn try_read_buf() { let (server, _) = listener.accept().await.unwrap(); let mut written = DATA.to_vec(); + // An accepted socket starts out assumed readable; a read that finds + // nothing clears that. + assert_eq!( + server.try_read(&mut [0; 1]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + // Track the server receiving data let mut readable = task::spawn(server.readable()); assert_pending!(readable.poll()); @@ -372,12 +389,7 @@ async fn try_read_buf() { #[cfg(not(target_os = "wasi"))] // WASI does not yet support `POLLHUP` or `POLLRDHUP` { - let mut count = 0; loop { - count += 1; - if count > 100 { - panic!("loop 4") - } let ready = server.ready(Interest::READABLE).await.unwrap(); if ready.is_read_closed() { diff --git a/tokio/tests/test_clock.rs b/tokio/tests/test_clock.rs index 891636fdb28..f5f643a7b59 100644 --- a/tokio/tests/test_clock.rs +++ b/tokio/tests/test_clock.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util" + ) +))] use tokio::time::{self, Duration, Instant}; diff --git a/tokio/tests/time_interval.rs b/tokio/tests/time_interval.rs index 7472a37123c..e65cb740560 100644 --- a/tokio/tests/time_interval.rs +++ b/tokio/tests/time_interval.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util" + ) +))] use std::pin::Pin; use std::task::{Context, Poll}; @@ -168,6 +176,18 @@ async fn skip() { check_interval_poll!(i, start, 1800); } +#[tokio::test] +async fn reset_doesnt_panic_max_duration() { + let mut interval = time::interval(Duration::MAX); + interval.reset(); +} + +#[tokio::test] +async fn reset_after_doesnt_panic_max_duration() { + let mut interval = time::interval(ms(1)); + interval.reset_after(Duration::MAX); +} + #[tokio::test(start_paused = true)] async fn reset() { let start = Instant::now(); diff --git a/tokio/tests/time_panic.rs b/tokio/tests/time_panic.rs index 918d02a416e..5aa651f35c3 100644 --- a/tokio/tests/time_panic.rs +++ b/tokio/tests/time_panic.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi doesn't support panic recovery + all(target_os = "emscripten", feature = "rt", feature = "time") +))] #![cfg(panic = "unwind")] use futures::future; @@ -22,21 +25,24 @@ fn rt_combinations() -> Vec { .unwrap(); rts.push(rt); - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .build() - .unwrap(); - rts.push(rt); + #[cfg(not(target_os = "emscripten"))] + { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + rts.push(rt); - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build() - .unwrap(); - rts.push(rt); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + rts.push(rt); + } - #[cfg(tokio_unstable)] + #[cfg(all(tokio_unstable, not(target_os = "emscripten")))] { let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) diff --git a/tokio/tests/time_pause.rs b/tokio/tests/time_pause.rs index be993b4c8dd..aed0037f222 100644 --- a/tokio/tests/time_pause.rs +++ b/tokio/tests/time_pause.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util" + ) +))] #![cfg(not(miri))] // Too slow on miri. use rand::SeedableRng; @@ -7,7 +15,7 @@ use rand::{rngs::StdRng, Rng}; use tokio::time::{self, Duration, Instant, Sleep}; use tokio_test::{assert_elapsed, assert_pending, assert_ready, assert_ready_eq, task}; -#[cfg(not(target_os = "wasi"))] +#[cfg(all(feature = "full", not(target_os = "wasi")))] use tokio_test::assert_err; use std::{ diff --git a/tokio/tests/time_sleep.rs b/tokio/tests/time_sleep.rs index b82b1cc6ae4..83af7027fe4 100644 --- a/tokio/tests/time_sleep.rs +++ b/tokio/tests/time_sleep.rs @@ -1,5 +1,13 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util" + ) +))] #![cfg(not(miri))] // Too slow on Miri. use std::future::Future; @@ -289,6 +297,39 @@ async fn no_out_of_bounds_close_to_max() { time::sleep(Duration::MAX - Duration::from_millis(1)).await; } +#[tokio::test(start_paused = true)] +async fn long_wait_sleep_does_not_break_other_timers() { + tokio::spawn(time::sleep(ms(10 << 36))); + + time::advance(ms((1 << 30) - 1)).await; + + let start = Instant::now(); + time::sleep(ms(10)).await; + assert_elapsed!(start, ms(10)); +} + +#[tokio::test(start_paused = true)] +async fn long_wait_sleep_does_not_corrupt_wheel() { + use futures::poll; + + tokio::spawn(time::sleep(ms(10 << 36))); + + time::advance(ms((1 << 30) - 1)).await; + + let mut first = Box::pin(time::sleep(ms(10))); + assert_pending!(poll!(first.as_mut())); + let mut second = Box::pin(time::sleep(ms(20))); + assert_pending!(poll!(second.as_mut())); + + time::advance(ms(30)).await; + + drop(first); + + time::advance(ms(1 << 31)).await; + + drop(second); +} + fn ms(n: u64) -> Duration { Duration::from_millis(n) } diff --git a/tokio/tests/time_timeout.rs b/tokio/tests/time_timeout.rs index ec871cf62fe..846cba1abdb 100644 --- a/tokio/tests/time_timeout.rs +++ b/tokio/tests/time_timeout.rs @@ -1,5 +1,14 @@ #![warn(rust_2018_idioms)] -#![cfg(feature = "full")] +#![cfg(any( + feature = "full", + all( + target_os = "emscripten", + feature = "rt", + feature = "macros", + feature = "test-util", + feature = "io-util" + ) +))] use tokio::sync::oneshot; use tokio::time::{self, timeout, timeout_at, Instant}; diff --git a/tokio/tests/time_wasm.rs b/tokio/tests/time_wasm.rs index 8e0483f2041..073df353de7 100644 --- a/tokio/tests/time_wasm.rs +++ b/tokio/tests/time_wasm.rs @@ -1,5 +1,9 @@ #![warn(rust_2018_idioms)] -#![cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] +#![cfg(all( + target_arch = "wasm32", + not(target_os = "wasi"), + not(target_os = "emscripten") +))] use wasm_bindgen_test::wasm_bindgen_test; diff --git a/tokio/tests/tracing_sync.rs b/tokio/tests/tracing_sync.rs index 7065282c44b..b39dbc47ca4 100644 --- a/tokio/tests/tracing_sync.rs +++ b/tokio/tests/tracing_sync.rs @@ -8,8 +8,8 @@ use tokio::sync; use tracing_mock::{expect, subscriber}; -#[tokio::test] -async fn test_barrier_creates_span() { +#[test] +fn test_barrier_creates_span() { let barrier_span = expect::span() .named("runtime.resource") .with_target("tokio::sync::barrier"); @@ -43,8 +43,8 @@ async fn test_barrier_creates_span() { handle.assert_finished(); } -#[tokio::test] -async fn test_mutex_creates_span() { +#[test] +fn test_mutex_creates_span() { let mutex_span = expect::span() .named("runtime.resource") .with_target("tokio::sync::mutex"); @@ -87,8 +87,8 @@ async fn test_mutex_creates_span() { handle.assert_finished(); } -#[tokio::test] -async fn test_oneshot_creates_span() { +#[test] +fn test_oneshot_creates_span() { let oneshot_span_id = expect::id(); let oneshot_span = expect::span() .with_id(oneshot_span_id.clone()) @@ -186,8 +186,8 @@ async fn test_oneshot_creates_span() { handle.assert_finished(); } -#[tokio::test] -async fn test_rwlock_creates_span() { +#[test] +fn test_rwlock_creates_span() { let rwlock_span = expect::span() .named("runtime.resource") .with_target("tokio::sync::rwlock"); @@ -242,8 +242,8 @@ async fn test_rwlock_creates_span() { handle.assert_finished(); } -#[tokio::test] -async fn test_semaphore_creates_span() { +#[test] +fn test_semaphore_creates_span() { let semaphore_span = expect::span() .named("runtime.resource") .with_target("tokio::sync::semaphore"); diff --git a/tokio/tests/tracing_task.rs b/tokio/tests/tracing_task.rs index 0cc9666aff2..9727c3307a6 100644 --- a/tokio/tests/tracing_task.rs +++ b/tokio/tests/tracing_task.rs @@ -10,166 +10,189 @@ use std::{mem, time::Duration}; use tokio::task; use tracing_mock::{expect, span::NewSpan, subscriber}; -#[tokio::test] -async fn task_spawn_creates_span() { - let task_span = expect::span() - .named("runtime.spawn") - .with_target("tokio::task"); - - let (subscriber, handle) = subscriber::mock() - .new_span(&task_span) - .enter(&task_span) - .exit(&task_span) - // The task span is entered once more when it gets dropped - .enter(&task_span) - .exit(&task_span) - .drop_span(task_span) - .run_with_handle(); - - { - let _guard = tracing::subscriber::set_default(subscriber); - tokio::spawn(futures::future::ready(())) - .await - .expect("failed to await join handle"); - } - - handle.assert_finished(); +static TEST_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn run_test(f: impl FnOnce() -> F) -> F::Output { + let _lock = TEST_MUTEX.lock().unwrap_or_else(|e| e.into_inner()); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(f()) } -#[tokio::test] -async fn task_spawn_loc_file_recorded() { - let task_span = expect::span() - .named("runtime.spawn") - .with_target("tokio::task") - .with_fields(expect::field("loc.file").with_value(&file!())); - - let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); - - { - let _guard = tracing::subscriber::set_default(subscriber); - - tokio::spawn(futures::future::ready(())) - .await - .expect("failed to await join handle"); - } +#[test] +fn task_spawn_creates_span() { + run_test(|| async { + let task_span = expect::span() + .named("runtime.spawn") + .with_target("tokio::task"); + + let (subscriber, handle) = subscriber::mock() + .new_span(&task_span) + .enter(&task_span) + .exit(&task_span) + // The task span is entered once more when it gets dropped + .enter(&task_span) + .exit(&task_span) + .drop_span(task_span) + .run_with_handle(); + + { + let _guard = tracing::subscriber::set_default(subscriber); + tokio::spawn(futures::future::ready(())) + .await + .expect("failed to await join handle"); + } - handle.assert_finished(); + handle.assert_finished(); + }); } -#[tokio::test] -async fn task_builder_name_recorded() { - let task_span = expect_task_named("test-task"); +#[test] +fn task_spawn_loc_file_recorded() { + run_test(|| async { + let task_span = expect::span() + .named("runtime.spawn") + .with_target("tokio::task") + .with_fields(expect::field("loc.file").with_value(&file!())); - let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); + let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); - { - let _guard = tracing::subscriber::set_default(subscriber); - task::Builder::new() - .name("test-task") - .spawn(futures::future::ready(())) - .unwrap() - .await - .expect("failed to await join handle"); - } + { + let _guard = tracing::subscriber::set_default(subscriber); - handle.assert_finished(); -} - -#[tokio::test] -async fn task_builder_loc_file_recorded() { - let task_span = expect::span() - .named("runtime.spawn") - .with_target("tokio::task") - .with_fields(expect::field("loc.file").with_value(&file!())); - - let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); + tokio::spawn(futures::future::ready(())) + .await + .expect("failed to await join handle"); + } - { - let _guard = tracing::subscriber::set_default(subscriber); + handle.assert_finished(); + }); +} - task::Builder::new() - .spawn(futures::future::ready(())) - .unwrap() - .await - .expect("failed to await join handle"); - } +#[test] +fn task_builder_name_recorded() { + run_test(|| async { + let task_span = expect_task_named("test-task"); + + let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); + + { + let _guard = tracing::subscriber::set_default(subscriber); + task::Builder::new() + .name("test-task") + .spawn(futures::future::ready(())) + .unwrap() + .await + .expect("failed to await join handle"); + } - handle.assert_finished(); + handle.assert_finished(); + }); } -#[tokio::test] -async fn task_spawn_sizes_recorded() { - let future = futures::future::ready(()); - let size = mem::size_of_val(&future) as u64; +#[test] +fn task_builder_loc_file_recorded() { + run_test(|| async { + let task_span = expect::span() + .named("runtime.spawn") + .with_target("tokio::task") + .with_fields(expect::field("loc.file").with_value(&file!())); - let task_span = expect::span() - .named("runtime.spawn") - .with_target("tokio::task") - // TODO(hds): check that original_size.bytes is NOT recorded when this can be done in - // tracing-mock without listing every other field. - .with_fields(expect::field("size.bytes").with_value(&size)); + let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); - let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); + { + let _guard = tracing::subscriber::set_default(subscriber); - { - let _guard = tracing::subscriber::set_default(subscriber); - - task::Builder::new() - .spawn(future) - .unwrap() - .await - .expect("failed to await join handle"); - } + task::Builder::new() + .spawn(futures::future::ready(())) + .unwrap() + .await + .expect("failed to await join handle"); + } - handle.assert_finished(); + handle.assert_finished(); + }); } -#[tokio::test] -async fn task_big_spawn_sizes_recorded() { - let future = { - async fn big() { - let mut a = [0_u8; N]; - for (idx, item) in a.iter_mut().enumerate() { - *item = (idx % 256) as u8; - } - tokio::time::sleep(Duration::from_millis(10)).await; - for (idx, item) in a.iter_mut().enumerate() { - assert_eq!(*item, (idx % 256) as u8); - } +#[test] +fn task_spawn_sizes_recorded() { + run_test(|| async { + let future = futures::future::ready(()); + let size = mem::size_of_val(&future) as u64; + + let task_span = expect::span() + .named("runtime.spawn") + .with_target("tokio::task") + // TODO(hds): check that original_size.bytes is NOT recorded when this can be done in + // tracing-mock without listing every other field. + .with_fields(expect::field("size.bytes").with_value(&size)); + + let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); + + { + let _guard = tracing::subscriber::set_default(subscriber); + + task::Builder::new() + .spawn(future) + .unwrap() + .await + .expect("failed to await join handle"); } - // This is larger than the release auto-boxing threshold - big::<20_000>() - }; - - fn boxed_size(_: &T) -> usize { - mem::size_of::>() - } - let size = mem::size_of_val(&future) as u64; - let boxed_size = boxed_size(&future); - - let task_span = expect::span() - .named("runtime.spawn") - .with_target("tokio::task") - .with_fields( - expect::field("size.bytes") - .with_value(&boxed_size) - .and(expect::field("original_size.bytes").with_value(&size)), - ); + handle.assert_finished(); + }); +} - let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); +#[test] +fn task_big_spawn_sizes_recorded() { + run_test(|| async { + let future = { + async fn big() { + let mut a = [0_u8; N]; + for (idx, item) in a.iter_mut().enumerate() { + *item = (idx % 256) as u8; + } + tokio::time::sleep(Duration::from_millis(10)).await; + for (idx, item) in a.iter_mut().enumerate() { + assert_eq!(*item, (idx % 256) as u8); + } + } - { - let _guard = tracing::subscriber::set_default(subscriber); + // This is larger than the release auto-boxing threshold + big::<20_000>() + }; - task::Builder::new() - .spawn(future) - .unwrap() - .await - .expect("failed to await join handle"); - } + fn boxed_size(_: &T) -> usize { + mem::size_of::>() + } + let size = mem::size_of_val(&future) as u64; + let boxed_size = boxed_size(&future); + + let task_span = expect::span() + .named("runtime.spawn") + .with_target("tokio::task") + .with_fields( + expect::field("size.bytes") + .with_value(&boxed_size) + .and(expect::field("original_size.bytes").with_value(&size)), + ); + + let (subscriber, handle) = subscriber::mock().new_span(task_span).run_with_handle(); + + { + let _guard = tracing::subscriber::set_default(subscriber); + + task::Builder::new() + .spawn(future) + .unwrap() + .await + .expect("failed to await join handle"); + } - handle.assert_finished(); + handle.assert_finished(); + }); } /// Expect a task with name diff --git a/tokio/tests/udp.rs b/tokio/tests/udp.rs index ef6d9df0e58..701dcd03688 100644 --- a/tokio/tests/udp.rs +++ b/tokio/tests/udp.rs @@ -56,10 +56,6 @@ async fn send_recv_poll() -> std::io::Result<()> { } #[tokio::test] -#[cfg_attr( - target_os = "wasi", - ignore = "temporarily disabled for WASI pending https://github.com/bytecodealliance/wasmtime/pull/13933" -)] async fn send_to_recv_closed_returns_err() -> std::io::Result<()> { let sender = UdpSocket::bind("127.0.0.1:0").await?; let receiver = UdpSocket::bind("127.0.0.1:0").await?; diff --git a/tokio/tests/uds_datagram.rs b/tokio/tests/uds_datagram.rs index 1b7e78326bf..2f3c45318f6 100644 --- a/tokio/tests/uds_datagram.rs +++ b/tokio/tests/uds_datagram.rs @@ -9,6 +9,7 @@ use tokio::try_join; use std::future::poll_fn; use std::io; use std::sync::Arc; +use std::task::Poll; async fn echo_server(socket: UnixDatagram) -> io::Result<()> { let mut recv_buf = vec![0u8; 1024]; @@ -423,3 +424,34 @@ async fn poll_ready() -> io::Result<()> { Ok(()) } + +// Both ends of a fresh pair are writable without a driver event, but not +// readable until the peer sends. +#[tokio::test(flavor = "current_thread")] +#[cfg_attr(miri, ignore)] // No SOCK_DGRAM for `socketpair` in miri. +async fn pair_starts_writable() -> io::Result<()> { + let (a, b) = UnixDatagram::pair()?; + + // Nothing has polled the driver since `pair()`. + assert!(poll_fn(|cx| Poll::Ready(a.poll_send_ready(cx))) + .await + .is_ready()); + a.try_send(b"hi")?; + b.try_send(b"yo")?; + + let (c, _d) = UnixDatagram::pair()?; + assert!(poll_fn(|cx| Poll::Ready(c.poll_recv_ready(cx))) + .await + .is_pending()); + assert_eq!( + c.try_recv(&mut [0u8; 1]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + + // The peer's datagram still arrives through the normal event path. + a.readable().await?; + let mut buf = [0u8; 8]; + assert_eq!(a.try_recv(&mut buf)?, 2); + assert_eq!(&buf[..2], b"yo"); + Ok(()) +} diff --git a/tokio/tests/uds_stream.rs b/tokio/tests/uds_stream.rs index 13cd671155a..b4f603c0d4d 100644 --- a/tokio/tests/uds_stream.rs +++ b/tokio/tests/uds_stream.rs @@ -111,6 +111,13 @@ async fn try_read_write() -> std::io::Result<()> { let (server, _) = listener.accept().await?; let mut written = msg.to_vec(); + // An accepted socket starts out assumed readable; a read that finds + // nothing clears that. + assert_eq!( + server.try_read(&mut [0; 1]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + // Track the server receiving data let mut readable = task::spawn(server.readable()); assert_pending!(readable.poll()); @@ -271,7 +278,10 @@ macro_rules! assert_not_writable_by_polling { async fn poll_read_ready() { let (mut client, mut server) = create_pair().await; - // Initial state - not readable. + // Initial state - an accepted socket is assumed readable until a read + // finds nothing. + assert_readable_by_polling!(server); + read_until_pending(&mut server); assert_not_readable_by_polling!(server); // There is data in the buffer - readable. @@ -348,6 +358,13 @@ async fn try_read_buf() -> std::io::Result<()> { let (server, _) = listener.accept().await?; let mut written = msg.to_vec(); + // An accepted socket starts out assumed readable; a read that finds + // nothing clears that. + assert_eq!( + server.try_read(&mut [0; 1]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + // Track the server receiving data let mut readable = task::spawn(server.readable()); assert_pending!(readable.poll()); @@ -473,3 +490,31 @@ async fn abstract_socket_name() { // `as_abstract_name` removes leading zero bytes assert_eq!(abstract_path_name, b"aaa"); } + +// Both ends of a fresh pair are writable without a driver event, but not +// readable until the peer writes. +#[tokio::test(flavor = "current_thread")] +async fn pair_starts_writable() -> io::Result<()> { + let (a, mut b) = UnixStream::pair()?; + + // Nothing has polled the driver since `pair()`. + let mut writable = task::spawn(a.writable()); + assert_ready_ok!(writable.poll()); + assert_eq!(a.try_write(b"hi")?, 2); + let mut write = task::spawn(b.write_all(b"yo")); + assert_ready_ok!(write.poll()); + drop(write); + + let (c, _d) = UnixStream::pair()?; + let mut readable = task::spawn(c.readable()); + assert_pending!(readable.poll()); + assert_eq!( + c.try_read(&mut [0u8; 1]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + + // The peer's bytes still arrive through the normal event path. + a.readable().await?; + assert_eq!(a.try_read(&mut [0u8; 8])?, 2); + Ok(()) +} diff --git a/tokio/tests/unwindsafe.rs b/tokio/tests/unwindsafe.rs index 8ab6654295b..161055058e6 100644 --- a/tokio/tests/unwindsafe.rs +++ b/tokio/tests/unwindsafe.rs @@ -1,5 +1,8 @@ #![warn(rust_2018_idioms)] -#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support panic recovery +#![cfg(any( + all(feature = "full", not(target_os = "wasi")), // Wasi does not support panic recovery + all(target_os = "emscripten", feature = "rt", feature = "sync") +))] use std::panic::{RefUnwindSafe, UnwindSafe}; @@ -14,6 +17,7 @@ fn join_handle_is_unwind_safe() { } #[test] +#[cfg(feature = "net")] fn net_types_are_unwind_safe() { is_unwind_safe::(); is_unwind_safe::(); @@ -22,7 +26,7 @@ fn net_types_are_unwind_safe() { } #[test] -#[cfg(unix)] +#[cfg(all(unix, feature = "net"))] fn unix_net_types_are_unwind_safe() { is_unwind_safe::(); is_unwind_safe::();