From 5f4a68dd04033091ae283f6ff233b4deefad4a50 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 26 Mar 2026 11:12:54 +0100 Subject: [PATCH 1/2] fuse: debug print requests when we hang in fuse_wait_aborted() Signed-off-by: Horst Birthelmer --- fs/fuse/dev.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 8c1b8df10bdce1..7fcd0bf4132bd4 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -429,7 +429,8 @@ static void request_wait_answer(struct fuse_req *req) * Either request is already in userspace, or it was forced. * Wait it out. */ - wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags)); + wait_event(req->waitq, + test_bit(FR_FINISHED, &req->flags)); } static void __fuse_request_send(struct fuse_req *req) @@ -2260,11 +2261,119 @@ void fuse_abort_conn(struct fuse_conn *fc) } EXPORT_SYMBOL_GPL(fuse_abort_conn); +static void fuse_debug_print_outstanding_reqs(struct fuse_conn *fc) +{ + struct fuse_dev *fud; + + pr_warn("FUSE: fuse_wait_aborted: num_waiting=%d (should be 0)\n", + atomic_read(&fc->num_waiting)); + +#ifdef CONFIG_FUSE_IO_URING + /* Print io_uring state if enabled */ + if (fc->ring) { + struct fuse_ring *ring = fc->ring; + + pr_warn("FUSE: io_uring enabled - queue_refs=%d ready=%d\n", + atomic_read(&ring->queue_refs), ring->ready); + } +#endif + + /* Print all outstanding requests - lockless for debug */ + list_for_each_entry(fud, &fc->devices, entry) { + struct fuse_pqueue *fpq = &fud->pq; + struct fuse_req *req; + int i; + + /* Print all requests on fpq->io */ + if (!list_empty(&fpq->io)) { + pr_warn("FUSE: Outstanding requests on fpq->io:\n"); + list_for_each_entry(req, &fpq->io, list) { +#ifdef CONFIG_FUSE_IO_URING + if (test_bit(FR_URING, &req->flags) && + req->ring_entry) { + struct fuse_ring_ent *ent = req->ring_entry; + + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d ring_ent=%p state=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags), + ent, ent->state); + } else { +#endif + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags)); +#ifdef CONFIG_FUSE_IO_URING + } +#endif + } + } + + /* Print all requests on fpq->processing */ + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + if (list_empty(&fpq->processing[i])) + continue; + + pr_warn("FUSE: Outstanding requests on fpq->processing[%d]:\n", + i); + list_for_each_entry(req, &fpq->processing[i], list) { +#ifdef CONFIG_FUSE_IO_URING + if (test_bit(FR_URING, &req->flags) && + req->ring_entry) { + struct fuse_ring_ent *ent = req->ring_entry; + + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d ring_ent=%p state=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags), + ent, ent->state); + } else { +#endif + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags)); +#ifdef CONFIG_FUSE_IO_URING + } +#endif + } + } + } +} + void fuse_wait_aborted(struct fuse_conn *fc) { + unsigned int timeout = 20; + /* matches implicit memory barrier in fuse_drop_waiting() */ smp_mb(); - wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); + +wait: + wait_event_timeout(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0, HZ * timeout); + + /* Debug: print info if we're waiting */ + if (atomic_read(&fc->num_waiting) > 0) { + fuse_debug_print_outstanding_reqs(fc); + timeout *= 3; + goto wait; + } fuse_uring_wait_stopped_queues(fc); } From 1f7eb8fe83074978d120f96269ee75da7c63a419 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 2 Apr 2026 08:14:42 +0200 Subject: [PATCH 2/2] fuse: fix io_uring connection abort leaving requests stuck Fix uninterruptible sleep (D state) hangs during FUSE filesystem teardown when using io_uring. The issue manifests as processes stuck waiting for requests that are never completed, particularly affecting force requests like FUSE_FLUSH or when requests are created after fuse_abort_conn() already finished. If on daemon exit io_uring_try_cancel_requests() runs and calls fuse_uring_cancel() which will teardown the entries by calling fuse_uring_entry_teardown() before fuse_abort_conn() then we end up in fuse_uring_abort with queue_refs == 0 and the queues are never stopped. If the queues are stopped all new requests will be rejected, but that does not happen, so all new calls are stuck. Signed-off-by: Horst Birthelmer --- fs/fuse/dev_uring.c | 3 +-- fs/fuse/dev_uring_i.h | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index ec91a33627d7f8..6b2de8b3e4ec8e 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -157,11 +157,10 @@ void fuse_uring_flush_bg(struct fuse_conn *fc) if (!queue) continue; - queue->stopped = true; - WARN_ON_ONCE(ring->fc->max_background != UINT_MAX); spin_lock(&queue->lock); spin_lock(&fc->bg_lock); + queue->stopped = true; fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); spin_unlock(&queue->lock); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 948e9c01aeaef5..778924ea342c76 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -178,10 +178,8 @@ static inline void fuse_uring_abort(struct fuse_conn *fc) if (ring == NULL) return; - if (atomic_read(&ring->queue_refs) > 0) { - fuse_uring_flush_bg(fc); - fuse_uring_stop_queues(ring); - } + fuse_uring_flush_bg(fc); + fuse_uring_stop_queues(ring); } static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc)