diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c587221a289c..9a01188ff45a 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2477,12 +2477,26 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, io_init_req_batch(&rb); while (!list_empty(done)) { + struct io_uring_cqe *cqe; + unsigned cflags; + req = list_first_entry(done, struct io_kiocb, inflight_entry); list_del(&req->inflight_entry); - - io_fill_cqe_req(req, req->result, io_put_rw_kbuf(req)); + cflags = io_put_rw_kbuf(req); (*nr_events)++; + cqe = io_get_cqe(ctx); + if (cqe) { + WRITE_ONCE(cqe->user_data, req->user_data); + WRITE_ONCE(cqe->res, req->result); + WRITE_ONCE(cqe->flags, cflags); + } else { + spin_lock(&ctx->completion_lock); + io_cqring_event_overflow(ctx, req->user_data, + req->result, cflags); + spin_unlock(&ctx->completion_lock); + } + if (req_ref_put_and_test(req)) io_req_free_batch(&rb, req, &ctx->submit_state); }