From e4b6d902a9e38f424ce118106ea4d1665b7951b5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:00 +0100 Subject: [PATCH 01/83] io_uring: improve sqpoll event/state handling As sqd->state changes rarely, don't check every event one by one but look them all at once. Add a helper function. Also don't go into event waiting sleeping with STOP flag set. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/645025f95c7eeec97f88ff497785f4f1d6f3966f.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fa8794c61af7..24c0042b0de7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6757,6 +6757,11 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) return submitted; } +static inline bool io_sqd_events_pending(struct io_sq_data *sqd) +{ + return READ_ONCE(sqd->state); +} + static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx) { /* Tell userspace we may need a wakeup call */ @@ -6815,6 +6820,24 @@ static void io_sqd_update_thread_idle(struct io_sq_data *sqd) sqd->sq_thread_idle = sq_thread_idle; } +static bool io_sqd_handle_event(struct io_sq_data *sqd) +{ + bool did_sig = false; + struct ksignal ksig; + + if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || + signal_pending(current)) { + mutex_unlock(&sqd->lock); + if (signal_pending(current)) + did_sig = get_signal(&ksig); + cond_resched(); + mutex_lock(&sqd->lock); + } + io_run_task_work(); + io_run_task_work_head(&sqd->park_task_work); + return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); +} + static int io_sq_thread(void *data) { struct io_sq_data *sqd = data; @@ -6836,29 +6859,17 @@ static int io_sq_thread(void *data) /* a user may had exited before the thread started */ io_run_task_work_head(&sqd->park_task_work); - while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { + while (1) { int ret; bool cap_entries, sqt_spin, needs_sched; - if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || - signal_pending(current)) { - bool did_sig = false; - - mutex_unlock(&sqd->lock); - if (signal_pending(current)) { - struct ksignal ksig; - - did_sig = get_signal(&ksig); - } - cond_resched(); - mutex_lock(&sqd->lock); - io_run_task_work(); - io_run_task_work_head(&sqd->park_task_work); - if (did_sig) + if (io_sqd_events_pending(sqd) || signal_pending(current)) { + if (io_sqd_handle_event(sqd)) break; timeout = jiffies + sqd->sq_thread_idle; continue; } + sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { @@ -6882,7 +6893,7 @@ static int io_sq_thread(void *data) } prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); - if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { + if (!io_sqd_events_pending(sqd)) { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); From aaa9f0f48172b190a835792abe63f8859372eeec Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:01 +0100 Subject: [PATCH 02/83] io_uring: improve sq_thread waiting check If SQPOLL task finds a ring requesting it to continue running, no need to set wake flag to rest of the rings as it will be cleared in a moment anyway, so hide it in a single sqd->ctx_list loop. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1ee5a696d9fd08645994c58ee147d149a8957d94.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 24c0042b0de7..85d0184c585d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6894,11 +6894,10 @@ static int io_sq_thread(void *data) prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); if (!io_sqd_events_pending(sqd)) { - list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) - io_ring_set_wakeup_flag(ctx); - needs_sched = true; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { + io_ring_set_wakeup_flag(ctx); + if ((ctx->flags & IORING_SETUP_IOPOLL) && !list_empty_careful(&ctx->iopoll_list)) { needs_sched = false; From 21f2fc080f8654ce60b3e9192ba3b596c6a2ead6 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:02 +0100 Subject: [PATCH 03/83] io_uring: remove unused park_task_work As sqpoll cancel via task_work is killed, remove everything related to park_task_work as it's not used anymore. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/310d8b76a2fbbf3e139373500e04ad9af7ee3dbb.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 85d0184c585d..d18be5afc403 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -288,7 +288,6 @@ struct io_sq_data { unsigned long state; struct completion exited; - struct callback_head *park_task_work; }; #define IO_IOPOLL_BATCH 8 @@ -6834,7 +6833,6 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd) mutex_lock(&sqd->lock); } io_run_task_work(); - io_run_task_work_head(&sqd->park_task_work); return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); } @@ -6856,9 +6854,6 @@ static int io_sq_thread(void *data) current->flags |= PF_NO_SETAFFINITY; mutex_lock(&sqd->lock); - /* a user may had exited before the thread started */ - io_run_task_work_head(&sqd->park_task_work); - while (1) { int ret; bool cap_entries, sqt_spin, needs_sched; @@ -6919,7 +6914,6 @@ static int io_sq_thread(void *data) } finish_wait(&sqd->wait, &wait); - io_run_task_work_head(&sqd->park_task_work); timeout = jiffies + sqd->sq_thread_idle; } @@ -6928,7 +6922,6 @@ static int io_sq_thread(void *data) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); io_run_task_work(); - io_run_task_work_head(&sqd->park_task_work); mutex_unlock(&sqd->lock); complete(&sqd->exited); From acfb381d9d714c657ff540099fa5a6fa98e71f07 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:03 +0100 Subject: [PATCH 04/83] io_uring: simplify waking sqo_sq_wait Going through submission in __io_sq_thread() and still having a full SQ is rather unexpected, so remove a check for SQ fullness and just wake up whoever wait on sqo_sq_wait. Also skip if it doesn't do submission in the first place, likely may to happen for SQPOLL sharing and/or IOPOLL. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e2e91751e87b1a39f8d63ef884aaff578123f61e.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d18be5afc403..3a7889939455 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6801,10 +6801,10 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) !(ctx->flags & IORING_SETUP_R_DISABLED)) ret = io_submit_sqes(ctx, to_submit); mutex_unlock(&ctx->uring_lock); - } - if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait)) - wake_up(&ctx->sqo_sq_wait); + if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) + wake_up(&ctx->sqo_sq_wait); + } return ret; } From 3dd0c97a9e011b11ce6bd245bacf58c57f6f7875 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:04 +0100 Subject: [PATCH 05/83] io_uring: get rid of files in exit cancel We don't match against files on cancellation anymore, so no need to drag around files_struct anymore, just pass a flag telling whether only inflight or all requests should be killed. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7bfc5409a78f8e2d6b27dec3293ec2d248677348.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 63 +++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3a7889939455..8b8d25216662 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1036,7 +1036,7 @@ static bool io_disarm_next(struct io_kiocb *req); static void io_uring_del_task_file(unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, - struct files_struct *files); + bool cancel_all); static void io_uring_cancel_sqpoll(struct io_sq_data *sqd); static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx); @@ -1105,15 +1105,14 @@ static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) percpu_ref_put(ref); } -static bool io_match_task(struct io_kiocb *head, - struct task_struct *task, - struct files_struct *files) +static bool io_match_task(struct io_kiocb *head, struct task_struct *task, + bool cancel_all) { struct io_kiocb *req; if (task && head->task != task) return false; - if (!files) + if (cancel_all) return true; io_for_each_link(req, head) { @@ -5256,7 +5255,7 @@ static bool io_poll_remove_one(struct io_kiocb *req) * Returns true if we found and killed one or more poll requests */ static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, - struct files_struct *files) + bool cancel_all) { struct hlist_node *tmp; struct io_kiocb *req; @@ -5268,7 +5267,7 @@ static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, list = &ctx->cancel_hash[i]; hlist_for_each_entry_safe(req, tmp, list, hash_node) { - if (io_match_task(req, tsk, files)) + if (io_match_task(req, tsk, cancel_all)) posted += io_poll_remove_one(req); } } @@ -8742,7 +8741,7 @@ static void io_ring_exit_work(struct work_struct *work) * as nobody else will be looking for them. */ do { - io_uring_try_cancel_requests(ctx, NULL, NULL); + io_uring_try_cancel_requests(ctx, NULL, true); if (ctx->sq_data) { struct io_sq_data *sqd = ctx->sq_data; struct task_struct *tsk; @@ -8793,14 +8792,14 @@ static void io_ring_exit_work(struct work_struct *work) /* Returns true if we found and killed one or more timeouts */ static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, - struct files_struct *files) + bool cancel_all) { struct io_kiocb *req, *tmp; int canceled = 0; spin_lock_irq(&ctx->completion_lock); list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { - if (io_match_task(req, tsk, files)) { + if (io_match_task(req, tsk, cancel_all)) { io_kill_timeout(req, -ECANCELED); canceled++; } @@ -8826,8 +8825,8 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_unregister_personality(ctx, index); mutex_unlock(&ctx->uring_lock); - io_kill_timeouts(ctx, NULL, NULL); - io_poll_remove_all(ctx, NULL, NULL); + io_kill_timeouts(ctx, NULL, true); + io_poll_remove_all(ctx, NULL, true); /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); @@ -8853,7 +8852,7 @@ static int io_uring_release(struct inode *inode, struct file *file) struct io_task_cancel { struct task_struct *task; - struct files_struct *files; + bool all; }; static bool io_cancel_task_cb(struct io_wq_work *work, void *data) @@ -8862,30 +8861,29 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data) struct io_task_cancel *cancel = data; bool ret; - if (cancel->files && (req->flags & REQ_F_LINK_TIMEOUT)) { + if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) { unsigned long flags; struct io_ring_ctx *ctx = req->ctx; /* protect against races with linked timeouts */ spin_lock_irqsave(&ctx->completion_lock, flags); - ret = io_match_task(req, cancel->task, cancel->files); + ret = io_match_task(req, cancel->task, cancel->all); spin_unlock_irqrestore(&ctx->completion_lock, flags); } else { - ret = io_match_task(req, cancel->task, cancel->files); + ret = io_match_task(req, cancel->task, cancel->all); } return ret; } static bool io_cancel_defer_files(struct io_ring_ctx *ctx, - struct task_struct *task, - struct files_struct *files) + struct task_struct *task, bool cancel_all) { struct io_defer_entry *de; LIST_HEAD(list); spin_lock_irq(&ctx->completion_lock); list_for_each_entry_reverse(de, &ctx->defer_list, list) { - if (io_match_task(de->req, task, files)) { + if (io_match_task(de->req, task, cancel_all)) { list_cut_position(&list, &ctx->defer_list, &de->list); break; } @@ -8929,9 +8927,9 @@ static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, - struct files_struct *files) + bool cancel_all) { - struct io_task_cancel cancel = { .task = task, .files = files, }; + struct io_task_cancel cancel = { .task = task, .all = cancel_all, }; struct io_uring_task *tctx = task ? task->io_uring : NULL; while (1) { @@ -8951,7 +8949,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } /* SQPOLL thread does its own polling */ - if ((!(ctx->flags & IORING_SETUP_SQPOLL) && !files) || + if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) || (ctx->sq_data && ctx->sq_data->thread == current)) { while (!list_empty_careful(&ctx->iopoll_list)) { io_iopoll_try_reap_events(ctx); @@ -8959,9 +8957,9 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } } - ret |= io_cancel_defer_files(ctx, task, files); - ret |= io_poll_remove_all(ctx, task, files); - ret |= io_kill_timeouts(ctx, task, files); + ret |= io_cancel_defer_files(ctx, task, cancel_all); + ret |= io_poll_remove_all(ctx, task, cancel_all); + ret |= io_kill_timeouts(ctx, task, cancel_all); ret |= io_run_task_work(); ret |= io_run_ctx_fallback(ctx); if (!ret) @@ -9067,7 +9065,7 @@ static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked) return percpu_counter_sum(&tctx->inflight); } -static void io_uring_try_cancel(struct files_struct *files) +static void io_uring_try_cancel(bool cancel_all) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; @@ -9078,7 +9076,7 @@ static void io_uring_try_cancel(struct files_struct *files) /* sqpoll task will cancel all its requests */ if (!ctx->sq_data) - io_uring_try_cancel_requests(ctx, current, files); + io_uring_try_cancel_requests(ctx, current, cancel_all); } } @@ -9104,7 +9102,7 @@ static void io_uring_cancel_sqpoll(struct io_sq_data *sqd) if (!inflight) break; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) - io_uring_try_cancel_requests(ctx, current, NULL); + io_uring_try_cancel_requests(ctx, current, true); prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); /* @@ -9128,6 +9126,7 @@ void __io_uring_cancel(struct files_struct *files) struct io_uring_task *tctx = current->io_uring; DEFINE_WAIT(wait); s64 inflight; + bool cancel_all = !files; if (tctx->io_wq) io_wq_exit_start(tctx->io_wq); @@ -9136,10 +9135,10 @@ void __io_uring_cancel(struct files_struct *files) atomic_inc(&tctx->in_idle); do { /* read completions before cancelations */ - inflight = tctx_inflight(tctx, !!files); + inflight = tctx_inflight(tctx, !cancel_all); if (!inflight) break; - io_uring_try_cancel(files); + io_uring_try_cancel(cancel_all); prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); /* @@ -9147,14 +9146,14 @@ void __io_uring_cancel(struct files_struct *files) * avoids a race where a completion comes in before we did * prepare_to_wait(). */ - if (inflight == tctx_inflight(tctx, !!files)) + if (inflight == tctx_inflight(tctx, !cancel_all)) schedule(); finish_wait(&tctx->wait, &wait); } while (1); atomic_dec(&tctx->in_idle); io_uring_clean_tctx(tctx); - if (!files) { + if (cancel_all) { /* for exec all current's requests should be gone, kill tctx */ __io_uring_free(current); } From 93d2bcd2cbfed2c714341f7a7ecd511aaedabd83 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:05 +0100 Subject: [PATCH 06/83] io_uring: make fail flag not link specific The main difference is in req_set_fail_links() renamed into req_set_fail(), which now sets REQ_F_FAIL_LINK/REQ_F_FAIL flag unconditional on whether it has been a link or not. It only matters in io_disarm_next(), which already handles it well, and all calls to it have a fast path checking REQ_F_LINK/HARDLINK. It looks cleaner, and sheds binary size text data bss dec hex filename 84235 12390 8 96633 17979 ./fs/io_uring.o 84151 12414 8 96573 1793d ./fs/io_uring.o Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e2224154dd6e53b665ac835d29436b177872fa10.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 87 +++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8b8d25216662..8a5fda76f7a0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -705,7 +705,7 @@ enum { REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, /* first byte is taken by user flags, shift it to not overlap */ - REQ_F_FAIL_LINK_BIT = 8, + REQ_F_FAIL_BIT = 8, REQ_F_INFLIGHT_BIT, REQ_F_CUR_POS_BIT, REQ_F_NOWAIT_BIT, @@ -741,7 +741,7 @@ enum { REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), /* fail rest of links */ - REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT), + REQ_F_FAIL = BIT(REQ_F_FAIL_BIT), /* on inflight list, should be cancelled and waited on exit reliably */ REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), /* read/write uses file position */ @@ -1122,10 +1122,9 @@ static bool io_match_task(struct io_kiocb *head, struct task_struct *task, return false; } -static inline void req_set_fail_links(struct io_kiocb *req) +static inline void req_set_fail(struct io_kiocb *req) { - if (req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; + req->flags |= REQ_F_FAIL; } static void io_ring_ctx_ref_free(struct percpu_ref *ref) @@ -1594,7 +1593,7 @@ static void io_req_complete_post(struct io_kiocb *req, long res, struct io_comp_state *cs = &ctx->submit_state.comp; if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { - if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK)) + if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) io_disarm_next(req); if (req->link) { io_req_task_queue(req->link); @@ -1650,7 +1649,7 @@ static inline void io_req_complete(struct io_kiocb *req, long res) static void io_req_complete_failed(struct io_kiocb *req, long res) { - req_set_fail_links(req); + req_set_fail(req); io_put_req(req); io_req_complete_post(req, res, 0); } @@ -1829,7 +1828,7 @@ static bool io_disarm_next(struct io_kiocb *req) if (likely(req->flags & REQ_F_LINK_TIMEOUT)) posted = io_kill_linked_timeout(req); - if (unlikely((req->flags & REQ_F_FAIL_LINK) && + if (unlikely((req->flags & REQ_F_FAIL) && !(req->flags & REQ_F_HARDLINK))) { posted |= (req->link != NULL); io_fail_links(req); @@ -1847,7 +1846,7 @@ static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) * dependencies to the next request. In case of failure, fail the rest * of the chain. */ - if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK)) { + if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; bool posted; @@ -2486,7 +2485,7 @@ static void __io_complete_rw(struct io_kiocb *req, long res, long res2, req->flags |= REQ_F_REISSUE; return; } - req_set_fail_links(req); + req_set_fail(req); } if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); @@ -2509,7 +2508,7 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) if (unlikely(res != req->result)) { if (!(res == -EAGAIN && io_rw_should_reissue(req) && io_resubmit_prep(req))) { - req_set_fail_links(req); + req_set_fail(req); req->flags |= REQ_F_DONT_REISSUE; } } @@ -2765,7 +2764,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, } else { int cflags = 0; - req_set_fail_links(req); + req_set_fail(req); if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); __io_req_complete(req, issue_flags, ret, cflags); @@ -3487,7 +3486,7 @@ static int io_renameat(struct io_kiocb *req, unsigned int issue_flags) req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -3531,7 +3530,7 @@ static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -3568,7 +3567,7 @@ static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) ret = __sys_shutdown_sock(sock, req->shutdown.how); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; #else @@ -3626,7 +3625,7 @@ static int io_tee(struct io_kiocb *req, unsigned int issue_flags) req->flags &= ~REQ_F_NEED_CLEANUP; if (ret != sp->len) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -3663,7 +3662,7 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags) req->flags &= ~REQ_F_NEED_CLEANUP; if (ret != sp->len) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -3716,7 +3715,7 @@ static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) end > 0 ? end : LLONG_MAX, req->sync.flags & IORING_FSYNC_DATASYNC); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -3745,7 +3744,7 @@ static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, req->sync.len); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -3864,7 +3863,7 @@ err: putname(req->open.filename); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -3936,7 +3935,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) if (head) ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); /* complete before unlock, IOPOLL may need the lock */ __io_req_complete(req, issue_flags, ret, 0); @@ -4027,7 +4026,7 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) __io_remove_buffers(ctx, head, p->bgid, -1U); } if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); /* complete before unlock, IOPOLL may need the lock */ __io_req_complete(req, issue_flags, ret, 0); io_ring_submit_unlock(ctx, !force_nonblock); @@ -4073,7 +4072,7 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) return -EAGAIN; if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; #else @@ -4109,7 +4108,7 @@ static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; #else @@ -4148,7 +4147,7 @@ static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4183,7 +4182,7 @@ static int io_statx(struct io_kiocb *req, unsigned int issue_flags) ctx->buffer); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -4241,7 +4240,7 @@ static int io_close(struct io_kiocb *req, unsigned int issue_flags) ret = filp_close(file, current->files); err: if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); if (file) fput(file); __io_req_complete(req, issue_flags, ret, 0); @@ -4274,7 +4273,7 @@ static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) ret = sync_file_range(req->file, req->sync.off, req->sync.len, req->sync.flags); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -4378,7 +4377,7 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < min_ret) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4420,7 +4419,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) ret = -EINTR; if (ret < min_ret) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4615,7 +4614,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, cflags); return 0; } @@ -4670,7 +4669,7 @@ out_free: if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_recv_kbuf(req); if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, cflags); return 0; } @@ -4709,7 +4708,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) if (ret < 0) { if (ret == -ERESTARTSYS) ret = -EINTR; - req_set_fail_links(req); + req_set_fail(req); } __io_req_complete(req, issue_flags, ret, 0); return 0; @@ -4773,7 +4772,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags) ret = -EINTR; out: if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -5244,7 +5243,7 @@ static bool io_poll_remove_one(struct io_kiocb *req) if (do_complete) { io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0); io_commit_cqring(req->ctx); - req_set_fail_links(req); + req_set_fail(req); io_put_req_deferred(req, 1); } @@ -5454,7 +5453,7 @@ static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) err: if (ret < 0) { spin_unlock_irq(&ctx->completion_lock); - req_set_fail_links(req); + req_set_fail(req); io_req_complete(req, ret); return 0; } @@ -5474,7 +5473,7 @@ err: if (!completing) { ret = io_poll_add(preq, issue_flags); if (ret < 0) { - req_set_fail_links(preq); + req_set_fail(preq); io_req_complete(preq, ret); } } @@ -5499,7 +5498,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); - req_set_fail_links(req); + req_set_fail(req); io_put_req(req); return HRTIMER_NORESTART; } @@ -5535,7 +5534,7 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) if (IS_ERR(req)) return PTR_ERR(req); - req_set_fail_links(req); + req_set_fail(req); io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0); io_put_req_deferred(req, 1); return 0; @@ -5614,7 +5613,7 @@ static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) spin_unlock_irq(&ctx->completion_lock); io_cqring_ev_posted(ctx); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_put_req(req); return 0; } @@ -5767,7 +5766,7 @@ done: io_cqring_ev_posted(ctx); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); } static int io_async_cancel_prep(struct io_kiocb *req, @@ -5824,7 +5823,7 @@ done: io_cqring_ev_posted(ctx); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); io_put_req(req); return 0; } @@ -5866,7 +5865,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) mutex_unlock(&ctx->uring_lock); if (ret < 0) - req_set_fail_links(req); + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -6569,7 +6568,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, fail_req: if (link->head) { /* fail even hard links since we don't submit */ - link->head->flags |= REQ_F_FAIL_LINK; + req_set_fail(link->head); io_req_complete_failed(link->head, -ECANCELED); link->head = NULL; } From b986af7e2df4f0871367c397ba61a542f37c0ab3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:06 +0100 Subject: [PATCH 07/83] io_uring: shuffle rarely used ctx fields There is a bunch of scattered around ctx fields that are almost never used, e.g. only on ring exit, plunge them to the end, better locality, better aesthetically. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/782ff94b00355923eae757d58b1a47821b5b46d4.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8a5fda76f7a0..6e19fe04a5d1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -368,9 +368,6 @@ struct io_ring_ctx { unsigned cached_cq_overflow; unsigned long sq_check_overflow; - /* hashed buffered write serialization */ - struct io_wq_hash *hash_map; - struct list_head defer_list; struct list_head timeout_list; struct list_head cq_overflow_list; @@ -387,9 +384,6 @@ struct io_ring_ctx { struct io_rings *rings; - /* Only used for accounting purposes */ - struct mm_struct *mm_account; - const struct cred *sq_creds; /* cred used for __io_sq_thread() */ struct io_sq_data *sq_data; /* if using sq thread polling */ @@ -410,14 +404,6 @@ struct io_ring_ctx { unsigned nr_user_bufs; struct io_mapped_ubuf **user_bufs; - struct user_struct *user; - - struct completion ref_comp; - -#if defined(CONFIG_UNIX) - struct socket *ring_sock; -#endif - struct xarray io_buffers; struct xarray personalities; @@ -461,12 +447,24 @@ struct io_ring_ctx { struct io_restriction restrictions; - /* exit task_work */ - struct callback_head *exit_task_work; - /* Keep this last, we don't need it for the fast path */ - struct work_struct exit_work; - struct list_head tctx_list; + struct { + #if defined(CONFIG_UNIX) + struct socket *ring_sock; + #endif + /* hashed buffered write serialization */ + struct io_wq_hash *hash_map; + + /* Only used for accounting purposes */ + struct user_struct *user; + struct mm_struct *mm_account; + + /* ctx exit and cancelation */ + struct callback_head *exit_task_work; + struct work_struct exit_work; + struct list_head tctx_list; + struct completion ref_comp; + }; }; struct io_uring_task { From b13a8918d395554ff9a8cee17d03ed45d805df24 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:07 +0100 Subject: [PATCH 08/83] io_uring: better locality for rsrc fields ring has two types of resource-related fields: used for request submission, and field needed for update/registration. Reshuffle them into these two groups for better locality and readability. The second group is not in the hot path, so it's natural to place them somewhere in the end. Also update an outdated comment. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/05b34795bb4440f4ec4510f08abd5a31830f8ca0.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6e19fe04a5d1..f628af3a3368 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -391,21 +391,17 @@ struct io_ring_ctx { struct list_head sqd_list; /* - * If used, fixed file set. Writers must ensure that ->refs is dead, - * readers must ensure that ->refs is alive as long as the file* is - * used. Only updated through io_uring_register(2). + * Fixed resources fast path, should be accessed only under uring_lock, + * and updated through io_uring_register(2) */ - struct io_rsrc_data *file_data; + struct io_rsrc_node *rsrc_node; + struct io_file_table file_table; unsigned nr_user_files; - - /* if used, fixed mapped user buffers */ - struct io_rsrc_data *buf_data; unsigned nr_user_bufs; struct io_mapped_ubuf **user_bufs; struct xarray io_buffers; - struct xarray personalities; u32 pers_next; @@ -437,16 +433,21 @@ struct io_ring_ctx { bool poll_multi_file; } ____cacheline_aligned_in_smp; - struct delayed_work rsrc_put_work; - struct llist_head rsrc_put_llist; - struct list_head rsrc_ref_list; - spinlock_t rsrc_ref_lock; - struct io_rsrc_node *rsrc_node; - struct io_rsrc_node *rsrc_backup_node; - struct io_mapped_ubuf *dummy_ubuf; - struct io_restriction restrictions; + /* slow path rsrc auxilary data, used by update/register */ + struct { + struct io_rsrc_node *rsrc_backup_node; + struct io_mapped_ubuf *dummy_ubuf; + struct io_rsrc_data *file_data; + struct io_rsrc_data *buf_data; + + struct delayed_work rsrc_put_work; + struct llist_head rsrc_put_llist; + struct list_head rsrc_ref_list; + spinlock_t rsrc_ref_lock; + }; + /* Keep this last, we don't need it for the fast path */ struct { #if defined(CONFIG_UNIX) From a566c5562d41b99f11c8224b2a3010e60ad93acf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:08 +0100 Subject: [PATCH 09/83] io_uring: remove dependency on ring->sq/cq_entries We have numbers of {sq,cq} entries cached in ctx, don't look up them in user-shared rings as 1) it may fetch additional cacheline 2) user may change it and so it's always error prone. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/745d31bc2da41283ddd0489ef784af5c8d6310e9.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f628af3a3368..169e95126acf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1356,7 +1356,7 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) { struct io_rings *r = ctx->rings; - return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries; + return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries; } static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) @@ -1374,7 +1374,7 @@ static inline struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) * control dependency is enough as we're using WRITE_ONCE to * fill the cq entry */ - if (__io_cqring_events(ctx) == rings->cq_ring_entries) + if (__io_cqring_events(ctx) == ctx->cq_entries) return NULL; tail = ctx->cached_cq_tail++; @@ -1427,11 +1427,10 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) /* Returns true if there are no backlogged entries after the flush */ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) { - struct io_rings *rings = ctx->rings; unsigned long flags; bool all_flushed, posted; - if (!force && __io_cqring_events(ctx) == rings->cq_ring_entries) + if (!force && __io_cqring_events(ctx) == ctx->cq_entries) return false; posted = false; From ea5ab3b579836d784357ae9cb5bf9d7242a645b9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:09 +0100 Subject: [PATCH 10/83] io_uring: deduce cq_mask from cq_entries No need to cache cq_mask, it's exactly cq_entries - 1, so just deduce it to not carry it around. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d439efad0503c8398451dae075e68a04362fbc8d.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 169e95126acf..a35a8dc5c930 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -362,7 +362,6 @@ struct io_ring_ctx { u32 *sq_array; unsigned cached_sq_head; unsigned sq_entries; - unsigned sq_mask; unsigned sq_thread_idle; unsigned cached_sq_dropped; unsigned cached_cq_overflow; @@ -408,7 +407,6 @@ struct io_ring_ctx { struct { unsigned cached_cq_tail; unsigned cq_entries; - unsigned cq_mask; atomic_t cq_timeouts; unsigned cq_last_tm_flush; unsigned cq_extra; @@ -1367,7 +1365,7 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) static inline struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; - unsigned tail; + unsigned tail, mask = ctx->cq_entries - 1; /* * writes to the cq entry need to come after reading head; the @@ -1378,7 +1376,7 @@ static inline struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) return NULL; tail = ctx->cached_cq_tail++; - return &rings->cqes[tail & ctx->cq_mask]; + return &rings->cqes[tail & mask]; } static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) @@ -6680,7 +6678,7 @@ static void io_commit_sqring(struct io_ring_ctx *ctx) static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) { u32 *sq_array = ctx->sq_array; - unsigned head; + unsigned head, mask = ctx->sq_entries - 1; /* * The cached sq head (or cq tail) serves two purposes: @@ -6690,7 +6688,7 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) * 2) allows the kernel side to track the head on its own, even * though the application is the one updating it. */ - head = READ_ONCE(sq_array[ctx->cached_sq_head++ & ctx->sq_mask]); + head = READ_ONCE(sq_array[ctx->cached_sq_head++ & mask]); if (likely(head < ctx->sq_entries)) return &ctx->sq_sqes[head]; @@ -9512,8 +9510,6 @@ static int io_allocate_scq_urings(struct io_ring_ctx *ctx, rings->cq_ring_mask = p->cq_entries - 1; rings->sq_ring_entries = p->sq_entries; rings->cq_ring_entries = p->cq_entries; - ctx->sq_mask = rings->sq_ring_mask; - ctx->cq_mask = rings->cq_ring_mask; size = array_size(sizeof(struct io_uring_sqe), p->sq_entries); if (size == SIZE_MAX) { From 8f6ed49a4443be35a11807695dbae2680f7ca6fc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:10 +0100 Subject: [PATCH 11/83] io_uring: kill cached_cq_overflow There are two copies of cq_overflow, shared with userspace and internal cached one. It was needed for DRAIN accounting, but now we have yet another knob to tune the accounting, i.e. cq_extra, and we can throw away the internal counter and just increment the one in the shared ring. If user modifies it as so never gets the right overflow value ever again, it's its problem, even though before we would have restored it back by next overflow. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8427965f5175dd051febc63804909861109ce859.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a35a8dc5c930..623978fd2883 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -364,7 +364,6 @@ struct io_ring_ctx { unsigned sq_entries; unsigned sq_thread_idle; unsigned cached_sq_dropped; - unsigned cached_cq_overflow; unsigned long sq_check_overflow; struct list_head defer_list; @@ -1199,13 +1198,20 @@ err: return NULL; } +static void io_account_cq_overflow(struct io_ring_ctx *ctx) +{ + struct io_rings *r = ctx->rings; + + WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1); + ctx->cq_extra--; +} + static bool req_need_defer(struct io_kiocb *req, u32 seq) { if (unlikely(req->flags & REQ_F_IO_DRAIN)) { struct io_ring_ctx *ctx = req->ctx; - return seq + ctx->cq_extra != ctx->cached_cq_tail - + READ_ONCE(ctx->cached_cq_overflow); + return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail; } return false; @@ -1444,8 +1450,8 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) if (cqe) memcpy(cqe, &ocqe->cqe, sizeof(*cqe)); else - WRITE_ONCE(ctx->rings->cq_overflow, - ++ctx->cached_cq_overflow); + io_account_cq_overflow(ctx); + posted = true; list_del(&ocqe->list); kfree(ocqe); @@ -1529,7 +1535,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, * or cannot allocate an overflow entry, then we need to drop it * on the floor. */ - WRITE_ONCE(ctx->rings->cq_overflow, ++ctx->cached_cq_overflow); + io_account_cq_overflow(ctx); return false; } if (list_empty(&ctx->cq_overflow_list)) { From d068b5068d43353a352b3ec92865f7045fdb213e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:11 +0100 Subject: [PATCH 12/83] io_uring: rename io_get_cqring Rename io_get_cqring() into io_get_cqe() for consistency with SQ, and just because the old name is not as clear. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a46a53e3f781de372f5632c184e61546b86515ce.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 623978fd2883..60230e7b75fa 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -11,7 +11,7 @@ * before writing the tail (using smp_load_acquire to read the tail will * do). It also needs a smp_mb() before updating CQ head (ordering the * entry load(s) with the head store), pairing with an implicit barrier - * through a control-dependency in io_get_cqring (smp_store_release to + * through a control-dependency in io_get_cqe (smp_store_release to * store head will do). Failure to do so could lead to reading invalid * CQ entries. * @@ -1368,7 +1368,7 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); } -static inline struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) +static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; unsigned tail, mask = ctx->cq_entries - 1; @@ -1440,7 +1440,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) posted = false; spin_lock_irqsave(&ctx->completion_lock, flags); while (!list_empty(&ctx->cq_overflow_list)) { - struct io_uring_cqe *cqe = io_get_cqring(ctx); + struct io_uring_cqe *cqe = io_get_cqe(ctx); struct io_overflow_cqe *ocqe; if (!cqe && !force) @@ -1562,7 +1562,7 @@ static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data * submission (by quite a lot). Increment the overflow count in * the ring. */ - cqe = io_get_cqring(ctx); + cqe = io_get_cqe(ctx); if (likely(cqe)) { WRITE_ONCE(cqe->user_data, user_data); WRITE_ONCE(cqe->res, res); From d0acdee296d42e700c16271d9f95085a9c897a53 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 16 May 2021 22:58:12 +0100 Subject: [PATCH 13/83] io_uring: don't bounce submit_state cachelines struct io_submit_state contains struct io_comp_state and so locked_free_*, that renders cachelines around ->locked_free* being invalidated on most non-inline completions, that may terrorise caches if submissions and completions are done by different tasks. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/290cb5412b76892e8631978ee8ab9db0c6290dd5.1621201931.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 60230e7b75fa..c42a2f3b7259 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -298,11 +298,8 @@ struct io_sq_data { struct io_comp_state { struct io_kiocb *reqs[IO_COMPL_BATCH]; unsigned int nr; - unsigned int locked_free_nr; /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; - /* IRQ completion list, under ->completion_lock */ - struct list_head locked_free_list; }; struct io_submit_link { @@ -379,6 +376,9 @@ struct io_ring_ctx { } ____cacheline_aligned_in_smp; struct io_submit_state submit_state; + /* IRQ completion list, under ->completion_lock */ + struct list_head locked_free_list; + unsigned int locked_free_nr; struct io_rings *rings; @@ -1189,7 +1189,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_llist_head(&ctx->rsrc_put_llist); INIT_LIST_HEAD(&ctx->tctx_list); INIT_LIST_HEAD(&ctx->submit_state.comp.free_list); - INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list); + INIT_LIST_HEAD(&ctx->locked_free_list); return ctx; err: kfree(ctx->dummy_ubuf); @@ -1592,8 +1592,6 @@ static void io_req_complete_post(struct io_kiocb *req, long res, * free_list cache. */ if (req_ref_put_and_test(req)) { - struct io_comp_state *cs = &ctx->submit_state.comp; - if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) io_disarm_next(req); @@ -1604,8 +1602,8 @@ static void io_req_complete_post(struct io_kiocb *req, long res, } io_dismantle_req(req); io_put_task(req->task, 1); - list_add(&req->compl.list, &cs->locked_free_list); - cs->locked_free_nr++; + list_add(&req->compl.list, &ctx->locked_free_list); + ctx->locked_free_nr++; } else { if (!percpu_ref_tryget(&ctx->refs)) req = NULL; @@ -1660,8 +1658,8 @@ static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, struct io_comp_state *cs) { spin_lock_irq(&ctx->completion_lock); - list_splice_init(&cs->locked_free_list, &cs->free_list); - cs->locked_free_nr = 0; + list_splice_init(&ctx->locked_free_list, &cs->free_list); + ctx->locked_free_nr = 0; spin_unlock_irq(&ctx->completion_lock); } @@ -1677,7 +1675,7 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) * locked cache, grab the lock and move them over to our submission * side cache. */ - if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH) + if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH) io_flush_cached_locked_reqs(ctx, cs); nr = state->free_reqs; From 40dad765c045ab6dbd481cc4f00d04953e77510c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 9 Jun 2021 15:26:54 +0100 Subject: [PATCH 14/83] io_uring: enable shmem/memfd memory registration Relax buffer registration restictions, which filters out file backed memory, and allow shmem/memfd as they have normal anonymous pages underneath. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index c42a2f3b7259..b93fa32172af 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8306,6 +8306,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, for (i = 0; i < nr_pages; i++) { struct vm_area_struct *vma = vmas[i]; + if (vma_is_shmem(vma)) + continue; if (vma->vm_file && !is_file_hugepages(vma->vm_file)) { ret = -EOPNOTSUPP; From 976517f162a05f4315b2373fd11585c395506259 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 9 Jun 2021 12:07:25 +0100 Subject: [PATCH 15/83] io_uring: fix blocking inline submission There is a complaint against sys_io_uring_enter() blocking if it submits stdin reads. The problem is in __io_file_supports_async(), which sees that it's a cdev and allows it to be processed inline. Punt char devices using generic rules of io_file_supports_async(), including checking for presence of *_iter() versions of rw callbacks. Apparently, it will affect most of cdevs with some exceptions like null and zero devices. Cc: stable@vger.kernel.org Reported-by: Birk Hirdman Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d60270856b8a4560a639ef5f76e55eb563633599.1623236455.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b93fa32172af..cdd9b53abbb2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2618,7 +2618,7 @@ static bool __io_file_supports_async(struct file *file, int rw) return true; return false; } - if (S_ISCHR(mode) || S_ISSOCK(mode)) + if (S_ISSOCK(mode)) return true; if (S_ISREG(mode)) { if (IS_ENABLED(CONFIG_BLOCK) && From c7f405d6fa36f778931881bfb1e12dd401d0bc62 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:12 +0100 Subject: [PATCH 16/83] io-wq: embed wqe ptr array into struct io_wq io-wq keeps an array of pointers to struct io_wqe, allocate this array as a part of struct io-wq, it's easier to code and saves an extra indirection for nearly each io-wq call. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1482c6a001923bbed662dc38a8a580fb08b1ed8c.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index b3e8624a37d0..1ca98fc7d52b 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -102,7 +102,6 @@ struct io_wqe { * Per io_wq state */ struct io_wq { - struct io_wqe **wqes; unsigned long state; free_work_fn *free_work; @@ -118,6 +117,8 @@ struct io_wq { struct hlist_node cpuhp_node; struct task_struct *task; + + struct io_wqe *wqes[]; }; static enum cpuhp_state io_wq_online; @@ -907,17 +908,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (WARN_ON_ONCE(!data->free_work || !data->do_work)) return ERR_PTR(-EINVAL); - wq = kzalloc(sizeof(*wq), GFP_KERNEL); + wq = kzalloc(struct_size(wq, wqes, nr_node_ids), GFP_KERNEL); if (!wq) return ERR_PTR(-ENOMEM); - - wq->wqes = kcalloc(nr_node_ids, sizeof(struct io_wqe *), GFP_KERNEL); - if (!wq->wqes) - goto err_wq; - ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node); if (ret) - goto err_wqes; + goto err_wq; refcount_inc(&data->hash->refs); wq->hash = data->hash; @@ -962,8 +958,6 @@ err: cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); for_each_node(node) kfree(wq->wqes[node]); -err_wqes: - kfree(wq->wqes); err_wq: kfree(wq); return ERR_PTR(ret); @@ -1036,7 +1030,6 @@ static void io_wq_destroy(struct io_wq *wq) kfree(wqe); } io_wq_put_hash(wq->hash); - kfree(wq->wqes); kfree(wq); } From 382cb030469db3d428ada09e7925f684ba9d61cf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:13 +0100 Subject: [PATCH 17/83] io-wq: remove unused io-wq refcounting iowq->refs is initialised to one and killed on exit, so it's not used and we can kill it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/401007393528ea7c102360e69a29b64498e15db2.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 1ca98fc7d52b..f058ea0bcae8 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -109,8 +109,6 @@ struct io_wq { struct io_wq_hash *hash; - refcount_t refs; - atomic_t worker_refs; struct completion worker_done; @@ -949,7 +947,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) } wq->task = get_task_struct(data->task); - refcount_set(&wq->refs, 1); atomic_set(&wq->worker_refs, 1); init_completion(&wq->worker_done); return wq; @@ -1038,8 +1035,7 @@ void io_wq_put_and_exit(struct io_wq *wq) WARN_ON_ONCE(!test_bit(IO_WQ_BIT_EXIT, &wq->state)); io_wq_exit_workers(wq); - if (refcount_dec_and_test(&wq->refs)) - io_wq_destroy(wq); + io_wq_destroy(wq); } static bool io_wq_worker_affinity(struct io_worker *worker, void *data) From cb3d8972c78ab0cdb55a30d6db927a3e0442b3f9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:14 +0100 Subject: [PATCH 18/83] io_uring: refactor io_iopoll_req_issued A simple refactoring of io_iopoll_req_issued(), move in_async inside so we don't pass it around and save on double checking it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1513bfde4f0c835be25ac69a82737ab0668d7665.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index cdd9b53abbb2..6c0b3f91e1ad 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2525,9 +2525,14 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) * find it from a io_do_iopoll() thread before the issuer is done * accessing the kiocb cookie. */ -static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async) +static void io_iopoll_req_issued(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; + const bool in_async = io_wq_current_is_worker(); + + /* workqueue context doesn't hold uring_lock, grab it now */ + if (unlikely(in_async)) + mutex_lock(&ctx->uring_lock); /* * Track whether we have multiple files in our lists. This will impact @@ -2554,14 +2559,19 @@ static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async) else list_add_tail(&req->inflight_entry, &ctx->iopoll_list); - /* - * If IORING_SETUP_SQPOLL is enabled, sqes are either handled in sq thread - * task context or in io worker task context. If current task context is - * sq thread, we don't need to check whether should wake up sq thread. - */ - if (in_async && (ctx->flags & IORING_SETUP_SQPOLL) && - wq_has_sleeper(&ctx->sq_data->wait)) - wake_up(&ctx->sq_data->wait); + if (unlikely(in_async)) { + /* + * If IORING_SETUP_SQPOLL is enabled, sqes are either handle + * in sq thread task context or in io worker task context. If + * current task context is sq thread, we don't need to check + * whether should wake up sq thread. + */ + if ((ctx->flags & IORING_SETUP_SQPOLL) && + wq_has_sleeper(&ctx->sq_data->wait)) + wake_up(&ctx->sq_data->wait); + + mutex_unlock(&ctx->uring_lock); + } } static inline void io_state_file_put(struct io_submit_state *state) @@ -6215,23 +6225,11 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if (creds) revert_creds(creds); - if (ret) return ret; - /* If the op doesn't have a file, we're not polling for it */ - if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) { - const bool in_async = io_wq_current_is_worker(); - - /* workqueue context doesn't hold uring_lock, grab it now */ - if (in_async) - mutex_lock(&ctx->uring_lock); - - io_iopoll_req_issued(req, in_async); - - if (in_async) - mutex_unlock(&ctx->uring_lock); - } + if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) + io_iopoll_req_issued(req); return 0; } From eef51daa72f745b6e771d18f6f37c7e5cd4ccdf1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:15 +0100 Subject: [PATCH 19/83] io_uring: rename function *task_file What at some moment was references to struct file used to control lifetimes of task/ctx is now just internal tctx structures/nodes, so rename outdated *task_file() routines into something more sensible. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e2fbce42932154c2631ce58ffbffaa232afe18d5.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6c0b3f91e1ad..41483ed8b8aa 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1029,7 +1029,7 @@ static const struct io_op_def io_op_defs[] = { }; static bool io_disarm_next(struct io_kiocb *req); -static void io_uring_del_task_file(unsigned long index); +static void io_uring_del_tctx_node(unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, bool cancel_all); @@ -8714,7 +8714,7 @@ static void io_tctx_exit_cb(struct callback_head *cb) * node. It'll be removed by the end of cancellation, just ignore it. */ if (!atomic_read(&tctx->in_idle)) - io_uring_del_task_file((unsigned long)work->ctx); + io_uring_del_tctx_node((unsigned long)work->ctx); complete(&work->completion); } @@ -8967,7 +8967,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } } -static int __io_uring_add_task_file(struct io_ring_ctx *ctx) +static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; @@ -9004,19 +9004,19 @@ static int __io_uring_add_task_file(struct io_ring_ctx *ctx) /* * Note that this task has used io_uring. We use it for cancelation purposes. */ -static inline int io_uring_add_task_file(struct io_ring_ctx *ctx) +static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) { struct io_uring_task *tctx = current->io_uring; if (likely(tctx && tctx->last == ctx)) return 0; - return __io_uring_add_task_file(ctx); + return __io_uring_add_tctx_node(ctx); } /* * Remove this io_uring_file -> task mapping. */ -static void io_uring_del_task_file(unsigned long index) +static void io_uring_del_tctx_node(unsigned long index) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; @@ -9046,7 +9046,7 @@ static void io_uring_clean_tctx(struct io_uring_task *tctx) unsigned long index; xa_for_each(&tctx->xa, index, node) - io_uring_del_task_file(index); + io_uring_del_tctx_node(index); if (wq) { /* * Must be after io_uring_del_task_file() (removes nodes under @@ -9330,7 +9330,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, } submitted = to_submit; } else if (to_submit) { - ret = io_uring_add_task_file(ctx); + ret = io_uring_add_tctx_node(ctx); if (unlikely(ret)) goto out; mutex_lock(&ctx->uring_lock); @@ -9540,7 +9540,7 @@ static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) if (fd < 0) return fd; - ret = io_uring_add_task_file(ctx); + ret = io_uring_add_tctx_node(ctx); if (ret) { put_unused_fd(fd); return ret; From 769e683715211ad3cbed5908a86b97dd54d60970 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:16 +0100 Subject: [PATCH 20/83] io-wq: don't repeat IO_WQ_BIT_EXIT check by worker io_wqe_worker()'s main loop does check IO_WQ_BIT_EXIT flag, so no need for a second test_bit at the end as it will immediately jump to the first check afterwards. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d6af4a51c86523a527fb5417c9fbc775c4b26497.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index f058ea0bcae8..8c13e23d4a8a 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -559,8 +559,7 @@ loop: if (ret) continue; /* timed out, exit unless we're the fixed worker */ - if (test_bit(IO_WQ_BIT_EXIT, &wq->state) || - !(worker->flags & IO_WORKER_F_FIXED)) + if (!(worker->flags & IO_WORKER_F_FIXED)) break; } From e587227b680f798dd74644d047dd52ddb36cb82c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:17 +0100 Subject: [PATCH 21/83] io-wq: simplify worker exiting io_worker_handle_work() already takes care of the empty list case and releases spinlock, so get rid of ugly conditional unlocking and unconditionally call handle_work() Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7521e485677f381036676943e876a0afecc23017.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 8c13e23d4a8a..2c37776c0280 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -565,10 +565,7 @@ loop: if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) { raw_spin_lock_irq(&wqe->lock); - if (!wq_list_empty(&wqe->work_list)) - io_worker_handle_work(worker); - else - raw_spin_unlock_irq(&wqe->lock); + io_worker_handle_work(worker); } io_worker_exit(worker); From d878c81610e187becff1454f36b63c59ec165566 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:18 +0100 Subject: [PATCH 22/83] io_uring: hide rsrc tag copy into generic helpers Make io_rsrc_data_alloc() taking care of rsrc tags loading on registration, so we don't need to repeat it for each new rsrc type. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5609680697bd09735de10561b75edb95283459da.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 55 +++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 41483ed8b8aa..6fda9fdee582 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7161,27 +7161,38 @@ static void io_rsrc_data_free(struct io_rsrc_data *data) kfree(data); } -static struct io_rsrc_data *io_rsrc_data_alloc(struct io_ring_ctx *ctx, - rsrc_put_fn *do_put, - unsigned nr) +static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, + u64 __user *utags, unsigned nr, + struct io_rsrc_data **pdata) { struct io_rsrc_data *data; + unsigned i; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) - return NULL; + return -ENOMEM; data->tags = kvcalloc(nr, sizeof(*data->tags), GFP_KERNEL); if (!data->tags) { kfree(data); - return NULL; + return -ENOMEM; + } + if (utags) { + for (i = 0; i < nr; i++) { + if (copy_from_user(&data->tags[i], &utags[i], + sizeof(data->tags[i]))) { + io_rsrc_data_free(data); + return -EFAULT; + } + } } atomic_set(&data->refs, 1); data->ctx = ctx; data->do_put = do_put; init_completion(&data->done); - return data; + *pdata = data; + return 0; } static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) @@ -7633,7 +7644,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, struct file *file; int fd, ret; unsigned i; - struct io_rsrc_data *file_data; if (ctx->file_data) return -EBUSY; @@ -7644,27 +7654,24 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, ret = io_rsrc_node_switch_start(ctx); if (ret) return ret; + ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args, + &ctx->file_data); + if (ret) + return ret; - file_data = io_rsrc_data_alloc(ctx, io_rsrc_file_put, nr_args); - if (!file_data) - return -ENOMEM; - ctx->file_data = file_data; ret = -ENOMEM; if (!io_alloc_file_tables(&ctx->file_table, nr_args)) goto out_free; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - u64 tag = 0; - - if ((tags && copy_from_user(&tag, &tags[i], sizeof(tag))) || - copy_from_user(&fd, &fds[i], sizeof(fd))) { + if (copy_from_user(&fd, &fds[i], sizeof(fd))) { ret = -EFAULT; goto out_fput; } /* allow sparse sets */ if (fd == -1) { ret = -EINVAL; - if (unlikely(tag)) + if (unlikely(ctx->file_data->tags[i])) goto out_fput; continue; } @@ -7685,7 +7692,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, fput(file); goto out_fput; } - ctx->file_data->tags[i] = tag; io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file); } @@ -8403,9 +8409,9 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ret = io_rsrc_node_switch_start(ctx); if (ret) return ret; - data = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, nr_args); - if (!data) - return -ENOMEM; + ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data); + if (ret) + return ret; ret = io_buffers_map_alloc(ctx, nr_args); if (ret) { io_rsrc_data_free(data); @@ -8413,19 +8419,13 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, } for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) { - u64 tag = 0; - - if (tags && copy_from_user(&tag, &tags[i], sizeof(tag))) { - ret = -EFAULT; - break; - } ret = io_copy_iov(ctx, &iov, arg, i); if (ret) break; ret = io_buffer_validate(&iov); if (ret) break; - if (!iov.iov_base && tag) { + if (!iov.iov_base && data->tags[i]) { ret = -EINVAL; break; } @@ -8434,7 +8434,6 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, &last_hpage); if (ret) break; - data->tags[i] = tag; } WARN_ON_ONCE(ctx->buf_data); From 157d257f99c15c43668a98f804e3e3e6eb956464 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:19 +0100 Subject: [PATCH 23/83] io_uring: remove rsrc put work irq save/restore io_rsrc_put_work() is executed by workqueue in non-irq context, so no need for irqsave/restore variants of spinlocking. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2a7f77220735f4ad404ac885b4d73bdf42d2f836.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6fda9fdee582..55e449d84b3e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7555,14 +7555,13 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node) if (prsrc->tag) { bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL; - unsigned long flags; io_ring_submit_lock(ctx, lock_ring); - spin_lock_irqsave(&ctx->completion_lock, flags); + spin_lock_irq(&ctx->completion_lock); io_cqring_fill_event(ctx, prsrc->tag, 0, 0); ctx->cq_extra++; io_commit_cqring(ctx); - spin_unlock_irqrestore(&ctx->completion_lock, flags); + spin_unlock_irq(&ctx->completion_lock); io_cqring_ev_posted(ctx); io_ring_submit_unlock(ctx, lock_ring); } From 9123c8ffce1610323ec9c0874fa0262353f41fc3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:20 +0100 Subject: [PATCH 24/83] io_uring: add helpers for 2 level table alloc Some parts like fixed file table use 2 level tables, factor out helpers for allocating/deallocating them as more users are to come. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1709212359cd82eb416d395f86fc78431ccfc0aa.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 73 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 55e449d84b3e..0fbf1946ac42 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7059,14 +7059,36 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; } -static void io_free_file_tables(struct io_file_table *table, unsigned nr_files) +static void io_free_page_table(void **table, size_t size) { - unsigned i, nr_tables = DIV_ROUND_UP(nr_files, IORING_MAX_FILES_TABLE); + unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); for (i = 0; i < nr_tables; i++) - kfree(table->files[i]); - kfree(table->files); - table->files = NULL; + kfree(table[i]); + kfree(table); +} + +static void **io_alloc_page_table(size_t size) +{ + unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); + size_t init_size = size; + void **table; + + table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL); + if (!table) + return NULL; + + for (i = 0; i < nr_tables; i++) { + unsigned int this_size = min(size, PAGE_SIZE); + + table[i] = kzalloc(this_size, GFP_KERNEL); + if (!table[i]) { + io_free_page_table(table, init_size); + return NULL; + } + size -= this_size; + } + return table; } static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx) @@ -7195,6 +7217,22 @@ static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, return 0; } +static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) +{ + size_t size = nr_files * sizeof(struct io_fixed_file); + + table->files = (struct io_fixed_file **)io_alloc_page_table(size); + return !!table->files; +} + +static void io_free_file_tables(struct io_file_table *table, unsigned nr_files) +{ + size_t size = nr_files * sizeof(struct io_fixed_file); + + io_free_page_table((void **)table->files, size); + table->files = NULL; +} + static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) { #if defined(CONFIG_UNIX) @@ -7456,31 +7494,6 @@ static int io_sqe_files_scm(struct io_ring_ctx *ctx) } #endif -static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) -{ - unsigned i, nr_tables = DIV_ROUND_UP(nr_files, IORING_MAX_FILES_TABLE); - - table->files = kcalloc(nr_tables, sizeof(*table->files), GFP_KERNEL); - if (!table->files) - return false; - - for (i = 0; i < nr_tables; i++) { - unsigned int this_files = min(nr_files, IORING_MAX_FILES_TABLE); - - table->files[i] = kcalloc(this_files, sizeof(*table->files[i]), - GFP_KERNEL); - if (!table->files[i]) - break; - nr_files -= this_files; - } - - if (i == nr_tables) - return true; - - io_free_file_tables(table, nr_tables * IORING_MAX_FILES_TABLE); - return false; -} - static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) { struct file *file = prsrc->file; From 2d091d62b1106e90f195599c67bf385ddedfc915 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:21 +0100 Subject: [PATCH 25/83] io_uring: don't vmalloc rsrc tags We don't really need vmalloc for keeping tags, it's not a hot path and is there out of convenience, so replace it with two level tables to not litter kernel virtual memory mappings. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/241a3422747113a8909e7e1030eb585d4a349e0d.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 52 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0fbf1946ac42..bc4d03b15aa6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -100,6 +100,10 @@ #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) +#define IO_RSRC_TAG_TABLE_SHIFT 9 +#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT) +#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1) + #define IORING_MAX_REG_BUFFERS (1U << 14) #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ @@ -243,7 +247,8 @@ typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); struct io_rsrc_data { struct io_ring_ctx *ctx; - u64 *tags; + u64 **tags; + unsigned int nr; rsrc_put_fn *do_put; atomic_t refs; struct completion done; @@ -7177,9 +7182,20 @@ static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ct return ret; } +static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) +{ + unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK; + unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT; + + return &data->tags[table_idx][off]; +} + static void io_rsrc_data_free(struct io_rsrc_data *data) { - kvfree(data->tags); + size_t size = data->nr * sizeof(data->tags[0][0]); + + if (data->tags) + io_free_page_table((void **)data->tags, size); kfree(data); } @@ -7188,33 +7204,37 @@ static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, struct io_rsrc_data **pdata) { struct io_rsrc_data *data; + int ret = -ENOMEM; unsigned i; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; - - data->tags = kvcalloc(nr, sizeof(*data->tags), GFP_KERNEL); + data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0])); if (!data->tags) { kfree(data); return -ENOMEM; } + + data->nr = nr; + data->ctx = ctx; + data->do_put = do_put; if (utags) { + ret = -EFAULT; for (i = 0; i < nr; i++) { - if (copy_from_user(&data->tags[i], &utags[i], - sizeof(data->tags[i]))) { - io_rsrc_data_free(data); - return -EFAULT; - } + if (copy_from_user(io_get_tag_slot(data, i), &utags[i], + sizeof(data->tags[i]))) + goto fail; } } atomic_set(&data->refs, 1); - data->ctx = ctx; - data->do_put = do_put; init_completion(&data->done); *pdata = data; return 0; +fail: + io_rsrc_data_free(data); + return ret; } static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) @@ -7683,7 +7703,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, /* allow sparse sets */ if (fd == -1) { ret = -EINVAL; - if (unlikely(ctx->file_data->tags[i])) + if (unlikely(*io_get_tag_slot(ctx->file_data, i))) goto out_fput; continue; } @@ -7781,7 +7801,7 @@ static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, if (!prsrc) return -ENOMEM; - prsrc->tag = data->tags[idx]; + prsrc->tag = *io_get_tag_slot(data, idx); prsrc->rsrc = rsrc; list_add(&prsrc->list, &node->rsrc_list); return 0; @@ -7851,7 +7871,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } - data->tags[up->offset + done] = tag; + *io_get_tag_slot(data, up->offset + done) = tag; io_fixed_file_set(file_slot, file); err = io_sqe_file_register(ctx, file, i); if (err) { @@ -8437,7 +8457,7 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ret = io_buffer_validate(&iov); if (ret) break; - if (!iov.iov_base && data->tags[i]) { + if (!iov.iov_base && *io_get_tag_slot(data, i)) { ret = -EINVAL; break; } @@ -8510,7 +8530,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, } ctx->user_bufs[i] = imu; - ctx->buf_data->tags[offset] = tag; + *io_get_tag_slot(ctx->buf_data, offset) = tag; } if (needs_switch) From 09899b19155a152f3ff4eb5c203232175d630fbc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:22 +0100 Subject: [PATCH 26/83] io_uring: cache task struct refs tctx in submission part is always synchronised because is executed from the task's context, so we can batch allocate tctx/task references and store them across syscall boundaries. It avoids enough of operations, including an atomic for getting task ref and a percpu_counter_add() function call, which still fallback to spinlock for large batching cases (around >=32). Should be good for SQPOLL submitting in small portions and coming at some moment bpf submissions. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/14b327b973410a3eec1f702ecf650e100513aca9.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bc4d03b15aa6..0ca0282e96b0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -110,6 +110,8 @@ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ IOSQE_BUFFER_SELECT) +#define IO_TCTX_REFS_CACHE_NR (1U << 10) + struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp; @@ -472,6 +474,7 @@ struct io_ring_ctx { struct io_uring_task { /* submission side */ + int cached_refs; struct xarray xa; struct wait_queue_head wait; const struct io_ring_ctx *last; @@ -6707,16 +6710,23 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { + struct io_uring_task *tctx; int submitted = 0; /* make sure SQ entry isn't read before tail */ nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx)); - if (!percpu_ref_tryget_many(&ctx->refs, nr)) return -EAGAIN; - percpu_counter_add(¤t->io_uring->inflight, nr); - refcount_add(nr, ¤t->usage); + tctx = current->io_uring; + tctx->cached_refs -= nr; + if (unlikely(tctx->cached_refs < 0)) { + unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR; + + percpu_counter_add(&tctx->inflight, refill); + refcount_add(refill, ¤t->usage); + tctx->cached_refs += refill; + } io_submit_state_start(&ctx->submit_state, nr); while (submitted < nr) { @@ -6742,12 +6752,10 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) if (unlikely(submitted != nr)) { int ref_used = (submitted == -EAGAIN) ? 0 : submitted; - struct io_uring_task *tctx = current->io_uring; int unused = nr - ref_used; + current->io_uring->cached_refs += unused; percpu_ref_put_many(&ctx->refs, unused); - percpu_counter_sub(&tctx->inflight, unused); - put_task_struct_many(current, unused); } io_submit_state_end(&ctx->submit_state, ctx); @@ -7929,7 +7937,7 @@ static int io_uring_alloc_task_context(struct task_struct *task, struct io_uring_task *tctx; int ret; - tctx = kmalloc(sizeof(*tctx), GFP_KERNEL); + tctx = kzalloc(sizeof(*tctx), GFP_KERNEL); if (unlikely(!tctx)) return -ENOMEM; @@ -7949,13 +7957,11 @@ static int io_uring_alloc_task_context(struct task_struct *task, xa_init(&tctx->xa); init_waitqueue_head(&tctx->wait); - tctx->last = NULL; atomic_set(&tctx->in_idle, 0); atomic_set(&tctx->inflight_tracked, 0); task->io_uring = tctx; spin_lock_init(&tctx->task_lock); INIT_WQ_LIST(&tctx->task_list); - tctx->task_state = 0; init_task_work(&tctx->task_work, tctx_task_work); return 0; } @@ -7966,6 +7972,7 @@ void __io_uring_free(struct task_struct *tsk) WARN_ON_ONCE(!xa_empty(&tctx->xa)); WARN_ON_ONCE(tctx->io_wq); + WARN_ON_ONCE(tctx->cached_refs); percpu_counter_destroy(&tctx->inflight); kfree(tctx); @@ -9110,6 +9117,16 @@ static void io_uring_try_cancel(bool cancel_all) } } +static void io_uring_drop_tctx_refs(struct task_struct *task) +{ + struct io_uring_task *tctx = task->io_uring; + unsigned int refs = tctx->cached_refs; + + tctx->cached_refs = 0; + percpu_counter_sub(&tctx->inflight, refs); + put_task_struct_many(task, refs); +} + /* should only be called by SQPOLL task */ static void io_uring_cancel_sqpoll(struct io_sq_data *sqd) { @@ -9125,6 +9142,7 @@ static void io_uring_cancel_sqpoll(struct io_sq_data *sqd) WARN_ON_ONCE(!sqd || sqd->thread != current); + io_uring_drop_tctx_refs(current); atomic_inc(&tctx->in_idle); do { /* read completions before cancelations */ @@ -9162,6 +9180,7 @@ void __io_uring_cancel(struct files_struct *files) io_wq_exit_start(tctx->io_wq); /* make sure overflow events are dropped */ + io_uring_drop_tctx_refs(current); atomic_inc(&tctx->in_idle); do { /* read completions before cancelations */ From 78cc687be9c5420d743346f78bb8af9d59a903f9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:23 +0100 Subject: [PATCH 27/83] io_uring: unify SQPOLL and user task cancellations Merge io_uring_cancel_sqpoll() and __io_uring_cancel() as it's easier to have a conditional ctx traverse inside than keeping them in sync. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/adfe24d6dad4a3883a40eee54352b8b65ac851bb.1623634181.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 93 ++++++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 61 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0ca0282e96b0..b49dc2d74e84 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1041,7 +1041,7 @@ static void io_uring_del_tctx_node(unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, bool cancel_all); -static void io_uring_cancel_sqpoll(struct io_sq_data *sqd); +static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx); static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data, @@ -6926,7 +6926,7 @@ static int io_sq_thread(void *data) timeout = jiffies + sqd->sq_thread_idle; } - io_uring_cancel_sqpoll(sqd); + io_uring_cancel_generic(true, sqd); sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); @@ -9102,21 +9102,6 @@ static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked) return percpu_counter_sum(&tctx->inflight); } -static void io_uring_try_cancel(bool cancel_all) -{ - struct io_uring_task *tctx = current->io_uring; - struct io_tctx_node *node; - unsigned long index; - - xa_for_each(&tctx->xa, index, node) { - struct io_ring_ctx *ctx = node->ctx; - - /* sqpoll task will cancel all its requests */ - if (!ctx->sq_data) - io_uring_try_cancel_requests(ctx, current, cancel_all); - } -} - static void io_uring_drop_tctx_refs(struct task_struct *task) { struct io_uring_task *tctx = task->io_uring; @@ -9127,59 +9112,24 @@ static void io_uring_drop_tctx_refs(struct task_struct *task) put_task_struct_many(task, refs); } -/* should only be called by SQPOLL task */ -static void io_uring_cancel_sqpoll(struct io_sq_data *sqd) +/* + * Find any io_uring ctx that this task has registered or done IO on, and cancel + * requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation. + */ +static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) { struct io_uring_task *tctx = current->io_uring; struct io_ring_ctx *ctx; s64 inflight; DEFINE_WAIT(wait); + WARN_ON_ONCE(sqd && sqd->thread != current); + if (!current->io_uring) return; if (tctx->io_wq) io_wq_exit_start(tctx->io_wq); - WARN_ON_ONCE(!sqd || sqd->thread != current); - - io_uring_drop_tctx_refs(current); - atomic_inc(&tctx->in_idle); - do { - /* read completions before cancelations */ - inflight = tctx_inflight(tctx, false); - if (!inflight) - break; - list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) - io_uring_try_cancel_requests(ctx, current, true); - - prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); - /* - * If we've seen completions, retry without waiting. This - * avoids a race where a completion comes in before we did - * prepare_to_wait(). - */ - if (inflight == tctx_inflight(tctx, false)) - schedule(); - finish_wait(&tctx->wait, &wait); - } while (1); - atomic_dec(&tctx->in_idle); -} - -/* - * Find any io_uring fd that this task has registered or done IO on, and cancel - * requests. - */ -void __io_uring_cancel(struct files_struct *files) -{ - struct io_uring_task *tctx = current->io_uring; - DEFINE_WAIT(wait); - s64 inflight; - bool cancel_all = !files; - - if (tctx->io_wq) - io_wq_exit_start(tctx->io_wq); - - /* make sure overflow events are dropped */ io_uring_drop_tctx_refs(current); atomic_inc(&tctx->in_idle); do { @@ -9187,9 +9137,25 @@ void __io_uring_cancel(struct files_struct *files) inflight = tctx_inflight(tctx, !cancel_all); if (!inflight) break; - io_uring_try_cancel(cancel_all); - prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); + if (!sqd) { + struct io_tctx_node *node; + unsigned long index; + + xa_for_each(&tctx->xa, index, node) { + /* sqpoll task will cancel all its requests */ + if (node->ctx->sq_data) + continue; + io_uring_try_cancel_requests(node->ctx, current, + cancel_all); + } + } else { + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) + io_uring_try_cancel_requests(ctx, current, + cancel_all); + } + + prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); /* * If we've seen completions, retry without waiting. This * avoids a race where a completion comes in before we did @@ -9208,6 +9174,11 @@ void __io_uring_cancel(struct files_struct *files) } } +void __io_uring_cancel(struct files_struct *files) +{ + io_uring_cancel_generic(!files, NULL); +} + static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff, size_t sz) { From aeab9506ef50d23b350d1822c324023c9e1cb783 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 02:36:24 +0100 Subject: [PATCH 28/83] io_uring: inline io_iter_do_read() There are only two calls in source code of io_iter_do_read(), the function is small and pretty hot though is failed to get inlined. Makr it as inline. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/25a26dae7660da73fbc2244b361b397ef43d3caf.1623634182.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b49dc2d74e84..d665c9419ad3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3253,7 +3253,7 @@ static bool io_rw_should_retry(struct io_kiocb *req) return true; } -static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) +static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) { if (req->file->f_op->read_iter) return call_read_iter(req->file, &req->rw.kiocb, iter); From fdd1dc316e8959b6730d733fba025a39dac7938f Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 15 Jun 2021 14:00:11 +0100 Subject: [PATCH 29/83] io_uring: Fix incorrect sizeof operator for copy_from_user call Static analysis is warning that the sizeof being used is should be of *data->tags[i] and not data->tags[i]. Although these are the same size on 64 bit systems it is not a portable assumption to assume this is true for all cases. Fix this by using a temporary pointer tag_slot to make the code a clearer. Addresses-Coverity: ("Sizeof not portable") Fixes: d878c81610e1 ("io_uring: hide rsrc tag copy into generic helpers") Signed-off-by: Colin Ian King Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20210615130011.57387-1-colin.king@canonical.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d665c9419ad3..3692bbc7bd01 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7230,8 +7230,10 @@ static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, if (utags) { ret = -EFAULT; for (i = 0; i < nr; i++) { - if (copy_from_user(io_get_tag_slot(data, i), &utags[i], - sizeof(data->tags[i]))) + u64 *tag_slot = io_get_tag_slot(data, i); + + if (copy_from_user(tag_slot, &utags[i], + sizeof(*tag_slot))) goto fail; } } From b1b2fc3574a6a94a1ab90271a7620345c39dc01f Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 15 Jun 2021 15:34:24 +0100 Subject: [PATCH 30/83] io-wq: remove redundant initialization of variable ret The variable ret is being initialized with a value that is never read, the assignment is redundant and can be removed. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20210615143424.60449-1-colin.king@canonical.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 2c37776c0280..e221aaab585c 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -896,7 +896,7 @@ static int io_wqe_hash_wake(struct wait_queue_entry *wait, unsigned mode, struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) { - int ret = -ENOMEM, node; + int ret, node; struct io_wq *wq; if (WARN_ON_ONCE(!data->free_work || !data->do_work)) From c7af47cf0fab5bad1fb8b250dfab8efc1f991559 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:20 +0100 Subject: [PATCH 31/83] io_uring: keep SQ pointers in a single cacheline sq_array and sq_sqes are always used together, however they are in different cachelines, where the borderline is right before cq_overflow_list is rather rarely touched. Move the fields together so it loads only one cacheline. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3ef2411a94874da06492506a8897eff679244f49.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3692bbc7bd01..8c501eed3813 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -364,6 +364,7 @@ struct io_ring_ctx { * array. */ u32 *sq_array; + struct io_uring_sqe *sq_sqes; unsigned cached_sq_head; unsigned sq_entries; unsigned sq_thread_idle; @@ -373,8 +374,6 @@ struct io_ring_ctx { struct list_head defer_list; struct list_head timeout_list; struct list_head cq_overflow_list; - - struct io_uring_sqe *sq_sqes; } ____cacheline_aligned_in_smp; struct { From b52ecf8cb5b5ccb8069adbdb82a68d3fa0f423db Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:21 +0100 Subject: [PATCH 32/83] io_uring: move ctx->flags from SQ cacheline ctx->flags are heavily used by both, completion and submission sides, so move it out from the ctx fields related to submissions. Instead, place it together with ctx->refs, because it's already cacheline-aligned and so pads lots of space, and both almost never change. Also, in most occasions they are accessed together as refs are taken at submission time and put back during completion. Do same with ctx->rings, where the pointer itself is never modified apart from ring init/free. Note: in percpu mode, struct percpu_ref doesn't modify the struct itself but takes indirection with ref->percpu_count_ptr. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4c48c173e63d35591383ba2b87e8b8e8dfdbd23d.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8c501eed3813..ba36eefdae2c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -341,17 +341,19 @@ struct io_submit_state { }; struct io_ring_ctx { + /* const or read-mostly hot data */ struct { struct percpu_ref refs; - } ____cacheline_aligned_in_smp; - struct { + struct io_rings *rings; unsigned int flags; unsigned int compat: 1; unsigned int drain_next: 1; unsigned int eventfd_async: 1; unsigned int restricted: 1; + } ____cacheline_aligned_in_smp; + struct { /* * Ring buffer of indices into array of io_uring_sqe, which is * mmapped by the application using the IORING_OFF_SQES offset. @@ -386,8 +388,6 @@ struct io_ring_ctx { struct list_head locked_free_list; unsigned int locked_free_nr; - struct io_rings *rings; - const struct cred *sq_creds; /* cred used for __io_sq_thread() */ struct io_sq_data *sq_data; /* if using sq thread polling */ From 7f1129d227ea54526380d0f37eb7b33ab9f200c1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:22 +0100 Subject: [PATCH 33/83] io_uring: shuffle more fields into SQ ctx section Since moving locked_free_* out of struct io_submit_state ctx->submit_state is accessed on submission side only, so move it into the submission section. Same goes for rsrc table pointers/nodes/etc., they must be taken and checked during submission because sync'ed by uring_lock, so move them there as well. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8a5899a50afc6ccca63249e716f580b246f3dec6.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ba36eefdae2c..a0720fb2565c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -353,6 +353,7 @@ struct io_ring_ctx { unsigned int restricted: 1; } ____cacheline_aligned_in_smp; + /* submission data */ struct { /* * Ring buffer of indices into array of io_uring_sqe, which is @@ -369,13 +370,27 @@ struct io_ring_ctx { struct io_uring_sqe *sq_sqes; unsigned cached_sq_head; unsigned sq_entries; - unsigned sq_thread_idle; unsigned cached_sq_dropped; unsigned long sq_check_overflow; - struct list_head defer_list; + + /* + * Fixed resources fast path, should be accessed only under + * uring_lock, and updated through io_uring_register(2) + */ + struct io_rsrc_node *rsrc_node; + struct io_file_table file_table; + unsigned nr_user_files; + unsigned nr_user_bufs; + struct io_mapped_ubuf **user_bufs; + + struct io_submit_state submit_state; struct list_head timeout_list; struct list_head cq_overflow_list; + struct xarray io_buffers; + struct xarray personalities; + u32 pers_next; + unsigned sq_thread_idle; } ____cacheline_aligned_in_smp; struct { @@ -383,7 +398,6 @@ struct io_ring_ctx { wait_queue_head_t wait; } ____cacheline_aligned_in_smp; - struct io_submit_state submit_state; /* IRQ completion list, under ->completion_lock */ struct list_head locked_free_list; unsigned int locked_free_nr; @@ -394,21 +408,6 @@ struct io_ring_ctx { struct wait_queue_head sqo_sq_wait; struct list_head sqd_list; - /* - * Fixed resources fast path, should be accessed only under uring_lock, - * and updated through io_uring_register(2) - */ - struct io_rsrc_node *rsrc_node; - - struct io_file_table file_table; - unsigned nr_user_files; - unsigned nr_user_bufs; - struct io_mapped_ubuf **user_bufs; - - struct xarray io_buffers; - struct xarray personalities; - u32 pers_next; - struct { unsigned cached_cq_tail; unsigned cq_entries; From 17d3aeb33cdae8c87a8ad97c4358a623a630e19a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:23 +0100 Subject: [PATCH 34/83] io_uring: refactor io_get_sqe() The line of io_get_sqe() evaluating @head consists of too many operations including READ_ONCE(), it's not convenient for probing. Refactor it also improving readability. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/866ad6e4ef4851c7c61f6b0e08dbd0a8d1abce84.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a0720fb2565c..c74a84a2532b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6685,8 +6685,8 @@ static void io_commit_sqring(struct io_ring_ctx *ctx) */ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) { - u32 *sq_array = ctx->sq_array; unsigned head, mask = ctx->sq_entries - 1; + unsigned sq_idx = ctx->cached_sq_head++ & mask; /* * The cached sq head (or cq tail) serves two purposes: @@ -6696,7 +6696,7 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) * 2) allows the kernel side to track the head on its own, even * though the application is the one updating it. */ - head = READ_ONCE(sq_array[ctx->cached_sq_head++ & mask]); + head = READ_ONCE(ctx->sq_array[sq_idx]); if (likely(head < ctx->sq_entries)) return &ctx->sq_sqes[head]; From 15641e427070f05fad2e9d74d191146d6514d30f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:24 +0100 Subject: [PATCH 35/83] io_uring: don't cache number of dropped SQEs Kill ->cached_sq_dropped and wire DRAIN sequence number correction via ->cq_extra, which is there exactly for that purpose. User visible dropped counter will be populated by incrementing it instead of keeping a copy, similarly as it was done not so long ago with cq_overflow. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/088aceb2707a534d531e2770267c4498e0507cc1.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c74a84a2532b..ecac362913cc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -370,7 +370,6 @@ struct io_ring_ctx { struct io_uring_sqe *sq_sqes; unsigned cached_sq_head; unsigned sq_entries; - unsigned cached_sq_dropped; unsigned long sq_check_overflow; struct list_head defer_list; @@ -5994,13 +5993,11 @@ static u32 io_get_sequence(struct io_kiocb *req) { struct io_kiocb *pos; struct io_ring_ctx *ctx = req->ctx; - u32 total_submitted, nr_reqs = 0; + u32 nr_reqs = 0; io_for_each_link(pos, req) nr_reqs++; - - total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped; - return total_submitted - nr_reqs; + return ctx->cached_sq_head - nr_reqs; } static int io_req_defer(struct io_kiocb *req) @@ -6701,8 +6698,9 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) return &ctx->sq_sqes[head]; /* drop invalid entries */ - ctx->cached_sq_dropped++; - WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped); + ctx->cq_extra--; + WRITE_ONCE(ctx->rings->sq_dropped, + READ_ONCE(ctx->rings->sq_dropped) + 1); return NULL; } From f18ee4cf0a277a0e3d043755046d5817d4ddd618 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:25 +0100 Subject: [PATCH 36/83] io_uring: optimise completion timeout flushing io_commit_cqring() might be very hot and we definitely don't want to touch ->timeout_list there, because 1) it's shared with the submission side so might lead to cache bouncing and 2) may need to load an extra cache line, especially for IRQ completions. We're interested in it at the completion side only when there are offset-mode timeouts, which are not so popular. Replace list_empty(->timeout_list) hot path check with a new one-way flag, which is set when we prepare the first offset-mode timeout. note: the flag sits in the same line as briefly used after ->rings Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e4892ec68b71a69f92ffbea4a1499be3ec0d463b.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ecac362913cc..b27734bc5ca3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -351,6 +351,7 @@ struct io_ring_ctx { unsigned int drain_next: 1; unsigned int eventfd_async: 1; unsigned int restricted: 1; + unsigned int off_timeout_used: 1; } ____cacheline_aligned_in_smp; /* submission data */ @@ -1318,12 +1319,12 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) { u32 seq; - if (list_empty(&ctx->timeout_list)) + if (likely(!ctx->off_timeout_used)) return; seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); - do { + while (!list_empty(&ctx->timeout_list)) { u32 events_needed, events_got; struct io_kiocb *req = list_first_entry(&ctx->timeout_list, struct io_kiocb, timeout.list); @@ -1345,8 +1346,7 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) list_del_init(&req->timeout.list); io_kill_timeout(req, 0); - } while (!list_empty(&ctx->timeout_list)); - + } ctx->cq_last_tm_flush = seq; } @@ -5651,6 +5651,8 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, return -EINVAL; req->timeout.off = off; + if (unlikely(off && !req->ctx->off_timeout_used)) + req->ctx->off_timeout_used = true; if (!req->async_data && io_alloc_async_data(req)) return -ENOMEM; From 5e159204d7edd5bd329e8cdb419dbd81d25751e0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:26 +0100 Subject: [PATCH 37/83] io_uring: small io_submit_sqe() optimisation submit_state.link is used only to assemble a link and not used for actual submission, so clear it before io_queue_sqe() in io_submit_sqe(), awhile it's hot and in caches and queueing doesn't spoil it. May also potentially help compiler with spilling or to do other optimisations. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1579939426f3ad6b55af3005b1389bbbed7d780d.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b27734bc5ca3..64fdeee01906 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6616,8 +6616,8 @@ fail_req: /* last request of a link, enqueue the link */ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { - io_queue_sqe(head); link->head = NULL; + io_queue_sqe(head); } } else { if (unlikely(ctx->drain_next)) { From 5ed7a37d21b369d03114dea12a1f16ae2e21baa8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:27 +0100 Subject: [PATCH 38/83] io_uring: clean up check_overflow flag There are no users of ->sq_check_overflow, only ->cq_check_overflow is used. Combine it and move out of completion related part of struct io_ring_ctx. A not so obvious benefit of it is fitting all completion side fields into a single cacheline. It was taking 2 lines before with 56B padding, and io_cqring_ev_posted*() were still touching both of them. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/25927394964df31d113e3c729416af573afff5f5.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 64fdeee01906..b9fd8adf67af 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -371,7 +371,6 @@ struct io_ring_ctx { struct io_uring_sqe *sq_sqes; unsigned cached_sq_head; unsigned sq_entries; - unsigned long sq_check_overflow; struct list_head defer_list; /* @@ -408,13 +407,14 @@ struct io_ring_ctx { struct wait_queue_head sqo_sq_wait; struct list_head sqd_list; + unsigned long check_cq_overflow; + struct { unsigned cached_cq_tail; unsigned cq_entries; atomic_t cq_timeouts; unsigned cq_last_tm_flush; unsigned cq_extra; - unsigned long cq_check_overflow; struct wait_queue_head cq_wait; struct fasync_struct *cq_fasync; struct eventfd_ctx *cq_ev_fd; @@ -1464,8 +1464,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) all_flushed = list_empty(&ctx->cq_overflow_list); if (all_flushed) { - clear_bit(0, &ctx->sq_check_overflow); - clear_bit(0, &ctx->cq_check_overflow); + clear_bit(0, &ctx->check_cq_overflow); ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW; } @@ -1481,7 +1480,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) { bool ret = true; - if (test_bit(0, &ctx->cq_check_overflow)) { + if (test_bit(0, &ctx->check_cq_overflow)) { /* iopoll syncs against uring_lock, not completion_lock */ if (ctx->flags & IORING_SETUP_IOPOLL) mutex_lock(&ctx->uring_lock); @@ -1544,8 +1543,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, return false; } if (list_empty(&ctx->cq_overflow_list)) { - set_bit(0, &ctx->sq_check_overflow); - set_bit(0, &ctx->cq_check_overflow); + set_bit(0, &ctx->check_cq_overflow); ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW; } ocqe->cqe.user_data = user_data; @@ -2391,7 +2389,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) * If we do, we can potentially be spinning for commands that * already triggered a CQE (eg in error). */ - if (test_bit(0, &ctx->cq_check_overflow)) + if (test_bit(0, &ctx->check_cq_overflow)) __io_cqring_overflow_flush(ctx, false); if (io_cqring_events(ctx)) goto out; @@ -6965,7 +6963,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, * Cannot safely flush overflowed CQEs from here, ensure we wake up * the task, and the next invocation will do it. */ - if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow)) + if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow)) return autoremove_wake_function(curr, mode, wake_flags, key); return -1; } @@ -6993,7 +6991,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, if (ret || io_should_wake(iowq)) return ret; /* let the caller flush overflows, retry */ - if (test_bit(0, &ctx->cq_check_overflow)) + if (test_bit(0, &ctx->check_cq_overflow)) return 1; *timeout = schedule_timeout(*timeout); @@ -8704,7 +8702,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) * Users may get EPOLLIN meanwhile seeing nothing in cqring, this * pushs them to do the flush. */ - if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow)) + if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow)) mask |= EPOLLIN | EPOLLRDNORM; return mask; From 311997b3fcddc2f169fff844bf6b48dbff0bb816 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:28 +0100 Subject: [PATCH 39/83] io_uring: wait heads renaming We use several wait_queue_head's for different purposes, but namings are confusing. First rename ctx->cq_wait into ctx->poll_wait, because this one is used for polling an io_uring instance. Then rename ctx->wait into ctx->cq_wait, which is responsible for CQE waiting. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/47b97a097780c86c67b20b6ccc4e077523dce682.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b9fd8adf67af..e19c9f7db2cf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -394,7 +394,7 @@ struct io_ring_ctx { struct { struct mutex uring_lock; - wait_queue_head_t wait; + wait_queue_head_t cq_wait; } ____cacheline_aligned_in_smp; /* IRQ completion list, under ->completion_lock */ @@ -415,7 +415,7 @@ struct io_ring_ctx { atomic_t cq_timeouts; unsigned cq_last_tm_flush; unsigned cq_extra; - struct wait_queue_head cq_wait; + struct wait_queue_head poll_wait; struct fasync_struct *cq_fasync; struct eventfd_ctx *cq_ev_fd; } ____cacheline_aligned_in_smp; @@ -1178,13 +1178,13 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) ctx->flags = p->flags; init_waitqueue_head(&ctx->sqo_sq_wait); INIT_LIST_HEAD(&ctx->sqd_list); - init_waitqueue_head(&ctx->cq_wait); + init_waitqueue_head(&ctx->poll_wait); INIT_LIST_HEAD(&ctx->cq_overflow_list); init_completion(&ctx->ref_comp); xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); - init_waitqueue_head(&ctx->wait); + init_waitqueue_head(&ctx->cq_wait); spin_lock_init(&ctx->completion_lock); INIT_LIST_HEAD(&ctx->iopoll_list); INIT_LIST_HEAD(&ctx->defer_list); @@ -1404,14 +1404,14 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) /* see waitqueue_active() comment */ smp_mb(); - if (waitqueue_active(&ctx->wait)) - wake_up(&ctx->wait); + if (waitqueue_active(&ctx->cq_wait)) + wake_up(&ctx->cq_wait); if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); if (io_should_trigger_evfd(ctx)) eventfd_signal(ctx->cq_ev_fd, 1); - if (waitqueue_active(&ctx->cq_wait)) { - wake_up_interruptible(&ctx->cq_wait); + if (waitqueue_active(&ctx->poll_wait)) { + wake_up_interruptible(&ctx->poll_wait); kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN); } } @@ -1422,13 +1422,13 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) smp_mb(); if (ctx->flags & IORING_SETUP_SQPOLL) { - if (waitqueue_active(&ctx->wait)) - wake_up(&ctx->wait); + if (waitqueue_active(&ctx->cq_wait)) + wake_up(&ctx->cq_wait); } if (io_should_trigger_evfd(ctx)) eventfd_signal(ctx->cq_ev_fd, 1); - if (waitqueue_active(&ctx->cq_wait)) { - wake_up_interruptible(&ctx->cq_wait); + if (waitqueue_active(&ctx->poll_wait)) { + wake_up_interruptible(&ctx->poll_wait); kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN); } } @@ -7056,10 +7056,10 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, ret = -EBUSY; break; } - prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, + prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); - finish_wait(&ctx->wait, &iowq.wq); + finish_wait(&ctx->cq_wait, &iowq.wq); cond_resched(); } while (ret > 0); @@ -8680,7 +8680,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) struct io_ring_ctx *ctx = file->private_data; __poll_t mask = 0; - poll_wait(file, &ctx->cq_wait, wait); + poll_wait(file, &ctx->poll_wait, wait); /* * synchronizes with barrier from wq_has_sleeper call in * io_commit_cqring From 0499e582aaff4e4072a760d1f31434acb50c7813 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:29 +0100 Subject: [PATCH 40/83] io_uring: move uring_lock location ->uring_lock is prevalently used for submission, even though it protects many other things like iopoll, registeration, selected bufs, and more. And it's placed together with ->cq_wait poked on completion and CQ waiting sides. Move them apart, ->uring_lock goes to the submission data, and cq_wait to completion related chunk. The last one requires some reshuffling so everything needed by io_cqring_ev_posted*() is in one cacheline. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/dea5e845caee4c98aa0922b46d713154d81f7bd8.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e19c9f7db2cf..74c8334d67a4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -356,6 +356,8 @@ struct io_ring_ctx { /* submission data */ struct { + struct mutex uring_lock; + /* * Ring buffer of indices into array of io_uring_sqe, which is * mmapped by the application using the IORING_OFF_SQES offset. @@ -392,11 +394,6 @@ struct io_ring_ctx { unsigned sq_thread_idle; } ____cacheline_aligned_in_smp; - struct { - struct mutex uring_lock; - wait_queue_head_t cq_wait; - } ____cacheline_aligned_in_smp; - /* IRQ completion list, under ->completion_lock */ struct list_head locked_free_list; unsigned int locked_free_nr; @@ -412,12 +409,13 @@ struct io_ring_ctx { struct { unsigned cached_cq_tail; unsigned cq_entries; - atomic_t cq_timeouts; - unsigned cq_last_tm_flush; - unsigned cq_extra; - struct wait_queue_head poll_wait; - struct fasync_struct *cq_fasync; struct eventfd_ctx *cq_ev_fd; + struct wait_queue_head poll_wait; + struct wait_queue_head cq_wait; + unsigned cq_extra; + atomic_t cq_timeouts; + struct fasync_struct *cq_fasync; + unsigned cq_last_tm_flush; } ____cacheline_aligned_in_smp; struct { From 76cc33d79175a1b224bf02d3ff6c7be53fc684d5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:30 +0100 Subject: [PATCH 41/83] io_uring: refactor io_req_defer() Rename io_req_defer() into io_drain_req() and refactor it uncoupling it from io_queue_sqe() error handling and preparing for coming optimisations. Also, prioritise non IOSQE_ASYNC path. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4f17dd56e7fbe52d1866f8acd8efe3284d2bebcb.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 74c8334d67a4..fc764e912844 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5998,7 +5998,7 @@ static u32 io_get_sequence(struct io_kiocb *req) return ctx->cached_sq_head - nr_reqs; } -static int io_req_defer(struct io_kiocb *req) +static bool io_drain_req(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; @@ -6008,27 +6008,29 @@ static int io_req_defer(struct io_kiocb *req) /* Still need defer if there is pending req in defer list. */ if (likely(list_empty_careful(&ctx->defer_list) && !(req->flags & REQ_F_IO_DRAIN))) - return 0; + return false; seq = io_get_sequence(req); /* Still a chance to pass the sequence check */ if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) - return 0; + return false; ret = io_req_prep_async(req); if (ret) return ret; io_prep_async_link(req); de = kmalloc(sizeof(*de), GFP_KERNEL); - if (!de) - return -ENOMEM; + if (!de) { + io_req_complete_failed(req, ret); + return true; + } spin_lock_irq(&ctx->completion_lock); if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { spin_unlock_irq(&ctx->completion_lock); kfree(de); io_queue_async_work(req); - return -EIOCBQUEUED; + return true; } trace_io_uring_defer(ctx, req, req->user_data); @@ -6036,7 +6038,7 @@ static int io_req_defer(struct io_kiocb *req) de->seq = seq; list_add_tail(&de->list, &ctx->defer_list); spin_unlock_irq(&ctx->completion_lock); - return -EIOCBQUEUED; + return true; } static void io_clean_op(struct io_kiocb *req) @@ -6447,21 +6449,18 @@ static void __io_queue_sqe(struct io_kiocb *req) static void io_queue_sqe(struct io_kiocb *req) { - int ret; + if (io_drain_req(req)) + return; - ret = io_req_defer(req); - if (ret) { - if (ret != -EIOCBQUEUED) { -fail_req: - io_req_complete_failed(req, ret); - } - } else if (req->flags & REQ_F_FORCE_ASYNC) { - ret = io_req_prep_async(req); - if (unlikely(ret)) - goto fail_req; - io_queue_async_work(req); - } else { + if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) { __io_queue_sqe(req); + } else { + int ret = io_req_prep_async(req); + + if (unlikely(ret)) + io_req_complete_failed(req, ret); + else + io_queue_async_work(req); } } From 441b8a7803bfa11af2355beea9a07720d4b5c03a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 14 Jun 2021 23:37:31 +0100 Subject: [PATCH 42/83] io_uring: optimise non-drain path Replace drain checks with one-way flag set upon seeing the first IOSQE_IO_DRAIN request. There are several places where it cuts cycles well: 1) It's much faster than the fast check with two conditions in io_drain_req() including pretty complex list_empty_careful(). 2) We can mark io_queue_sqe() inline now, that's a huge win. 3) It replaces timeout and drain checks in io_commit_cqring() with a single flags test. Also great not touching ->defer_list there without a reason so limiting cache bouncing. It adds a small amount of overhead to drain path, but it's negligible. The main nuisance is that once it meets any DRAIN request in io_uring instance lifetime it will _always_ go through a slower path, so drain-less and offset-mode timeout less applications are preferable. The overhead in that case would be not big, but it's worth to bear in mind. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/98d2fff8c4da5144bb0d08499f591d4768128ea3.1623709150.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 57 +++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fc764e912844..ab9a16c89545 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -352,6 +352,7 @@ struct io_ring_ctx { unsigned int eventfd_async: 1; unsigned int restricted: 1; unsigned int off_timeout_used: 1; + unsigned int drain_used: 1; } ____cacheline_aligned_in_smp; /* submission data */ @@ -1299,9 +1300,9 @@ static void io_kill_timeout(struct io_kiocb *req, int status) } } -static void __io_queue_deferred(struct io_ring_ctx *ctx) +static void io_queue_deferred(struct io_ring_ctx *ctx) { - do { + while (!list_empty(&ctx->defer_list)) { struct io_defer_entry *de = list_first_entry(&ctx->defer_list, struct io_defer_entry, list); @@ -1310,17 +1311,12 @@ static void __io_queue_deferred(struct io_ring_ctx *ctx) list_del_init(&de->list); io_req_task_queue(de->req); kfree(de); - } while (!list_empty(&ctx->defer_list)); + } } static void io_flush_timeouts(struct io_ring_ctx *ctx) { - u32 seq; - - if (likely(!ctx->off_timeout_used)) - return; - - seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); + u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); while (!list_empty(&ctx->timeout_list)) { u32 events_needed, events_got; @@ -1350,13 +1346,14 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) static void io_commit_cqring(struct io_ring_ctx *ctx) { - io_flush_timeouts(ctx); - + if (unlikely(ctx->off_timeout_used || ctx->drain_used)) { + if (ctx->off_timeout_used) + io_flush_timeouts(ctx); + if (ctx->drain_used) + io_queue_deferred(ctx); + } /* order cqe stores with ring update */ smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); - - if (unlikely(!list_empty(&ctx->defer_list))) - __io_queue_deferred(ctx); } static inline bool io_sqring_full(struct io_ring_ctx *ctx) @@ -6447,9 +6444,9 @@ static void __io_queue_sqe(struct io_kiocb *req) io_queue_linked_timeout(linked_timeout); } -static void io_queue_sqe(struct io_kiocb *req) +static inline void io_queue_sqe(struct io_kiocb *req) { - if (io_drain_req(req)) + if (unlikely(req->ctx->drain_used) && io_drain_req(req)) return; if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) { @@ -6573,6 +6570,23 @@ fail_req: io_req_complete_failed(req, ret); return ret; } + + if (unlikely(req->flags & REQ_F_IO_DRAIN)) { + ctx->drain_used = true; + + /* + * Taking sequential execution of a link, draining both sides + * of the link also fullfils IOSQE_IO_DRAIN semantics for all + * requests in the link. So, it drains the head and the + * next after the link request. The last one is done via + * drain_next flag to persist the effect across calls. + */ + if (link->head) { + link->head->flags |= REQ_F_IO_DRAIN; + ctx->drain_next = 1; + } + } + ret = io_req_prep(req, sqe); if (unlikely(ret)) goto fail_req; @@ -6591,17 +6605,6 @@ fail_req: if (link->head) { struct io_kiocb *head = link->head; - /* - * Taking sequential execution of a link, draining both sides - * of the link also fullfils IOSQE_IO_DRAIN semantics for all - * requests in the link. So, it drains the head and the - * next after the link request. The last one is done via - * drain_next flag to persist the effect across calls. - */ - if (req->flags & REQ_F_IO_DRAIN) { - head->flags |= REQ_F_IO_DRAIN; - ctx->drain_next = 1; - } ret = io_req_prep_async(req); if (unlikely(ret)) goto fail_req; From dd9ae8a0b2985ead64dfcfa2f9a0ce5efa1480aa Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Fri, 4 Jun 2021 17:42:56 +0100 Subject: [PATCH 43/83] io_uring: Fix comment of io_get_sqe The sqe_ptr argument has been gone since 709b302faddf (io_uring: simplify io_get_sqring, 2020-04-08), made the return value of the function. Update the comment accordingly. Signed-off-by: Fam Zheng Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20210604164256.12242-1-fam.zheng@bytedance.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ab9a16c89545..807a1abe69c5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6673,7 +6673,7 @@ static void io_commit_sqring(struct io_ring_ctx *ctx) } /* - * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory + * Fetch an sqe, if one is available. Note this returns a pointer to memory * that is mapped by userspace. This means that care needs to be taken to * ensure that reads are stable, as we cannot rely on userspace always * being a good citizen. If members of the sqe are validated and then later From 27f6b318dea2d7ccccc9dca416e59431838c2929 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 15 Jun 2021 13:20:13 +0100 Subject: [PATCH 44/83] io_uring: fix min types mismatch in table alloc fs/io_uring.c: In function 'io_alloc_page_table': include/linux/minmax.h:20:28: warning: comparison of distinct pointer types lacks a cast Cast everything to size_t using min_t. Reported-by: Stephen Rothwell Fixes: 9123c8ffce16 ("io_uring: add helpers for 2 level table alloc") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/50f420a956bca070a43810d4a805293ed54f39d8.1623759527.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 807a1abe69c5..6292b8da0a75 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7088,7 +7088,7 @@ static void **io_alloc_page_table(size_t size) return NULL; for (i = 0; i < nr_tables; i++) { - unsigned int this_size = min(size, PAGE_SIZE); + unsigned int this_size = min_t(size_t, size, PAGE_SIZE); table[i] = kzalloc(this_size, GFP_KERNEL); if (!table[i]) { From 10c669040e9b3538e1732c8d40729636b17ce9dd Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 15 Jun 2021 16:47:56 +0100 Subject: [PATCH 45/83] io_uring: switch !DRAIN fast path when possible ->drain_used is one way, which is not optimal if users use DRAIN but very rarely. However, we can just clear it in io_drain_req() when all drained before requests are gone. Also rename the flag to reflect the change and be more clear about it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7f37a240857546a94df6348507edddacab150460.1623772051.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6292b8da0a75..25106cf7e57c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -352,7 +352,7 @@ struct io_ring_ctx { unsigned int eventfd_async: 1; unsigned int restricted: 1; unsigned int off_timeout_used: 1; - unsigned int drain_used: 1; + unsigned int drain_active: 1; } ____cacheline_aligned_in_smp; /* submission data */ @@ -1346,10 +1346,10 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) static void io_commit_cqring(struct io_ring_ctx *ctx) { - if (unlikely(ctx->off_timeout_used || ctx->drain_used)) { + if (unlikely(ctx->off_timeout_used || ctx->drain_active)) { if (ctx->off_timeout_used) io_flush_timeouts(ctx); - if (ctx->drain_used) + if (ctx->drain_active) io_queue_deferred(ctx); } /* order cqe stores with ring update */ @@ -6004,8 +6004,10 @@ static bool io_drain_req(struct io_kiocb *req) /* Still need defer if there is pending req in defer list. */ if (likely(list_empty_careful(&ctx->defer_list) && - !(req->flags & REQ_F_IO_DRAIN))) + !(req->flags & REQ_F_IO_DRAIN))) { + ctx->drain_active = false; return false; + } seq = io_get_sequence(req); /* Still a chance to pass the sequence check */ @@ -6446,7 +6448,7 @@ static void __io_queue_sqe(struct io_kiocb *req) static inline void io_queue_sqe(struct io_kiocb *req) { - if (unlikely(req->ctx->drain_used) && io_drain_req(req)) + if (unlikely(req->ctx->drain_active) && io_drain_req(req)) return; if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) { @@ -6572,7 +6574,7 @@ fail_req: } if (unlikely(req->flags & REQ_F_IO_DRAIN)) { - ctx->drain_used = true; + ctx->drain_active = true; /* * Taking sequential execution of a link, draining both sides From 3c19966d3710dbe5a44658c532052f11d797aecb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 15 Jun 2021 16:47:57 +0100 Subject: [PATCH 46/83] io_uring: shove more drain bits out of hot path Place all drain_next logic into io_drain_req(), so it's never executed if there was no drained requests before. The only thing we need is to set ->drain_active if we see a request with IOSQE_IO_DRAIN, do that in io_init_req() where flags are definitely in registers. Also, all drain-related code is encapsulated in io_drain_req(), makes it cleaner. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/68bf4f7395ddaafbf1a26bd97b57d57d45a9f900.1623772051.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 25106cf7e57c..f63fc79df4eb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5997,11 +5997,31 @@ static u32 io_get_sequence(struct io_kiocb *req) static bool io_drain_req(struct io_kiocb *req) { + struct io_kiocb *pos; struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; int ret; u32 seq; + /* + * If we need to drain a request in the middle of a link, drain the + * head request and the next request/link after the current link. + * Considering sequential execution of links, IOSQE_IO_DRAIN will be + * maintained for every request of our link. + */ + if (ctx->drain_next) { + req->flags |= REQ_F_IO_DRAIN; + ctx->drain_next = false; + } + /* not interested in head, start from the first linked */ + io_for_each_link(pos, req->link) { + if (pos->flags & REQ_F_IO_DRAIN) { + ctx->drain_next = true; + req->flags |= REQ_F_IO_DRAIN; + break; + } + } + /* Still need defer if there is pending req in defer list. */ if (likely(list_empty_careful(&ctx->defer_list) && !(req->flags & REQ_F_IO_DRAIN))) { @@ -6522,6 +6542,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if ((sqe_flags & IOSQE_BUFFER_SELECT) && !io_op_defs[req->opcode].buffer_select) return -EOPNOTSUPP; + if (unlikely(sqe_flags & IOSQE_IO_DRAIN)) + ctx->drain_active = true; personality = READ_ONCE(sqe->personality); if (personality) { @@ -6573,22 +6595,6 @@ fail_req: return ret; } - if (unlikely(req->flags & REQ_F_IO_DRAIN)) { - ctx->drain_active = true; - - /* - * Taking sequential execution of a link, draining both sides - * of the link also fullfils IOSQE_IO_DRAIN semantics for all - * requests in the link. So, it drains the head and the - * next after the link request. The last one is done via - * drain_next flag to persist the effect across calls. - */ - if (link->head) { - link->head->flags |= REQ_F_IO_DRAIN; - ctx->drain_next = 1; - } - } - ret = io_req_prep(req, sqe); if (unlikely(ret)) goto fail_req; @@ -6620,10 +6626,6 @@ fail_req: io_queue_sqe(head); } } else { - if (unlikely(ctx->drain_next)) { - req->flags |= REQ_F_IO_DRAIN; - ctx->drain_next = 0; - } if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { link->head = req; link->last = req; From 2335f6f5ddf2f4621395fac5fa4b53d075828cc1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 15 Jun 2021 16:47:58 +0100 Subject: [PATCH 47/83] io_uring: optimise io_commit_cqring() In most cases io_commit_cqring() is just an smp_store_release(), and it's hot enough, especially for IRQ rw, to want it to save on a function call. Mark it inline and extract a non-inlined slow path doing drain and timeout flushing. The inlined part is pretty slim to not cause binary bloating. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7350f8b6b92caa50a48a80be39909f0d83eddd93.1623772051.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f63fc79df4eb..16156a655d8b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1344,14 +1344,18 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) ctx->cq_last_tm_flush = seq; } -static void io_commit_cqring(struct io_ring_ctx *ctx) +static void __io_commit_cqring_flush(struct io_ring_ctx *ctx) { - if (unlikely(ctx->off_timeout_used || ctx->drain_active)) { - if (ctx->off_timeout_used) - io_flush_timeouts(ctx); - if (ctx->drain_active) - io_queue_deferred(ctx); - } + if (ctx->off_timeout_used) + io_flush_timeouts(ctx); + if (ctx->drain_active) + io_queue_deferred(ctx); +} + +static inline void io_commit_cqring(struct io_ring_ctx *ctx) +{ + if (unlikely(ctx->off_timeout_used || ctx->drain_active)) + __io_commit_cqring_flush(ctx); /* order cqe stores with ring update */ smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); } From 236daeae3616b1c62ce1a9f8a348d576ec9e22d9 Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Mon, 31 May 2021 02:36:37 -0400 Subject: [PATCH 48/83] io_uring: Add to traces the req pointer when available The req pointer uniquely identify a specific request. Having it in traces can provide valuable insights that is not possible to have if the calling process is reusing the same user_data value. Reviewed-by: Pavel Begunkov Signed-off-by: Olivier Langlois Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 ++--- include/trace/events/io_uring.h | 71 ++++++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 16156a655d8b..d916eb2cef09 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5073,7 +5073,7 @@ static void io_async_task_func(struct callback_head *cb) struct async_poll *apoll = req->apoll; struct io_ring_ctx *ctx = req->ctx; - trace_io_uring_task_run(req->ctx, req->opcode, req->user_data); + trace_io_uring_task_run(req->ctx, req, req->opcode, req->user_data); if (io_poll_rewait(req, &apoll->poll)) { spin_unlock_irq(&ctx->completion_lock); @@ -5206,8 +5206,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req) return false; } spin_unlock_irq(&ctx->completion_lock); - trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask, - apoll->poll.events); + trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data, + mask, apoll->poll.events); return true; } @@ -6604,8 +6604,9 @@ fail_req: goto fail_req; /* don't need @sqe from now on */ - trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, - true, ctx->flags & IORING_SETUP_SQPOLL); + trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data, + req->flags, true, + ctx->flags & IORING_SETUP_SQPOLL); /* * If we already have a head request, queue this one for async diff --git a/include/trace/events/io_uring.h b/include/trace/events/io_uring.h index abb8b24744fd..12addad1f837 100644 --- a/include/trace/events/io_uring.h +++ b/include/trace/events/io_uring.h @@ -323,8 +323,10 @@ TRACE_EVENT(io_uring_complete, * io_uring_submit_sqe - called before submitting one SQE * * @ctx: pointer to a ring context structure + * @req: pointer to a submitted request * @opcode: opcode of request * @user_data: user data associated with the request + * @flags request flags * @force_nonblock: whether a context blocking or not * @sq_thread: true if sq_thread has submitted this SQE * @@ -333,41 +335,60 @@ TRACE_EVENT(io_uring_complete, */ TRACE_EVENT(io_uring_submit_sqe, - TP_PROTO(void *ctx, u8 opcode, u64 user_data, bool force_nonblock, - bool sq_thread), + TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, u32 flags, + bool force_nonblock, bool sq_thread), - TP_ARGS(ctx, opcode, user_data, force_nonblock, sq_thread), + TP_ARGS(ctx, req, opcode, user_data, flags, force_nonblock, sq_thread), TP_STRUCT__entry ( __field( void *, ctx ) + __field( void *, req ) __field( u8, opcode ) __field( u64, user_data ) + __field( u32, flags ) __field( bool, force_nonblock ) __field( bool, sq_thread ) ), TP_fast_assign( __entry->ctx = ctx; + __entry->req = req; __entry->opcode = opcode; __entry->user_data = user_data; + __entry->flags = flags; __entry->force_nonblock = force_nonblock; __entry->sq_thread = sq_thread; ), - TP_printk("ring %p, op %d, data 0x%llx, non block %d, sq_thread %d", - __entry->ctx, __entry->opcode, - (unsigned long long) __entry->user_data, - __entry->force_nonblock, __entry->sq_thread) + TP_printk("ring %p, req %p, op %d, data 0x%llx, flags %u, " + "non block %d, sq_thread %d", __entry->ctx, __entry->req, + __entry->opcode, (unsigned long long)__entry->user_data, + __entry->flags, __entry->force_nonblock, __entry->sq_thread) ); +/* + * io_uring_poll_arm - called after arming a poll wait if successful + * + * @ctx: pointer to a ring context structure + * @req: pointer to the armed request + * @opcode: opcode of request + * @user_data: user data associated with the request + * @mask: request poll events mask + * @events: registered events of interest + * + * Allows to track which fds are waiting for and what are the events of + * interest. + */ TRACE_EVENT(io_uring_poll_arm, - TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask, int events), + TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, + int mask, int events), - TP_ARGS(ctx, opcode, user_data, mask, events), + TP_ARGS(ctx, req, opcode, user_data, mask, events), TP_STRUCT__entry ( __field( void *, ctx ) + __field( void *, req ) __field( u8, opcode ) __field( u64, user_data ) __field( int, mask ) @@ -376,16 +397,17 @@ TRACE_EVENT(io_uring_poll_arm, TP_fast_assign( __entry->ctx = ctx; + __entry->req = req; __entry->opcode = opcode; __entry->user_data = user_data; __entry->mask = mask; __entry->events = events; ), - TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x, events 0x%x", - __entry->ctx, __entry->opcode, - (unsigned long long) __entry->user_data, - __entry->mask, __entry->events) + TP_printk("ring %p, req %p, op %d, data 0x%llx, mask 0x%x, events 0x%x", + __entry->ctx, __entry->req, __entry->opcode, + (unsigned long long) __entry->user_data, + __entry->mask, __entry->events) ); TRACE_EVENT(io_uring_poll_wake, @@ -440,27 +462,40 @@ TRACE_EVENT(io_uring_task_add, __entry->mask) ); +/* + * io_uring_task_run - called when task_work_run() executes the poll events + * notification callbacks + * + * @ctx: pointer to a ring context structure + * @req: pointer to the armed request + * @opcode: opcode of request + * @user_data: user data associated with the request + * + * Allows to track when notified poll events are processed + */ TRACE_EVENT(io_uring_task_run, - TP_PROTO(void *ctx, u8 opcode, u64 user_data), + TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data), - TP_ARGS(ctx, opcode, user_data), + TP_ARGS(ctx, req, opcode, user_data), TP_STRUCT__entry ( __field( void *, ctx ) + __field( void *, req ) __field( u8, opcode ) __field( u64, user_data ) ), TP_fast_assign( __entry->ctx = ctx; + __entry->req = req; __entry->opcode = opcode; __entry->user_data = user_data; ), - TP_printk("ring %p, op %d, data 0x%llx", - __entry->ctx, __entry->opcode, - (unsigned long long) __entry->user_data) + TP_printk("ring %p, req %p, op %d, data 0x%llx", + __entry->ctx, __entry->req, __entry->opcode, + (unsigned long long) __entry->user_data) ); #endif /* _TRACE_IO_URING_H */ From 3d7b7b5285f0a8e73e332f3d7c7b2ca1e46309d7 Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Mon, 31 May 2021 02:54:15 -0400 Subject: [PATCH 49/83] io_uring: minor clean up in trace events definition Fix tabulation to make nice columns Reviewed-by: Pavel Begunkov Signed-off-by: Olivier Langlois Signed-off-by: Jens Axboe --- include/trace/events/io_uring.h | 35 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/include/trace/events/io_uring.h b/include/trace/events/io_uring.h index 12addad1f837..e4e44a2b4aa9 100644 --- a/include/trace/events/io_uring.h +++ b/include/trace/events/io_uring.h @@ -12,11 +12,11 @@ struct io_wq_work; /** * io_uring_create - called after a new io_uring context was prepared * - * @fd: corresponding file descriptor - * @ctx: pointer to a ring context structure + * @fd: corresponding file descriptor + * @ctx: pointer to a ring context structure * @sq_entries: actual SQ size * @cq_entries: actual CQ size - * @flags: SQ ring flags, provided to io_uring_setup(2) + * @flags: SQ ring flags, provided to io_uring_setup(2) * * Allows to trace io_uring creation and provide pointer to a context, that can * be used later to find correlated events. @@ -52,12 +52,12 @@ TRACE_EVENT(io_uring_create, * io_uring_register - called after a buffer/file/eventfd was successfully * registered for a ring * - * @ctx: pointer to a ring context structure - * @opcode: describes which operation to perform + * @ctx: pointer to a ring context structure + * @opcode: describes which operation to perform * @nr_user_files: number of registered files * @nr_user_bufs: number of registered buffers * @cq_ev_fd: whether eventfs registered or not - * @ret: return code + * @ret: return code * * Allows to trace fixed files/buffers/eventfds, that could be registered to * avoid an overhead of getting references to them for every operation. This @@ -142,16 +142,16 @@ TRACE_EVENT(io_uring_queue_async_work, TP_ARGS(ctx, rw, req, work, flags), TP_STRUCT__entry ( - __field( void *, ctx ) - __field( int, rw ) - __field( void *, req ) + __field( void *, ctx ) + __field( int, rw ) + __field( void *, req ) __field( struct io_wq_work *, work ) __field( unsigned int, flags ) ), TP_fast_assign( __entry->ctx = ctx; - __entry->rw = rw; + __entry->rw = rw; __entry->req = req; __entry->work = work; __entry->flags = flags; @@ -196,10 +196,10 @@ TRACE_EVENT(io_uring_defer, /** * io_uring_link - called before the io_uring request added into link_list of - * another request + * another request * - * @ctx: pointer to a ring context structure - * @req: pointer to a linked request + * @ctx: pointer to a ring context structure + * @req: pointer to a linked request * @target_req: pointer to a previous request, that would contain @req * * Allows to track linked requests, to understand dependencies between requests @@ -212,8 +212,8 @@ TRACE_EVENT(io_uring_link, TP_ARGS(ctx, req, target_req), TP_STRUCT__entry ( - __field( void *, ctx ) - __field( void *, req ) + __field( void *, ctx ) + __field( void *, req ) __field( void *, target_req ) ), @@ -244,7 +244,7 @@ TRACE_EVENT(io_uring_cqring_wait, TP_ARGS(ctx, min_events), TP_STRUCT__entry ( - __field( void *, ctx ) + __field( void *, ctx ) __field( int, min_events ) ), @@ -272,7 +272,7 @@ TRACE_EVENT(io_uring_fail_link, TP_ARGS(req, link), TP_STRUCT__entry ( - __field( void *, req ) + __field( void *, req ) __field( void *, link ) ), @@ -318,7 +318,6 @@ TRACE_EVENT(io_uring_complete, __entry->res, __entry->cflags) ); - /** * io_uring_submit_sqe - called before submitting one SQE * From ec16d35b6c9d8c89b3b7327a52c4972a7e4281d3 Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Mon, 31 May 2021 02:54:59 -0400 Subject: [PATCH 50/83] io-wq: remove header files not needed anymore mm related header files are not needed for io-wq module. remove them for a small clean-up. Reviewed-by: Pavel Begunkov Signed-off-by: Olivier Langlois Signed-off-by: Jens Axboe --- fs/io-wq.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index e221aaab585c..897b94530b57 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include #include From 0e03496d1967abf1ebb151a24318c07d07f41f7f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Jun 2021 10:08:11 -0600 Subject: [PATCH 51/83] io-wq: use private CPU mask In preparation for allowing user specific CPU masks for IO thread creation, switch to using a mask embedded in the per-node wqe structure. Signed-off-by: Jens Axboe --- fs/io-wq.c | 56 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 897b94530b57..2af8e1df4646 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -94,6 +94,8 @@ struct io_wqe { struct io_wq *wq; struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; + + cpumask_var_t cpu_mask; }; /* @@ -638,7 +640,7 @@ fail: tsk->pf_io_worker = worker; worker->task = tsk; - set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node)); + set_cpus_allowed_ptr(tsk, wqe->cpu_mask); tsk->flags |= PF_NO_SETAFFINITY; raw_spin_lock_irq(&wqe->lock); @@ -922,6 +924,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); if (!wqe) goto err; + if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL)) + goto err; + cpumask_copy(wqe->cpu_mask, cpumask_of_node(node)); wq->wqes[node] = wqe; wqe->node = alloc_node; wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND; @@ -947,8 +952,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) err: io_wq_put_hash(data->hash); cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); - for_each_node(node) + for_each_node(node) { + if (!wq->wqes[node]) + continue; + free_cpumask_var(wq->wqes[node]->cpu_mask); kfree(wq->wqes[node]); + } err_wq: kfree(wq); return ERR_PTR(ret); @@ -1018,6 +1027,7 @@ static void io_wq_destroy(struct io_wq *wq) .cancel_all = true, }; io_wqe_cancel_pending_work(wqe, &match); + free_cpumask_var(wqe->cpu_mask); kfree(wqe); } io_wq_put_hash(wq->hash); @@ -1032,23 +1042,49 @@ void io_wq_put_and_exit(struct io_wq *wq) io_wq_destroy(wq); } +struct online_data { + unsigned int cpu; + bool online; +}; + static bool io_wq_worker_affinity(struct io_worker *worker, void *data) { - set_cpus_allowed_ptr(worker->task, cpumask_of_node(worker->wqe->node)); + struct online_data *od = data; + if (od->online) + cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask); + else + cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask); return false; } +static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online) +{ + struct online_data od = { + .cpu = cpu, + .online = online + }; + int i; + + rcu_read_lock(); + for_each_node(i) + io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od); + rcu_read_unlock(); + return 0; +} + static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) { struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); - int i; - rcu_read_lock(); - for_each_node(i) - io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL); - rcu_read_unlock(); - return 0; + return __io_wq_cpu_online(wq, cpu, true); +} + +static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) +{ + struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); + + return __io_wq_cpu_online(wq, cpu, false); } static __init int io_wq_init(void) @@ -1056,7 +1092,7 @@ static __init int io_wq_init(void) int ret; ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", - io_wq_cpu_online, NULL); + io_wq_cpu_online, io_wq_cpu_offline); if (ret < 0) return ret; io_wq_online = ret; From fe76421d1da1dcdb3a2cd8428ac40106bff28bc0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Jun 2021 10:19:54 -0600 Subject: [PATCH 52/83] io_uring: allow user configurable IO thread CPU affinity io-wq defaults to per-node masks for IO workers. This works fine by default, but isn't particularly handy for workloads that prefer more specific affinities, for either performance or isolation reasons. This adds IORING_REGISTER_IOWQ_AFF that allows the user to pass in a CPU mask that is then applied to IO thread workers, and an IORING_UNREGISTER_IOWQ_AFF that simply resets the masks back to the default of per-node. Note that no care is given to existing IO threads, they will need to go through a reschedule before the affinity is correct if they are already running or sleeping. Signed-off-by: Jens Axboe --- fs/io-wq.c | 17 ++++++++++++ fs/io-wq.h | 2 ++ fs/io_uring.c | 51 +++++++++++++++++++++++++++++++++++ include/uapi/linux/io_uring.h | 4 +++ 4 files changed, 74 insertions(+) diff --git a/fs/io-wq.c b/fs/io-wq.c index 2af8e1df4646..bb4d3ee9592e 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -1087,6 +1087,23 @@ static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) return __io_wq_cpu_online(wq, cpu, false); } +int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask) +{ + int i; + + rcu_read_lock(); + for_each_node(i) { + struct io_wqe *wqe = wq->wqes[i]; + + if (mask) + cpumask_copy(wqe->cpu_mask, mask); + else + cpumask_copy(wqe->cpu_mask, cpumask_of_node(i)); + } + rcu_read_unlock(); + return 0; +} + static __init int io_wq_init(void) { int ret; diff --git a/fs/io-wq.h b/fs/io-wq.h index af2df0680ee2..02299cdcf55c 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -128,6 +128,8 @@ void io_wq_put_and_exit(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); +int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask); + static inline bool io_wq_is_hashed(struct io_wq_work *work) { return work->flags & IO_WQ_WORK_HASHED; diff --git a/fs/io_uring.c b/fs/io_uring.c index d916eb2cef09..46a25a7cb70a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9983,6 +9983,43 @@ static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, return -EINVAL; } +static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, + unsigned len) +{ + struct io_uring_task *tctx = current->io_uring; + cpumask_var_t new_mask; + int ret; + + if (!tctx || !tctx->io_wq) + return -EINVAL; + + if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) + return -ENOMEM; + + cpumask_clear(new_mask); + if (len > cpumask_size()) + len = cpumask_size(); + + if (copy_from_user(new_mask, arg, len)) { + free_cpumask_var(new_mask); + return -EFAULT; + } + + ret = io_wq_cpu_affinity(tctx->io_wq, new_mask); + free_cpumask_var(new_mask); + return ret; +} + +static int io_unregister_iowq_aff(struct io_ring_ctx *ctx) +{ + struct io_uring_task *tctx = current->io_uring; + + if (!tctx || !tctx->io_wq) + return -EINVAL; + + return io_wq_cpu_affinity(tctx->io_wq, NULL); +} + static bool io_register_op_must_quiesce(int op) { switch (op) { @@ -9998,6 +10035,8 @@ static bool io_register_op_must_quiesce(int op) case IORING_REGISTER_FILES_UPDATE2: case IORING_REGISTER_BUFFERS2: case IORING_REGISTER_BUFFERS_UPDATE: + case IORING_REGISTER_IOWQ_AFF: + case IORING_UNREGISTER_IOWQ_AFF: return false; default: return true; @@ -10137,6 +10176,18 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ret = io_register_rsrc_update(ctx, arg, nr_args, IORING_RSRC_BUFFER); break; + case IORING_REGISTER_IOWQ_AFF: + ret = -EINVAL; + if (!arg || !nr_args) + break; + ret = io_register_iowq_aff(ctx, arg, nr_args); + break; + case IORING_UNREGISTER_IOWQ_AFF: + ret = -EINVAL; + if (arg || nr_args) + break; + ret = io_unregister_iowq_aff(ctx); + break; default: ret = -EINVAL; break; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 162ff99ed2cb..f1f9ac114b51 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -306,6 +306,10 @@ enum { IORING_REGISTER_BUFFERS2 = 15, IORING_REGISTER_BUFFERS_UPDATE = 16, + /* set/clear io-wq thread affinities */ + IORING_REGISTER_IOWQ_AFF = 17, + IORING_UNREGISTER_IOWQ_AFF = 18, + /* this goes last */ IORING_REGISTER_LAST }; From e6ab8991c5d0b0deae0961dc22c0edd1dee328f5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:13:59 +0100 Subject: [PATCH 53/83] io_uring: fix false WARN_ONCE WARNING: CPU: 1 PID: 11749 at fs/io-wq.c:244 io_wqe_wake_worker fs/io-wq.c:244 [inline] WARNING: CPU: 1 PID: 11749 at fs/io-wq.c:244 io_wqe_enqueue+0x7f6/0x910 fs/io-wq.c:751 A WARN_ON_ONCE() in io_wqe_wake_worker() can be triggered by a valid userspace setup. Replace it with pr_warn. Reported-by: syzbot+ea2f1484cffe5109dc10@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f7ede342c3342c4c26668f5168e2993e38bbd99c.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index bb4d3ee9592e..843d4a7bcd6e 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -240,7 +240,8 @@ static void io_wqe_wake_worker(struct io_wqe *wqe, struct io_wqe_acct *acct) * Most likely an attempt to queue unbounded work on an io_wq that * wasn't setup with any unbounded workers. */ - WARN_ON_ONCE(!acct->max_workers); + if (unlikely(!acct->max_workers)) + pr_warn_once("io-wq is not configured for unbound workers"); rcu_read_lock(); ret = io_wqe_activate_free_worker(wqe); @@ -901,6 +902,8 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (WARN_ON_ONCE(!data->free_work || !data->do_work)) return ERR_PTR(-EINVAL); + if (WARN_ON_ONCE(!bounded)) + return ERR_PTR(-EINVAL); wq = kzalloc(struct_size(wq, wqes, nr_node_ids), GFP_KERNEL); if (!wq) From 2a2758f26df519fab011f49d53440382dda8e1a5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:00 +0100 Subject: [PATCH 54/83] io_uring: refactor io_submit_flush_completions() struct io_comp_state is always contained in struct io_ring_ctx, don't pass them into io_submit_flush_completions() separately, it makes the interface cleaner and simplifies it for the compiler. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/44d6ca57003a82484338e95197024dbd65a1b376.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 46a25a7cb70a..1e5ffc602e1f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1059,8 +1059,7 @@ static void __io_queue_sqe(struct io_kiocb *req); static void io_rsrc_put_work(struct work_struct *work); static void io_req_task_queue(struct io_kiocb *req); -static void io_submit_flush_completions(struct io_comp_state *cs, - struct io_ring_ctx *ctx); +static void io_submit_flush_completions(struct io_ring_ctx *ctx); static bool io_poll_remove_waitqs(struct io_kiocb *req); static int io_req_prep_async(struct io_kiocb *req); @@ -1879,7 +1878,7 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx) return; if (ctx->submit_state.comp.nr) { mutex_lock(&ctx->uring_lock); - io_submit_flush_completions(&ctx->submit_state.comp, ctx); + io_submit_flush_completions(ctx); mutex_unlock(&ctx->uring_lock); } percpu_ref_put(&ctx->refs); @@ -2127,9 +2126,9 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, list_add(&req->compl.list, &state->comp.free_list); } -static void io_submit_flush_completions(struct io_comp_state *cs, - struct io_ring_ctx *ctx) +static void io_submit_flush_completions(struct io_ring_ctx *ctx) { + struct io_comp_state *cs = &ctx->submit_state.comp; int i, nr = cs->nr; struct io_kiocb *req; struct req_batch rb; @@ -6451,7 +6450,7 @@ static void __io_queue_sqe(struct io_kiocb *req) cs->reqs[cs->nr++] = req; if (cs->nr == ARRAY_SIZE(cs->reqs)) - io_submit_flush_completions(cs, ctx); + io_submit_flush_completions(ctx); } else { io_put_req(req); } @@ -6651,7 +6650,7 @@ static void io_submit_state_end(struct io_submit_state *state, if (state->link.head) io_queue_sqe(state->link.head); if (state->comp.nr) - io_submit_flush_completions(&state->comp, ctx); + io_submit_flush_completions(ctx); if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); From c10d1f986b4e2a906862148c77a97f186cc08b9e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:01 +0100 Subject: [PATCH 55/83] io_uring: move creds from io-wq work to io_kiocb io-wq now doesn't have anything to do with creds now, so move ->creds from struct io_wq_work into request (aka struct io_kiocb). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8520c72ab8b8f4b96db12a228a2ab4c094ae64e1.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.h | 1 - fs/io_uring.c | 24 +++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/fs/io-wq.h b/fs/io-wq.h index 02299cdcf55c..3999ee58ff26 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -87,7 +87,6 @@ static inline void wq_list_del(struct io_wq_work_list *list, struct io_wq_work { struct io_wq_work_node list; - const struct cred *creds; unsigned flags; }; diff --git a/fs/io_uring.c b/fs/io_uring.c index 1e5ffc602e1f..1b8d0ad9a16d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -851,6 +851,8 @@ struct io_kiocb { struct hlist_node hash_node; struct async_poll *apoll; struct io_wq_work work; + const struct cred *creds; + /* store used ubuf, so we can prevent reloading */ struct io_mapped_ubuf *imu; }; @@ -1234,8 +1236,8 @@ static void io_prep_async_work(struct io_kiocb *req) const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_ring_ctx *ctx = req->ctx; - if (!req->work.creds) - req->work.creds = get_current_cred(); + if (!req->creds) + req->creds = get_current_cred(); req->work.list.next = NULL; req->work.flags = 0; @@ -1745,9 +1747,9 @@ static void io_dismantle_req(struct io_kiocb *req) percpu_ref_put(req->fixed_rsrc_refs); if (req->async_data) kfree(req->async_data); - if (req->work.creds) { - put_cred(req->work.creds); - req->work.creds = NULL; + if (req->creds) { + put_cred(req->creds); + req->creds = NULL; } } @@ -6139,8 +6141,8 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) const struct cred *creds = NULL; int ret; - if (req->work.creds && req->work.creds != current_cred()) - creds = override_creds(req->work.creds); + if (req->creds && req->creds != current_cred()) + creds = override_creds(req->creds); switch (req->opcode) { case IORING_OP_NOP: @@ -6532,7 +6534,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, atomic_set(&req->refs, 2); req->task = current; req->result = 0; - req->work.creds = NULL; + req->creds = NULL; /* enforce forwards compatibility on users */ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) @@ -6550,10 +6552,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, personality = READ_ONCE(sqe->personality); if (personality) { - req->work.creds = xa_load(&ctx->personalities, personality); - if (!req->work.creds) + req->creds = xa_load(&ctx->personalities, personality); + if (!req->creds) return -EINVAL; - get_cred(req->work.creds); + get_cred(req->creds); } state = &ctx->submit_state; From b8e64b530011162adda0e176150774d22326c50c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:02 +0100 Subject: [PATCH 56/83] io_uring: track request creds with a flag Currently, if req->creds is not NULL, then there are creds assigned. Track the invariant with a new flag in req->flags. No need to clear the field at init, and also cleanup can be efficiently moved into io_clean_op(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5f8baeb8d3b909487f555542350e2eac97005556.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1b8d0ad9a16d..af93f790a1da 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -718,6 +718,7 @@ enum { REQ_F_COMPLETE_INLINE_BIT, REQ_F_REISSUE_BIT, REQ_F_DONT_REISSUE_BIT, + REQ_F_CREDS_BIT, /* keep async read/write and isreg together and in order */ REQ_F_ASYNC_READ_BIT, REQ_F_ASYNC_WRITE_BIT, @@ -771,6 +772,8 @@ enum { REQ_F_ASYNC_WRITE = BIT(REQ_F_ASYNC_WRITE_BIT), /* regular file */ REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), + /* has creds assigned */ + REQ_F_CREDS = BIT(REQ_F_CREDS_BIT), }; struct async_poll { @@ -1236,8 +1239,10 @@ static void io_prep_async_work(struct io_kiocb *req) const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_ring_ctx *ctx = req->ctx; - if (!req->creds) + if (!(req->flags & REQ_F_CREDS)) { + req->flags |= REQ_F_CREDS; req->creds = get_current_cred(); + } req->work.list.next = NULL; req->work.flags = 0; @@ -1623,7 +1628,7 @@ static void io_req_complete_post(struct io_kiocb *req, long res, static inline bool io_req_needs_clean(struct io_kiocb *req) { return req->flags & (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | - REQ_F_POLLED | REQ_F_INFLIGHT); + REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS); } static void io_req_complete_state(struct io_kiocb *req, long res, @@ -1747,10 +1752,6 @@ static void io_dismantle_req(struct io_kiocb *req) percpu_ref_put(req->fixed_rsrc_refs); if (req->async_data) kfree(req->async_data); - if (req->creds) { - put_cred(req->creds); - req->creds = NULL; - } } /* must to be called somewhat shortly after putting a request */ @@ -6133,6 +6134,10 @@ static void io_clean_op(struct io_kiocb *req) atomic_dec(&tctx->inflight_tracked); req->flags &= ~REQ_F_INFLIGHT; } + if (req->flags & REQ_F_CREDS) { + put_cred(req->creds); + req->flags &= ~REQ_F_CREDS; + } } static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) @@ -6141,7 +6146,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) const struct cred *creds = NULL; int ret; - if (req->creds && req->creds != current_cred()) + if ((req->flags & REQ_F_CREDS) && req->creds != current_cred()) creds = override_creds(req->creds); switch (req->opcode) { @@ -6534,7 +6539,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, atomic_set(&req->refs, 2); req->task = current; req->result = 0; - req->creds = NULL; /* enforce forwards compatibility on users */ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) @@ -6556,6 +6560,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (!req->creds) return -EINVAL; get_cred(req->creds); + req->flags |= REQ_F_CREDS; } state = &ctx->submit_state; From 1dacb4df4ebe61ec2005d7ab82ee38ffa7125ee7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:03 +0100 Subject: [PATCH 57/83] io_uring: simplify iovec freeing in io_clean_op() We don't get REQ_F_NEED_CLEANUP for rw unless there is ->free_iovec set, so remove the optimisation of NULL checking it inline, kfree() will take care if that would ever be the case. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a233dc655d3d45bd4f69b73d55a61de46d914415.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index af93f790a1da..8543f1c37e2b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6092,8 +6092,8 @@ static void io_clean_op(struct io_kiocb *req) case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE: { struct io_async_rw *io = req->async_data; - if (io->free_iovec) - kfree(io->free_iovec); + + kfree(io->free_iovec); break; } case IORING_OP_RECVMSG: From c854357bc1b965e1e261c612d5be1297dfb3e406 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:04 +0100 Subject: [PATCH 58/83] io_uring: clean all flags in io_clean_op() at once Clean all flags in io_clean_op() in the end in one operation, will save us a couple of operation and binary size. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b8efe1f022a037f74e7fe497c69fb554d59bfeaf.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8543f1c37e2b..474705a97f29 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -109,6 +109,8 @@ #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ IOSQE_BUFFER_SELECT) +#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ + REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS) #define IO_TCTX_REFS_CACHE_NR (1U << 10) @@ -1627,8 +1629,7 @@ static void io_req_complete_post(struct io_kiocb *req, long res, static inline bool io_req_needs_clean(struct io_kiocb *req) { - return req->flags & (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | - REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS); + return req->flags & IO_REQ_CLEAN_FLAGS; } static void io_req_complete_state(struct io_kiocb *req, long res, @@ -6080,7 +6081,6 @@ static void io_clean_op(struct io_kiocb *req) kfree(req->sr_msg.kbuf); break; } - req->flags &= ~REQ_F_BUFFER_SELECTED; } if (req->flags & REQ_F_NEED_CLEANUP) { @@ -6121,7 +6121,6 @@ static void io_clean_op(struct io_kiocb *req) putname(req->unlink.filename); break; } - req->flags &= ~REQ_F_NEED_CLEANUP; } if ((req->flags & REQ_F_POLLED) && req->apoll) { kfree(req->apoll->double_poll); @@ -6132,12 +6131,11 @@ static void io_clean_op(struct io_kiocb *req) struct io_uring_task *tctx = req->task->io_uring; atomic_dec(&tctx->inflight_tracked); - req->flags &= ~REQ_F_INFLIGHT; } - if (req->flags & REQ_F_CREDS) { + if (req->flags & REQ_F_CREDS) put_cred(req->creds); - req->flags &= ~REQ_F_CREDS; - } + + req->flags &= ~IO_REQ_CLEAN_FLAGS; } static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) From a3dbdf54da80326fd12bc11ad75ecd699a82374f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:05 +0100 Subject: [PATCH 59/83] io_uring: refactor io_get_sequence() Clean up io_get_sequence() and add a comment describing the magic around sequence correction. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f55dc409936b8afa4698d24b8677a34d31077ccb.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 474705a97f29..c41e9a925fa2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5993,13 +5993,12 @@ static int io_req_prep_async(struct io_kiocb *req) static u32 io_get_sequence(struct io_kiocb *req) { - struct io_kiocb *pos; - struct io_ring_ctx *ctx = req->ctx; - u32 nr_reqs = 0; + u32 seq = req->ctx->cached_sq_head; - io_for_each_link(pos, req) - nr_reqs++; - return ctx->cached_sq_head - nr_reqs; + /* need original cached_sq_head, but it was increased for each req */ + io_for_each_link(req, req) + seq--; + return seq; } static bool io_drain_req(struct io_kiocb *req) From 3f18407dc6f2db0968daaa36c39a772c2c9f8ea7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:06 +0100 Subject: [PATCH 60/83] io_uring: inline __tctx_task_work() Inline __tctx_task_work() into tctx_task_work() in preparation for further optimisations. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f9c05c4bc9763af7bd8e25ebc3c5f7b6f69148f8.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 67 ++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c41e9a925fa2..dc71850d7a49 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1888,48 +1888,43 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx) percpu_ref_put(&ctx->refs); } -static bool __tctx_task_work(struct io_uring_task *tctx) -{ - struct io_ring_ctx *ctx = NULL; - struct io_wq_work_list list; - struct io_wq_work_node *node; - - if (wq_list_empty(&tctx->task_list)) - return false; - - spin_lock_irq(&tctx->task_lock); - list = tctx->task_list; - INIT_WQ_LIST(&tctx->task_list); - spin_unlock_irq(&tctx->task_lock); - - node = list.first; - while (node) { - struct io_wq_work_node *next = node->next; - struct io_kiocb *req; - - req = container_of(node, struct io_kiocb, io_task_work.node); - if (req->ctx != ctx) { - ctx_flush_and_put(ctx); - ctx = req->ctx; - percpu_ref_get(&ctx->refs); - } - - req->task_work.func(&req->task_work); - node = next; - } - - ctx_flush_and_put(ctx); - return list.first != NULL; -} - static void tctx_task_work(struct callback_head *cb) { - struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); + struct io_uring_task *tctx = container_of(cb, struct io_uring_task, + task_work); clear_bit(0, &tctx->task_state); - while (__tctx_task_work(tctx)) + while (!wq_list_empty(&tctx->task_list)) { + struct io_ring_ctx *ctx = NULL; + struct io_wq_work_list list; + struct io_wq_work_node *node; + + spin_lock_irq(&tctx->task_lock); + list = tctx->task_list; + INIT_WQ_LIST(&tctx->task_list); + spin_unlock_irq(&tctx->task_lock); + + node = list.first; + while (node) { + struct io_wq_work_node *next = node->next; + struct io_kiocb *req = container_of(node, struct io_kiocb, + io_task_work.node); + + if (req->ctx != ctx) { + ctx_flush_and_put(ctx); + ctx = req->ctx; + percpu_ref_get(&ctx->refs); + } + req->task_work.func(&req->task_work); + node = next; + } + + ctx_flush_and_put(ctx); + if (!list.first) + break; cond_resched(); + } } static int io_req_task_work_add(struct io_kiocb *req) From ebd0df2e63426bbd9ed50966e888c87eac88fc30 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:07 +0100 Subject: [PATCH 61/83] io_uring: optimise task_work submit flushing tctx_task_work() tries to fetch a next batch of requests, but before it would flush completions from the previous batch that may be sub-optimal. E.g. io_req_task_queue() executes a head of the link where all the linked may be enqueued through the same io_req_task_queue(). And there are more cases for that. Do the flushing at the end, so it can cache completions of several waves of a single tctx_task_work(), and do the flush at the very end. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3cac83934e4fbce520ff8025c3524398b3ae0270.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index dc71850d7a49..49f06484ba0e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1890,13 +1890,13 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx) static void tctx_task_work(struct callback_head *cb) { + struct io_ring_ctx *ctx = NULL; struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); clear_bit(0, &tctx->task_state); while (!wq_list_empty(&tctx->task_list)) { - struct io_ring_ctx *ctx = NULL; struct io_wq_work_list list; struct io_wq_work_node *node; @@ -1920,11 +1920,12 @@ static void tctx_task_work(struct callback_head *cb) node = next; } - ctx_flush_and_put(ctx); if (!list.first) break; cond_resched(); } + + ctx_flush_and_put(ctx); } static int io_req_task_work_add(struct io_kiocb *req) From c6538be9e4883d1371adaff45712b1b2172773dd Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:08 +0100 Subject: [PATCH 62/83] io_uring: refactor tctx task_work list splicing We don't need a full copy of tctx->task_list in tctx_task_work(), but only a first one, so just assign node directly. Taking into account that task_works are run in a context of a task, it's very unlikely to first see non-empty tctx->task_list and then splice it empty, can only happen with task_work cancellations that is not-normal slow path anyway. Hence, get rid of the check in the end, it's there not for validity but "performance" purposes. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d076c83fedb8253baf43acb23b8fafd7c5da1714.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 49f06484ba0e..51db0d80b67b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1897,15 +1897,13 @@ static void tctx_task_work(struct callback_head *cb) clear_bit(0, &tctx->task_state); while (!wq_list_empty(&tctx->task_list)) { - struct io_wq_work_list list; struct io_wq_work_node *node; spin_lock_irq(&tctx->task_lock); - list = tctx->task_list; + node = tctx->task_list.first; INIT_WQ_LIST(&tctx->task_list); spin_unlock_irq(&tctx->task_lock); - node = list.first; while (node) { struct io_wq_work_node *next = node->next; struct io_kiocb *req = container_of(node, struct io_kiocb, @@ -1919,9 +1917,6 @@ static void tctx_task_work(struct callback_head *cb) req->task_work.func(&req->task_work); node = next; } - - if (!list.first) - break; cond_resched(); } From 16f72070386fca59312bde696cff917bb04b183e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:09 +0100 Subject: [PATCH 63/83] io_uring: don't resched with empty task_list Entering tctx_task_work() with empty task_list is a strange scenario, that can happen only on rare occasion during task exit, so let's not check for task_list emptiness in advance and do it do-while style. The code still correct for the empty case, just would do extra work about which we don't care. Do extra step and do the check before cond_resched(), so we don't resched if have nothing to execute. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c4173e288e69793d03c7d7ce826f9d28afba718a.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 51db0d80b67b..55bc348ed8fe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1896,7 +1896,7 @@ static void tctx_task_work(struct callback_head *cb) clear_bit(0, &tctx->task_state); - while (!wq_list_empty(&tctx->task_list)) { + while (1) { struct io_wq_work_node *node; spin_lock_irq(&tctx->task_lock); @@ -1917,6 +1917,8 @@ static void tctx_task_work(struct callback_head *cb) req->task_work.func(&req->task_work); node = next; } + if (wq_list_empty(&tctx->task_list)) + break; cond_resched(); } From 7a778f9dc32deae4f748903f6f9169dc01cbcd28 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 17 Jun 2021 18:14:10 +0100 Subject: [PATCH 64/83] io_uring: improve in tctx_task_work() resubmission If task_state is cleared, io_req_task_work_add() will go the slow path adding a task_work, setting the task_state, waking up the task and so on. Not to mention it's expensive. tctx_task_work() first clears the state and then executes all the work items queued, so if any of them resubmits or adds new task_work items, it would unnecessarily go through the slow path of io_req_task_work_add(). Let's clear the ->task_state at the end. We still have to check ->task_list for emptiness afterward to synchronise with io_req_task_work_add(), do that, and set the state back if we're going to retry, because clearing not-ours task_state on the next iteration would be buggy. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1ef72cdac7022adf0cd7ce4bfe3bb5c82a62eb93.1623949695.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 55bc348ed8fe..fc8637f591a6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1894,8 +1894,6 @@ static void tctx_task_work(struct callback_head *cb) struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); - clear_bit(0, &tctx->task_state); - while (1) { struct io_wq_work_node *node; @@ -1917,8 +1915,14 @@ static void tctx_task_work(struct callback_head *cb) req->task_work.func(&req->task_work); node = next; } - if (wq_list_empty(&tctx->task_list)) - break; + if (wq_list_empty(&tctx->task_list)) { + clear_bit(0, &tctx->task_state); + if (wq_list_empty(&tctx->task_list)) + break; + /* another tctx_task_work() is enqueued, yield */ + if (test_and_set_bit(0, &tctx->task_state)) + break; + } cond_resched(); } From 997135017716c33f3405e86cca5da9567b40a08e Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Wed, 23 Jun 2021 11:50:11 -0700 Subject: [PATCH 65/83] io_uring: Fix race condition when sqp thread goes to sleep If an asynchronous completion happens before the task is preparing itself to wait and set its state to TASK_INTERRUPTIBLE, the completion will not wake up the sqp thread. Cc: stable@vger.kernel.org Signed-off-by: Olivier Langlois Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/d1419dc32ec6a97b453bee34dc03fa6a02797142.1624473200.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fc8637f591a6..7c545fa66f31 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6902,7 +6902,7 @@ static int io_sq_thread(void *data) } prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); - if (!io_sqd_events_pending(sqd)) { + if (!io_sqd_events_pending(sqd) && !io_run_task_work()) { needs_sched = true; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { io_ring_set_wakeup_flag(ctx); From 4ce8ad95f0afe927e9a29e7ad491274ebe3a8a7b Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Wed, 23 Jun 2021 11:50:18 -0700 Subject: [PATCH 66/83] io_uring: Create define to modify a SQPOLL parameter The magic number used to cap the number of entries extracted from an io_uring instance SQ before moving to the other instances is an interesting parameter to experiment with. A define has been created to make it easy to change its value from a single location. Signed-off-by: Olivier Langlois Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/b401640063e77ad3e9f921e09c9b3ac10a8bb923.1624473200.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7c545fa66f31..e7997f9bf879 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -89,6 +89,7 @@ #define IORING_MAX_ENTRIES 32768 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) +#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 /* * Shift of 9 is 512 entries, or exactly one page on 64-bit archs @@ -6797,8 +6798,8 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) to_submit = io_sqring_entries(ctx); /* if we're handling multiple rings, cap submit size for fairness */ - if (cap_entries && to_submit > 8) - to_submit = 8; + if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE) + to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE; if (!list_empty(&ctx->iopoll_list) || to_submit) { unsigned nr_events = 0; From 948e19479cb649587165243c6cc12d113c9cbbe0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:09:55 +0100 Subject: [PATCH 67/83] io_uring: don't change sqpoll creds if not needed SQPOLL doesn't need to change creds if it's not submitting requests. Move creds overriding into __io_sq_thread() after checking if there are SQEs pending. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c54368da2357ac539e0a333f7cfff70d5fb045b2.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e7997f9bf879..0cfbdee5da1a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6803,6 +6803,10 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) if (!list_empty(&ctx->iopoll_list) || to_submit) { unsigned nr_events = 0; + const struct cred *creds = NULL; + + if (ctx->sq_creds != current_cred()) + creds = override_creds(ctx->sq_creds); mutex_lock(&ctx->uring_lock); if (!list_empty(&ctx->iopoll_list)) @@ -6819,6 +6823,8 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) wake_up(&ctx->sqo_sq_wait); + if (creds) + revert_creds(creds); } return ret; @@ -6870,7 +6876,6 @@ static int io_sq_thread(void *data) mutex_lock(&sqd->lock); while (1) { - int ret; bool cap_entries, sqt_spin, needs_sched; if (io_sqd_events_pending(sqd) || signal_pending(current)) { @@ -6883,13 +6888,8 @@ static int io_sq_thread(void *data) sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { - const struct cred *creds = NULL; + int ret = __io_sq_thread(ctx, cap_entries); - if (ctx->sq_creds != current_cred()) - creds = override_creds(ctx->sq_creds); - ret = __io_sq_thread(ctx, cap_entries); - if (creds) - revert_creds(creds); if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; } From 1a924a808208c1880ef9f36b6bf98d27af045f06 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:09:56 +0100 Subject: [PATCH 68/83] io_uring: refactor io_sq_thread() Move needs_sched declaration into the block where it's used, so it's harder to misuse/wrongfully reuse. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e4a07db1353ee38b924dd1b45394cf8e746130b4.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0cfbdee5da1a..8b69982aa9e2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6876,7 +6876,7 @@ static int io_sq_thread(void *data) mutex_lock(&sqd->lock); while (1) { - bool cap_entries, sqt_spin, needs_sched; + bool cap_entries, sqt_spin = false; if (io_sqd_events_pending(sqd) || signal_pending(current)) { if (io_sqd_handle_event(sqd)) @@ -6885,7 +6885,6 @@ static int io_sq_thread(void *data) continue; } - sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { int ret = __io_sq_thread(ctx, cap_entries); @@ -6904,7 +6903,8 @@ static int io_sq_thread(void *data) prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); if (!io_sqd_events_pending(sqd) && !io_run_task_work()) { - needs_sched = true; + bool needs_sched = true; + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { io_ring_set_wakeup_flag(ctx); From fe7e325750299126b9cc86d3071af594b46c4518 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:09:57 +0100 Subject: [PATCH 69/83] io_uring: fix code style problems Fix a bunch of problems mostly found by checkpatch.pl Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/cfaf9a2f27b43934144fe9422a916bd327099f44.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8b69982aa9e2..7639bf3627c8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -173,7 +173,7 @@ struct io_rings { * Written by the application, shouldn't be modified by the * kernel. */ - u32 cq_flags; + u32 cq_flags; /* * Number of completion events lost because the queue was full; * this should be avoided by the application by making sure @@ -857,7 +857,7 @@ struct io_kiocb { struct hlist_node hash_node; struct async_poll *apoll; struct io_wq_work work; - const struct cred *creds; + const struct cred *creds; /* store used ubuf, so we can prevent reloading */ struct io_mapped_ubuf *imu; @@ -1707,7 +1707,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; - BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); + BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH); if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; @@ -2769,7 +2769,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, else io_rw_done(kiocb, ret); - if (check_reissue && req->flags & REQ_F_REISSUE) { + if (check_reissue && (req->flags & REQ_F_REISSUE)) { req->flags &= ~REQ_F_REISSUE; if (io_resubmit_prep(req)) { req_ref_get(req); @@ -3591,7 +3591,7 @@ static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) static int __io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_splice* sp = &req->splice; + struct io_splice *sp = &req->splice; unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) @@ -3645,7 +3645,7 @@ static int io_tee(struct io_kiocb *req, unsigned int issue_flags) static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_splice* sp = &req->splice; + struct io_splice *sp = &req->splice; sp->off_in = READ_ONCE(sqe->splice_off_in); sp->off_out = READ_ONCE(sqe->off); @@ -8567,6 +8567,7 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) ctx->cq_ev_fd = eventfd_ctx_fdget(fd); if (IS_ERR(ctx->cq_ev_fd)) { int ret = PTR_ERR(ctx->cq_ev_fd); + ctx->cq_ev_fd = NULL; return ret; } @@ -9347,9 +9348,8 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, io_cqring_overflow_flush(ctx, false); ret = -EOWNERDEAD; - if (unlikely(ctx->sq_data->thread == NULL)) { + if (unlikely(ctx->sq_data->thread == NULL)) goto out; - } if (flags & IORING_ENTER_SQ_WAKEUP) wake_up(&ctx->sq_data->wait); if (flags & IORING_ENTER_SQ_WAIT) { From 16340eab61a3ed1b5c983c19cfa9f51929b2beeb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:09:58 +0100 Subject: [PATCH 70/83] io_uring: update sqe layout build checks Add missing BUILD_BUG_SQE_ELEM() for ->buf_group verifying that SQE layout doesn't change. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1f9d21bd74599b856b3a632be4c23ffa184a3ef0.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7639bf3627c8..fe543d639ab0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10267,6 +10267,7 @@ static int __init io_uring_init(void) BUILD_BUG_SQE_ELEM(28, __u32, splice_flags); BUILD_BUG_SQE_ELEM(32, __u64, user_data); BUILD_BUG_SQE_ELEM(40, __u16, buf_index); + BUILD_BUG_SQE_ELEM(40, __u16, buf_group); BUILD_BUG_SQE_ELEM(42, __u16, personality); BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); @@ -10279,6 +10280,7 @@ static int __init io_uring_init(void) BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int)); + req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); return 0; From 9ba6a1c06279ce499fcf755d8134d679a1f3b4ed Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:09:59 +0100 Subject: [PATCH 71/83] io_uring: simplify struct io_uring_sqe layout Flatten struct io_uring_sqe, the last union is exactly 64B, so move them out of union { struct { ... }}, and decrease __pad2 size. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2e21ef7aed136293d654450bc3088973a8adc730.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index f1f9ac114b51..79126d5cd289 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -46,21 +46,17 @@ struct io_uring_sqe { __u32 unlink_flags; }; __u64 user_data; /* data to be passed back at completion time */ + /* pack this to avoid bogus arm OABI complaints */ union { - struct { - /* pack this to avoid bogus arm OABI complaints */ - union { - /* index into fixed buffers, if used */ - __u16 buf_index; - /* for grouped buffer selection */ - __u16 buf_group; - } __attribute__((packed)); - /* personality to use, if used */ - __u16 personality; - __s32 splice_fd_in; - }; - __u64 __pad2[3]; - }; + /* index into fixed buffers, if used */ + __u16 buf_index; + /* for grouped buffer selection */ + __u16 buf_group; + } __attribute__((packed)); + /* personality to use, if used */ + __u16 personality; + __s32 splice_fd_in; + __u64 __pad2[2]; }; enum { From 12dcb58ac785ee678f577e1502d966b538375aae Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 24 Jun 2021 15:10:00 +0100 Subject: [PATCH 72/83] io_uring: refactor io_openat2() Put do_filp_open() fail path of io_openat2() under a single if, deduplicating put_unused_fd(), making it look better and helping the hot path. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f4c84d25c049d0af2adc19c703bbfef607200209.1624543113.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fe543d639ab0..d7a68a2f2ec6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3851,27 +3851,26 @@ static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) goto err; file = do_filp_open(req->open.dfd, req->open.filename, &op); - /* only retry if RESOLVE_CACHED wasn't already set by application */ - if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) && - file == ERR_PTR(-EAGAIN)) { + if (IS_ERR(file)) { /* - * We could hang on to this 'fd', but seems like marginal - * gain for something that is now known to be a slower path. - * So just put it, and we'll get a new one when we retry. + * We could hang on to this 'fd' on retrying, but seems like + * marginal gain for something that is now known to be a slower + * path. So just put it, and we'll get a new one when we retry. */ put_unused_fd(ret); - return -EAGAIN; + + ret = PTR_ERR(file); + /* only retry if RESOLVE_CACHED wasn't already set by application */ + if (ret == -EAGAIN && + (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK))) + return -EAGAIN; + goto err; } - if (IS_ERR(file)) { - put_unused_fd(ret); - ret = PTR_ERR(file); - } else { - if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) - file->f_flags &= ~O_NONBLOCK; - fsnotify_open(file); - fd_install(ret, file); - } + if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) + file->f_flags &= ~O_NONBLOCK; + fsnotify_open(file); + fd_install(ret, file); err: putname(req->open.filename); req->flags &= ~REQ_F_NEED_CLEANUP; From ed7eb2592286ead7d3bfdf8adf65e65392167cc4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 23 Jun 2021 09:04:13 -0600 Subject: [PATCH 73/83] io_uring: add IOPOLL and reserved field checks to IORING_OP_RENAMEAT We can't support IOPOLL with non-pollable request types, and we should check for unused/reserved fields like we do for other request types. Fixes: 80a261fd0032 ("io_uring: add support for IORING_OP_RENAMEAT") Cc: stable@vger.kernel.org Reported-by: Dmitry Kadashev Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index d7a68a2f2ec6..02eda0575c56 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3463,6 +3463,10 @@ static int io_renameat_prep(struct io_kiocb *req, struct io_rename *ren = &req->rename; const char __user *oldf, *newf; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; + if (sqe->ioprio || sqe->buf_index) + return -EINVAL; if (unlikely(req->flags & REQ_F_FIXED_FILE)) return -EBADF; From 22634bc5620d29765e5199c7b230a372c7ddcda2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 23 Jun 2021 09:07:45 -0600 Subject: [PATCH 74/83] io_uring: add IOPOLL and reserved field checks to IORING_OP_UNLINKAT We can't support IOPOLL with non-pollable request types, and we should check for unused/reserved fields like we do for other request types. Fixes: 14a1143b68ee ("io_uring: add support for IORING_OP_UNLINKAT") Cc: stable@vger.kernel.org Reported-by: Dmitry Kadashev Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 02eda0575c56..a0bb3cc7d3bb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3514,6 +3514,10 @@ static int io_unlinkat_prep(struct io_kiocb *req, struct io_unlink *un = &req->unlink; const char __user *fname; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; + if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index) + return -EINVAL; if (unlikely(req->flags & REQ_F_FIXED_FILE)) return -EBADF; From 59b735aeeb0f23a760bc21f1c5a1ab6c79e9fe0e Mon Sep 17 00:00:00 2001 From: Olivier Langlois Date: Tue, 22 Jun 2021 05:17:39 -0700 Subject: [PATCH 75/83] io_uring: reduce latency by reissueing the operation It is quite frequent that when an operation fails and returns EAGAIN, the data becomes available between that failure and the call to vfs_poll() done by io_arm_poll_handler(). Detecting the situation and reissuing the operation is much faster than going ahead and push the operation to the io-wq. Performance improvement testing has been performed with: Single thread, 1 TCP connection receiving a 5 Mbps stream, no sqpoll. 4 measurements have been taken: 1. The time it takes to process a read request when data is already available 2. The time it takes to process by calling twice io_issue_sqe() after vfs_poll() indicated that data was available 3. The time it takes to execute io_queue_async_work() 4. The time it takes to complete a read request asynchronously 2.25% of all the read operations did use the new path. ready data (baseline) avg 3657.94182918628 min 580 max 20098 stddev 1213.15975908162 reissue completion average 7882.67567567568 min 2316 max 28811 stddev 1982.79172973284 insert io-wq time average 8983.82276995305 min 3324 max 87816 stddev 2551.60056552038 async time completion average 24670.4758861127 min 10758 max 102612 stddev 3483.92416873804 Conclusion: On average reissuing the sqe with the patch code is 1.1uSec faster and in the worse case scenario 59uSec faster than placing the request on io-wq On average completion time by reissuing the sqe with the patch code is 16.79uSec faster and in the worse case scenario 73.8uSec faster than async completion. Signed-off-by: Olivier Langlois Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/9e8441419bb1b8f3c3fcc607b2713efecdef2136.1624364038.git.olivier@trillion01.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a0bb3cc7d3bb..ab30c0a0c09b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5160,7 +5160,13 @@ static __poll_t __io_arm_poll_handler(struct io_kiocb *req, return mask; } -static bool io_arm_poll_handler(struct io_kiocb *req) +enum { + IO_APOLL_OK, + IO_APOLL_ABORTED, + IO_APOLL_READY +}; + +static int io_arm_poll_handler(struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_ring_ctx *ctx = req->ctx; @@ -5170,22 +5176,22 @@ static bool io_arm_poll_handler(struct io_kiocb *req) int rw; if (!req->file || !file_can_poll(req->file)) - return false; + return IO_APOLL_ABORTED; if (req->flags & REQ_F_POLLED) - return false; + return IO_APOLL_ABORTED; if (def->pollin) rw = READ; else if (def->pollout) rw = WRITE; else - return false; + return IO_APOLL_ABORTED; /* if we can't nonblock try, then no point in arming a poll handler */ if (!io_file_supports_async(req, rw)) - return false; + return IO_APOLL_ABORTED; apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); if (unlikely(!apoll)) - return false; + return IO_APOLL_ABORTED; apoll->double_poll = NULL; req->flags |= REQ_F_POLLED; @@ -5211,12 +5217,14 @@ static bool io_arm_poll_handler(struct io_kiocb *req) if (ret || ipt.error) { io_poll_remove_double(req); spin_unlock_irq(&ctx->completion_lock); - return false; + if (ret) + return IO_APOLL_READY; + return IO_APOLL_ABORTED; } spin_unlock_irq(&ctx->completion_lock); trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data, mask, apoll->poll.events); - return true; + return IO_APOLL_OK; } static bool __io_poll_remove_one(struct io_kiocb *req, @@ -6445,6 +6453,7 @@ static void __io_queue_sqe(struct io_kiocb *req) struct io_kiocb *linked_timeout = io_prep_linked_timeout(req); int ret; +issue_sqe: ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); /* @@ -6464,12 +6473,16 @@ static void __io_queue_sqe(struct io_kiocb *req) io_put_req(req); } } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) { - if (!io_arm_poll_handler(req)) { + switch (io_arm_poll_handler(req)) { + case IO_APOLL_READY: + goto issue_sqe; + case IO_APOLL_ABORTED: /* * Queued up for async execution, worker will release * submit reference when the iocb is actually submitted. */ io_queue_async_work(req); + break; } } else { io_req_complete_failed(req, ret); From b2d9c3da77115b5172749dec20312651e67e0adf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 26 Jun 2021 21:40:44 +0100 Subject: [PATCH 76/83] io_uring: refactor io_arm_poll_handler() gcc 11 goes a weird path and duplicates most of io_arm_poll_handler() for READ and WRITE cases. Help it and move all pollin vs pollout specific bits under a single if-else, so there is no temptation for this kind of unfolding. before vs after: text data bss dec hex filename 85362 12650 8 98020 17ee4 ./fs/io_uring.o 85186 12650 8 97844 17e34 ./fs/io_uring.o Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1deea0037293a922a0358e2958384b2e42437885.1624739600.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ab30c0a0c09b..bfba7558ea86 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5172,19 +5172,29 @@ static int io_arm_poll_handler(struct io_kiocb *req) struct io_ring_ctx *ctx = req->ctx; struct async_poll *apoll; struct io_poll_table ipt; - __poll_t mask, ret; + __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI; int rw; if (!req->file || !file_can_poll(req->file)) return IO_APOLL_ABORTED; if (req->flags & REQ_F_POLLED) return IO_APOLL_ABORTED; - if (def->pollin) - rw = READ; - else if (def->pollout) - rw = WRITE; - else + if (!def->pollin && !def->pollout) return IO_APOLL_ABORTED; + + if (def->pollin) { + rw = READ; + mask |= POLLIN | POLLRDNORM; + + /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ + if ((req->opcode == IORING_OP_RECVMSG) && + (req->sr_msg.msg_flags & MSG_ERRQUEUE)) + mask &= ~POLLIN; + } else { + rw = WRITE; + mask |= POLLOUT | POLLWRNORM; + } + /* if we can't nonblock try, then no point in arming a poll handler */ if (!io_file_supports_async(req, rw)) return IO_APOLL_ABORTED; @@ -5193,23 +5203,8 @@ static int io_arm_poll_handler(struct io_kiocb *req) if (unlikely(!apoll)) return IO_APOLL_ABORTED; apoll->double_poll = NULL; - - req->flags |= REQ_F_POLLED; req->apoll = apoll; - - mask = EPOLLONESHOT; - if (def->pollin) - mask |= POLLIN | POLLRDNORM; - if (def->pollout) - mask |= POLLOUT | POLLWRNORM; - - /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ - if ((req->opcode == IORING_OP_RECVMSG) && - (req->sr_msg.msg_flags & MSG_ERRQUEUE)) - mask &= ~POLLIN; - - mask |= POLLERR | POLLPRI; - + req->flags |= REQ_F_POLLED; ipt.pt._qproc = io_async_queue_proc; ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, From dd432ea5204eeb92a2abf246ce518e68679da772 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 26 Jun 2021 21:40:45 +0100 Subject: [PATCH 77/83] io_uring: mainstream sqpoll task_work running task_works are widely used, so place io_run_task_work() directly into the main path of io_sq_thread(), and remove it from other places where it's not needed anymore. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/24eb5e35d519c590d3dffbd694b4c61a5fe49029.1624739600.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bfba7558ea86..80b7a6f04841 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6868,7 +6868,6 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd) cond_resched(); mutex_lock(&sqd->lock); } - io_run_task_work(); return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); } @@ -6897,7 +6896,6 @@ static int io_sq_thread(void *data) if (io_sqd_handle_event(sqd)) break; timeout = jiffies + sqd->sq_thread_idle; - continue; } cap_entries = !list_is_singular(&sqd->ctx_list); @@ -6907,9 +6905,10 @@ static int io_sq_thread(void *data) if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; } + if (io_run_task_work()) + sqt_spin = true; if (sqt_spin || !time_after(jiffies, timeout)) { - io_run_task_work(); cond_resched(); if (sqt_spin) timeout = jiffies + sqd->sq_thread_idle; @@ -6917,7 +6916,7 @@ static int io_sq_thread(void *data) } prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); - if (!io_sqd_events_pending(sqd) && !io_run_task_work()) { + if (!io_sqd_events_pending(sqd) && !current->task_works) { bool needs_sched = true; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { From e5dc480d4ed9884274e95c757fa2d2e9cc1047ee Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 26 Jun 2021 21:40:46 +0100 Subject: [PATCH 78/83] io_uring: remove not needed PF_EXITING check Since cancellation got moved before exit_signals(), there is no one left who can call io_run_task_work() with PF_EXIING set, so remove the check. Note that __io_req_task_submit() still needs a similar check. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f7f305ececb1e6044ea649fb983ca754805bb884.1624739600.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 80b7a6f04841..ce88ad58955a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2235,12 +2235,6 @@ static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req) static inline bool io_run_task_work(void) { - /* - * Not safe to run on exiting task, and the task_work handling will - * not add work to such a task. - */ - if (unlikely(current->flags & PF_EXITING)) - return false; if (current->task_works) { __set_current_state(TASK_RUNNING); task_work_run(); @@ -9020,7 +9014,8 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, ret |= io_cancel_defer_files(ctx, task, cancel_all); ret |= io_poll_remove_all(ctx, task, cancel_all); ret |= io_kill_timeouts(ctx, task, cancel_all); - ret |= io_run_task_work(); + if (task) + ret |= io_run_task_work(); ret |= io_run_ctx_fallback(ctx); if (!ret) break; From 4cfb25bf8877c947e5ae4875e387babe87e12afa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 26 Jun 2021 21:40:47 +0100 Subject: [PATCH 79/83] io_uring: optimise hot path restricted checks Move likely/unlikely from io_check_restriction() to specifically ctx->restricted check, because doesn't do what it supposed to and make the common path take an extra jump. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/22bf70d0a543dfc935d7276bdc73081784e30698.1624739600.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ce88ad58955a..a5df65f6f9ab 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6506,7 +6506,7 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx, struct io_kiocb *req, unsigned int sqe_flags) { - if (!ctx->restricted) + if (likely(!ctx->restricted)) return true; if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) @@ -6549,7 +6549,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, return -EINVAL; if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; - if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) + if (!io_check_restriction(ctx, req, sqe_flags)) return -EACCES; if ((sqe_flags & IOSQE_BUFFER_SELECT) && From 5182ed2e332e8e11fa3c1649ef6d6546ccca64d0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 26 Jun 2021 21:40:48 +0100 Subject: [PATCH 80/83] io_uring: refactor io_submit_flush_completions Don't init req_batch before we actually need it. Also, add a small clean up for req declaration. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/ad85512e12bd3a20d521e9782750300970e5afc8.1624739600.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a5df65f6f9ab..b1620fbd69eb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2132,22 +2132,22 @@ static void io_submit_flush_completions(struct io_ring_ctx *ctx) { struct io_comp_state *cs = &ctx->submit_state.comp; int i, nr = cs->nr; - struct io_kiocb *req; struct req_batch rb; - io_init_req_batch(&rb); spin_lock_irq(&ctx->completion_lock); for (i = 0; i < nr; i++) { - req = cs->reqs[i]; + struct io_kiocb *req = cs->reqs[i]; + __io_cqring_fill_event(ctx, req->user_data, req->result, req->compl.cflags); } io_commit_cqring(ctx); spin_unlock_irq(&ctx->completion_lock); - io_cqring_ev_posted(ctx); + + io_init_req_batch(&rb); for (i = 0; i < nr; i++) { - req = cs->reqs[i]; + struct io_kiocb *req = cs->reqs[i]; /* submission and completion refs */ if (req_ref_sub_and_test(req, 2)) From 99ebe4efbd3882422db1fd6a1b477291ea8bdab7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 26 Jun 2021 21:40:49 +0100 Subject: [PATCH 81/83] io_uring: pre-initialise some of req fields Most of requests are allocated from an internal cache, so it's waste of time fully initialising them every time. Instead, let's pre-init some of the fields we can during initial allocation (e.g. kmalloc(), see io_alloc_req()) and keep them valid on request recycling. There are four of them in this patch: ->ctx is always stays the same ->link is NULL on free, it's an invariant ->result is not even needed to init, just a precaution ->async_data we now clean in io_dismantle_req() as it's likely to never be allocated. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/892ba0e71309bba9fe9e0142472330bbf9d8f05d.1624739600.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b1620fbd69eb..b14de92832e1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1711,7 +1711,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; - int ret; + int ret, i; if (io_flush_cached_reqs(ctx)) goto got_req; @@ -1729,6 +1729,20 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) return NULL; ret = 1; } + + /* + * Don't initialise the fields below on every allocation, but + * do that in advance and keep valid on free. + */ + for (i = 0; i < ret; i++) { + struct io_kiocb *req = state->reqs[i]; + + req->ctx = ctx; + req->link = NULL; + req->async_data = NULL; + /* not necessary, but safer to zero */ + req->result = 0; + } state->free_reqs = ret; } got_req: @@ -1752,8 +1766,10 @@ static void io_dismantle_req(struct io_kiocb *req) io_put_file(req->file); if (req->fixed_rsrc_refs) percpu_ref_put(req->fixed_rsrc_refs); - if (req->async_data) + if (req->async_data) { kfree(req->async_data); + req->async_data = NULL; + } } /* must to be called somewhat shortly after putting a request */ @@ -6534,15 +6550,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, /* same numerical values with corresponding REQ_F_*, safe to copy */ req->flags = sqe_flags = READ_ONCE(sqe->flags); req->user_data = READ_ONCE(sqe->user_data); - req->async_data = NULL; req->file = NULL; - req->ctx = ctx; - req->link = NULL; req->fixed_rsrc_refs = NULL; /* one is dropped after submission, the other at completion */ atomic_set(&req->refs, 2); req->task = current; - req->result = 0; /* enforce forwards compatibility on users */ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) From 915b3dde9b72cb4f531b04208daafcd0a257b847 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Mon, 28 Jun 2021 05:37:30 +0800 Subject: [PATCH 82/83] io_uring: spin in iopoll() only when reqs are in a single queue We currently spin in iopoll() when requests to be iopolled are for same file(device), while one device may have multiple hardware queues. given an example: hw_queue_0 | hw_queue_1 req(30us) req(10us) If we first spin on iopolling for the hw_queue_0. the avg latency would be (30us + 30us) / 2 = 30us. While if we do round robin, the avg latency would be (30us + 10us) / 2 = 20us since we reap the request in hw_queue_1 in time. So it's better to do spinning only when requests are in same hardware queue. Signed-off-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b14de92832e1..67099bb99a02 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -434,7 +434,7 @@ struct io_ring_ctx { struct list_head iopoll_list; struct hlist_head *cancel_hash; unsigned cancel_hash_bits; - bool poll_multi_file; + bool poll_multi_queue; } ____cacheline_aligned_in_smp; struct io_restriction restrictions; @@ -2314,7 +2314,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, * Only spin for completions if we don't have multiple devices hanging * off our complete list, and we're under the requested amount. */ - spin = !ctx->poll_multi_file && *nr_events < min; + spin = !ctx->poll_multi_queue && *nr_events < min; ret = 0; list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) { @@ -2553,14 +2553,22 @@ static void io_iopoll_req_issued(struct io_kiocb *req) * different devices. */ if (list_empty(&ctx->iopoll_list)) { - ctx->poll_multi_file = false; - } else if (!ctx->poll_multi_file) { + ctx->poll_multi_queue = false; + } else if (!ctx->poll_multi_queue) { struct io_kiocb *list_req; + unsigned int queue_num0, queue_num1; list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, inflight_entry); - if (list_req->file != req->file) - ctx->poll_multi_file = true; + + if (list_req->file != req->file) { + ctx->poll_multi_queue = true; + } else { + queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie); + queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie); + if (queue_num0 != queue_num1) + ctx->poll_multi_queue = true; + } } /* From e149bd742b2db6a63fc078b1ea6843dc9b22678d Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Mon, 28 Jun 2021 05:48:05 +0800 Subject: [PATCH 83/83] io_uring: code clean for kiocb_done() A simple code clean for kiocb_done() Signed-off-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 67099bb99a02..e55b21fc0ab2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2782,7 +2782,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, if (req->flags & REQ_F_CUR_POS) req->file->f_pos = kiocb->ki_pos; - if (ret >= 0 && kiocb->ki_complete == io_complete_rw) + if (ret >= 0 && check_reissue) __io_complete_rw(req, ret, 0, issue_flags); else io_rw_done(kiocb, ret);