Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-22 12:11:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

io_uring: ensure iopoll runs local task work as well

Browse Source 
Combine the two checks we have for task_work running and whether or not
we need to shuffle the mutex into one, so we unify how task_work is run
in the iopoll loop. This helps ensure that local task_work is run when
needed, and also optimizes that path to avoid a mutex shuffle if it's
not needed.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2022-09-03 09:52:01 -06:00
parent 8ac5d85a89
commit dac6a0eae7
2 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
io_uring/io_uring.c
View File

@ -1428,12 +1428,18 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
* forever, while the workqueue is stuck trying to acquire the * forever, while the workqueue is stuck trying to acquire the
* very same mutex. * very same mutex.
*/ */
if (wq_list_empty(&ctx->iopoll_list)) { if (wq_list_empty(&ctx->iopoll_list) ||
io_task_work_pending(ctx)) {
if (!llist_empty(&ctx->work_llist))
__io_run_local_work(ctx, true);
if (task_work_pending(current) ||
wq_list_empty(&ctx->iopoll_list)) {
u32 tail = ctx->cached_cq_tail; u32 tail = ctx->cached_cq_tail;
mutex_unlock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock);
ret = io_run_task_work_ctx(ctx); ret = io_run_task_work();
mutex_lock(&ctx->uring_lock); mutex_lock(&ctx->uring_lock);
if (ret < 0) if (ret < 0)
break; break;
@ -1442,11 +1448,6 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
wq_list_empty(&ctx->iopoll_list)) wq_list_empty(&ctx->iopoll_list))
break; break;
} }
if (task_work_pending(current)) {
mutex_unlock(&ctx->uring_lock);
io_run_task_work();
mutex_lock(&ctx->uring_lock);
} }
ret = io_do_iopoll(ctx, !min); ret = io_do_iopoll(ctx, !min);
if (ret < 0) if (ret < 0)

6
io_uring/io_uring.h
View File

@ -236,6 +236,12 @@ static inline int io_run_task_work(void)
return 0; return 0;
} }
static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
{
return test_thread_flag(TIF_NOTIFY_SIGNAL) ||
!wq_list_empty(&ctx->work_llist);
}
static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
{ {
int ret = 0; int ret = 0;