Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-25 13:41:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

io_uring: convert poll path to use io_cmd_type

Browse Source 
Remove struct io_poll_iocb from io_kiocb, and convert the poll path to
use the io_cmd_type approach instead.

While at it, rename io_poll_iocb to io_poll which is consistent with the
other request type private structures.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2022-05-24 09:13:46 -06:00
parent 3c306fb2f9
commit 8d4388d116
1 changed files with 28 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
io_uring/io_uring.c
View File

@ -556,7 +556,7 @@ struct io_uring_task {
* First field must be the file pointer in all the * First field must be the file pointer in all the
* iocb unions! See also 'struct kiocb' in <linux/fs.h> * iocb unions! See also 'struct kiocb' in <linux/fs.h>
*/ */
struct io_poll_iocb { struct io_poll {
struct file *file; struct file *file;
struct wait_queue_head *head; struct wait_queue_head *head;
__poll_t events; __poll_t events;
@ -919,8 +919,8 @@ enum {
}; };
struct async_poll { struct async_poll {
struct io_poll_iocb poll; struct io_poll poll;
struct io_poll_iocb *double_poll; struct io_poll *double_poll;
}; };
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked);
@ -976,7 +976,6 @@ struct io_kiocb {
*/ */
struct file *file; struct file *file;
struct io_cmd_data cmd; struct io_cmd_data cmd;
struct io_poll_iocb poll;
struct io_poll_update poll_update; struct io_poll_update poll_update;
struct io_accept accept; struct io_accept accept;
struct io_sync sync; struct io_sync sync;
@ -6579,7 +6578,7 @@ static void io_poll_mark_cancelled(struct io_kiocb *req)
atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);
} }
static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) static struct io_poll *io_poll_get_double(struct io_kiocb *req)
{ {
/* pure poll stashes this in ->async_data, poll driven retry elsewhere */ /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
if (req->opcode == IORING_OP_POLL_ADD) if (req->opcode == IORING_OP_POLL_ADD)
@ -6587,10 +6586,10 @@ static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
return req->apoll->double_poll; return req->apoll->double_poll;
} }
static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) static struct io_poll *io_poll_get_single(struct io_kiocb *req)
{ {
if (req->opcode == IORING_OP_POLL_ADD) if (req->opcode == IORING_OP_POLL_ADD)
return &req->poll; return io_kiocb_to_cmd(req);
return &req->apoll->poll; return &req->apoll->poll;
} }
@ -6603,7 +6602,7 @@ static void io_poll_req_insert(struct io_kiocb *req)
hlist_add_head(&req->hash_node, list); hlist_add_head(&req->hash_node, list);
} }
static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, static void io_init_poll_iocb(struct io_poll *poll, __poll_t events,
wait_queue_func_t wake_func) wait_queue_func_t wake_func)
{ {
poll->head = NULL; poll->head = NULL;
@ -6614,7 +6613,7 @@ static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
init_waitqueue_func_entry(&poll->wait, wake_func); init_waitqueue_func_entry(&poll->wait, wake_func);
} }
static inline void io_poll_remove_entry(struct io_poll_iocb *poll) static inline void io_poll_remove_entry(struct io_poll *poll)
{ {
struct wait_queue_head *head = smp_load_acquire(&poll->head); struct wait_queue_head *head = smp_load_acquire(&poll->head);
@ -6740,7 +6739,9 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
return; return;
if (!ret) { if (!ret) {
req->cqe.res = mangle_poll(req->cqe.res & req->poll.events); struct io_poll *poll = io_kiocb_to_cmd(req);
req->cqe.res = mangle_poll(req->cqe.res & poll->events);
} else { } else {
req->cqe.res = ret; req->cqe.res = ret;
req_set_fail(req); req_set_fail(req);
@ -6816,8 +6817,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
void *key) void *key)
{ {
struct io_kiocb *req = wqe_to_req(wait); struct io_kiocb *req = wqe_to_req(wait);
struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, struct io_poll *poll = container_of(wait, struct io_poll, wait);
wait);
__poll_t mask = key_to_poll(key); __poll_t mask = key_to_poll(key);
if (unlikely(mask & POLLFREE)) { if (unlikely(mask & POLLFREE)) {
@ -6863,20 +6863,20 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
return 1; return 1;
} }
static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,
struct wait_queue_head *head, struct wait_queue_head *head,
struct io_poll_iocb **poll_ptr) struct io_poll **poll_ptr)
{ {
struct io_kiocb *req = pt->req; struct io_kiocb *req = pt->req;
unsigned long wqe_private = (unsigned long) req; unsigned long wqe_private = (unsigned long) req;
/* /*
* The file being polled uses multiple waitqueues for poll handling * The file being polled uses multiple waitqueues for poll handling
* (e.g. one for read, one for write). Setup a separate io_poll_iocb * (e.g. one for read, one for write). Setup a separate io_poll
* if this happens. * if this happens.
*/ */
if (unlikely(pt->nr_entries)) { if (unlikely(pt->nr_entries)) {
struct io_poll_iocb *first = poll; struct io_poll *first = poll;
/* double add on the same waitqueue head, ignore */ /* double add on the same waitqueue head, ignore */
if (first->head == head) if (first->head == head)
@ -6918,13 +6918,14 @@ static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
struct poll_table_struct *p) struct poll_table_struct *p)
{ {
struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
struct io_poll *poll = io_kiocb_to_cmd(pt->req);
__io_queue_proc(&pt->req->poll, pt, head, __io_queue_proc(poll, pt, head,
(struct io_poll_iocb **) &pt->req->async_data); (struct io_poll **) &pt->req->async_data);
} }
static int __io_arm_poll_handler(struct io_kiocb *req, static int __io_arm_poll_handler(struct io_kiocb *req,
struct io_poll_iocb *poll, struct io_poll *poll,
struct io_poll_table *ipt, __poll_t mask) struct io_poll_table *ipt, __poll_t mask)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
@ -7207,7 +7208,7 @@ static int io_poll_remove_prep(struct io_kiocb *req,
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{ {
struct io_poll_iocb *poll = &req->poll; struct io_poll *poll = io_kiocb_to_cmd(req);
u32 flags; u32 flags;
if (sqe->buf_index || sqe->off || sqe->addr) if (sqe->buf_index || sqe->off || sqe->addr)
@ -7225,13 +7226,13 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_poll_iocb *poll = &req->poll; struct io_poll *poll = io_kiocb_to_cmd(req);
struct io_poll_table ipt; struct io_poll_table ipt;
int ret; int ret;
ipt.pt._qproc = io_poll_queue_proc; ipt.pt._qproc = io_poll_queue_proc;
ret = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events); ret = __io_arm_poll_handler(req, poll, &ipt, poll->events);
if (!ret && ipt.error) if (!ret && ipt.error)
req_set_fail(req); req_set_fail(req);
ret = ret ?: ipt.error; ret = ret ?: ipt.error;
@ -7260,9 +7261,11 @@ static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
if (req->poll_update.update_events || req->poll_update.update_user_data) { if (req->poll_update.update_events || req->poll_update.update_user_data) {
/* only mask one event flags, keep behavior flags */ /* only mask one event flags, keep behavior flags */
if (req->poll_update.update_events) { if (req->poll_update.update_events) {
preq->poll.events &= ~0xffff; struct io_poll *poll = io_kiocb_to_cmd(preq);
preq->poll.events |= req->poll_update.events & 0xffff;
preq->poll.events |= IO_POLL_UNMASK; poll->events &= ~0xffff;
poll->events |= req->poll_update.events & 0xffff;
poll->events |= IO_POLL_UNMASK;
} }
if (req->poll_update.update_user_data) if (req->poll_update.update_user_data)
preq->cqe.user_data = req->poll_update.new_user_data; preq->cqe.user_data = req->poll_update.new_user_data;