leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

io_uring: optimise out req->opcode reloading

Browse Source 
Looking at the assembly, the compiler decided to reload req->opcode in
io_op_defs[opcode].needs_file instead of one it had in a register, so
store it in a temp variable so it can be optimised out. Also move the
personality block later, it's better for spilling/etc. as it only
depends on @sqe, which we're keeping anyway.

By the way, zero req->opcode if it over IORING_OP_LAST, not a problem,
at the moment but is safer.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/6ba869f5f8b7b0f991c87fdf089f0abf87cbe06b.1633532552.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2021-10-06 16:06:49 +01:00 committed by Jens Axboe
parent 5a158c6b0d
commit 4a04d1d148
1 changed files with 17 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
fs/io_uring.c
View File

@ -6975,9 +6975,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
{
unsigned int sqe_flags;
int personality;
u8 opcode;
/* req is partially pre-initialised, see io_preinit_req() */
req->opcode = READ_ONCE(sqe->opcode);
req->opcode = opcode = READ_ONCE(sqe->opcode);
/* same numerical values with corresponding REQ_F_*, safe to copy */
req->flags = sqe_flags = READ_ONCE(sqe->flags);
req->user_data = READ_ONCE(sqe->user_data);
@ -6985,14 +6986,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
req->fixed_rsrc_refs = NULL;
req->task = current;
if (unlikely(req->opcode >= IORING_OP_LAST))
if (unlikely(opcode >= IORING_OP_LAST)) {
req->opcode = 0;
return -EINVAL;
}
if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
/* enforce forwards compatibility on users */
if (sqe_flags & ~SQE_VALID_FLAGS)
return -EINVAL;
if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
!io_op_defs[req->opcode].buffer_select)
!io_op_defs[opcode].buffer_select)
return -EOPNOTSUPP;
if (sqe_flags & IOSQE_IO_DRAIN)
io_init_req_drain(req);
@ -7011,23 +7014,14 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
}
}
personality = READ_ONCE(sqe->personality);
if (personality) {
req->creds = xa_load(&ctx->personalities, personality);
if (!req->creds)
return -EINVAL;
get_cred(req->creds);
req->flags |= REQ_F_CREDS;
}
if (io_op_defs[req->opcode].needs_file) {
if (io_op_defs[opcode].needs_file) {
struct io_submit_state *state = &ctx->submit_state;
/*
* Plug now if we have more than 2 IO left after this, and the
* target is potentially a read/write to block based storage.
*/
if (state->need_plug && io_op_defs[req->opcode].plug) {
if (state->need_plug && io_op_defs[opcode].plug) {
state->plug_started = true;
state->need_plug = false;
blk_start_plug(&state->plug);
@ -7039,6 +7033,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
return -EBADF;
}
personality = READ_ONCE(sqe->personality);
if (personality) {
req->creds = xa_load(&ctx->personalities, personality);
if (!req->creds)
return -EINVAL;
get_cred(req->creds);
req->flags |= REQ_F_CREDS;
}
return io_req_prep(req, sqe);
}