io_uring: remove io_identity
We are no longer grabbing state, so no need to maintain an IO identity that we COW if there are changes. Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe
parent
44526bedc2
commit
4379bf8bd7
26
fs/io-wq.c
26
fs/io-wq.c
@ -53,6 +53,9 @@ struct io_worker {
|
||||
struct io_wq_work *cur_work;
|
||||
spinlock_t lock;
|
||||
|
||||
const struct cred *cur_creds;
|
||||
const struct cred *saved_creds;
|
||||
|
||||
struct rcu_head rcu;
|
||||
};
|
||||
|
||||
@ -171,6 +174,11 @@ static void io_worker_exit(struct io_worker *worker)
|
||||
worker->flags = 0;
|
||||
preempt_enable();
|
||||
|
||||
if (worker->saved_creds) {
|
||||
revert_creds(worker->saved_creds);
|
||||
worker->cur_creds = worker->saved_creds = NULL;
|
||||
}
|
||||
|
||||
raw_spin_lock_irq(&wqe->lock);
|
||||
hlist_nulls_del_rcu(&worker->nulls_node);
|
||||
list_del_rcu(&worker->all_list);
|
||||
@ -312,6 +320,10 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker)
|
||||
worker->flags |= IO_WORKER_F_FREE;
|
||||
hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
|
||||
}
|
||||
if (worker->saved_creds) {
|
||||
revert_creds(worker->saved_creds);
|
||||
worker->cur_creds = worker->saved_creds = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static inline unsigned int io_get_work_hash(struct io_wq_work *work)
|
||||
@ -359,6 +371,18 @@ static void io_flush_signals(void)
|
||||
}
|
||||
}
|
||||
|
||||
static void io_wq_switch_creds(struct io_worker *worker,
|
||||
struct io_wq_work *work)
|
||||
{
|
||||
const struct cred *old_creds = override_creds(work->creds);
|
||||
|
||||
worker->cur_creds = work->creds;
|
||||
if (worker->saved_creds)
|
||||
put_cred(old_creds); /* creds set by previous switch */
|
||||
else
|
||||
worker->saved_creds = old_creds;
|
||||
}
|
||||
|
||||
static void io_assign_current_work(struct io_worker *worker,
|
||||
struct io_wq_work *work)
|
||||
{
|
||||
@ -407,6 +431,8 @@ get_next:
|
||||
unsigned int hash = io_get_work_hash(work);
|
||||
|
||||
next_hashed = wq_next_work(work);
|
||||
if (work->creds && worker->cur_creds != work->creds)
|
||||
io_wq_switch_creds(worker, work);
|
||||
wq->do_work(work);
|
||||
io_assign_current_work(worker, NULL);
|
||||
|
||||
|
||||
@ -78,7 +78,7 @@ static inline void wq_list_del(struct io_wq_work_list *list,
|
||||
|
||||
struct io_wq_work {
|
||||
struct io_wq_work_node list;
|
||||
struct io_identity *identity;
|
||||
const struct cred *creds;
|
||||
unsigned flags;
|
||||
};
|
||||
|
||||
|
||||
104
fs/io_uring.c
104
fs/io_uring.c
@ -1094,7 +1094,7 @@ static bool io_match_task(struct io_kiocb *head,
|
||||
continue;
|
||||
if (req->file && req->file->f_op == &io_uring_fops)
|
||||
return true;
|
||||
if (req->work.identity->files == files)
|
||||
if (req->task->files == files)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@ -1218,31 +1218,6 @@ static inline void req_set_fail_links(struct io_kiocb *req)
|
||||
req->flags |= REQ_F_FAIL_LINK;
|
||||
}
|
||||
|
||||
/*
|
||||
* None of these are dereferenced, they are simply used to check if any of
|
||||
* them have changed. If we're under current and check they are still the
|
||||
* same, we're fine to grab references to them for actual out-of-line use.
|
||||
*/
|
||||
static void io_init_identity(struct io_identity *id)
|
||||
{
|
||||
id->files = current->files;
|
||||
id->mm = current->mm;
|
||||
#ifdef CONFIG_BLK_CGROUP
|
||||
rcu_read_lock();
|
||||
id->blkcg_css = blkcg_css();
|
||||
rcu_read_unlock();
|
||||
#endif
|
||||
id->creds = current_cred();
|
||||
id->nsproxy = current->nsproxy;
|
||||
id->fs = current->fs;
|
||||
id->fsize = rlimit(RLIMIT_FSIZE);
|
||||
#ifdef CONFIG_AUDIT
|
||||
id->loginuid = current->loginuid;
|
||||
id->sessionid = current->sessionid;
|
||||
#endif
|
||||
refcount_set(&id->count, 1);
|
||||
}
|
||||
|
||||
static inline void __io_req_init_async(struct io_kiocb *req)
|
||||
{
|
||||
memset(&req->work, 0, sizeof(req->work));
|
||||
@ -1255,17 +1230,10 @@ static inline void __io_req_init_async(struct io_kiocb *req)
|
||||
*/
|
||||
static inline void io_req_init_async(struct io_kiocb *req)
|
||||
{
|
||||
struct io_uring_task *tctx = current->io_uring;
|
||||
|
||||
if (req->flags & REQ_F_WORK_INITIALIZED)
|
||||
return;
|
||||
|
||||
__io_req_init_async(req);
|
||||
|
||||
/* Grab a ref if this isn't our static identity */
|
||||
req->work.identity = tctx->identity;
|
||||
if (tctx->identity != &tctx->__identity)
|
||||
refcount_inc(&req->work.identity->count);
|
||||
}
|
||||
|
||||
static void io_ring_ctx_ref_free(struct percpu_ref *ref)
|
||||
@ -1350,19 +1318,15 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
|
||||
return false;
|
||||
}
|
||||
|
||||
static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req)
|
||||
{
|
||||
if (req->work.identity == &tctx->__identity)
|
||||
return;
|
||||
if (refcount_dec_and_test(&req->work.identity->count))
|
||||
kfree(req->work.identity);
|
||||
}
|
||||
|
||||
static void io_req_clean_work(struct io_kiocb *req)
|
||||
{
|
||||
if (!(req->flags & REQ_F_WORK_INITIALIZED))
|
||||
return;
|
||||
|
||||
if (req->work.creds) {
|
||||
put_cred(req->work.creds);
|
||||
req->work.creds = NULL;
|
||||
}
|
||||
if (req->flags & REQ_F_INFLIGHT) {
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
struct io_uring_task *tctx = req->task->io_uring;
|
||||
@ -1377,7 +1341,6 @@ static void io_req_clean_work(struct io_kiocb *req)
|
||||
}
|
||||
|
||||
req->flags &= ~REQ_F_WORK_INITIALIZED;
|
||||
io_put_identity(req->task->io_uring, req);
|
||||
}
|
||||
|
||||
static void io_req_track_inflight(struct io_kiocb *req)
|
||||
@ -1411,6 +1374,8 @@ static void io_prep_async_work(struct io_kiocb *req)
|
||||
if (def->unbound_nonreg_file)
|
||||
req->work.flags |= IO_WQ_WORK_UNBOUND;
|
||||
}
|
||||
if (!req->work.creds)
|
||||
req->work.creds = get_current_cred();
|
||||
}
|
||||
|
||||
static void io_prep_async_link(struct io_kiocb *req)
|
||||
@ -6376,9 +6341,9 @@ static void __io_queue_sqe(struct io_kiocb *req)
|
||||
const struct cred *old_creds = NULL;
|
||||
int ret;
|
||||
|
||||
if ((req->flags & REQ_F_WORK_INITIALIZED) &&
|
||||
req->work.identity->creds != current_cred())
|
||||
old_creds = override_creds(req->work.identity->creds);
|
||||
if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
|
||||
req->work.creds != current_cred())
|
||||
old_creds = override_creds(req->work.creds);
|
||||
|
||||
ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
|
||||
|
||||
@ -6508,16 +6473,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
|
||||
|
||||
id = READ_ONCE(sqe->personality);
|
||||
if (id) {
|
||||
struct io_identity *iod;
|
||||
|
||||
iod = idr_find(&ctx->personality_idr, id);
|
||||
if (unlikely(!iod))
|
||||
return -EINVAL;
|
||||
refcount_inc(&iod->count);
|
||||
|
||||
__io_req_init_async(req);
|
||||
get_cred(iod->creds);
|
||||
req->work.identity = iod;
|
||||
req->work.creds = idr_find(&ctx->personality_idr, id);
|
||||
if (unlikely(!req->work.creds))
|
||||
return -EINVAL;
|
||||
get_cred(req->work.creds);
|
||||
}
|
||||
|
||||
state = &ctx->submit_state;
|
||||
@ -7936,8 +7896,6 @@ static int io_uring_alloc_task_context(struct task_struct *task,
|
||||
tctx->last = NULL;
|
||||
atomic_set(&tctx->in_idle, 0);
|
||||
tctx->sqpoll = false;
|
||||
io_init_identity(&tctx->__identity);
|
||||
tctx->identity = &tctx->__identity;
|
||||
task->io_uring = tctx;
|
||||
spin_lock_init(&tctx->task_lock);
|
||||
INIT_WQ_LIST(&tctx->task_list);
|
||||
@ -7951,9 +7909,6 @@ void __io_uring_free(struct task_struct *tsk)
|
||||
struct io_uring_task *tctx = tsk->io_uring;
|
||||
|
||||
WARN_ON_ONCE(!xa_empty(&tctx->xa));
|
||||
WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1);
|
||||
if (tctx->identity != &tctx->__identity)
|
||||
kfree(tctx->identity);
|
||||
percpu_counter_destroy(&tctx->inflight);
|
||||
kfree(tctx);
|
||||
tsk->io_uring = NULL;
|
||||
@ -8593,13 +8548,11 @@ static int io_uring_fasync(int fd, struct file *file, int on)
|
||||
|
||||
static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
|
||||
{
|
||||
struct io_identity *iod;
|
||||
const struct cred *creds;
|
||||
|
||||
iod = idr_remove(&ctx->personality_idr, id);
|
||||
if (iod) {
|
||||
put_cred(iod->creds);
|
||||
if (refcount_dec_and_test(&iod->count))
|
||||
kfree(iod);
|
||||
creds = idr_remove(&ctx->personality_idr, id);
|
||||
if (creds) {
|
||||
put_cred(creds);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -9300,8 +9253,7 @@ out_fput:
|
||||
#ifdef CONFIG_PROC_FS
|
||||
static int io_uring_show_cred(int id, void *p, void *data)
|
||||
{
|
||||
struct io_identity *iod = p;
|
||||
const struct cred *cred = iod->creds;
|
||||
const struct cred *cred = p;
|
||||
struct seq_file *m = data;
|
||||
struct user_namespace *uns = seq_user_ns(m);
|
||||
struct group_info *gi;
|
||||
@ -9732,21 +9684,15 @@ out:
|
||||
|
||||
static int io_register_personality(struct io_ring_ctx *ctx)
|
||||
{
|
||||
struct io_identity *id;
|
||||
const struct cred *creds;
|
||||
int ret;
|
||||
|
||||
id = kmalloc(sizeof(*id), GFP_KERNEL);
|
||||
if (unlikely(!id))
|
||||
return -ENOMEM;
|
||||
creds = get_current_cred();
|
||||
|
||||
io_init_identity(id);
|
||||
id->creds = get_current_cred();
|
||||
|
||||
ret = idr_alloc_cyclic(&ctx->personality_idr, id, 1, USHRT_MAX, GFP_KERNEL);
|
||||
if (ret < 0) {
|
||||
put_cred(id->creds);
|
||||
kfree(id);
|
||||
}
|
||||
ret = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
|
||||
USHRT_MAX, GFP_KERNEL);
|
||||
if (ret < 0)
|
||||
put_cred(creds);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@ -5,23 +5,6 @@
|
||||
#include <linux/sched.h>
|
||||
#include <linux/xarray.h>
|
||||
|
||||
struct io_identity {
|
||||
struct files_struct *files;
|
||||
struct mm_struct *mm;
|
||||
#ifdef CONFIG_BLK_CGROUP
|
||||
struct cgroup_subsys_state *blkcg_css;
|
||||
#endif
|
||||
const struct cred *creds;
|
||||
struct nsproxy *nsproxy;
|
||||
struct fs_struct *fs;
|
||||
unsigned long fsize;
|
||||
#ifdef CONFIG_AUDIT
|
||||
kuid_t loginuid;
|
||||
unsigned int sessionid;
|
||||
#endif
|
||||
refcount_t count;
|
||||
};
|
||||
|
||||
struct io_wq_work_node {
|
||||
struct io_wq_work_node *next;
|
||||
};
|
||||
@ -38,8 +21,6 @@ struct io_uring_task {
|
||||
struct file *last;
|
||||
void *io_wq;
|
||||
struct percpu_counter inflight;
|
||||
struct io_identity __identity;
|
||||
struct io_identity *identity;
|
||||
atomic_t in_idle;
|
||||
bool sqpoll;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user