leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

io_uring/net: make page accounting more consistent

Browse Source 
Make network page accounting more consistent with how buffer
registration is working, i.e. account all memory to ctx->user.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/4aacfe64bbb81b27f9ecf5d5c219c69a07e5aa56.1658742118.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-07-25 10:52:05 +01:00 committed by Jens Axboe
parent 2e32ba5607
commit 6a9ce66f4d
5 changed files with 37 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
io_uring/net.c
View File

@ -985,7 +985,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
&msg.msg_iter);
if (unlikely(ret))
return ret;
ret = mm_account_pinned_pages(&notif->uarg.mmp, zc->len);
ret = io_notif_account_mem(notif, zc->len);
if (unlikely(ret))
return ret;
}

9
io_uring/notif.c
View File

@ -14,12 +14,10 @@ static void __io_notif_complete_tw(struct callback_head *cb)
struct io_notif *notif = container_of(cb, struct io_notif, task_work);
struct io_rsrc_node *rsrc_node = notif->rsrc_node;
struct io_ring_ctx *ctx = notif->ctx;
struct mmpin *mmp = &notif->uarg.mmp;
if (mmp->user) {
atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
free_uid(mmp->user);
mmp->user = NULL;
if (notif->account_pages && ctx->user) {
__io_unaccount_mem(ctx->user, notif->account_pages);
notif->account_pages = 0;
}
if (likely(notif->task)) {
io_put_task(notif->task, 1);
@ -121,6 +119,7 @@ struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx,
notif->ctx = ctx;
notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
notif->uarg.callback = io_uring_tx_zerocopy_callback;
notif->account_pages = 0;
}
notif->seq = slot->seq++;

19
io_uring/notif.h
View File

@ -5,6 +5,8 @@
#include <net/sock.h>
#include <linux/nospec.h>
#include "rsrc.h"
#define IO_NOTIF_SPLICE_BATCH 32
#define IORING_MAX_NOTIF_SLOTS (1U << 10)
@ -23,6 +25,8 @@ struct io_notif {
/* hook into ctx->notif_list and ctx->notif_list_locked */
struct list_head cache_node;
unsigned long account_pages;
union {
struct callback_head task_work;
struct work_struct commit_work;
@ -85,3 +89,18 @@ static inline void io_notif_slot_flush_submit(struct io_notif_slot *slot,
}
io_notif_slot_flush(slot);
}
static inline int io_notif_account_mem(struct io_notif *notif, unsigned len)
{
struct io_ring_ctx *ctx = notif->ctx;
unsigned nr_pages = (len >> PAGE_SHIFT) + 2;
int ret;
if (ctx->user) {
ret = __io_account_mem(ctx->user, nr_pages);
if (ret)
return ret;
notif->account_pages += nr_pages;
}
return 0;
}

12
io_uring/rsrc.c
View File

@ -44,17 +44,13 @@ void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
}
}
static inline void __io_unaccount_mem(struct user_struct *user,
unsigned long nr_pages)
{
atomic_long_sub(nr_pages, &user->locked_vm);
}
static inline int __io_account_mem(struct user_struct *user,
unsigned long nr_pages)
int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
{
unsigned long page_limit, cur_pages, new_pages;
if (!nr_pages)
return 0;
/* Don't allow more pages than we can safely lock */
page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

9
io_uring/rsrc.h
View File

@ -169,4 +169,13 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags);
int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
static inline void __io_unaccount_mem(struct user_struct *user,
unsigned long nr_pages)
{
atomic_long_sub(nr_pages, &user->locked_vm);
}
#endif