Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-23 12:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

bpf: Remove usage of kmem_cache from bpf_mem_cache.

Browse Source 
For bpf_mem_cache based hash maps the following stress test:
for (i = 1; i <= 512; i <<= 1)
  for (j = 1; j <= 1 << 18; j <<= 1)
    fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, i, j, 2, 0);
creates many kmem_cache-s that are not mergeable in debug kernels
and consume unnecessary amount of memory.
Turned out bpf_mem_cache's free_list logic does batching well,
so usage of kmem_cache for fixes size allocations doesn't bring
any performance benefits vs normal kmalloc.
Hence get rid of kmem_cache in bpf_mem_cache.
That saves memory, speeds up map create/destroy operations,
while maintains hash map update/delete performance.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20220902211058.60789-16-alexei.starovoitov@gmail.com
This commit is contained in:
Alexei Starovoitov 2022-09-02 14:10:57 -07:00 committed by Daniel Borkmann
parent 02cc5aa29e
commit bfc03c15be
1 changed files with 14 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
kernel/bpf/memalloc.c
View File

@ -91,17 +91,13 @@ struct bpf_mem_cache {
*/
struct llist_head free_llist_extra;
/* kmem_cache != NULL when bpf_mem_alloc was created for specific
* element size.
*/
struct kmem_cache *kmem_cache;
struct irq_work refill_work;
struct obj_cgroup *objcg;
int unit_size;
/* count of objects in free_llist */
int free_cnt;
int low_watermark, high_watermark, batch;
bool percpu;
int percpu_size;
struct rcu_head rcu;
struct llist_head free_by_rcu;
@ -134,8 +130,8 @@ static void *__alloc(struct bpf_mem_cache *c, int node)
*/
gfp_t flags = GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT;
if (c->percpu) {
void **obj = kmem_cache_alloc_node(c->kmem_cache, flags, node);
if (c->percpu_size) {
void **obj = kmalloc_node(c->percpu_size, flags, node);
void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
if (!obj || !pptr) {
@ -147,9 +143,6 @@ static void *__alloc(struct bpf_mem_cache *c, int node)
return obj;
}
if (c->kmem_cache)
return kmem_cache_alloc_node(c->kmem_cache, flags, node);
return kmalloc_node(c->unit_size, flags, node);
}
@ -207,16 +200,13 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node)
static void free_one(struct bpf_mem_cache *c, void *obj)
{
if (c->percpu) {
if (c->percpu_size) {
free_percpu(((void **)obj)[1]);
kmem_cache_free(c->kmem_cache, obj);
kfree(obj);
return;
}
if (c->kmem_cache)
kmem_cache_free(c->kmem_cache, obj);
else
kfree(obj);
kfree(obj);
}
static void __free_rcu(struct rcu_head *head)
@ -356,7 +346,7 @@ static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu)
alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu));
}
/* When size != 0 create kmem_cache and bpf_mem_cache for each cpu.
/* When size != 0 bpf_mem_cache for each cpu.
* This is typical bpf hash map use case when all elements have equal size.
*
* When size == 0 allocate 11 bpf_mem_cache-s for each cpu, then rely on
@ -368,40 +358,29 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096};
struct bpf_mem_caches *cc, __percpu *pcc;
struct bpf_mem_cache *c, __percpu *pc;
struct kmem_cache *kmem_cache = NULL;
struct obj_cgroup *objcg = NULL;
char buf[32];
int cpu, i, unit_size;
int cpu, i, unit_size, percpu_size = 0;
if (size) {
pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
if (!pc)
return -ENOMEM;
if (percpu) {
unit_size = size;
if (percpu)
/* room for llist_node and per-cpu pointer */
size = LLIST_NODE_SZ + sizeof(void *);
} else {
percpu_size = LLIST_NODE_SZ + sizeof(void *);
else
size += LLIST_NODE_SZ; /* room for llist_node */
unit_size = size;
}
unit_size = size;
snprintf(buf, sizeof(buf), "bpf-%u", size);
kmem_cache = kmem_cache_create(buf, size, 8, 0, NULL);
if (!kmem_cache) {
free_percpu(pc);
return -ENOMEM;
}
#ifdef CONFIG_MEMCG_KMEM
objcg = get_obj_cgroup_from_current();
#endif
for_each_possible_cpu(cpu) {
c = per_cpu_ptr(pc, cpu);
c->kmem_cache = kmem_cache;
c->unit_size = unit_size;
c->objcg = objcg;
c->percpu = percpu;
c->percpu_size = percpu_size;
prefill_mem_cache(c, cpu);
}
ma->cache = pc;
@ -461,8 +440,7 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)
c = per_cpu_ptr(ma->cache, cpu);
drain_mem_cache(c);
}
/* kmem_cache and memcg are the same across cpus */
kmem_cache_destroy(c->kmem_cache);
/* objcg is the same across cpus */
if (c->objcg)
obj_cgroup_put(c->objcg);
/* c->waiting_for_gp list was drained, but __free_rcu might