mirror of
https://github.com/torvalds/linux.git
synced 2024-11-29 15:41:36 +00:00
Merge branch 'slab/next' into for-linus
Conflicts: mm/slub.c
This commit is contained in:
Pekka Enberg
commit
bfb91fb650
@ -37,9 +37,7 @@ enum stat_item {
|
||||
|
||||
struct kmem_cache_cpu {
|
||||
void **freelist; /* Pointer to next available object */
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
unsigned long tid; /* Globally unique transaction id */
|
||||
#endif
|
||||
struct page *page; /* The slab from which we are allocating */
|
||||
int node; /* The node of the page (or -1 for debug) */
|
||||
#ifdef CONFIG_SLUB_STATS
|
||||
@ -179,7 +177,8 @@ static __always_inline int kmalloc_index(size_t size)
|
||||
if (size <= 4 * 1024) return 12;
|
||||
/*
|
||||
* The following is only needed to support architectures with a larger page
|
||||
* size than 4k.
|
||||
* size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
|
||||
* size we would have to go up to 128k.
|
||||
*/
|
||||
if (size <= 8 * 1024) return 13;
|
||||
if (size <= 16 * 1024) return 14;
|
||||
@ -190,7 +189,8 @@ static __always_inline int kmalloc_index(size_t size)
|
||||
if (size <= 512 * 1024) return 19;
|
||||
if (size <= 1024 * 1024) return 20;
|
||||
if (size <= 2 * 1024 * 1024) return 21;
|
||||
return -1;
|
||||
BUG();
|
||||
return -1; /* Will never be reached */
|
||||
|
||||
/*
|
||||
* What we really wanted to do and cannot do because of compiler issues is:
|
||||
|
||||
165
mm/slub.c
165
mm/slub.c
@ -261,6 +261,18 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
|
||||
return *(void **)(object + s->offset);
|
||||
}
|
||||
|
||||
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
|
||||
{
|
||||
void *p;
|
||||
|
||||
#ifdef CONFIG_DEBUG_PAGEALLOC
|
||||
probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
|
||||
#else
|
||||
p = get_freepointer(s, object);
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
|
||||
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
|
||||
{
|
||||
*(void **)(object + s->offset) = fp;
|
||||
@ -271,10 +283,6 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
|
||||
for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
|
||||
__p += (__s)->size)
|
||||
|
||||
/* Scan freelist */
|
||||
#define for_each_free_object(__p, __s, __free) \
|
||||
for (__p = (__free); __p; __p = get_freepointer((__s), __p))
|
||||
|
||||
/* Determine object index from a given position */
|
||||
static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
|
||||
{
|
||||
@ -331,6 +339,21 @@ static inline int oo_objects(struct kmem_cache_order_objects x)
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SLUB_DEBUG
|
||||
/*
|
||||
* Determine a map of object in use on a page.
|
||||
*
|
||||
* Slab lock or node listlock must be held to guarantee that the page does
|
||||
* not vanish from under us.
|
||||
*/
|
||||
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
|
||||
{
|
||||
void *p;
|
||||
void *addr = page_address(page);
|
||||
|
||||
for (p = page->freelist; p; p = get_freepointer(s, p))
|
||||
set_bit(slab_index(p, s, addr), map);
|
||||
}
|
||||
|
||||
/*
|
||||
* Debug settings:
|
||||
*/
|
||||
@ -1487,7 +1510,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
|
||||
int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
|
||||
|
||||
page = get_partial_node(get_node(s, searchnode));
|
||||
if (page || node != -1)
|
||||
if (page || node != NUMA_NO_NODE)
|
||||
return page;
|
||||
|
||||
return get_any_partial(s, flags);
|
||||
@ -1540,7 +1563,6 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
#ifdef CONFIG_PREEMPT
|
||||
/*
|
||||
* Calculate the next globally unique transaction for disambiguiation
|
||||
@ -1600,17 +1622,12 @@ static inline void note_cmpxchg_failure(const char *n,
|
||||
stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void init_kmem_cache_cpus(struct kmem_cache *s)
|
||||
{
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu)
|
||||
per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
|
||||
#endif
|
||||
|
||||
}
|
||||
/*
|
||||
* Remove the cpu slab
|
||||
@ -1643,9 +1660,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
|
||||
page->inuse--;
|
||||
}
|
||||
c->page = NULL;
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
c->tid = next_tid(c->tid);
|
||||
#endif
|
||||
unfreeze_slab(s, page, tail);
|
||||
}
|
||||
|
||||
@ -1779,8 +1794,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
|
||||
unsigned long addr, struct kmem_cache_cpu *c)
|
||||
{
|
||||
void **object;
|
||||
struct page *new;
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
struct page *page;
|
||||
unsigned long flags;
|
||||
|
||||
local_irq_save(flags);
|
||||
@ -1791,38 +1805,36 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
|
||||
* pointer.
|
||||
*/
|
||||
c = this_cpu_ptr(s->cpu_slab);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* We handle __GFP_ZERO in the caller */
|
||||
gfpflags &= ~__GFP_ZERO;
|
||||
|
||||
if (!c->page)
|
||||
page = c->page;
|
||||
if (!page)
|
||||
goto new_slab;
|
||||
|
||||
slab_lock(c->page);
|
||||
slab_lock(page);
|
||||
if (unlikely(!node_match(c, node)))
|
||||
goto another_slab;
|
||||
|
||||
stat(s, ALLOC_REFILL);
|
||||
|
||||
load_freelist:
|
||||
object = c->page->freelist;
|
||||
object = page->freelist;
|
||||
if (unlikely(!object))
|
||||
goto another_slab;
|
||||
if (kmem_cache_debug(s))
|
||||
goto debug;
|
||||
|
||||
c->freelist = get_freepointer(s, object);
|
||||
c->page->inuse = c->page->objects;
|
||||
c->page->freelist = NULL;
|
||||
c->node = page_to_nid(c->page);
|
||||
page->inuse = page->objects;
|
||||
page->freelist = NULL;
|
||||
|
||||
unlock_out:
|
||||
slab_unlock(c->page);
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
slab_unlock(page);
|
||||
c->tid = next_tid(c->tid);
|
||||
local_irq_restore(flags);
|
||||
#endif
|
||||
stat(s, ALLOC_SLOWPATH);
|
||||
return object;
|
||||
|
||||
@ -1830,10 +1842,11 @@ another_slab:
|
||||
deactivate_slab(s, c);
|
||||
|
||||
new_slab:
|
||||
new = get_partial(s, gfpflags, node);
|
||||
if (new) {
|
||||
c->page = new;
|
||||
page = get_partial(s, gfpflags, node);
|
||||
if (page) {
|
||||
stat(s, ALLOC_FROM_PARTIAL);
|
||||
c->node = page_to_nid(page);
|
||||
c->page = page;
|
||||
goto load_freelist;
|
||||
}
|
||||
|
||||
@ -1841,33 +1854,35 @@ new_slab:
|
||||
if (gfpflags & __GFP_WAIT)
|
||||
local_irq_enable();
|
||||
|
||||
new = new_slab(s, gfpflags, node);
|
||||
page = new_slab(s, gfpflags, node);
|
||||
|
||||
if (gfpflags & __GFP_WAIT)
|
||||
local_irq_disable();
|
||||
|
||||
if (new) {
|
||||
if (page) {
|
||||
c = __this_cpu_ptr(s->cpu_slab);
|
||||
stat(s, ALLOC_SLAB);
|
||||
if (c->page)
|
||||
flush_slab(s, c);
|
||||
slab_lock(new);
|
||||
__SetPageSlubFrozen(new);
|
||||
c->page = new;
|
||||
|
||||
slab_lock(page);
|
||||
__SetPageSlubFrozen(page);
|
||||
c->node = page_to_nid(page);
|
||||
c->page = page;
|
||||
goto load_freelist;
|
||||
}
|
||||
if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
|
||||
slab_out_of_memory(s, gfpflags, node);
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_restore(flags);
|
||||
#endif
|
||||
return NULL;
|
||||
debug:
|
||||
if (!alloc_debug_processing(s, c->page, object, addr))
|
||||
if (!alloc_debug_processing(s, page, object, addr))
|
||||
goto another_slab;
|
||||
|
||||
c->page->inuse++;
|
||||
c->page->freelist = get_freepointer(s, object);
|
||||
page->inuse++;
|
||||
page->freelist = get_freepointer(s, object);
|
||||
deactivate_slab(s, c);
|
||||
c->page = NULL;
|
||||
c->node = NUMA_NO_NODE;
|
||||
goto unlock_out;
|
||||
}
|
||||
@ -1887,20 +1902,12 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
|
||||
{
|
||||
void **object;
|
||||
struct kmem_cache_cpu *c;
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
unsigned long tid;
|
||||
#else
|
||||
unsigned long flags;
|
||||
#endif
|
||||
|
||||
if (slab_pre_alloc_hook(s, gfpflags))
|
||||
return NULL;
|
||||
|
||||
#ifndef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_save(flags);
|
||||
#else
|
||||
redo:
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Must read kmem_cache cpu data via this cpu ptr. Preemption is
|
||||
@ -1910,7 +1917,6 @@ redo:
|
||||
*/
|
||||
c = __this_cpu_ptr(s->cpu_slab);
|
||||
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
/*
|
||||
* The transaction ids are globally unique per cpu and per operation on
|
||||
* a per cpu queue. Thus they can be guarantee that the cmpxchg_double
|
||||
@ -1919,7 +1925,6 @@ redo:
|
||||
*/
|
||||
tid = c->tid;
|
||||
barrier();
|
||||
#endif
|
||||
|
||||
object = c->freelist;
|
||||
if (unlikely(!object || !node_match(c, node)))
|
||||
@ -1927,7 +1932,6 @@ redo:
|
||||
object = __slab_alloc(s, gfpflags, node, addr, c);
|
||||
|
||||
else {
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
/*
|
||||
* The cmpxchg will only match if there was no additional
|
||||
* operation and if we are on the right processor.
|
||||
@ -1943,21 +1947,14 @@ redo:
|
||||
if (unlikely(!irqsafe_cpu_cmpxchg_double(
|
||||
s->cpu_slab->freelist, s->cpu_slab->tid,
|
||||
object, tid,
|
||||
get_freepointer(s, object), next_tid(tid)))) {
|
||||
get_freepointer_safe(s, object), next_tid(tid)))) {
|
||||
|
||||
note_cmpxchg_failure("slab_alloc", s, tid);
|
||||
goto redo;
|
||||
}
|
||||
#else
|
||||
c->freelist = get_freepointer(s, object);
|
||||
#endif
|
||||
stat(s, ALLOC_FASTPATH);
|
||||
}
|
||||
|
||||
#ifndef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_restore(flags);
|
||||
#endif
|
||||
|
||||
if (unlikely(gfpflags & __GFP_ZERO) && object)
|
||||
memset(object, 0, s->objsize);
|
||||
|
||||
@ -2034,18 +2031,15 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
|
||||
{
|
||||
void *prior;
|
||||
void **object = (void *)x;
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
unsigned long flags;
|
||||
|
||||
local_irq_save(flags);
|
||||
#endif
|
||||
slab_lock(page);
|
||||
stat(s, FREE_SLOWPATH);
|
||||
|
||||
if (kmem_cache_debug(s))
|
||||
goto debug;
|
||||
if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
|
||||
goto out_unlock;
|
||||
|
||||
checks_ok:
|
||||
prior = page->freelist;
|
||||
set_freepointer(s, object, prior);
|
||||
page->freelist = object;
|
||||
@ -2070,9 +2064,7 @@ checks_ok:
|
||||
|
||||
out_unlock:
|
||||
slab_unlock(page);
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_restore(flags);
|
||||
#endif
|
||||
return;
|
||||
|
||||
slab_empty:
|
||||
@ -2084,17 +2076,9 @@ slab_empty:
|
||||
stat(s, FREE_REMOVE_PARTIAL);
|
||||
}
|
||||
slab_unlock(page);
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_restore(flags);
|
||||
#endif
|
||||
stat(s, FREE_SLAB);
|
||||
discard_slab(s, page);
|
||||
return;
|
||||
|
||||
debug:
|
||||
if (!free_debug_processing(s, page, x, addr))
|
||||
goto out_unlock;
|
||||
goto checks_ok;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -2113,20 +2097,11 @@ static __always_inline void slab_free(struct kmem_cache *s,
|
||||
{
|
||||
void **object = (void *)x;
|
||||
struct kmem_cache_cpu *c;
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
unsigned long tid;
|
||||
#else
|
||||
unsigned long flags;
|
||||
#endif
|
||||
|
||||
slab_free_hook(s, x);
|
||||
|
||||
#ifndef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_save(flags);
|
||||
|
||||
#else
|
||||
redo:
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Determine the currently cpus per cpu slab.
|
||||
@ -2136,15 +2111,12 @@ redo:
|
||||
*/
|
||||
c = __this_cpu_ptr(s->cpu_slab);
|
||||
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
tid = c->tid;
|
||||
barrier();
|
||||
#endif
|
||||
|
||||
if (likely(page == c->page && c->node != NUMA_NO_NODE)) {
|
||||
if (likely(page == c->page)) {
|
||||
set_freepointer(s, object, c->freelist);
|
||||
|
||||
#ifdef CONFIG_CMPXCHG_LOCAL
|
||||
if (unlikely(!irqsafe_cpu_cmpxchg_double(
|
||||
s->cpu_slab->freelist, s->cpu_slab->tid,
|
||||
c->freelist, tid,
|
||||
@ -2153,16 +2125,10 @@ redo:
|
||||
note_cmpxchg_failure("slab_free", s, tid);
|
||||
goto redo;
|
||||
}
|
||||
#else
|
||||
c->freelist = object;
|
||||
#endif
|
||||
stat(s, FREE_FASTPATH);
|
||||
} else
|
||||
__slab_free(s, page, x, addr);
|
||||
|
||||
#ifndef CONFIG_CMPXCHG_LOCAL
|
||||
local_irq_restore(flags);
|
||||
#endif
|
||||
}
|
||||
|
||||
void kmem_cache_free(struct kmem_cache *s, void *x)
|
||||
@ -2673,9 +2639,8 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page,
|
||||
return;
|
||||
slab_err(s, page, "%s", text);
|
||||
slab_lock(page);
|
||||
for_each_free_object(p, s, page->freelist)
|
||||
set_bit(slab_index(p, s, addr), map);
|
||||
|
||||
get_map(s, page, map);
|
||||
for_each_object(p, s, addr, page->objects) {
|
||||
|
||||
if (!test_bit(slab_index(p, s, addr), map)) {
|
||||
@ -3203,7 +3168,7 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
|
||||
list_for_each_entry(p, &n->partial, lru)
|
||||
p->slab = s;
|
||||
|
||||
#ifdef CONFIG_SLAB_DEBUG
|
||||
#ifdef CONFIG_SLUB_DEBUG
|
||||
list_for_each_entry(p, &n->full, lru)
|
||||
p->slab = s;
|
||||
#endif
|
||||
@ -3610,10 +3575,11 @@ static int validate_slab(struct kmem_cache *s, struct page *page,
|
||||
/* Now we know that a valid freelist exists */
|
||||
bitmap_zero(map, page->objects);
|
||||
|
||||
for_each_free_object(p, s, page->freelist) {
|
||||
set_bit(slab_index(p, s, addr), map);
|
||||
if (!check_object(s, page, p, SLUB_RED_INACTIVE))
|
||||
return 0;
|
||||
get_map(s, page, map);
|
||||
for_each_object(p, s, addr, page->objects) {
|
||||
if (test_bit(slab_index(p, s, addr), map))
|
||||
if (!check_object(s, page, p, SLUB_RED_INACTIVE))
|
||||
return 0;
|
||||
}
|
||||
|
||||
for_each_object(p, s, addr, page->objects)
|
||||
@ -3821,8 +3787,7 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s,
|
||||
void *p;
|
||||
|
||||
bitmap_zero(map, page->objects);
|
||||
for_each_free_object(p, s, page->freelist)
|
||||
set_bit(slab_index(p, s, addr), map);
|
||||
get_map(s, page, map);
|
||||
|
||||
for_each_object(p, s, addr, page->objects)
|
||||
if (!test_bit(slab_index(p, s, addr), map))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user