2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* linux/mm/slab.c
|
|
|
|
* Written by Mark Hemment, 1996/97.
|
|
|
|
* (markhe@nextd.demon.co.uk)
|
|
|
|
*
|
|
|
|
* kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
|
|
|
|
*
|
|
|
|
* Major cleanup, different bufctl logic, per-cpu arrays
|
|
|
|
* (c) 2000 Manfred Spraul
|
|
|
|
*
|
|
|
|
* Cleanup, make the head arrays unconditional, preparation for NUMA
|
|
|
|
* (c) 2002 Manfred Spraul
|
|
|
|
*
|
|
|
|
* An implementation of the Slab Allocator as described in outline in;
|
|
|
|
* UNIX Internals: The New Frontiers by Uresh Vahalia
|
|
|
|
* Pub: Prentice Hall ISBN 0-13-101908-2
|
|
|
|
* or with a little more detail in;
|
|
|
|
* The Slab Allocator: An Object-Caching Kernel Memory Allocator
|
|
|
|
* Jeff Bonwick (Sun Microsystems).
|
|
|
|
* Presented at: USENIX Summer 1994 Technical Conference
|
|
|
|
*
|
|
|
|
* The memory is organized in caches, one cache for each object type.
|
|
|
|
* (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
|
|
|
|
* Each cache consists out of many slabs (they are small (usually one
|
|
|
|
* page long) and always contiguous), and each slab contains multiple
|
|
|
|
* initialized objects.
|
|
|
|
*
|
|
|
|
* This means, that your constructor is used only for newly allocated
|
2007-10-19 23:27:18 +00:00
|
|
|
* slabs and you must pass objects with the same initializations to
|
2005-04-16 22:20:36 +00:00
|
|
|
* kmem_cache_free.
|
|
|
|
*
|
|
|
|
* Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
|
|
|
|
* normal). If you need a special memory type, then must create a new
|
|
|
|
* cache for that memory type.
|
|
|
|
*
|
|
|
|
* In order to reduce fragmentation, the slabs are sorted in 3 groups:
|
|
|
|
* full slabs with 0 free objects
|
|
|
|
* partial slabs
|
|
|
|
* empty slabs with no allocated objects
|
|
|
|
*
|
|
|
|
* If partial slabs exist, then new allocations come from these slabs,
|
|
|
|
* otherwise from empty slabs or new slabs are allocated.
|
|
|
|
*
|
|
|
|
* kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
|
|
|
|
* during kmem_cache_destroy(). The caller must prevent concurrent allocs.
|
|
|
|
*
|
|
|
|
* Each cache has a short per-cpu head array, most allocs
|
|
|
|
* and frees go into that array, and if that array overflows, then 1/2
|
|
|
|
* of the entries in the array are given back into the global cache.
|
|
|
|
* The head array is strictly LIFO and should improve the cache hit rates.
|
|
|
|
* On SMP, it additionally reduces the spinlock operations.
|
|
|
|
*
|
2006-03-22 08:08:11 +00:00
|
|
|
* The c_cpuarray may not be read with enabled local interrupts -
|
2005-04-16 22:20:36 +00:00
|
|
|
* it's changed with a smp_call_function().
|
|
|
|
*
|
|
|
|
* SMP synchronization:
|
|
|
|
* constructors and destructors are called without any locking.
|
2006-02-01 11:05:50 +00:00
|
|
|
* Several members in struct kmem_cache and struct slab never change, they
|
2005-04-16 22:20:36 +00:00
|
|
|
* are accessed without any locking.
|
|
|
|
* The per-cpu arrays are never accessed from the wrong cpu, no locking,
|
|
|
|
* and local interrupts are disabled so slab code is preempt-safe.
|
|
|
|
* The non-constant members are protected with a per-cache irq spinlock.
|
|
|
|
*
|
|
|
|
* Many thanks to Mark Hemment, who wrote another per-cpu slab patch
|
|
|
|
* in 2000 - many ideas in the current implementation are derived from
|
|
|
|
* his patch.
|
|
|
|
*
|
|
|
|
* Further notes from the original documentation:
|
|
|
|
*
|
|
|
|
* 11 April '97. Started multi-threading - markhe
|
2012-07-06 20:25:12 +00:00
|
|
|
* The global cache-chain is protected by the mutex 'slab_mutex'.
|
2005-04-16 22:20:36 +00:00
|
|
|
* The sem is only needed when accessing/extending the cache-chain, which
|
|
|
|
* can never happen inside an interrupt (kmem_cache_create(),
|
|
|
|
* kmem_cache_shrink() and kmem_cache_reap()).
|
|
|
|
*
|
|
|
|
* At present, each engine can be growing a cache. This should be blocked.
|
|
|
|
*
|
2005-09-09 20:03:32 +00:00
|
|
|
* 15 March 2005. NUMA slab allocator.
|
|
|
|
* Shai Fultheim <shai@scalex86.org>.
|
|
|
|
* Shobhit Dayal <shobhit@calsoftinc.com>
|
|
|
|
* Alok N Kataria <alokk@calsoftinc.com>
|
|
|
|
* Christoph Lameter <christoph@lameter.com>
|
|
|
|
*
|
|
|
|
* Modified the slab allocator to be node aware on NUMA systems.
|
|
|
|
* Each node has its own list of partial, free and full slabs.
|
|
|
|
* All object allocations for a node occur from node specific slab lists.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/mm.h>
|
2006-06-27 09:53:52 +00:00
|
|
|
#include <linux/poison.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/compiler.h>
|
[PATCH] cpuset memory spread slab cache implementation
Provide the slab cache infrastructure to support cpuset memory spreading.
See the previous patches, cpuset_mem_spread, for an explanation of cpuset
memory spreading.
This patch provides a slab cache SLAB_MEM_SPREAD flag. If set in the
kmem_cache_create() call defining a slab cache, then any task marked with the
process state flag PF_MEMSPREAD will spread memory page allocations for that
cache over all the allowed nodes, instead of preferring the local (faulting)
node.
On systems not configured with CONFIG_NUMA, this results in no change to the
page allocation code path for slab caches.
On systems with cpusets configured in the kernel, but the "memory_spread"
cpuset option not enabled for the current tasks cpuset, this adds a call to a
cpuset routine and failed bit test of the processor state flag PF_SPREAD_SLAB.
For tasks so marked, a second inline test is done for the slab cache flag
SLAB_MEM_SPREAD, and if that is set and if the allocation is not
in_interrupt(), this adds a call to to a cpuset routine that computes which of
the tasks mems_allowed nodes should be preferred for this allocation.
==> This patch adds another hook into the performance critical
code path to allocating objects from the slab cache, in the
____cache_alloc() chunk, below. The next patch optimizes this
hook, reducing the impact of the combined mempolicy plus memory
spreading hooks on this critical code path to a single check
against the tasks task_struct flags word.
This patch provides the generic slab flags and logic needed to apply memory
spreading to a particular slab.
A subsequent patch will mark a few specific slab caches for this placement
policy.
Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-24 11:16:07 +00:00
|
|
|
#include <linux/cpuset.h>
|
2008-10-05 20:59:10 +00:00
|
|
|
#include <linux/proc_fs.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/kallsyms.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/rcupdate.h>
|
2005-06-23 07:09:02 +00:00
|
|
|
#include <linux/string.h>
|
2006-12-07 04:36:41 +00:00
|
|
|
#include <linux/uaccess.h>
|
2005-09-09 20:03:32 +00:00
|
|
|
#include <linux/nodemask.h>
|
2009-06-11 12:22:40 +00:00
|
|
|
#include <linux/kmemleak.h>
|
2006-01-19 01:42:36 +00:00
|
|
|
#include <linux/mempolicy.h>
|
2006-01-19 01:42:33 +00:00
|
|
|
#include <linux/mutex.h>
|
2006-12-08 10:39:44 +00:00
|
|
|
#include <linux/fault-inject.h>
|
2006-06-27 09:54:55 +00:00
|
|
|
#include <linux/rtmutex.h>
|
2006-12-13 08:34:27 +00:00
|
|
|
#include <linux/reciprocal_div.h>
|
2008-04-30 07:55:01 +00:00
|
|
|
#include <linux/debugobjects.h>
|
2008-05-09 18:35:53 +00:00
|
|
|
#include <linux/kmemcheck.h>
|
2010-03-28 02:40:47 +00:00
|
|
|
#include <linux/memory.h>
|
2011-05-20 19:50:29 +00:00
|
|
|
#include <linux/prefetch.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-07-31 23:44:30 +00:00
|
|
|
#include <net/sock.h>
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
|
2012-01-09 22:15:42 +00:00
|
|
|
#include <trace/events/kmem.h>
|
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
#include "internal.h"
|
|
|
|
|
2012-12-18 22:22:46 +00:00
|
|
|
#include "slab.h"
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2007-05-06 21:50:16 +00:00
|
|
|
* DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
|
2005-04-16 22:20:36 +00:00
|
|
|
* 0 for faster, smaller code (especially in the critical paths).
|
|
|
|
*
|
|
|
|
* STATS - 1 to collect stats for /proc/slabinfo.
|
|
|
|
* 0 for faster, smaller code (especially in the critical paths).
|
|
|
|
*
|
|
|
|
* FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB
|
|
|
|
#define DEBUG 1
|
|
|
|
#define STATS 1
|
|
|
|
#define FORCED_DEBUG 1
|
|
|
|
#else
|
|
|
|
#define DEBUG 0
|
|
|
|
#define STATS 0
|
|
|
|
#define FORCED_DEBUG 0
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Shouldn't this be in a header file somewhere? */
|
|
|
|
#define BYTES_PER_WORD sizeof(void *)
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-05 01:26:44 +00:00
|
|
|
#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#ifndef ARCH_KMALLOC_FLAGS
|
|
|
|
#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
|
|
|
|
#endif
|
|
|
|
|
2013-12-02 08:49:41 +00:00
|
|
|
#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
|
|
|
|
<= SLAB_OBJ_MIN_SIZE) ? 1 : 0)
|
|
|
|
|
|
|
|
#if FREELIST_BYTE_INDEX
|
|
|
|
typedef unsigned char freelist_idx_t;
|
|
|
|
#else
|
|
|
|
typedef unsigned short freelist_idx_t;
|
|
|
|
#endif
|
|
|
|
|
2014-05-05 20:20:04 +00:00
|
|
|
#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
|
2013-12-02 08:49:41 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* struct array_cache
|
|
|
|
*
|
|
|
|
* Purpose:
|
|
|
|
* - LIFO ordering, to hand out cache-warm objects from _alloc
|
|
|
|
* - reduce the number of linked list operations
|
|
|
|
* - reduce spinlock operations
|
|
|
|
*
|
|
|
|
* The limit is stored in the per-cpu structure to reduce the data cache
|
|
|
|
* footprint.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
struct array_cache {
|
|
|
|
unsigned int avail;
|
|
|
|
unsigned int limit;
|
|
|
|
unsigned int batchcount;
|
|
|
|
unsigned int touched;
|
2007-10-17 06:30:05 +00:00
|
|
|
void *entry[]; /*
|
2006-03-22 08:08:11 +00:00
|
|
|
* Must have this definition in here for the proper
|
|
|
|
* alignment of array_cache. Also simplifies accessing
|
|
|
|
* the entries.
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache {
|
|
|
|
spinlock_t lock;
|
|
|
|
struct array_cache ac;
|
|
|
|
};
|
|
|
|
|
2005-09-09 20:03:32 +00:00
|
|
|
/*
|
|
|
|
* Need this for bootstrapping a per node allocator.
|
|
|
|
*/
|
2014-10-09 22:26:27 +00:00
|
|
|
#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
|
2013-01-10 19:14:19 +00:00
|
|
|
static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
|
2005-09-09 20:03:32 +00:00
|
|
|
#define CACHE_CACHE 0
|
2014-10-09 22:26:27 +00:00
|
|
|
#define SIZE_NODE (MAX_NUMNODES)
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2006-06-30 08:55:45 +00:00
|
|
|
static int drain_freelist(struct kmem_cache *cache,
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n, int tofree);
|
2006-06-30 08:55:45 +00:00
|
|
|
static void free_block(struct kmem_cache *cachep, void **objpp, int len,
|
2014-08-06 23:04:25 +00:00
|
|
|
int node, struct list_head *list);
|
|
|
|
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
|
2009-06-10 16:40:04 +00:00
|
|
|
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
|
2006-11-22 14:55:48 +00:00
|
|
|
static void cache_reap(struct work_struct *unused);
|
2006-06-30 08:55:45 +00:00
|
|
|
|
2016-05-20 00:10:26 +00:00
|
|
|
static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
|
|
|
|
void **list);
|
|
|
|
static inline void fixup_slab_list(struct kmem_cache *cachep,
|
|
|
|
struct kmem_cache_node *n, struct page *page,
|
|
|
|
void **list);
|
2006-06-23 09:03:46 +00:00
|
|
|
static int slab_early_init = 1;
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
static void kmem_cache_node_init(struct kmem_cache_node *parent)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
|
|
|
INIT_LIST_HEAD(&parent->slabs_full);
|
|
|
|
INIT_LIST_HEAD(&parent->slabs_partial);
|
|
|
|
INIT_LIST_HEAD(&parent->slabs_free);
|
|
|
|
parent->shared = NULL;
|
|
|
|
parent->alien = NULL;
|
2006-02-05 07:27:56 +00:00
|
|
|
parent->colour_next = 0;
|
2005-09-09 20:03:32 +00:00
|
|
|
spin_lock_init(&parent->list_lock);
|
|
|
|
parent->free_objects = 0;
|
|
|
|
parent->free_touched = 0;
|
|
|
|
}
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
#define MAKE_LIST(cachep, listp, slab, nodeid) \
|
|
|
|
do { \
|
|
|
|
INIT_LIST_HEAD(listp); \
|
2014-08-06 23:04:11 +00:00
|
|
|
list_splice(&get_node(cachep, nodeid)->slab, listp); \
|
2005-09-09 20:03:32 +00:00
|
|
|
} while (0)
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
|
|
|
|
do { \
|
2005-09-09 20:03:32 +00:00
|
|
|
MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
|
|
|
|
MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
|
|
|
|
MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
|
|
|
|
} while (0)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
#define CFLGS_OBJFREELIST_SLAB (0x40000000UL)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define CFLGS_OFF_SLAB (0x80000000UL)
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
#define OBJFREELIST_SLAB(x) ((x)->flags & CFLGS_OBJFREELIST_SLAB)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
|
|
|
|
|
|
|
|
#define BATCHREFILL_LIMIT 16
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Optimization question: fewer reaps means less probability for unnessary
|
|
|
|
* cpucache drain/refill cycles.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2005-11-08 15:44:08 +00:00
|
|
|
* OTOH the cpuarrays can contain lots of objects,
|
2005-04-16 22:20:36 +00:00
|
|
|
* which could lock up otherwise freeable slabs.
|
|
|
|
*/
|
2014-03-30 09:02:20 +00:00
|
|
|
#define REAPTIMEOUT_AC (2*HZ)
|
|
|
|
#define REAPTIMEOUT_NODE (4*HZ)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#if STATS
|
|
|
|
#define STATS_INC_ACTIVE(x) ((x)->num_active++)
|
|
|
|
#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
|
|
|
|
#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
|
|
|
|
#define STATS_INC_GROWN(x) ((x)->grown++)
|
2006-06-30 08:55:45 +00:00
|
|
|
#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
|
2006-03-22 08:08:11 +00:00
|
|
|
#define STATS_SET_HIGH(x) \
|
|
|
|
do { \
|
|
|
|
if ((x)->num_active > (x)->high_mark) \
|
|
|
|
(x)->high_mark = (x)->num_active; \
|
|
|
|
} while (0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define STATS_INC_ERR(x) ((x)->errors++)
|
|
|
|
#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
|
2005-09-09 20:03:32 +00:00
|
|
|
#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
|
2006-04-11 05:52:54 +00:00
|
|
|
#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
|
2006-03-22 08:08:11 +00:00
|
|
|
#define STATS_SET_FREEABLE(x, i) \
|
|
|
|
do { \
|
|
|
|
if ((x)->max_freeable < i) \
|
|
|
|
(x)->max_freeable = i; \
|
|
|
|
} while (0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
|
|
|
|
#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
|
|
|
|
#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
|
|
|
|
#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
|
|
|
|
#else
|
|
|
|
#define STATS_INC_ACTIVE(x) do { } while (0)
|
|
|
|
#define STATS_DEC_ACTIVE(x) do { } while (0)
|
|
|
|
#define STATS_INC_ALLOCED(x) do { } while (0)
|
|
|
|
#define STATS_INC_GROWN(x) do { } while (0)
|
2010-08-10 00:19:03 +00:00
|
|
|
#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define STATS_SET_HIGH(x) do { } while (0)
|
|
|
|
#define STATS_INC_ERR(x) do { } while (0)
|
|
|
|
#define STATS_INC_NODEALLOCS(x) do { } while (0)
|
2005-09-09 20:03:32 +00:00
|
|
|
#define STATS_INC_NODEFREES(x) do { } while (0)
|
2006-04-11 05:52:54 +00:00
|
|
|
#define STATS_INC_ACOVERFLOW(x) do { } while (0)
|
2006-03-22 08:08:11 +00:00
|
|
|
#define STATS_SET_FREEABLE(x, i) do { } while (0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define STATS_INC_ALLOCHIT(x) do { } while (0)
|
|
|
|
#define STATS_INC_ALLOCMISS(x) do { } while (0)
|
|
|
|
#define STATS_INC_FREEHIT(x) do { } while (0)
|
|
|
|
#define STATS_INC_FREEMISS(x) do { } while (0)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* memory layout of objects:
|
2005-04-16 22:20:36 +00:00
|
|
|
* 0 : objp
|
2006-02-01 11:05:42 +00:00
|
|
|
* 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
|
2005-04-16 22:20:36 +00:00
|
|
|
* the end of an object is aligned with the end of the real
|
|
|
|
* allocation. Catches writes behind the end of the allocation.
|
2006-02-01 11:05:42 +00:00
|
|
|
* cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
|
2005-04-16 22:20:36 +00:00
|
|
|
* redzone word.
|
2006-02-01 11:05:42 +00:00
|
|
|
* cachep->obj_offset: The real object.
|
2012-06-13 15:24:57 +00:00
|
|
|
* cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
|
|
|
|
* cachep->size - 1* BYTES_PER_WORD: last caller address
|
2006-03-22 08:08:11 +00:00
|
|
|
* [BYTES_PER_WORD long]
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2006-02-01 11:05:50 +00:00
|
|
|
static int obj_offset(struct kmem_cache *cachep)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-02-01 11:05:42 +00:00
|
|
|
return cachep->obj_offset;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
return (unsigned long long*) (objp + obj_offset(cachep) -
|
|
|
|
sizeof(unsigned long long));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
2012-06-13 15:24:57 +00:00
|
|
|
return (unsigned long long *)(objp + cachep->size -
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
sizeof(unsigned long long) -
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-05 01:26:44 +00:00
|
|
|
REDZONE_ALIGN);
|
2012-06-13 15:24:57 +00:00
|
|
|
return (unsigned long long *) (objp + cachep->size -
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
sizeof(unsigned long long));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void **dbg_userword(struct kmem_cache *cachep, void *objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
BUG_ON(!(cachep->flags & SLAB_STORE_USER));
|
2012-06-13 15:24:57 +00:00
|
|
|
return (void **)(objp + cachep->size - BYTES_PER_WORD);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
2006-02-01 11:05:42 +00:00
|
|
|
#define obj_offset(x) 0
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
|
|
|
|
#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
|
2005-04-16 22:20:36 +00:00
|
|
|
#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
#ifdef CONFIG_DEBUG_SLAB_LEAK
|
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
static inline bool is_store_user_clean(struct kmem_cache *cachep)
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
{
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
return atomic_read(&cachep->store_user_clean) == 1;
|
|
|
|
}
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
static inline void set_store_user_clean(struct kmem_cache *cachep)
|
|
|
|
{
|
|
|
|
atomic_set(&cachep->store_user_clean, 1);
|
|
|
|
}
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
static inline void set_store_user_dirty(struct kmem_cache *cachep)
|
|
|
|
{
|
|
|
|
if (is_store_user_clean(cachep))
|
|
|
|
atomic_set(&cachep->store_user_clean, 0);
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
static inline void set_store_user_dirty(struct kmem_cache *cachep) {}
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2011-10-19 05:09:28 +00:00
|
|
|
* Do not go above this order unless 0 objects fit into the slab or
|
|
|
|
* overridden on the command line.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2011-10-19 05:09:24 +00:00
|
|
|
#define SLAB_MAX_ORDER_HI 1
|
|
|
|
#define SLAB_MAX_ORDER_LO 0
|
|
|
|
static int slab_max_order = SLAB_MAX_ORDER_LO;
|
2011-10-19 05:09:28 +00:00
|
|
|
static bool slab_max_order_set __initdata;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-02-01 11:05:49 +00:00
|
|
|
static inline struct kmem_cache *virt_to_cache(const void *obj)
|
|
|
|
{
|
2007-05-06 21:49:41 +00:00
|
|
|
struct page *page = virt_to_head_page(obj);
|
2012-06-13 15:24:56 +00:00
|
|
|
return page->slab_cache;
|
2006-02-01 11:05:49 +00:00
|
|
|
}
|
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
|
2006-03-22 08:08:10 +00:00
|
|
|
unsigned int idx)
|
|
|
|
{
|
2013-10-24 01:07:49 +00:00
|
|
|
return page->s_mem + cache->size * idx;
|
2006-03-22 08:08:10 +00:00
|
|
|
}
|
|
|
|
|
2006-12-13 08:34:27 +00:00
|
|
|
/*
|
2012-06-13 15:24:57 +00:00
|
|
|
* We want to avoid an expensive divide : (offset / cache->size)
|
|
|
|
* Using the fact that size is a constant for a particular cache,
|
|
|
|
* we can replace (offset / cache->size) by
|
2006-12-13 08:34:27 +00:00
|
|
|
* reciprocal_divide(offset, cache->reciprocal_buffer_size)
|
|
|
|
*/
|
|
|
|
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
|
2013-10-24 01:07:49 +00:00
|
|
|
const struct page *page, void *obj)
|
2006-03-22 08:08:10 +00:00
|
|
|
{
|
2013-10-24 01:07:49 +00:00
|
|
|
u32 offset = (obj - page->s_mem);
|
2006-12-13 08:34:27 +00:00
|
|
|
return reciprocal_divide(offset, cache->reciprocal_buffer_size);
|
2006-03-22 08:08:10 +00:00
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:09 +00:00
|
|
|
#define BOOT_CPUCACHE_ENTRIES 1
|
2005-04-16 22:20:36 +00:00
|
|
|
/* internal cache of cache description objs */
|
2012-09-05 00:20:33 +00:00
|
|
|
static struct kmem_cache kmem_cache_boot = {
|
2006-01-08 09:00:37 +00:00
|
|
|
.batchcount = 1,
|
|
|
|
.limit = BOOT_CPUCACHE_ENTRIES,
|
|
|
|
.shared = 1,
|
2012-06-13 15:24:57 +00:00
|
|
|
.size = sizeof(struct kmem_cache),
|
2006-01-08 09:00:37 +00:00
|
|
|
.name = "kmem_cache",
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
2009-10-29 13:34:13 +00:00
|
|
|
static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:26:27 +00:00
|
|
|
return this_cpu_ptr(cachep->cpu_cache);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Calculate the number of objects and left-over bytes for a given buffer size.
|
|
|
|
*/
|
2016-03-15 21:54:53 +00:00
|
|
|
static unsigned int cache_estimate(unsigned long gfporder, size_t buffer_size,
|
|
|
|
unsigned long flags, size_t *left_over)
|
2006-02-01 11:05:45 +00:00
|
|
|
{
|
2016-03-15 21:54:53 +00:00
|
|
|
unsigned int num;
|
2006-02-01 11:05:45 +00:00
|
|
|
size_t slab_size = PAGE_SIZE << gfporder;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-02-01 11:05:45 +00:00
|
|
|
/*
|
|
|
|
* The slab management structure can be either off the slab or
|
|
|
|
* on it. For the latter case, the memory allocated for a
|
|
|
|
* slab is used for:
|
|
|
|
*
|
|
|
|
* - @buffer_size bytes for each object
|
2016-03-15 21:54:30 +00:00
|
|
|
* - One freelist_idx_t for each object
|
|
|
|
*
|
|
|
|
* We don't need to consider alignment of freelist because
|
|
|
|
* freelist will be at the end of slab page. The objects will be
|
|
|
|
* at the correct alignment.
|
2006-02-01 11:05:45 +00:00
|
|
|
*
|
|
|
|
* If the slab management structure is off the slab, then the
|
|
|
|
* alignment will already be calculated into the size. Because
|
|
|
|
* the slabs are all pages aligned, the objects will be at the
|
|
|
|
* correct alignment when allocated.
|
|
|
|
*/
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (flags & (CFLGS_OBJFREELIST_SLAB | CFLGS_OFF_SLAB)) {
|
2016-03-15 21:54:53 +00:00
|
|
|
num = slab_size / buffer_size;
|
2016-03-15 21:54:30 +00:00
|
|
|
*left_over = slab_size % buffer_size;
|
2006-02-01 11:05:45 +00:00
|
|
|
} else {
|
2016-03-15 21:54:53 +00:00
|
|
|
num = slab_size / (buffer_size + sizeof(freelist_idx_t));
|
2016-03-15 21:54:30 +00:00
|
|
|
*left_over = slab_size %
|
|
|
|
(buffer_size + sizeof(freelist_idx_t));
|
2006-02-01 11:05:45 +00:00
|
|
|
}
|
2016-03-15 21:54:53 +00:00
|
|
|
|
|
|
|
return num;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-09-11 19:49:38 +00:00
|
|
|
#if DEBUG
|
2008-04-30 07:55:07 +00:00
|
|
|
#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
static void __slab_error(const char *function, struct kmem_cache *cachep,
|
|
|
|
char *msg)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("slab error in %s(): cache `%s': %s\n",
|
2006-01-08 09:00:37 +00:00
|
|
|
function, cachep->name, msg);
|
2005-04-16 22:20:36 +00:00
|
|
|
dump_stack();
|
2013-01-21 06:47:39 +00:00
|
|
|
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2012-09-11 19:49:38 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-12-07 04:32:16 +00:00
|
|
|
/*
|
|
|
|
* By default on NUMA we use alien caches to stage the freeing of
|
|
|
|
* objects allocated from other nodes. This causes massive memory
|
|
|
|
* inefficiencies when using fake NUMA setup to split memory into a
|
|
|
|
* large number of small nodes, so it can be disabled on the command
|
|
|
|
* line
|
|
|
|
*/
|
|
|
|
|
|
|
|
static int use_alien_caches __read_mostly = 1;
|
|
|
|
static int __init noaliencache_setup(char *s)
|
|
|
|
{
|
|
|
|
use_alien_caches = 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("noaliencache", noaliencache_setup);
|
|
|
|
|
2011-10-19 05:09:28 +00:00
|
|
|
static int __init slab_max_order_setup(char *str)
|
|
|
|
{
|
|
|
|
get_option(&str, &slab_max_order);
|
|
|
|
slab_max_order = slab_max_order < 0 ? 0 :
|
|
|
|
min(slab_max_order, MAX_ORDER - 1);
|
|
|
|
slab_max_order_set = true;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("slab_max_order=", slab_max_order_setup);
|
|
|
|
|
2006-03-10 01:33:54 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
/*
|
|
|
|
* Special reaping functions for NUMA systems called from cache_reap().
|
|
|
|
* These take care of doing round robin flushing of alien caches (containing
|
|
|
|
* objects freed on different nodes from which they were allocated) and the
|
|
|
|
* flushing of remote pcps by calling drain_node_pages.
|
|
|
|
*/
|
2009-10-29 13:34:13 +00:00
|
|
|
static DEFINE_PER_CPU(unsigned long, slab_reap_node);
|
2006-03-10 01:33:54 +00:00
|
|
|
|
|
|
|
static void init_reap_node(int cpu)
|
|
|
|
{
|
2016-05-20 00:10:58 +00:00
|
|
|
per_cpu(slab_reap_node, cpu) = next_node_in(cpu_to_mem(cpu),
|
|
|
|
node_online_map);
|
2006-03-10 01:33:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void next_reap_node(void)
|
|
|
|
{
|
2010-12-08 15:22:55 +00:00
|
|
|
int node = __this_cpu_read(slab_reap_node);
|
2006-03-10 01:33:54 +00:00
|
|
|
|
2016-05-20 00:10:58 +00:00
|
|
|
node = next_node_in(node, node_online_map);
|
2010-12-08 15:22:55 +00:00
|
|
|
__this_cpu_write(slab_reap_node, node);
|
2006-03-10 01:33:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
#define init_reap_node(cpu) do { } while (0)
|
|
|
|
#define next_reap_node(void) do { } while (0)
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
|
|
|
|
* via the workqueue/eventd.
|
|
|
|
* Add the CPU number into the expiration time to minimize the possibility of
|
|
|
|
* the CPUs getting into lockstep and contending for the global cache chain
|
|
|
|
* lock.
|
|
|
|
*/
|
2013-06-19 18:53:51 +00:00
|
|
|
static void start_cpu_timer(int cpu)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-10-29 13:34:13 +00:00
|
|
|
struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* When this gets called from do_initcalls via cpucache_init(),
|
|
|
|
* init_workqueues() has already run, so keventd will be setup
|
|
|
|
* at that time.
|
|
|
|
*/
|
2006-11-22 14:54:01 +00:00
|
|
|
if (keventd_up() && reap_work->work.func == NULL) {
|
2006-03-10 01:33:54 +00:00
|
|
|
init_reap_node(cpu);
|
2012-08-21 20:18:23 +00:00
|
|
|
INIT_DEFERRABLE_WORK(reap_work, cache_reap);
|
2006-12-10 10:21:28 +00:00
|
|
|
schedule_delayed_work_on(cpu, reap_work,
|
|
|
|
__round_jiffies_relative(HZ, cpu));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-06 23:04:27 +00:00
|
|
|
static void init_arraycache(struct array_cache *ac, int limit, int batch)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-06-11 12:22:40 +00:00
|
|
|
/*
|
|
|
|
* The array_cache structures contain pointers to free object.
|
2011-03-31 01:57:33 +00:00
|
|
|
* However, when such objects are allocated or transferred to another
|
2009-06-11 12:22:40 +00:00
|
|
|
* cache the pointers are not cleared and they could be counted as
|
|
|
|
* valid references during a kmemleak scan. Therefore, kmemleak must
|
|
|
|
* not scan such objects.
|
|
|
|
*/
|
2014-08-06 23:04:27 +00:00
|
|
|
kmemleak_no_scan(ac);
|
|
|
|
if (ac) {
|
|
|
|
ac->avail = 0;
|
|
|
|
ac->limit = limit;
|
|
|
|
ac->batchcount = batch;
|
|
|
|
ac->touched = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2014-08-06 23:04:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct array_cache *alloc_arraycache(int node, int entries,
|
|
|
|
int batchcount, gfp_t gfp)
|
|
|
|
{
|
2014-08-06 23:04:40 +00:00
|
|
|
size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
|
2014-08-06 23:04:27 +00:00
|
|
|
struct array_cache *ac = NULL;
|
|
|
|
|
|
|
|
ac = kmalloc_node(memsize, gfp, node);
|
|
|
|
init_arraycache(ac, entries, batchcount);
|
|
|
|
return ac;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
static noinline void cache_free_pfmemalloc(struct kmem_cache *cachep,
|
|
|
|
struct page *page, void *objp)
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
{
|
2016-03-15 21:54:56 +00:00
|
|
|
struct kmem_cache_node *n;
|
|
|
|
int page_node;
|
|
|
|
LIST_HEAD(list);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
page_node = page_to_nid(page);
|
|
|
|
n = get_node(cachep, page_node);
|
2012-07-31 23:44:30 +00:00
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
spin_lock(&n->list_lock);
|
|
|
|
free_block(cachep, &objp, 1, page_node, &list);
|
|
|
|
spin_unlock(&n->list_lock);
|
2012-07-31 23:44:30 +00:00
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
}
|
|
|
|
|
2006-03-25 11:06:44 +00:00
|
|
|
/*
|
|
|
|
* Transfer objects in one arraycache to another.
|
|
|
|
* Locking must be handled by the caller.
|
|
|
|
*
|
|
|
|
* Return the number of entries transferred.
|
|
|
|
*/
|
|
|
|
static int transfer_objects(struct array_cache *to,
|
|
|
|
struct array_cache *from, unsigned int max)
|
|
|
|
{
|
|
|
|
/* Figure out how many entries to transfer */
|
2010-10-26 21:22:23 +00:00
|
|
|
int nr = min3(from->avail, max, to->limit - to->avail);
|
2006-03-25 11:06:44 +00:00
|
|
|
|
|
|
|
if (!nr)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
memcpy(to->entry + to->avail, from->entry + from->avail -nr,
|
|
|
|
sizeof(void *) *nr);
|
|
|
|
|
|
|
|
from->avail -= nr;
|
|
|
|
to->avail += nr;
|
|
|
|
return nr;
|
|
|
|
}
|
|
|
|
|
2006-09-27 08:50:08 +00:00
|
|
|
#ifndef CONFIG_NUMA
|
|
|
|
|
|
|
|
#define drain_alien_cache(cachep, alien) do { } while (0)
|
2013-01-10 19:14:19 +00:00
|
|
|
#define reap_alien(cachep, n) do { } while (0)
|
2006-09-27 08:50:08 +00:00
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
static inline struct alien_cache **alloc_alien_cache(int node,
|
|
|
|
int limit, gfp_t gfp)
|
2006-09-27 08:50:08 +00:00
|
|
|
{
|
2016-05-20 00:10:05 +00:00
|
|
|
return NULL;
|
2006-09-27 08:50:08 +00:00
|
|
|
}
|
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
static inline void free_alien_cache(struct alien_cache **ac_ptr)
|
2006-09-27 08:50:08 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void *alternate_node_alloc(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2006-12-07 04:32:30 +00:00
|
|
|
static inline void *____cache_alloc_node(struct kmem_cache *cachep,
|
2006-09-27 08:50:08 +00:00
|
|
|
gfp_t flags, int nodeid)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 22:46:55 +00:00
|
|
|
static inline gfp_t gfp_exact_node(gfp_t flags)
|
|
|
|
{
|
mm: thp: set THP defrag by default to madvise and add a stall-free defrag option
THP defrag is enabled by default to direct reclaim/compact but not wake
kswapd in the event of a THP allocation failure. The problem is that
THP allocation requests potentially enter reclaim/compaction. This
potentially incurs a severe stall that is not guaranteed to be offset by
reduced TLB misses. While there has been considerable effort to reduce
the impact of reclaim/compaction, it is still a high cost and workloads
that should fit in memory fail to do so. Specifically, a simple
anon/file streaming workload will enter direct reclaim on NUMA at least
even though the working set size is 80% of RAM. It's been years and
it's time to throw in the towel.
First, this patch defines THP defrag as follows;
madvise: A failed allocation will direct reclaim/compact if the application requests it
never: Neither reclaim/compact nor wake kswapd
defer: A failed allocation will wake kswapd/kcompactd
always: A failed allocation will direct reclaim/compact (historical behaviour)
khugepaged defrag will enter direct/reclaim but not wake kswapd.
Next it sets the default defrag option to be "madvise" to only enter
direct reclaim/compaction for applications that specifically requested
it.
Lastly, it removes a check from the page allocator slowpath that is
related to __GFP_THISNODE to allow "defer" to work. The callers that
really cares are slub/slab and they are updated accordingly. The slab
one may be surprising because it also corrects a comment as kswapd was
never woken up by that path.
This means that a THP fault will no longer stall for most applications
by default and the ideal for most users that get THP if they are
immediately available. There are still options for users that prefer a
stall at startup of a new application by either restoring historical
behaviour with "always" or pick a half-way point with "defer" where
kswapd does some of the work in the background and wakes kcompactd if
necessary. THP defrag for khugepaged remains enabled and will enter
direct/reclaim but no wakeup kswapd or kcompactd.
After this patch a THP allocation failure will quickly fallback and rely
on khugepaged to recover the situation at some time in the future. In
some cases, this will reduce THP usage but the benefit of THP is hard to
measure and not a universal win where as a stall to reclaim/compaction
is definitely measurable and can be painful.
The first test for this is using "usemem" to read a large file and write
a large anonymous mapping (to avoid the zero page) multiple times. The
total size of the mappings is 80% of RAM and the benchmark simply
measures how long it takes to complete. It uses multiple threads to see
if that is a factor. On UMA, the performance is almost identical so is
not reported but on NUMA, we see this
usemem
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Amean System-1 102.86 ( 0.00%) 46.81 ( 54.50%)
Amean System-4 37.85 ( 0.00%) 34.02 ( 10.12%)
Amean System-7 48.12 ( 0.00%) 46.89 ( 2.56%)
Amean System-12 51.98 ( 0.00%) 56.96 ( -9.57%)
Amean System-21 80.16 ( 0.00%) 79.05 ( 1.39%)
Amean System-30 110.71 ( 0.00%) 107.17 ( 3.20%)
Amean System-48 127.98 ( 0.00%) 124.83 ( 2.46%)
Amean Elapsd-1 185.84 ( 0.00%) 105.51 ( 43.23%)
Amean Elapsd-4 26.19 ( 0.00%) 25.58 ( 2.33%)
Amean Elapsd-7 21.65 ( 0.00%) 21.62 ( 0.16%)
Amean Elapsd-12 18.58 ( 0.00%) 17.94 ( 3.43%)
Amean Elapsd-21 17.53 ( 0.00%) 16.60 ( 5.33%)
Amean Elapsd-30 17.45 ( 0.00%) 17.13 ( 1.84%)
Amean Elapsd-48 15.40 ( 0.00%) 15.27 ( 0.82%)
For a single thread, the benchmark completes 43.23% faster with this
patch applied with smaller benefits as the thread increases. Similar,
notice the large reduction in most cases in system CPU usage. The
overall CPU time is
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
User 10357.65 10438.33
System 3988.88 3543.94
Elapsed 2203.01 1634.41
Which is substantial. Now, the reclaim figures
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 128458477 278352931
Major Faults 2174976 225
Swap Ins 16904701 0
Swap Outs 17359627 0
Allocation stalls 43611 0
DMA allocs 0 0
DMA32 allocs 19832646 19448017
Normal allocs 614488453 580941839
Movable allocs 0 0
Direct pages scanned 24163800 0
Kswapd pages scanned 0 0
Kswapd pages reclaimed 0 0
Direct pages reclaimed 20691346 0
Compaction stalls 42263 0
Compaction success 938 0
Compaction failures 41325 0
This patch eliminates almost all swapping and direct reclaim activity.
There is still overhead but it's from NUMA balancing which does not
identify that it's pointless trying to do anything with this workload.
I also tried the thpscale benchmark which forces a corner case where
compaction can be used heavily and measures the latency of whether base
or huge pages were used
thpscale Fault Latencies
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Amean fault-base-1 5288.84 ( 0.00%) 2817.12 ( 46.73%)
Amean fault-base-3 6365.53 ( 0.00%) 3499.11 ( 45.03%)
Amean fault-base-5 6526.19 ( 0.00%) 4363.06 ( 33.15%)
Amean fault-base-7 7142.25 ( 0.00%) 4858.08 ( 31.98%)
Amean fault-base-12 13827.64 ( 0.00%) 10292.11 ( 25.57%)
Amean fault-base-18 18235.07 ( 0.00%) 13788.84 ( 24.38%)
Amean fault-base-24 21597.80 ( 0.00%) 24388.03 (-12.92%)
Amean fault-base-30 26754.15 ( 0.00%) 19700.55 ( 26.36%)
Amean fault-base-32 26784.94 ( 0.00%) 19513.57 ( 27.15%)
Amean fault-huge-1 4223.96 ( 0.00%) 2178.57 ( 48.42%)
Amean fault-huge-3 2194.77 ( 0.00%) 2149.74 ( 2.05%)
Amean fault-huge-5 2569.60 ( 0.00%) 2346.95 ( 8.66%)
Amean fault-huge-7 3612.69 ( 0.00%) 2997.70 ( 17.02%)
Amean fault-huge-12 3301.75 ( 0.00%) 6727.02 (-103.74%)
Amean fault-huge-18 6696.47 ( 0.00%) 6685.72 ( 0.16%)
Amean fault-huge-24 8000.72 ( 0.00%) 9311.43 (-16.38%)
Amean fault-huge-30 13305.55 ( 0.00%) 9750.45 ( 26.72%)
Amean fault-huge-32 9981.71 ( 0.00%) 10316.06 ( -3.35%)
The average time to fault pages is substantially reduced in the majority
of caseds but with the obvious caveat that fewer THPs are actually used
in this adverse workload
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Percentage huge-1 0.71 ( 0.00%) 14.04 (1865.22%)
Percentage huge-3 10.77 ( 0.00%) 33.05 (206.85%)
Percentage huge-5 60.39 ( 0.00%) 38.51 (-36.23%)
Percentage huge-7 45.97 ( 0.00%) 34.57 (-24.79%)
Percentage huge-12 68.12 ( 0.00%) 40.07 (-41.17%)
Percentage huge-18 64.93 ( 0.00%) 47.82 (-26.35%)
Percentage huge-24 62.69 ( 0.00%) 44.23 (-29.44%)
Percentage huge-30 43.49 ( 0.00%) 55.38 ( 27.34%)
Percentage huge-32 50.72 ( 0.00%) 51.90 ( 2.35%)
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 37429143 47564000
Major Faults 1916 1558
Swap Ins 1466 1079
Swap Outs 2936863 149626
Allocation stalls 62510 3
DMA allocs 0 0
DMA32 allocs 6566458 6401314
Normal allocs 216361697 216538171
Movable allocs 0 0
Direct pages scanned 25977580 17998
Kswapd pages scanned 0 3638931
Kswapd pages reclaimed 0 207236
Direct pages reclaimed 8833714 88
Compaction stalls 103349 5
Compaction success 270 4
Compaction failures 103079 1
Note again that while this does swap as it's an aggressive workload, the
direct relcim activity and allocation stalls is substantially reduced.
There is some kswapd activity but ftrace showed that the kswapd activity
was due to normal wakeups from 4K pages being allocated.
Compaction-related stalls and activity are almost eliminated.
I also tried the stutter benchmark. For this, I do not have figures for
NUMA but it's something that does impact UMA so I'll report what is
available
stutter
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Min mmap 7.3571 ( 0.00%) 7.3438 ( 0.18%)
1st-qrtle mmap 7.5278 ( 0.00%) 17.9200 (-138.05%)
2nd-qrtle mmap 7.6818 ( 0.00%) 21.6055 (-181.25%)
3rd-qrtle mmap 11.0889 ( 0.00%) 21.8881 (-97.39%)
Max-90% mmap 27.8978 ( 0.00%) 22.1632 ( 20.56%)
Max-93% mmap 28.3202 ( 0.00%) 22.3044 ( 21.24%)
Max-95% mmap 28.5600 ( 0.00%) 22.4580 ( 21.37%)
Max-99% mmap 29.6032 ( 0.00%) 25.5216 ( 13.79%)
Max mmap 4109.7289 ( 0.00%) 4813.9832 (-17.14%)
Mean mmap 12.4474 ( 0.00%) 19.3027 (-55.07%)
This benchmark is trying to fault an anonymous mapping while there is a
heavy IO load -- a scenario that desktop users used to complain about
frequently. This shows a mix because the ideal case of mapping with THP
is not hit as often. However, note that 99% of the mappings complete
13.79% faster. The CPU usage here is particularly interesting
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
User 67.50 0.99
System 1327.88 91.30
Elapsed 2079.00 2128.98
And once again we look at the reclaim figures
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 335241922 1314582827
Major Faults 715 819
Swap Ins 0 0
Swap Outs 0 0
Allocation stalls 532723 0
DMA allocs 0 0
DMA32 allocs 1822364341 1177950222
Normal allocs 1815640808 1517844854
Movable allocs 0 0
Direct pages scanned 21892772 0
Kswapd pages scanned 20015890 41879484
Kswapd pages reclaimed 19961986 41822072
Direct pages reclaimed 21892741 0
Compaction stalls 1065755 0
Compaction success 514 0
Compaction failures 1065241 0
Allocation stalls and all direct reclaim activity is eliminated as well
as compaction-related stalls.
THP gives impressive gains in some cases but only if they are quickly
available. We're not going to reach the point where they are completely
free so lets take the costs out of the fast paths finally and defer the
cost to kswapd, kcompactd and khugepaged where it belongs.
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 21:19:23 +00:00
|
|
|
return flags & ~__GFP_NOFAIL;
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 22:46:55 +00:00
|
|
|
}
|
|
|
|
|
2006-09-27 08:50:08 +00:00
|
|
|
#else /* CONFIG_NUMA */
|
|
|
|
|
2006-12-07 04:32:30 +00:00
|
|
|
static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
|
2006-03-24 11:16:08 +00:00
|
|
|
static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
|
2006-01-19 01:42:36 +00:00
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
static struct alien_cache *__alloc_alien_cache(int node, int entries,
|
|
|
|
int batch, gfp_t gfp)
|
|
|
|
{
|
2014-08-06 23:04:40 +00:00
|
|
|
size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache *alc = NULL;
|
|
|
|
|
|
|
|
alc = kmalloc_node(memsize, gfp, node);
|
|
|
|
init_arraycache(&alc->ac, entries, batch);
|
2014-08-06 23:04:31 +00:00
|
|
|
spin_lock_init(&alc->lock);
|
2014-08-06 23:04:29 +00:00
|
|
|
return alc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache **alc_ptr;
|
2014-08-06 23:04:40 +00:00
|
|
|
size_t memsize = sizeof(void *) * nr_node_ids;
|
2005-09-09 20:03:32 +00:00
|
|
|
int i;
|
|
|
|
|
|
|
|
if (limit > 1)
|
|
|
|
limit = 12;
|
2014-08-06 23:04:29 +00:00
|
|
|
alc_ptr = kzalloc_node(memsize, gfp, node);
|
|
|
|
if (!alc_ptr)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
for_each_node(i) {
|
|
|
|
if (i == node || !node_online(i))
|
|
|
|
continue;
|
|
|
|
alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
|
|
|
|
if (!alc_ptr[i]) {
|
|
|
|
for (i--; i >= 0; i--)
|
|
|
|
kfree(alc_ptr[i]);
|
|
|
|
kfree(alc_ptr);
|
|
|
|
return NULL;
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
}
|
2014-08-06 23:04:29 +00:00
|
|
|
return alc_ptr;
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
static void free_alien_cache(struct alien_cache **alc_ptr)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
if (!alc_ptr)
|
2005-09-09 20:03:32 +00:00
|
|
|
return;
|
|
|
|
for_each_node(i)
|
2014-08-06 23:04:29 +00:00
|
|
|
kfree(alc_ptr[i]);
|
|
|
|
kfree(alc_ptr);
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void __drain_alien_cache(struct kmem_cache *cachep,
|
2014-08-06 23:04:33 +00:00
|
|
|
struct array_cache *ac, int node,
|
|
|
|
struct list_head *list)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2014-08-06 23:04:11 +00:00
|
|
|
struct kmem_cache_node *n = get_node(cachep, node);
|
2005-09-09 20:03:32 +00:00
|
|
|
|
|
|
|
if (ac->avail) {
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock(&n->list_lock);
|
2006-03-25 11:06:45 +00:00
|
|
|
/*
|
|
|
|
* Stuff objects into the remote nodes shared array first.
|
|
|
|
* That way we could avoid the overhead of putting the objects
|
|
|
|
* into the free lists and getting them back later.
|
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
if (n->shared)
|
|
|
|
transfer_objects(n->shared, ac, ac->limit);
|
2006-03-25 11:06:45 +00:00
|
|
|
|
2014-08-06 23:04:33 +00:00
|
|
|
free_block(cachep, ac->entry, ac->avail, node, list);
|
2005-09-09 20:03:32 +00:00
|
|
|
ac->avail = 0;
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-10 01:33:54 +00:00
|
|
|
/*
|
|
|
|
* Called from cache_reap() to regularly drain alien caches round robin.
|
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
|
2006-03-10 01:33:54 +00:00
|
|
|
{
|
2010-12-08 15:22:55 +00:00
|
|
|
int node = __this_cpu_read(slab_reap_node);
|
2006-03-10 01:33:54 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
if (n->alien) {
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache *alc = n->alien[node];
|
|
|
|
struct array_cache *ac;
|
|
|
|
|
|
|
|
if (alc) {
|
|
|
|
ac = &alc->ac;
|
2014-08-06 23:04:31 +00:00
|
|
|
if (ac->avail && spin_trylock_irq(&alc->lock)) {
|
2014-08-06 23:04:33 +00:00
|
|
|
LIST_HEAD(list);
|
|
|
|
|
|
|
|
__drain_alien_cache(cachep, ac, node, &list);
|
2014-08-06 23:04:31 +00:00
|
|
|
spin_unlock_irq(&alc->lock);
|
2014-08-06 23:04:33 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2014-08-06 23:04:29 +00:00
|
|
|
}
|
2006-03-10 01:33:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
static void drain_alien_cache(struct kmem_cache *cachep,
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache **alien)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2006-01-08 09:00:37 +00:00
|
|
|
int i = 0;
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache *alc;
|
2005-09-09 20:03:32 +00:00
|
|
|
struct array_cache *ac;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
for_each_online_node(i) {
|
2014-08-06 23:04:29 +00:00
|
|
|
alc = alien[i];
|
|
|
|
if (alc) {
|
2014-08-06 23:04:33 +00:00
|
|
|
LIST_HEAD(list);
|
|
|
|
|
2014-08-06 23:04:29 +00:00
|
|
|
ac = &alc->ac;
|
2014-08-06 23:04:31 +00:00
|
|
|
spin_lock_irqsave(&alc->lock, flags);
|
2014-08-06 23:04:33 +00:00
|
|
|
__drain_alien_cache(cachep, ac, i, &list);
|
2014-08-06 23:04:31 +00:00
|
|
|
spin_unlock_irqrestore(&alc->lock, flags);
|
2014-08-06 23:04:33 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2006-06-23 09:03:05 +00:00
|
|
|
|
2014-10-09 22:26:09 +00:00
|
|
|
static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
|
|
|
|
int node, int page_node)
|
2006-06-23 09:03:05 +00:00
|
|
|
{
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache *alien = NULL;
|
|
|
|
struct array_cache *ac;
|
2014-08-06 23:04:25 +00:00
|
|
|
LIST_HEAD(list);
|
2006-10-06 07:43:52 +00:00
|
|
|
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
2006-06-23 09:03:05 +00:00
|
|
|
STATS_INC_NODEFREES(cachep);
|
2014-10-09 22:26:09 +00:00
|
|
|
if (n->alien && n->alien[page_node]) {
|
|
|
|
alien = n->alien[page_node];
|
2014-08-06 23:04:29 +00:00
|
|
|
ac = &alien->ac;
|
2014-08-06 23:04:31 +00:00
|
|
|
spin_lock(&alien->lock);
|
2014-08-06 23:04:29 +00:00
|
|
|
if (unlikely(ac->avail == ac->limit)) {
|
2006-06-23 09:03:05 +00:00
|
|
|
STATS_INC_ACOVERFLOW(cachep);
|
2014-10-09 22:26:09 +00:00
|
|
|
__drain_alien_cache(cachep, ac, page_node, &list);
|
2006-06-23 09:03:05 +00:00
|
|
|
}
|
2016-03-15 21:54:56 +00:00
|
|
|
ac->entry[ac->avail++] = objp;
|
2014-08-06 23:04:31 +00:00
|
|
|
spin_unlock(&alien->lock);
|
2014-08-06 23:04:33 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2006-06-23 09:03:05 +00:00
|
|
|
} else {
|
2014-10-09 22:26:09 +00:00
|
|
|
n = get_node(cachep, page_node);
|
2014-08-06 23:04:11 +00:00
|
|
|
spin_lock(&n->list_lock);
|
2014-10-09 22:26:09 +00:00
|
|
|
free_block(cachep, &objp, 1, page_node, &list);
|
2014-08-06 23:04:11 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
2014-08-06 23:04:25 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2006-06-23 09:03:05 +00:00
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
2014-10-09 22:26:09 +00:00
|
|
|
|
|
|
|
static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
|
|
|
|
{
|
|
|
|
int page_node = page_to_nid(virt_to_page(objp));
|
|
|
|
int node = numa_mem_id();
|
|
|
|
/*
|
|
|
|
* Make sure we are not freeing a object from another node to the array
|
|
|
|
* cache on this cpu.
|
|
|
|
*/
|
|
|
|
if (likely(node == page_node))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return __cache_free_alien(cachep, objp, node, page_node);
|
|
|
|
}
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 22:46:55 +00:00
|
|
|
|
|
|
|
/*
|
mm: thp: set THP defrag by default to madvise and add a stall-free defrag option
THP defrag is enabled by default to direct reclaim/compact but not wake
kswapd in the event of a THP allocation failure. The problem is that
THP allocation requests potentially enter reclaim/compaction. This
potentially incurs a severe stall that is not guaranteed to be offset by
reduced TLB misses. While there has been considerable effort to reduce
the impact of reclaim/compaction, it is still a high cost and workloads
that should fit in memory fail to do so. Specifically, a simple
anon/file streaming workload will enter direct reclaim on NUMA at least
even though the working set size is 80% of RAM. It's been years and
it's time to throw in the towel.
First, this patch defines THP defrag as follows;
madvise: A failed allocation will direct reclaim/compact if the application requests it
never: Neither reclaim/compact nor wake kswapd
defer: A failed allocation will wake kswapd/kcompactd
always: A failed allocation will direct reclaim/compact (historical behaviour)
khugepaged defrag will enter direct/reclaim but not wake kswapd.
Next it sets the default defrag option to be "madvise" to only enter
direct reclaim/compaction for applications that specifically requested
it.
Lastly, it removes a check from the page allocator slowpath that is
related to __GFP_THISNODE to allow "defer" to work. The callers that
really cares are slub/slab and they are updated accordingly. The slab
one may be surprising because it also corrects a comment as kswapd was
never woken up by that path.
This means that a THP fault will no longer stall for most applications
by default and the ideal for most users that get THP if they are
immediately available. There are still options for users that prefer a
stall at startup of a new application by either restoring historical
behaviour with "always" or pick a half-way point with "defer" where
kswapd does some of the work in the background and wakes kcompactd if
necessary. THP defrag for khugepaged remains enabled and will enter
direct/reclaim but no wakeup kswapd or kcompactd.
After this patch a THP allocation failure will quickly fallback and rely
on khugepaged to recover the situation at some time in the future. In
some cases, this will reduce THP usage but the benefit of THP is hard to
measure and not a universal win where as a stall to reclaim/compaction
is definitely measurable and can be painful.
The first test for this is using "usemem" to read a large file and write
a large anonymous mapping (to avoid the zero page) multiple times. The
total size of the mappings is 80% of RAM and the benchmark simply
measures how long it takes to complete. It uses multiple threads to see
if that is a factor. On UMA, the performance is almost identical so is
not reported but on NUMA, we see this
usemem
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Amean System-1 102.86 ( 0.00%) 46.81 ( 54.50%)
Amean System-4 37.85 ( 0.00%) 34.02 ( 10.12%)
Amean System-7 48.12 ( 0.00%) 46.89 ( 2.56%)
Amean System-12 51.98 ( 0.00%) 56.96 ( -9.57%)
Amean System-21 80.16 ( 0.00%) 79.05 ( 1.39%)
Amean System-30 110.71 ( 0.00%) 107.17 ( 3.20%)
Amean System-48 127.98 ( 0.00%) 124.83 ( 2.46%)
Amean Elapsd-1 185.84 ( 0.00%) 105.51 ( 43.23%)
Amean Elapsd-4 26.19 ( 0.00%) 25.58 ( 2.33%)
Amean Elapsd-7 21.65 ( 0.00%) 21.62 ( 0.16%)
Amean Elapsd-12 18.58 ( 0.00%) 17.94 ( 3.43%)
Amean Elapsd-21 17.53 ( 0.00%) 16.60 ( 5.33%)
Amean Elapsd-30 17.45 ( 0.00%) 17.13 ( 1.84%)
Amean Elapsd-48 15.40 ( 0.00%) 15.27 ( 0.82%)
For a single thread, the benchmark completes 43.23% faster with this
patch applied with smaller benefits as the thread increases. Similar,
notice the large reduction in most cases in system CPU usage. The
overall CPU time is
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
User 10357.65 10438.33
System 3988.88 3543.94
Elapsed 2203.01 1634.41
Which is substantial. Now, the reclaim figures
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 128458477 278352931
Major Faults 2174976 225
Swap Ins 16904701 0
Swap Outs 17359627 0
Allocation stalls 43611 0
DMA allocs 0 0
DMA32 allocs 19832646 19448017
Normal allocs 614488453 580941839
Movable allocs 0 0
Direct pages scanned 24163800 0
Kswapd pages scanned 0 0
Kswapd pages reclaimed 0 0
Direct pages reclaimed 20691346 0
Compaction stalls 42263 0
Compaction success 938 0
Compaction failures 41325 0
This patch eliminates almost all swapping and direct reclaim activity.
There is still overhead but it's from NUMA balancing which does not
identify that it's pointless trying to do anything with this workload.
I also tried the thpscale benchmark which forces a corner case where
compaction can be used heavily and measures the latency of whether base
or huge pages were used
thpscale Fault Latencies
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Amean fault-base-1 5288.84 ( 0.00%) 2817.12 ( 46.73%)
Amean fault-base-3 6365.53 ( 0.00%) 3499.11 ( 45.03%)
Amean fault-base-5 6526.19 ( 0.00%) 4363.06 ( 33.15%)
Amean fault-base-7 7142.25 ( 0.00%) 4858.08 ( 31.98%)
Amean fault-base-12 13827.64 ( 0.00%) 10292.11 ( 25.57%)
Amean fault-base-18 18235.07 ( 0.00%) 13788.84 ( 24.38%)
Amean fault-base-24 21597.80 ( 0.00%) 24388.03 (-12.92%)
Amean fault-base-30 26754.15 ( 0.00%) 19700.55 ( 26.36%)
Amean fault-base-32 26784.94 ( 0.00%) 19513.57 ( 27.15%)
Amean fault-huge-1 4223.96 ( 0.00%) 2178.57 ( 48.42%)
Amean fault-huge-3 2194.77 ( 0.00%) 2149.74 ( 2.05%)
Amean fault-huge-5 2569.60 ( 0.00%) 2346.95 ( 8.66%)
Amean fault-huge-7 3612.69 ( 0.00%) 2997.70 ( 17.02%)
Amean fault-huge-12 3301.75 ( 0.00%) 6727.02 (-103.74%)
Amean fault-huge-18 6696.47 ( 0.00%) 6685.72 ( 0.16%)
Amean fault-huge-24 8000.72 ( 0.00%) 9311.43 (-16.38%)
Amean fault-huge-30 13305.55 ( 0.00%) 9750.45 ( 26.72%)
Amean fault-huge-32 9981.71 ( 0.00%) 10316.06 ( -3.35%)
The average time to fault pages is substantially reduced in the majority
of caseds but with the obvious caveat that fewer THPs are actually used
in this adverse workload
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Percentage huge-1 0.71 ( 0.00%) 14.04 (1865.22%)
Percentage huge-3 10.77 ( 0.00%) 33.05 (206.85%)
Percentage huge-5 60.39 ( 0.00%) 38.51 (-36.23%)
Percentage huge-7 45.97 ( 0.00%) 34.57 (-24.79%)
Percentage huge-12 68.12 ( 0.00%) 40.07 (-41.17%)
Percentage huge-18 64.93 ( 0.00%) 47.82 (-26.35%)
Percentage huge-24 62.69 ( 0.00%) 44.23 (-29.44%)
Percentage huge-30 43.49 ( 0.00%) 55.38 ( 27.34%)
Percentage huge-32 50.72 ( 0.00%) 51.90 ( 2.35%)
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 37429143 47564000
Major Faults 1916 1558
Swap Ins 1466 1079
Swap Outs 2936863 149626
Allocation stalls 62510 3
DMA allocs 0 0
DMA32 allocs 6566458 6401314
Normal allocs 216361697 216538171
Movable allocs 0 0
Direct pages scanned 25977580 17998
Kswapd pages scanned 0 3638931
Kswapd pages reclaimed 0 207236
Direct pages reclaimed 8833714 88
Compaction stalls 103349 5
Compaction success 270 4
Compaction failures 103079 1
Note again that while this does swap as it's an aggressive workload, the
direct relcim activity and allocation stalls is substantially reduced.
There is some kswapd activity but ftrace showed that the kswapd activity
was due to normal wakeups from 4K pages being allocated.
Compaction-related stalls and activity are almost eliminated.
I also tried the stutter benchmark. For this, I do not have figures for
NUMA but it's something that does impact UMA so I'll report what is
available
stutter
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Min mmap 7.3571 ( 0.00%) 7.3438 ( 0.18%)
1st-qrtle mmap 7.5278 ( 0.00%) 17.9200 (-138.05%)
2nd-qrtle mmap 7.6818 ( 0.00%) 21.6055 (-181.25%)
3rd-qrtle mmap 11.0889 ( 0.00%) 21.8881 (-97.39%)
Max-90% mmap 27.8978 ( 0.00%) 22.1632 ( 20.56%)
Max-93% mmap 28.3202 ( 0.00%) 22.3044 ( 21.24%)
Max-95% mmap 28.5600 ( 0.00%) 22.4580 ( 21.37%)
Max-99% mmap 29.6032 ( 0.00%) 25.5216 ( 13.79%)
Max mmap 4109.7289 ( 0.00%) 4813.9832 (-17.14%)
Mean mmap 12.4474 ( 0.00%) 19.3027 (-55.07%)
This benchmark is trying to fault an anonymous mapping while there is a
heavy IO load -- a scenario that desktop users used to complain about
frequently. This shows a mix because the ideal case of mapping with THP
is not hit as often. However, note that 99% of the mappings complete
13.79% faster. The CPU usage here is particularly interesting
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
User 67.50 0.99
System 1327.88 91.30
Elapsed 2079.00 2128.98
And once again we look at the reclaim figures
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 335241922 1314582827
Major Faults 715 819
Swap Ins 0 0
Swap Outs 0 0
Allocation stalls 532723 0
DMA allocs 0 0
DMA32 allocs 1822364341 1177950222
Normal allocs 1815640808 1517844854
Movable allocs 0 0
Direct pages scanned 21892772 0
Kswapd pages scanned 20015890 41879484
Kswapd pages reclaimed 19961986 41822072
Direct pages reclaimed 21892741 0
Compaction stalls 1065755 0
Compaction success 514 0
Compaction failures 1065241 0
Allocation stalls and all direct reclaim activity is eliminated as well
as compaction-related stalls.
THP gives impressive gains in some cases but only if they are quickly
available. We're not going to reach the point where they are completely
free so lets take the costs out of the fast paths finally and defer the
cost to kswapd, kcompactd and khugepaged where it belongs.
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 21:19:23 +00:00
|
|
|
* Construct gfp mask to allocate from a specific node but do not reclaim or
|
|
|
|
* warn about failures.
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 22:46:55 +00:00
|
|
|
*/
|
|
|
|
static inline gfp_t gfp_exact_node(gfp_t flags)
|
|
|
|
{
|
mm: thp: set THP defrag by default to madvise and add a stall-free defrag option
THP defrag is enabled by default to direct reclaim/compact but not wake
kswapd in the event of a THP allocation failure. The problem is that
THP allocation requests potentially enter reclaim/compaction. This
potentially incurs a severe stall that is not guaranteed to be offset by
reduced TLB misses. While there has been considerable effort to reduce
the impact of reclaim/compaction, it is still a high cost and workloads
that should fit in memory fail to do so. Specifically, a simple
anon/file streaming workload will enter direct reclaim on NUMA at least
even though the working set size is 80% of RAM. It's been years and
it's time to throw in the towel.
First, this patch defines THP defrag as follows;
madvise: A failed allocation will direct reclaim/compact if the application requests it
never: Neither reclaim/compact nor wake kswapd
defer: A failed allocation will wake kswapd/kcompactd
always: A failed allocation will direct reclaim/compact (historical behaviour)
khugepaged defrag will enter direct/reclaim but not wake kswapd.
Next it sets the default defrag option to be "madvise" to only enter
direct reclaim/compaction for applications that specifically requested
it.
Lastly, it removes a check from the page allocator slowpath that is
related to __GFP_THISNODE to allow "defer" to work. The callers that
really cares are slub/slab and they are updated accordingly. The slab
one may be surprising because it also corrects a comment as kswapd was
never woken up by that path.
This means that a THP fault will no longer stall for most applications
by default and the ideal for most users that get THP if they are
immediately available. There are still options for users that prefer a
stall at startup of a new application by either restoring historical
behaviour with "always" or pick a half-way point with "defer" where
kswapd does some of the work in the background and wakes kcompactd if
necessary. THP defrag for khugepaged remains enabled and will enter
direct/reclaim but no wakeup kswapd or kcompactd.
After this patch a THP allocation failure will quickly fallback and rely
on khugepaged to recover the situation at some time in the future. In
some cases, this will reduce THP usage but the benefit of THP is hard to
measure and not a universal win where as a stall to reclaim/compaction
is definitely measurable and can be painful.
The first test for this is using "usemem" to read a large file and write
a large anonymous mapping (to avoid the zero page) multiple times. The
total size of the mappings is 80% of RAM and the benchmark simply
measures how long it takes to complete. It uses multiple threads to see
if that is a factor. On UMA, the performance is almost identical so is
not reported but on NUMA, we see this
usemem
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Amean System-1 102.86 ( 0.00%) 46.81 ( 54.50%)
Amean System-4 37.85 ( 0.00%) 34.02 ( 10.12%)
Amean System-7 48.12 ( 0.00%) 46.89 ( 2.56%)
Amean System-12 51.98 ( 0.00%) 56.96 ( -9.57%)
Amean System-21 80.16 ( 0.00%) 79.05 ( 1.39%)
Amean System-30 110.71 ( 0.00%) 107.17 ( 3.20%)
Amean System-48 127.98 ( 0.00%) 124.83 ( 2.46%)
Amean Elapsd-1 185.84 ( 0.00%) 105.51 ( 43.23%)
Amean Elapsd-4 26.19 ( 0.00%) 25.58 ( 2.33%)
Amean Elapsd-7 21.65 ( 0.00%) 21.62 ( 0.16%)
Amean Elapsd-12 18.58 ( 0.00%) 17.94 ( 3.43%)
Amean Elapsd-21 17.53 ( 0.00%) 16.60 ( 5.33%)
Amean Elapsd-30 17.45 ( 0.00%) 17.13 ( 1.84%)
Amean Elapsd-48 15.40 ( 0.00%) 15.27 ( 0.82%)
For a single thread, the benchmark completes 43.23% faster with this
patch applied with smaller benefits as the thread increases. Similar,
notice the large reduction in most cases in system CPU usage. The
overall CPU time is
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
User 10357.65 10438.33
System 3988.88 3543.94
Elapsed 2203.01 1634.41
Which is substantial. Now, the reclaim figures
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 128458477 278352931
Major Faults 2174976 225
Swap Ins 16904701 0
Swap Outs 17359627 0
Allocation stalls 43611 0
DMA allocs 0 0
DMA32 allocs 19832646 19448017
Normal allocs 614488453 580941839
Movable allocs 0 0
Direct pages scanned 24163800 0
Kswapd pages scanned 0 0
Kswapd pages reclaimed 0 0
Direct pages reclaimed 20691346 0
Compaction stalls 42263 0
Compaction success 938 0
Compaction failures 41325 0
This patch eliminates almost all swapping and direct reclaim activity.
There is still overhead but it's from NUMA balancing which does not
identify that it's pointless trying to do anything with this workload.
I also tried the thpscale benchmark which forces a corner case where
compaction can be used heavily and measures the latency of whether base
or huge pages were used
thpscale Fault Latencies
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Amean fault-base-1 5288.84 ( 0.00%) 2817.12 ( 46.73%)
Amean fault-base-3 6365.53 ( 0.00%) 3499.11 ( 45.03%)
Amean fault-base-5 6526.19 ( 0.00%) 4363.06 ( 33.15%)
Amean fault-base-7 7142.25 ( 0.00%) 4858.08 ( 31.98%)
Amean fault-base-12 13827.64 ( 0.00%) 10292.11 ( 25.57%)
Amean fault-base-18 18235.07 ( 0.00%) 13788.84 ( 24.38%)
Amean fault-base-24 21597.80 ( 0.00%) 24388.03 (-12.92%)
Amean fault-base-30 26754.15 ( 0.00%) 19700.55 ( 26.36%)
Amean fault-base-32 26784.94 ( 0.00%) 19513.57 ( 27.15%)
Amean fault-huge-1 4223.96 ( 0.00%) 2178.57 ( 48.42%)
Amean fault-huge-3 2194.77 ( 0.00%) 2149.74 ( 2.05%)
Amean fault-huge-5 2569.60 ( 0.00%) 2346.95 ( 8.66%)
Amean fault-huge-7 3612.69 ( 0.00%) 2997.70 ( 17.02%)
Amean fault-huge-12 3301.75 ( 0.00%) 6727.02 (-103.74%)
Amean fault-huge-18 6696.47 ( 0.00%) 6685.72 ( 0.16%)
Amean fault-huge-24 8000.72 ( 0.00%) 9311.43 (-16.38%)
Amean fault-huge-30 13305.55 ( 0.00%) 9750.45 ( 26.72%)
Amean fault-huge-32 9981.71 ( 0.00%) 10316.06 ( -3.35%)
The average time to fault pages is substantially reduced in the majority
of caseds but with the obvious caveat that fewer THPs are actually used
in this adverse workload
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Percentage huge-1 0.71 ( 0.00%) 14.04 (1865.22%)
Percentage huge-3 10.77 ( 0.00%) 33.05 (206.85%)
Percentage huge-5 60.39 ( 0.00%) 38.51 (-36.23%)
Percentage huge-7 45.97 ( 0.00%) 34.57 (-24.79%)
Percentage huge-12 68.12 ( 0.00%) 40.07 (-41.17%)
Percentage huge-18 64.93 ( 0.00%) 47.82 (-26.35%)
Percentage huge-24 62.69 ( 0.00%) 44.23 (-29.44%)
Percentage huge-30 43.49 ( 0.00%) 55.38 ( 27.34%)
Percentage huge-32 50.72 ( 0.00%) 51.90 ( 2.35%)
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 37429143 47564000
Major Faults 1916 1558
Swap Ins 1466 1079
Swap Outs 2936863 149626
Allocation stalls 62510 3
DMA allocs 0 0
DMA32 allocs 6566458 6401314
Normal allocs 216361697 216538171
Movable allocs 0 0
Direct pages scanned 25977580 17998
Kswapd pages scanned 0 3638931
Kswapd pages reclaimed 0 207236
Direct pages reclaimed 8833714 88
Compaction stalls 103349 5
Compaction success 270 4
Compaction failures 103079 1
Note again that while this does swap as it's an aggressive workload, the
direct relcim activity and allocation stalls is substantially reduced.
There is some kswapd activity but ftrace showed that the kswapd activity
was due to normal wakeups from 4K pages being allocated.
Compaction-related stalls and activity are almost eliminated.
I also tried the stutter benchmark. For this, I do not have figures for
NUMA but it's something that does impact UMA so I'll report what is
available
stutter
4.4.0 4.4.0
kcompactd-v1r1 nodefrag-v1r3
Min mmap 7.3571 ( 0.00%) 7.3438 ( 0.18%)
1st-qrtle mmap 7.5278 ( 0.00%) 17.9200 (-138.05%)
2nd-qrtle mmap 7.6818 ( 0.00%) 21.6055 (-181.25%)
3rd-qrtle mmap 11.0889 ( 0.00%) 21.8881 (-97.39%)
Max-90% mmap 27.8978 ( 0.00%) 22.1632 ( 20.56%)
Max-93% mmap 28.3202 ( 0.00%) 22.3044 ( 21.24%)
Max-95% mmap 28.5600 ( 0.00%) 22.4580 ( 21.37%)
Max-99% mmap 29.6032 ( 0.00%) 25.5216 ( 13.79%)
Max mmap 4109.7289 ( 0.00%) 4813.9832 (-17.14%)
Mean mmap 12.4474 ( 0.00%) 19.3027 (-55.07%)
This benchmark is trying to fault an anonymous mapping while there is a
heavy IO load -- a scenario that desktop users used to complain about
frequently. This shows a mix because the ideal case of mapping with THP
is not hit as often. However, note that 99% of the mappings complete
13.79% faster. The CPU usage here is particularly interesting
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
User 67.50 0.99
System 1327.88 91.30
Elapsed 2079.00 2128.98
And once again we look at the reclaim figures
4.4.0 4.4.0
kcompactd-v1r1nodefrag-v1r3
Minor Faults 335241922 1314582827
Major Faults 715 819
Swap Ins 0 0
Swap Outs 0 0
Allocation stalls 532723 0
DMA allocs 0 0
DMA32 allocs 1822364341 1177950222
Normal allocs 1815640808 1517844854
Movable allocs 0 0
Direct pages scanned 21892772 0
Kswapd pages scanned 20015890 41879484
Kswapd pages reclaimed 19961986 41822072
Direct pages reclaimed 21892741 0
Compaction stalls 1065755 0
Compaction success 514 0
Compaction failures 1065241 0
Allocation stalls and all direct reclaim activity is eliminated as well
as compaction-related stalls.
THP gives impressive gains in some cases but only if they are quickly
available. We're not going to reach the point where they are completely
free so lets take the costs out of the fast paths finally and defer the
cost to kswapd, kcompactd and khugepaged where it belongs.
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 21:19:23 +00:00
|
|
|
return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 22:46:55 +00:00
|
|
|
}
|
2005-09-09 20:03:32 +00:00
|
|
|
#endif
|
|
|
|
|
2016-05-20 00:10:11 +00:00
|
|
|
static int init_cache_node(struct kmem_cache *cachep, int node, gfp_t gfp)
|
|
|
|
{
|
|
|
|
struct kmem_cache_node *n;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set up the kmem_cache_node for cpu before we can
|
|
|
|
* begin anything. Make sure some other cpu on this
|
|
|
|
* node has not already allocated this
|
|
|
|
*/
|
|
|
|
n = get_node(cachep, node);
|
|
|
|
if (n) {
|
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount +
|
|
|
|
cachep->num;
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
|
|
|
|
if (!n)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
kmem_cache_node_init(n);
|
|
|
|
n->next_reap = jiffies + REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
|
|
|
|
|
|
|
n->free_limit =
|
|
|
|
(1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The kmem_cache_nodes don't come and go as CPUs
|
|
|
|
* come and go. slab_mutex is sufficient
|
|
|
|
* protection here.
|
|
|
|
*/
|
|
|
|
cachep->node[node] = n;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-08-23 12:53:19 +00:00
|
|
|
#if (defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)) || defined(CONFIG_SMP)
|
2010-03-28 02:40:47 +00:00
|
|
|
/*
|
2013-01-10 19:14:19 +00:00
|
|
|
* Allocates and initializes node for a node on each slab cache, used for
|
2013-01-10 19:14:19 +00:00
|
|
|
* either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
|
2010-03-28 02:40:47 +00:00
|
|
|
* will be allocated off-node since memory is not yet online for the new node.
|
2013-01-10 19:14:19 +00:00
|
|
|
* When hotplugging memory or a cpu, existing node are not replaced if
|
2010-03-28 02:40:47 +00:00
|
|
|
* already in use.
|
|
|
|
*
|
2012-07-06 20:25:12 +00:00
|
|
|
* Must hold slab_mutex.
|
2010-03-28 02:40:47 +00:00
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
static int init_cache_node_node(int node)
|
2010-03-28 02:40:47 +00:00
|
|
|
{
|
2016-05-20 00:10:11 +00:00
|
|
|
int ret;
|
2010-03-28 02:40:47 +00:00
|
|
|
struct kmem_cache *cachep;
|
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2016-05-20 00:10:11 +00:00
|
|
|
ret = init_cache_node(cachep, node, GFP_KERNEL);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2010-03-28 02:40:47 +00:00
|
|
|
}
|
2016-05-20 00:10:11 +00:00
|
|
|
|
2010-03-28 02:40:47 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2016-08-23 12:53:19 +00:00
|
|
|
#endif
|
2010-03-28 02:40:47 +00:00
|
|
|
|
2016-05-20 00:10:14 +00:00
|
|
|
static int setup_kmem_cache_node(struct kmem_cache *cachep,
|
|
|
|
int node, gfp_t gfp, bool force_change)
|
|
|
|
{
|
|
|
|
int ret = -ENOMEM;
|
|
|
|
struct kmem_cache_node *n;
|
|
|
|
struct array_cache *old_shared = NULL;
|
|
|
|
struct array_cache *new_shared = NULL;
|
|
|
|
struct alien_cache **new_alien = NULL;
|
|
|
|
LIST_HEAD(list);
|
|
|
|
|
|
|
|
if (use_alien_caches) {
|
|
|
|
new_alien = alloc_alien_cache(node, cachep->limit, gfp);
|
|
|
|
if (!new_alien)
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cachep->shared) {
|
|
|
|
new_shared = alloc_arraycache(node,
|
|
|
|
cachep->shared * cachep->batchcount, 0xbaadf00d, gfp);
|
|
|
|
if (!new_shared)
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = init_cache_node(cachep, node, gfp);
|
|
|
|
if (ret)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
n = get_node(cachep, node);
|
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
if (n->shared && force_change) {
|
|
|
|
free_block(cachep, n->shared->entry,
|
|
|
|
n->shared->avail, node, &list);
|
|
|
|
n->shared->avail = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!n->shared || force_change) {
|
|
|
|
old_shared = n->shared;
|
|
|
|
n->shared = new_shared;
|
|
|
|
new_shared = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!n->alien) {
|
|
|
|
n->alien = new_alien;
|
|
|
|
new_alien = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
|
|
|
slabs_destroy(cachep, &list);
|
|
|
|
|
mm/slab: lockless decision to grow cache
To check whether free objects exist or not precisely, we need to grab a
lock. But, accuracy isn't that important because race window would be
even small and if there is too much free object, cache reaper would reap
it. So, this patch makes the check for free object exisistence not to
hold a lock. This will reduce lock contention in heavily allocation
case.
Note that until now, n->shared can be freed during the processing by
writing slabinfo, but, with some trick in this patch, we can access it
freely within interrupt disabled period.
Below is the result of concurrent allocation/free in slab allocation
benchmark made by Christoph a long time ago. I make the output simpler.
The number shows cycle count during alloc/free respectively so less is
better.
* Before
Kmalloc N*alloc N*free(32): Average=248/966
Kmalloc N*alloc N*free(64): Average=261/949
Kmalloc N*alloc N*free(128): Average=314/1016
Kmalloc N*alloc N*free(256): Average=741/1061
Kmalloc N*alloc N*free(512): Average=1246/1152
Kmalloc N*alloc N*free(1024): Average=2437/1259
Kmalloc N*alloc N*free(2048): Average=4980/1800
Kmalloc N*alloc N*free(4096): Average=9000/2078
* After
Kmalloc N*alloc N*free(32): Average=344/792
Kmalloc N*alloc N*free(64): Average=347/882
Kmalloc N*alloc N*free(128): Average=390/959
Kmalloc N*alloc N*free(256): Average=393/1067
Kmalloc N*alloc N*free(512): Average=683/1229
Kmalloc N*alloc N*free(1024): Average=1295/1325
Kmalloc N*alloc N*free(2048): Average=2513/1664
Kmalloc N*alloc N*free(4096): Average=4742/2172
It shows that allocation performance decreases for the object size up to
128 and it may be due to extra checks in cache_alloc_refill(). But,
with considering improvement of free performance, net result looks the
same. Result for other size class looks very promising, roughly, 50%
performance improvement.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 00:10:31 +00:00
|
|
|
/*
|
|
|
|
* To protect lockless access to n->shared during irq disabled context.
|
|
|
|
* If n->shared isn't NULL in irq disabled context, accessing to it is
|
|
|
|
* guaranteed to be valid until irq is re-enabled, because it will be
|
|
|
|
* freed after synchronize_sched().
|
|
|
|
*/
|
|
|
|
if (force_change)
|
|
|
|
synchronize_sched();
|
|
|
|
|
2016-05-20 00:10:14 +00:00
|
|
|
fail:
|
|
|
|
kfree(old_shared);
|
|
|
|
kfree(new_shared);
|
|
|
|
free_alien_cache(new_alien);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-08-23 12:53:19 +00:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
2013-06-19 18:53:51 +00:00
|
|
|
static void cpuup_canceled(long cpu)
|
2007-10-18 10:05:09 +00:00
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n = NULL;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
int node = cpu_to_mem(cpu);
|
2009-03-13 04:19:46 +00:00
|
|
|
const struct cpumask *mask = cpumask_of_node(node);
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2007-10-18 10:05:09 +00:00
|
|
|
struct array_cache *nc;
|
|
|
|
struct array_cache *shared;
|
2014-08-06 23:04:29 +00:00
|
|
|
struct alien_cache **alien;
|
2014-08-06 23:04:25 +00:00
|
|
|
LIST_HEAD(list);
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 19:14:19 +00:00
|
|
|
if (!n)
|
2014-10-09 22:26:27 +00:00
|
|
|
continue;
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
/* Free limit for this kmem_cache_node */
|
|
|
|
n->free_limit -= cachep->batchcount;
|
2014-10-09 22:26:27 +00:00
|
|
|
|
|
|
|
/* cpu is dead; no one can alloc from it. */
|
|
|
|
nc = per_cpu_ptr(cachep->cpu_cache, cpu);
|
|
|
|
if (nc) {
|
2014-08-06 23:04:25 +00:00
|
|
|
free_block(cachep, nc->entry, nc->avail, node, &list);
|
2014-10-09 22:26:27 +00:00
|
|
|
nc->avail = 0;
|
|
|
|
}
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2009-12-17 17:43:12 +00:00
|
|
|
if (!cpumask_empty(mask)) {
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2014-10-09 22:26:27 +00:00
|
|
|
goto free_slab;
|
2007-10-18 10:05:09 +00:00
|
|
|
}
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
shared = n->shared;
|
2007-10-18 10:05:09 +00:00
|
|
|
if (shared) {
|
|
|
|
free_block(cachep, shared->entry,
|
2014-08-06 23:04:25 +00:00
|
|
|
shared->avail, node, &list);
|
2013-01-10 19:14:19 +00:00
|
|
|
n->shared = NULL;
|
2007-10-18 10:05:09 +00:00
|
|
|
}
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
alien = n->alien;
|
|
|
|
n->alien = NULL;
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2007-10-18 10:05:09 +00:00
|
|
|
|
|
|
|
kfree(shared);
|
|
|
|
if (alien) {
|
|
|
|
drain_alien_cache(cachep, alien);
|
|
|
|
free_alien_cache(alien);
|
|
|
|
}
|
2014-10-09 22:26:27 +00:00
|
|
|
|
|
|
|
free_slab:
|
2014-08-06 23:04:25 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2007-10-18 10:05:09 +00:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* In the previous loop, all the objects were freed to
|
|
|
|
* the respective cache's slabs, now we can go ahead and
|
|
|
|
* shrink each nodelist to its limit.
|
|
|
|
*/
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 19:14:19 +00:00
|
|
|
if (!n)
|
2007-10-18 10:05:09 +00:00
|
|
|
continue;
|
2016-05-20 00:10:08 +00:00
|
|
|
drain_freelist(cachep, n, INT_MAX);
|
2007-10-18 10:05:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-06-19 18:53:51 +00:00
|
|
|
static int cpuup_prepare(long cpu)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-02-01 11:05:50 +00:00
|
|
|
struct kmem_cache *cachep;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
int node = cpu_to_mem(cpu);
|
2010-03-28 02:40:47 +00:00
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-18 10:05:09 +00:00
|
|
|
/*
|
|
|
|
* We need to do this right in the beginning since
|
|
|
|
* alloc_arraycache's are going to use this list.
|
|
|
|
* kmalloc_node allows us to add the slab to the right
|
2013-01-10 19:14:19 +00:00
|
|
|
* kmem_cache_node and not this cpu's kmem_cache_node
|
2007-10-18 10:05:09 +00:00
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
err = init_cache_node_node(node);
|
2010-03-28 02:40:47 +00:00
|
|
|
if (err < 0)
|
|
|
|
goto bad;
|
2007-10-18 10:05:09 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Now we can go ahead with allocating the shared arrays and
|
|
|
|
* array caches
|
|
|
|
*/
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2016-05-20 00:10:14 +00:00
|
|
|
err = setup_kmem_cache_node(cachep, node, GFP_KERNEL, false);
|
|
|
|
if (err)
|
|
|
|
goto bad;
|
2007-10-18 10:05:09 +00:00
|
|
|
}
|
2009-11-23 20:01:15 +00:00
|
|
|
|
2007-10-18 10:05:09 +00:00
|
|
|
return 0;
|
|
|
|
bad:
|
2007-10-18 10:05:11 +00:00
|
|
|
cpuup_canceled(cpu);
|
2007-10-18 10:05:09 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2016-08-23 12:53:19 +00:00
|
|
|
int slab_prepare_cpu(unsigned int cpu)
|
2007-10-18 10:05:09 +00:00
|
|
|
{
|
2016-08-23 12:53:19 +00:00
|
|
|
int err;
|
2007-10-18 10:05:09 +00:00
|
|
|
|
2016-08-23 12:53:19 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
|
|
|
err = cpuup_prepare(cpu);
|
|
|
|
mutex_unlock(&slab_mutex);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This is called for a failed online attempt and for a successful
|
|
|
|
* offline.
|
|
|
|
*
|
|
|
|
* Even if all the cpus of a node are down, we don't free the
|
|
|
|
* kmem_list3 of any cache. This to avoid a race between cpu_down, and
|
|
|
|
* a kmalloc allocation from another cpu for memory from the node of
|
|
|
|
* the cpu going down. The list3 structure is usually allocated from
|
|
|
|
* kmem_cache_create() and gets destroyed at kmem_cache_destroy().
|
|
|
|
*/
|
|
|
|
int slab_dead_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
mutex_lock(&slab_mutex);
|
|
|
|
cpuup_canceled(cpu);
|
|
|
|
mutex_unlock(&slab_mutex);
|
|
|
|
return 0;
|
|
|
|
}
|
[PATCH] mm: slab: eliminate lock_cpu_hotplug from slab
Here's an attempt towards doing away with lock_cpu_hotplug in the slab
subsystem. This approach also fixes a bug which shows up when cpus are
being offlined/onlined and slab caches are being tuned simultaneously.
http://marc.theaimsgroup.com/?l=linux-kernel&m=116098888100481&w=2
The patch has been stress tested overnight on a 2 socket 4 core AMD box with
repeated cpu online and offline, while dbench and kernbench process are
running, and slab caches being tuned at the same time.
There were no lockdep warnings either. (This test on 2,6.18 as 2.6.19-rc
crashes at __drain_pages
http://marc.theaimsgroup.com/?l=linux-kernel&m=116172164217678&w=2 )
The approach here is to hold cache_chain_mutex from CPU_UP_PREPARE until
CPU_ONLINE (similar in approach as worqueue_mutex) . Slab code sensitive
to cpu_online_map (kmem_cache_create, kmem_cache_destroy, slabinfo_write,
__cache_shrink) is already serialized with cache_chain_mutex. (This patch
lengthens cache_chain_mutex hold time at kmem_cache_destroy to cover this).
This patch also takes the cache_chain_sem at kmem_cache_shrink to protect
sanity of cpu_online_map at __cache_shrink, as viewed by slab.
(kmem_cache_shrink->__cache_shrink->drain_cpu_caches). But, really,
kmem_cache_shrink is used at just one place in the acpi subsystem! Do we
really need to keep kmem_cache_shrink at all?
Another note. Looks like a cpu hotplug event can send CPU_UP_CANCELED to
a registered subsystem even if the subsystem did not receive CPU_UP_PREPARE.
This could be due to a subsystem registered for notification earlier than
the current subsystem crapping out with NOTIFY_BAD. Badness can occur with
in the CPU_UP_CANCELED code path at slab if this happens (The same would
apply for workqueue.c as well). To overcome this, we might have to use either
a) a per subsystem flag and avoid handling of CPU_UP_CANCELED, or
b) Use a special notifier events like LOCK_ACQUIRE/RELEASE as Gautham was
using in his experiments, or
c) Do not send CPU_UP_CANCELED to a subsystem which did not receive
CPU_UP_PREPARE.
I would prefer c).
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Signed-off-by: Shai Fultheim <shai@scalex86.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 04:32:14 +00:00
|
|
|
#endif
|
2016-08-23 12:53:19 +00:00
|
|
|
|
|
|
|
static int slab_online_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
start_cpu_timer(cpu);
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2016-08-23 12:53:19 +00:00
|
|
|
static int slab_offline_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Shutdown cache reaper. Note that the slab_mutex is held so
|
|
|
|
* that if cache_reap() is invoked it cannot do anything
|
|
|
|
* expensive but will only modify reap_work and reschedule the
|
|
|
|
* timer.
|
|
|
|
*/
|
|
|
|
cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
|
|
|
|
/* Now the cache_reaper is guaranteed to be not running. */
|
|
|
|
per_cpu(slab_reap_work, cpu).work.func = NULL;
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-03-28 02:40:47 +00:00
|
|
|
#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
|
|
|
|
/*
|
|
|
|
* Drains freelist for a node on each slab cache, used for memory hot-remove.
|
|
|
|
* Returns -EBUSY if all objects cannot be drained so that the node is not
|
|
|
|
* removed.
|
|
|
|
*
|
2012-07-06 20:25:12 +00:00
|
|
|
* Must hold slab_mutex.
|
2010-03-28 02:40:47 +00:00
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
static int __meminit drain_cache_node_node(int node)
|
2010-03-28 02:40:47 +00:00
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
int ret = 0;
|
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2010-03-28 02:40:47 +00:00
|
|
|
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 19:14:19 +00:00
|
|
|
if (!n)
|
2010-03-28 02:40:47 +00:00
|
|
|
continue;
|
|
|
|
|
2016-05-20 00:10:08 +00:00
|
|
|
drain_freelist(cachep, n, INT_MAX);
|
2010-03-28 02:40:47 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
if (!list_empty(&n->slabs_full) ||
|
|
|
|
!list_empty(&n->slabs_partial)) {
|
2010-03-28 02:40:47 +00:00
|
|
|
ret = -EBUSY;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __meminit slab_memory_callback(struct notifier_block *self,
|
|
|
|
unsigned long action, void *arg)
|
|
|
|
{
|
|
|
|
struct memory_notify *mnb = arg;
|
|
|
|
int ret = 0;
|
|
|
|
int nid;
|
|
|
|
|
|
|
|
nid = mnb->status_change_nid;
|
|
|
|
if (nid < 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
case MEM_GOING_ONLINE:
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
2013-01-10 19:14:19 +00:00
|
|
|
ret = init_cache_node_node(nid);
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_unlock(&slab_mutex);
|
2010-03-28 02:40:47 +00:00
|
|
|
break;
|
|
|
|
case MEM_GOING_OFFLINE:
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
2013-01-10 19:14:19 +00:00
|
|
|
ret = drain_cache_node_node(nid);
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_unlock(&slab_mutex);
|
2010-03-28 02:40:47 +00:00
|
|
|
break;
|
|
|
|
case MEM_ONLINE:
|
|
|
|
case MEM_OFFLINE:
|
|
|
|
case MEM_CANCEL_ONLINE:
|
|
|
|
case MEM_CANCEL_OFFLINE:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
out:
|
2011-03-22 23:30:49 +00:00
|
|
|
return notifier_from_errno(ret);
|
2010-03-28 02:40:47 +00:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
|
|
|
|
|
2005-09-09 20:03:32 +00:00
|
|
|
/*
|
2013-01-10 19:14:19 +00:00
|
|
|
* swap the static kmem_cache_node with kmalloced memory
|
2005-09-09 20:03:32 +00:00
|
|
|
*/
|
2013-01-10 19:12:17 +00:00
|
|
|
static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
|
2010-03-28 02:40:47 +00:00
|
|
|
int nodeid)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2013-01-10 19:12:17 +00:00
|
|
|
struct kmem_cache_node *ptr;
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2013-01-10 19:12:17 +00:00
|
|
|
ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
|
2005-09-09 20:03:32 +00:00
|
|
|
BUG_ON(!ptr);
|
|
|
|
|
2013-01-10 19:12:17 +00:00
|
|
|
memcpy(ptr, list, sizeof(struct kmem_cache_node));
|
2006-07-03 07:25:28 +00:00
|
|
|
/*
|
|
|
|
* Do not assume that spinlocks can be initialized via memcpy:
|
|
|
|
*/
|
|
|
|
spin_lock_init(&ptr->list_lock);
|
|
|
|
|
2005-09-09 20:03:32 +00:00
|
|
|
MAKE_ALL_LISTS(cachep, ptr, nodeid);
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep->node[nodeid] = ptr;
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
|
2008-01-25 06:20:51 +00:00
|
|
|
/*
|
2013-01-10 19:14:19 +00:00
|
|
|
* For setting up all the kmem_cache_node for cache whose buffer_size is same as
|
|
|
|
* size of kmem_cache_node.
|
2008-01-25 06:20:51 +00:00
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
static void __init set_up_node(struct kmem_cache *cachep, int index)
|
2008-01-25 06:20:51 +00:00
|
|
|
{
|
|
|
|
int node;
|
|
|
|
|
|
|
|
for_each_online_node(node) {
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep->node[node] = &init_kmem_cache_node[index + node];
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep->node[node]->next_reap = jiffies +
|
2014-03-30 09:02:20 +00:00
|
|
|
REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
2008-01-25 06:20:51 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Initialisation. Called after the page allocator have been initialised and
|
|
|
|
* before smp_init().
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
void __init kmem_cache_init(void)
|
|
|
|
{
|
2005-09-09 20:03:32 +00:00
|
|
|
int i;
|
|
|
|
|
2013-10-24 01:07:42 +00:00
|
|
|
BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
|
|
|
|
sizeof(struct rcu_head));
|
2012-09-05 00:20:33 +00:00
|
|
|
kmem_cache = &kmem_cache_boot;
|
|
|
|
|
2016-05-20 00:10:05 +00:00
|
|
|
if (!IS_ENABLED(CONFIG_NUMA) || num_possible_nodes() == 1)
|
2007-05-02 17:27:18 +00:00
|
|
|
use_alien_caches = 0;
|
|
|
|
|
2012-11-28 16:23:01 +00:00
|
|
|
for (i = 0; i < NUM_INIT_LISTS; i++)
|
2013-01-10 19:14:19 +00:00
|
|
|
kmem_cache_node_init(&init_kmem_cache_node[i]);
|
2012-11-28 16:23:01 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Fragmentation resistance on low memory - only use bigger
|
2011-10-19 05:09:28 +00:00
|
|
|
* page orders on machines with more than 32MB of memory if
|
|
|
|
* not overridden on the command line.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2011-10-19 05:09:28 +00:00
|
|
|
if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
|
2011-10-19 05:09:24 +00:00
|
|
|
slab_max_order = SLAB_MAX_ORDER_HI;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* Bootstrap is tricky, because several objects are allocated
|
|
|
|
* from caches that do not exist yet:
|
2012-09-05 00:20:33 +00:00
|
|
|
* 1) initialize the kmem_cache cache: it contains the struct
|
|
|
|
* kmem_cache structures of all caches, except kmem_cache itself:
|
|
|
|
* kmem_cache is statically allocated.
|
2005-09-09 20:03:32 +00:00
|
|
|
* Initially an __init data area is used for the head array and the
|
2013-01-10 19:14:19 +00:00
|
|
|
* kmem_cache_node structures, it's replaced with a kmalloc allocated
|
2005-09-09 20:03:32 +00:00
|
|
|
* array at the end of the bootstrap.
|
2005-04-16 22:20:36 +00:00
|
|
|
* 2) Create the first kmalloc cache.
|
2006-02-01 11:05:50 +00:00
|
|
|
* The struct kmem_cache for the new cache is allocated normally.
|
2005-09-09 20:03:32 +00:00
|
|
|
* An __init data area is used for the head array.
|
|
|
|
* 3) Create the remaining kmalloc caches, with minimally sized
|
|
|
|
* head arrays.
|
2012-09-05 00:20:33 +00:00
|
|
|
* 4) Replace the __init data head arrays for kmem_cache and the first
|
2005-04-16 22:20:36 +00:00
|
|
|
* kmalloc cache with kmalloc allocated arrays.
|
2013-01-10 19:14:19 +00:00
|
|
|
* 5) Replace the __init data for kmem_cache_node for kmem_cache and
|
2005-09-09 20:03:32 +00:00
|
|
|
* the other cache's with kmalloc allocated memory.
|
|
|
|
* 6) Resize the head arrays of the kmalloc caches to their final sizes.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2012-09-05 00:20:33 +00:00
|
|
|
/* 1) create the kmem_cache */
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-05-06 21:49:29 +00:00
|
|
|
/*
|
2011-07-20 17:04:23 +00:00
|
|
|
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
|
2007-05-06 21:49:29 +00:00
|
|
|
*/
|
2012-11-28 16:23:09 +00:00
|
|
|
create_boot_cache(kmem_cache, "kmem_cache",
|
2014-10-09 22:26:27 +00:00
|
|
|
offsetof(struct kmem_cache, node) +
|
2013-01-10 19:12:17 +00:00
|
|
|
nr_node_ids * sizeof(struct kmem_cache_node *),
|
2012-11-28 16:23:09 +00:00
|
|
|
SLAB_HWCACHE_ALIGN);
|
|
|
|
list_add(&kmem_cache->list, &slab_caches);
|
2014-10-09 22:26:27 +00:00
|
|
|
slab_state = PARTIAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
2014-10-09 22:26:27 +00:00
|
|
|
* Initialize the caches that provide memory for the kmem_cache_node
|
|
|
|
* structures first. Without this, further allocations will bug.
|
2005-09-09 20:03:32 +00:00
|
|
|
*/
|
2014-10-09 22:26:27 +00:00
|
|
|
kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
|
2013-01-10 19:14:19 +00:00
|
|
|
kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
|
2014-10-09 22:26:27 +00:00
|
|
|
slab_state = PARTIAL_NODE;
|
2015-06-24 23:55:57 +00:00
|
|
|
setup_kmalloc_cache_index_table();
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2006-06-23 09:03:46 +00:00
|
|
|
slab_early_init = 0;
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
/* 5) Replace the bootstrap kmem_cache_node */
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2006-10-06 07:43:52 +00:00
|
|
|
int nid;
|
|
|
|
|
2008-01-24 13:49:54 +00:00
|
|
|
for_each_online_node(nid) {
|
2013-01-10 19:14:19 +00:00
|
|
|
init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
|
2008-01-25 06:20:51 +00:00
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
init_list(kmalloc_caches[INDEX_NODE],
|
2013-01-10 19:14:19 +00:00
|
|
|
&init_kmem_cache_node[SIZE_NODE + nid], nid);
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:12:17 +00:00
|
|
|
create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
|
2009-06-12 12:58:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __init kmem_cache_init_late(void)
|
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
|
2012-07-06 20:25:11 +00:00
|
|
|
slab_state = UP;
|
2011-11-28 20:12:40 +00:00
|
|
|
|
2009-06-12 12:58:59 +00:00
|
|
|
/* 6) resize the head arrays to their final sizes */
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
|
|
|
list_for_each_entry(cachep, &slab_caches, list)
|
2009-06-12 12:58:59 +00:00
|
|
|
if (enable_cpucache(cachep, GFP_NOWAIT))
|
|
|
|
BUG();
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_unlock(&slab_mutex);
|
2006-09-26 06:31:38 +00:00
|
|
|
|
2012-07-06 20:25:11 +00:00
|
|
|
/* Done! */
|
|
|
|
slab_state = FULL;
|
|
|
|
|
2010-03-28 02:40:47 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
/*
|
|
|
|
* Register a memory hotplug callback that initializes and frees
|
2013-01-10 19:14:19 +00:00
|
|
|
* node.
|
2010-03-28 02:40:47 +00:00
|
|
|
*/
|
|
|
|
hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
|
|
|
|
#endif
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* The reap timers are started later, with a module init call: That part
|
|
|
|
* of the kernel is not yet operational.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init cpucache_init(void)
|
|
|
|
{
|
2016-08-23 12:53:19 +00:00
|
|
|
int ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Register the timers that return unneeded pages to the page allocator
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2016-08-23 12:53:19 +00:00
|
|
|
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "SLAB online",
|
|
|
|
slab_online_cpu, slab_offline_cpu);
|
|
|
|
WARN_ON(ret < 0);
|
slab: move FULL state transition to an initcall
During kmem_cache_init_late(), we transition to the LATE state,
and after some more work, to the FULL state, its last state
This is quite different from slub, that will only transition to
its last state (previously SYSFS), in a (late)initcall, after a lot
more of the kernel is ready.
This means that in slab, we have no way to taking actions dependent
on the initialization of other pieces of the kernel that are supposed
to start way after kmem_init_late(), such as cgroups initialization.
To achieve more consistency in this behavior, that patch only
transitions to the UP state in kmem_init_late. In my analysis,
setup_cpu_cache() should be happy to test for >= UP, instead of
== FULL. It also has passed some tests I've made.
We then only mark FULL state after the reap timers are in place,
meaning that no further setup is expected.
Signed-off-by: Glauber Costa <glommer@parallels.com>
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2012-06-20 20:59:18 +00:00
|
|
|
|
|
|
|
/* Done! */
|
2012-07-06 20:25:11 +00:00
|
|
|
slab_state = FULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
__initcall(cpucache_init);
|
|
|
|
|
2012-03-09 20:27:27 +00:00
|
|
|
static noinline void
|
|
|
|
slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
|
|
|
|
{
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 23:06:36 +00:00
|
|
|
#if DEBUG
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2012-03-09 20:27:27 +00:00
|
|
|
unsigned long flags;
|
|
|
|
int node;
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 23:06:36 +00:00
|
|
|
static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
|
|
|
|
DEFAULT_RATELIMIT_BURST);
|
|
|
|
|
|
|
|
if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
|
|
|
|
return;
|
2012-03-09 20:27:27 +00:00
|
|
|
|
2016-03-15 21:56:33 +00:00
|
|
|
pr_warn("SLAB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
|
|
|
|
nodeid, gfpflags, &gfpflags);
|
|
|
|
pr_warn(" cache: %s, object size: %d, order: %d\n",
|
2012-06-13 15:24:57 +00:00
|
|
|
cachep->name, cachep->size, cachep->gfporder);
|
2012-03-09 20:27:27 +00:00
|
|
|
|
2014-08-06 23:04:11 +00:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2012-03-09 20:27:27 +00:00
|
|
|
unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
|
|
|
|
unsigned long active_slabs = 0, num_slabs = 0;
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock_irqsave(&n->list_lock, flags);
|
2013-10-24 01:07:49 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru) {
|
2012-03-09 20:27:27 +00:00
|
|
|
active_objs += cachep->num;
|
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-24 01:07:49 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru) {
|
|
|
|
active_objs += page->active;
|
2012-03-09 20:27:27 +00:00
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-24 01:07:49 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_free, lru)
|
2012-03-09 20:27:27 +00:00
|
|
|
num_slabs++;
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
free_objects += n->free_objects;
|
|
|
|
spin_unlock_irqrestore(&n->list_lock, flags);
|
2012-03-09 20:27:27 +00:00
|
|
|
|
|
|
|
num_slabs += active_slabs;
|
|
|
|
num_objs = num_slabs * cachep->num;
|
2016-03-15 21:56:33 +00:00
|
|
|
pr_warn(" node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
|
2012-03-09 20:27:27 +00:00
|
|
|
node, active_slabs, num_slabs, active_objs, num_objs,
|
|
|
|
free_objects);
|
|
|
|
}
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 23:06:36 +00:00
|
|
|
#endif
|
2012-03-09 20:27:27 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2014-08-06 23:04:46 +00:00
|
|
|
* Interface to system's page allocator. No need to hold the
|
|
|
|
* kmem_cache_node ->list_lock.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* If we requested dmaable memory, we will get it. Even if we
|
|
|
|
* did not request dmaable memory, we might get it, but that
|
|
|
|
* would be relatively rare and ignorable.
|
|
|
|
*/
|
2013-10-24 01:07:38 +00:00
|
|
|
static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
|
|
|
|
int nodeid)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct page *page;
|
2006-06-23 09:03:17 +00:00
|
|
|
int nr_pages;
|
2006-09-27 08:50:08 +00:00
|
|
|
|
2012-06-14 12:17:21 +00:00
|
|
|
flags |= cachep->allocflags;
|
2007-10-16 08:25:52 +00:00
|
|
|
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
|
|
|
|
flags |= __GFP_RECLAIMABLE;
|
2006-06-23 09:03:17 +00:00
|
|
|
|
mm: rename alloc_pages_exact_node() to __alloc_pages_node()
alloc_pages_exact_node() was introduced in commit 6484eb3e2a81 ("page
allocator: do not check NUMA node ID when the caller knows the node is
valid") as an optimized variant of alloc_pages_node(), that doesn't
fallback to current node for nid == NUMA_NO_NODE. Unfortunately the
name of the function can easily suggest that the allocation is
restricted to the given node and fails otherwise. In truth, the node is
only preferred, unless __GFP_THISNODE is passed among the gfp flags.
The misleading name has lead to mistakes in the past, see for example
commits 5265047ac301 ("mm, thp: really limit transparent hugepage
allocation to local node") and b360edb43f8e ("mm, mempolicy:
migrate_to_node should only migrate to node").
Another issue with the name is that there's a family of
alloc_pages_exact*() functions where 'exact' means exact size (instead
of page order), which leads to more confusion.
To prevent further mistakes, this patch effectively renames
alloc_pages_exact_node() to __alloc_pages_node() to better convey that
it's an optimized variant of alloc_pages_node() not intended for general
usage. Both functions get described in comments.
It has been also considered to really provide a convenience function for
allocations restricted to a node, but the major opinion seems to be that
__GFP_THISNODE already provides that functionality and we shouldn't
duplicate the API needlessly. The number of users would be small
anyway.
Existing callers of alloc_pages_exact_node() are simply converted to
call __alloc_pages_node(), with the exception of sba_alloc_coherent()
which open-codes the check for NUMA_NO_NODE, so it is converted to use
alloc_pages_node() instead. This means it no longer performs some
VM_BUG_ON checks, and since the current check for nid in
alloc_pages_node() uses a 'nid < 0' comparison (which includes
NUMA_NO_NODE), it may hide wrong values which would be previously
exposed.
Both differences will be rectified by the next patch.
To sum up, this patch makes no functional changes, except temporarily
hiding potentially buggy callers. Restricting the checks in
alloc_pages_node() is left for the next patch which can in turn expose
more existing buggy callers.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Robin Holt <robinmholt@gmail.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Gleb Natapov <gleb@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Cliff Whickman <cpw@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 22:03:50 +00:00
|
|
|
page = __alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
|
2012-03-09 20:27:27 +00:00
|
|
|
if (!page) {
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 23:06:36 +00:00
|
|
|
slab_out_of_memory(cachep, flags, nodeid);
|
2005-04-16 22:20:36 +00:00
|
|
|
return NULL;
|
2012-03-09 20:27:27 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
memcg: unify slab and other kmem pages charging
We have memcg_kmem_charge and memcg_kmem_uncharge methods for charging and
uncharging kmem pages to memcg, but currently they are not used for
charging slab pages (i.e. they are only used for charging pages allocated
with alloc_kmem_pages). The only reason why the slab subsystem uses
special helpers, memcg_charge_slab and memcg_uncharge_slab, is that it
needs to charge to the memcg of kmem cache while memcg_charge_kmem charges
to the memcg that the current task belongs to.
To remove this diversity, this patch adds an extra argument to
__memcg_kmem_charge that can be a pointer to a memcg or NULL. If it is
not NULL, the function tries to charge to the memcg it points to,
otherwise it charge to the current context. Next, it makes the slab
subsystem use this function to charge slab pages.
Since memcg_charge_kmem and memcg_uncharge_kmem helpers are now used only
in __memcg_kmem_charge and __memcg_kmem_uncharge, they are inlined. Since
__memcg_kmem_charge stores a pointer to the memcg in the page struct, we
don't need memcg_uncharge_slab anymore and can use free_kmem_pages.
Besides, one can now detect which memcg a slab page belongs to by reading
/proc/kpagecgroup.
Note, this patch switches slab to charge-after-alloc design. Since this
design is already used for all other memcg charges, it should not make any
difference.
[hannes@cmpxchg.org: better to have an outer function than a magic parameter for the memcg lookup]
Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-06 02:49:01 +00:00
|
|
|
if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
|
|
|
|
__free_pages(page, cachep->gfporder);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2006-06-23 09:03:17 +00:00
|
|
|
nr_pages = (1 << cachep->gfporder);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
|
2006-09-26 06:31:51 +00:00
|
|
|
add_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_RECLAIMABLE, nr_pages);
|
|
|
|
else
|
|
|
|
add_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_UNRECLAIMABLE, nr_pages);
|
2016-03-15 21:54:56 +00:00
|
|
|
|
2013-10-24 01:07:44 +00:00
|
|
|
__SetPageSlab(page);
|
2016-03-15 21:54:56 +00:00
|
|
|
/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
|
|
|
|
if (sk_memalloc_socks() && page_is_pfmemalloc(page))
|
2013-10-24 01:07:44 +00:00
|
|
|
SetPageSlabPfmemalloc(page);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
|
2008-11-25 15:55:53 +00:00
|
|
|
if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
|
|
|
|
kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
|
|
|
|
|
|
|
|
if (cachep->ctor)
|
|
|
|
kmemcheck_mark_uninitialized_pages(page, nr_pages);
|
|
|
|
else
|
|
|
|
kmemcheck_mark_unallocated_pages(page, nr_pages);
|
|
|
|
}
|
2008-05-09 18:35:53 +00:00
|
|
|
|
2013-10-24 01:07:38 +00:00
|
|
|
return page;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Interface to system's page release.
|
|
|
|
*/
|
2013-10-24 01:07:38 +00:00
|
|
|
static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2016-03-17 21:17:35 +00:00
|
|
|
int order = cachep->gfporder;
|
|
|
|
unsigned long nr_freed = (1 << order);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-17 21:17:35 +00:00
|
|
|
kmemcheck_free_shadow(page, order);
|
2008-05-09 18:35:53 +00:00
|
|
|
|
2006-09-26 06:31:51 +00:00
|
|
|
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
|
|
|
|
sub_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_RECLAIMABLE, nr_freed);
|
|
|
|
else
|
|
|
|
sub_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_UNRECLAIMABLE, nr_freed);
|
2013-10-24 01:07:37 +00:00
|
|
|
|
2013-10-24 01:07:44 +00:00
|
|
|
BUG_ON(!PageSlab(page));
|
2013-10-24 01:07:37 +00:00
|
|
|
__ClearPageSlabPfmemalloc(page);
|
2013-10-24 01:07:44 +00:00
|
|
|
__ClearPageSlab(page);
|
2013-10-24 01:07:49 +00:00
|
|
|
page_mapcount_reset(page);
|
|
|
|
page->mapping = NULL;
|
2012-12-18 22:22:50 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (current->reclaim_state)
|
|
|
|
current->reclaim_state->reclaimed_slab += nr_freed;
|
2016-03-17 21:17:35 +00:00
|
|
|
memcg_uncharge_slab(page, order, cachep);
|
|
|
|
__free_pages(page, order);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void kmem_rcu_free(struct rcu_head *head)
|
|
|
|
{
|
2013-10-24 01:07:42 +00:00
|
|
|
struct kmem_cache *cachep;
|
|
|
|
struct page *page;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-10-24 01:07:42 +00:00
|
|
|
page = container_of(head, struct page, rcu_head);
|
|
|
|
cachep = page->slab_cache;
|
|
|
|
|
|
|
|
kmem_freepages(cachep, page);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
2016-03-15 21:54:21 +00:00
|
|
|
static bool is_debug_pagealloc_cache(struct kmem_cache *cachep)
|
|
|
|
{
|
|
|
|
if (debug_pagealloc_enabled() && OFF_SLAB(cachep) &&
|
|
|
|
(cachep->size % PAGE_SIZE) == 0)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2006-02-01 11:05:50 +00:00
|
|
|
static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
|
2006-01-08 09:00:37 +00:00
|
|
|
unsigned long caller)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-06-13 15:24:58 +00:00
|
|
|
int size = cachep->object_size;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-02-01 11:05:42 +00:00
|
|
|
addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-01-08 09:00:37 +00:00
|
|
|
if (size < 5 * sizeof(unsigned long))
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
|
|
|
|
2006-01-08 09:00:37 +00:00
|
|
|
*addr++ = 0x12345678;
|
|
|
|
*addr++ = caller;
|
|
|
|
*addr++ = smp_processor_id();
|
|
|
|
size -= 3 * sizeof(unsigned long);
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
unsigned long *sptr = &caller;
|
|
|
|
unsigned long svalue;
|
|
|
|
|
|
|
|
while (!kstack_end(sptr)) {
|
|
|
|
svalue = *sptr++;
|
|
|
|
if (kernel_text_address(svalue)) {
|
2006-01-08 09:00:37 +00:00
|
|
|
*addr++ = svalue;
|
2005-04-16 22:20:36 +00:00
|
|
|
size -= sizeof(unsigned long);
|
|
|
|
if (size <= sizeof(unsigned long))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2006-01-08 09:00:37 +00:00
|
|
|
*addr++ = 0x87654321;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-03-15 21:54:21 +00:00
|
|
|
|
|
|
|
static void slab_kernel_map(struct kmem_cache *cachep, void *objp,
|
|
|
|
int map, unsigned long caller)
|
|
|
|
{
|
|
|
|
if (!is_debug_pagealloc_cache(cachep))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (caller)
|
|
|
|
store_stackinfo(cachep, objp, caller);
|
|
|
|
|
|
|
|
kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp,
|
|
|
|
int map, unsigned long caller) {}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-06-13 15:24:58 +00:00
|
|
|
int size = cachep->object_size;
|
2006-02-01 11:05:42 +00:00
|
|
|
addr = &((char *)addr)[obj_offset(cachep)];
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
memset(addr, val, size);
|
2006-01-08 09:00:37 +00:00
|
|
|
*(unsigned char *)(addr + size - 1) = POISON_END;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dump_line(char *data, int offset, int limit)
|
|
|
|
{
|
|
|
|
int i;
|
2006-09-29 08:59:51 +00:00
|
|
|
unsigned char error = 0;
|
|
|
|
int bad_count = 0;
|
|
|
|
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("%03x: ", offset);
|
2006-09-29 08:59:51 +00:00
|
|
|
for (i = 0; i < limit; i++) {
|
|
|
|
if (data[offset + i] != POISON_FREE) {
|
|
|
|
error = data[offset + i];
|
|
|
|
bad_count++;
|
|
|
|
}
|
|
|
|
}
|
2011-07-29 16:22:13 +00:00
|
|
|
print_hex_dump(KERN_CONT, "", 0, 16, 1,
|
|
|
|
&data[offset], limit, 1);
|
2006-09-29 08:59:51 +00:00
|
|
|
|
|
|
|
if (bad_count == 1) {
|
|
|
|
error ^= POISON_FREE;
|
|
|
|
if (!(error & (error - 1))) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Single bit error detected. Probably bad RAM.\n");
|
2006-09-29 08:59:51 +00:00
|
|
|
#ifdef CONFIG_X86
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Run memtest86+ or a similar memory test tool.\n");
|
2006-09-29 08:59:51 +00:00
|
|
|
#else
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Run a memory test tool.\n");
|
2006-09-29 08:59:51 +00:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int i, size;
|
|
|
|
char *realobj;
|
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Redzone: 0x%llx/0x%llx\n",
|
|
|
|
*dbg_redzone1(cachep, objp),
|
|
|
|
*dbg_redzone2(cachep, objp));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (cachep->flags & SLAB_STORE_USER) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Last user: [<%p>](%pSR)\n",
|
2012-12-12 18:19:12 +00:00
|
|
|
*dbg_userword(cachep, objp),
|
|
|
|
*dbg_userword(cachep, objp));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2006-02-01 11:05:42 +00:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2012-06-13 15:24:58 +00:00
|
|
|
size = cachep->object_size;
|
2006-01-08 09:00:37 +00:00
|
|
|
for (i = 0; i < size && lines; i += 16, lines--) {
|
2005-04-16 22:20:36 +00:00
|
|
|
int limit;
|
|
|
|
limit = 16;
|
2006-01-08 09:00:37 +00:00
|
|
|
if (i + limit > size)
|
|
|
|
limit = size - i;
|
2005-04-16 22:20:36 +00:00
|
|
|
dump_line(realobj, i, limit);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void check_poison_obj(struct kmem_cache *cachep, void *objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
char *realobj;
|
|
|
|
int size, i;
|
|
|
|
int lines = 0;
|
|
|
|
|
2016-03-15 21:54:21 +00:00
|
|
|
if (is_debug_pagealloc_cache(cachep))
|
|
|
|
return;
|
|
|
|
|
2006-02-01 11:05:42 +00:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2012-06-13 15:24:58 +00:00
|
|
|
size = cachep->object_size;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-01-08 09:00:37 +00:00
|
|
|
for (i = 0; i < size; i++) {
|
2005-04-16 22:20:36 +00:00
|
|
|
char exp = POISON_FREE;
|
2006-01-08 09:00:37 +00:00
|
|
|
if (i == size - 1)
|
2005-04-16 22:20:36 +00:00
|
|
|
exp = POISON_END;
|
|
|
|
if (realobj[i] != exp) {
|
|
|
|
int limit;
|
|
|
|
/* Mismatch ! */
|
|
|
|
/* Print header */
|
|
|
|
if (lines == 0) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Slab corruption (%s): %s start=%p, len=%d\n",
|
|
|
|
print_tainted(), cachep->name,
|
|
|
|
realobj, size);
|
2005-04-16 22:20:36 +00:00
|
|
|
print_objinfo(cachep, objp, 0);
|
|
|
|
}
|
|
|
|
/* Hexdump the affected line */
|
2006-01-08 09:00:37 +00:00
|
|
|
i = (i / 16) * 16;
|
2005-04-16 22:20:36 +00:00
|
|
|
limit = 16;
|
2006-01-08 09:00:37 +00:00
|
|
|
if (i + limit > size)
|
|
|
|
limit = size - i;
|
2005-04-16 22:20:36 +00:00
|
|
|
dump_line(realobj, i, limit);
|
|
|
|
i += 16;
|
|
|
|
lines++;
|
|
|
|
/* Limit to 5 lines */
|
|
|
|
if (lines > 5)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (lines != 0) {
|
|
|
|
/* Print some data about the neighboring objects, if they
|
|
|
|
* exist:
|
|
|
|
*/
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page = virt_to_head_page(objp);
|
2006-03-22 08:08:10 +00:00
|
|
|
unsigned int objnr;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
objnr = obj_to_index(cachep, page, objp);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (objnr) {
|
2013-10-24 01:07:49 +00:00
|
|
|
objp = index_to_obj(cachep, page, objnr - 1);
|
2006-02-01 11:05:42 +00:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Prev obj: start=%p, len=%d\n", realobj, size);
|
2005-04-16 22:20:36 +00:00
|
|
|
print_objinfo(cachep, objp, 2);
|
|
|
|
}
|
2006-01-08 09:00:37 +00:00
|
|
|
if (objnr + 1 < cachep->num) {
|
2013-10-24 01:07:49 +00:00
|
|
|
objp = index_to_obj(cachep, page, objnr + 1);
|
2006-02-01 11:05:42 +00:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("Next obj: start=%p, len=%d\n", realobj, size);
|
2005-04-16 22:20:36 +00:00
|
|
|
print_objinfo(cachep, objp, 2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-02-01 11:05:46 +00:00
|
|
|
#if DEBUG
|
2013-10-24 01:07:49 +00:00
|
|
|
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
|
|
|
|
struct page *page)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int i;
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
|
|
|
|
if (OBJFREELIST_SLAB(cachep) && cachep->flags & SLAB_POISON) {
|
|
|
|
poison_obj(cachep, page->freelist - obj_offset(cachep),
|
|
|
|
POISON_FREE);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
for (i = 0; i < cachep->num; i++) {
|
2013-10-24 01:07:49 +00:00
|
|
|
void *objp = index_to_obj(cachep, page, i);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_POISON) {
|
|
|
|
check_poison_obj(cachep, objp);
|
2016-03-15 21:54:21 +00:00
|
|
|
slab_kernel_map(cachep, objp, 1, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
|
|
|
if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
|
2016-03-17 21:19:47 +00:00
|
|
|
slab_error(cachep, "start of a freed object was overwritten");
|
2005-04-16 22:20:36 +00:00
|
|
|
if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
|
2016-03-17 21:19:47 +00:00
|
|
|
slab_error(cachep, "end of a freed object was overwritten");
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
2006-02-01 11:05:46 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#else
|
2013-10-24 01:07:49 +00:00
|
|
|
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
|
|
|
|
struct page *page)
|
2006-02-01 11:05:46 +00:00
|
|
|
{
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2006-03-22 08:08:14 +00:00
|
|
|
/**
|
|
|
|
* slab_destroy - destroy and release all objects in a slab
|
|
|
|
* @cachep: cache pointer being destroyed
|
2014-01-27 17:57:08 +00:00
|
|
|
* @page: page pointer being destroyed
|
2006-03-22 08:08:14 +00:00
|
|
|
*
|
2014-08-06 23:04:46 +00:00
|
|
|
* Destroy all the objs in a slab page, and release the mem back to the system.
|
|
|
|
* Before calling the slab page must have been unlinked from the cache. The
|
|
|
|
* kmem_cache_node ->list_lock is not held/needed.
|
2006-02-01 11:05:46 +00:00
|
|
|
*/
|
2013-10-24 01:07:49 +00:00
|
|
|
static void slab_destroy(struct kmem_cache *cachep, struct page *page)
|
2006-02-01 11:05:46 +00:00
|
|
|
{
|
2013-10-30 10:04:01 +00:00
|
|
|
void *freelist;
|
2006-02-01 11:05:46 +00:00
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
freelist = page->freelist;
|
|
|
|
slab_destroy_debugcheck(cachep, page);
|
2015-11-07 00:29:44 +00:00
|
|
|
if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
|
|
|
|
call_rcu(&page->rcu_head, kmem_rcu_free);
|
|
|
|
else
|
2013-10-24 01:07:38 +00:00
|
|
|
kmem_freepages(cachep, page);
|
2013-10-24 01:07:42 +00:00
|
|
|
|
|
|
|
/*
|
2013-10-24 01:07:49 +00:00
|
|
|
* From now on, we don't use freelist
|
2013-10-24 01:07:42 +00:00
|
|
|
* although actual page can be freed in rcu context
|
|
|
|
*/
|
|
|
|
if (OFF_SLAB(cachep))
|
2013-10-24 01:07:49 +00:00
|
|
|
kmem_cache_free(cachep->freelist_cache, freelist);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2014-08-06 23:04:25 +00:00
|
|
|
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct page *page, *n;
|
|
|
|
|
|
|
|
list_for_each_entry_safe(page, n, list, lru) {
|
|
|
|
list_del(&page->lru);
|
|
|
|
slab_destroy(cachep, page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-01-08 09:00:36 +00:00
|
|
|
/**
|
2006-02-01 11:05:52 +00:00
|
|
|
* calculate_slab_order - calculate size (page order) of slabs
|
|
|
|
* @cachep: pointer to the cache that is being created
|
|
|
|
* @size: size of objects to be created in this cache.
|
|
|
|
* @flags: slab allocation flags
|
|
|
|
*
|
|
|
|
* Also calculates the number of objects per slab.
|
2006-01-08 09:00:36 +00:00
|
|
|
*
|
|
|
|
* This could be made much more intelligent. For now, try to avoid using
|
|
|
|
* high order pages for slabs. When the gfp() functions are more friendly
|
|
|
|
* towards high-order requests, this should be changed.
|
|
|
|
*/
|
2006-03-22 08:08:11 +00:00
|
|
|
static size_t calculate_slab_order(struct kmem_cache *cachep,
|
2016-03-15 21:54:30 +00:00
|
|
|
size_t size, unsigned long flags)
|
2006-01-08 09:00:36 +00:00
|
|
|
{
|
|
|
|
size_t left_over = 0;
|
2006-03-07 01:44:43 +00:00
|
|
|
int gfporder;
|
2006-01-08 09:00:36 +00:00
|
|
|
|
2007-05-17 05:11:01 +00:00
|
|
|
for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
|
2006-01-08 09:00:36 +00:00
|
|
|
unsigned int num;
|
|
|
|
size_t remainder;
|
|
|
|
|
2016-03-15 21:54:53 +00:00
|
|
|
num = cache_estimate(gfporder, size, flags, &remainder);
|
2006-01-08 09:00:36 +00:00
|
|
|
if (!num)
|
|
|
|
continue;
|
2006-03-07 01:44:43 +00:00
|
|
|
|
2013-12-02 08:49:41 +00:00
|
|
|
/* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
|
|
|
|
if (num > SLAB_OBJ_MAX_NUM)
|
|
|
|
break;
|
|
|
|
|
2006-06-02 13:44:58 +00:00
|
|
|
if (flags & CFLGS_OFF_SLAB) {
|
2016-03-15 21:54:41 +00:00
|
|
|
struct kmem_cache *freelist_cache;
|
|
|
|
size_t freelist_size;
|
|
|
|
|
|
|
|
freelist_size = num * sizeof(freelist_idx_t);
|
|
|
|
freelist_cache = kmalloc_slab(freelist_size, 0u);
|
|
|
|
if (!freelist_cache)
|
|
|
|
continue;
|
|
|
|
|
2006-06-02 13:44:58 +00:00
|
|
|
/*
|
2016-03-15 21:54:41 +00:00
|
|
|
* Needed to avoid possible looping condition
|
2016-05-20 00:10:26 +00:00
|
|
|
* in cache_grow_begin()
|
2006-06-02 13:44:58 +00:00
|
|
|
*/
|
2016-03-15 21:54:41 +00:00
|
|
|
if (OFF_SLAB(freelist_cache))
|
|
|
|
continue;
|
2006-06-02 13:44:58 +00:00
|
|
|
|
2016-03-15 21:54:41 +00:00
|
|
|
/* check if off slab has enough benefit */
|
|
|
|
if (freelist_cache->size > cachep->size / 2)
|
|
|
|
continue;
|
2006-06-02 13:44:58 +00:00
|
|
|
}
|
2006-01-08 09:00:36 +00:00
|
|
|
|
2006-03-07 01:44:43 +00:00
|
|
|
/* Found something acceptable - save it away */
|
2006-01-08 09:00:36 +00:00
|
|
|
cachep->num = num;
|
2006-03-07 01:44:43 +00:00
|
|
|
cachep->gfporder = gfporder;
|
2006-01-08 09:00:36 +00:00
|
|
|
left_over = remainder;
|
|
|
|
|
2006-03-08 18:33:05 +00:00
|
|
|
/*
|
|
|
|
* A VFS-reclaimable slab tends to have most allocations
|
|
|
|
* as GFP_NOFS and we really don't want to have to be allocating
|
|
|
|
* higher-order pages when we are unable to shrink dcache.
|
|
|
|
*/
|
|
|
|
if (flags & SLAB_RECLAIM_ACCOUNT)
|
|
|
|
break;
|
|
|
|
|
2006-01-08 09:00:36 +00:00
|
|
|
/*
|
|
|
|
* Large number of objects is good, but very large slabs are
|
|
|
|
* currently bad for the gfp()s.
|
|
|
|
*/
|
2011-10-19 05:09:24 +00:00
|
|
|
if (gfporder >= slab_max_order)
|
2006-01-08 09:00:36 +00:00
|
|
|
break;
|
|
|
|
|
2006-03-07 01:44:43 +00:00
|
|
|
/*
|
|
|
|
* Acceptable internal fragmentation?
|
|
|
|
*/
|
2006-03-22 08:08:11 +00:00
|
|
|
if (left_over * 8 <= (PAGE_SIZE << gfporder))
|
2006-01-08 09:00:36 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
return left_over;
|
|
|
|
}
|
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
static struct array_cache __percpu *alloc_kmem_cache_cpus(
|
|
|
|
struct kmem_cache *cachep, int entries, int batchcount)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
size_t size;
|
|
|
|
struct array_cache __percpu *cpu_cache;
|
|
|
|
|
|
|
|
size = sizeof(void *) * entries + sizeof(struct array_cache);
|
2014-10-13 22:51:01 +00:00
|
|
|
cpu_cache = __alloc_percpu(size, sizeof(void *));
|
2014-10-09 22:26:27 +00:00
|
|
|
|
|
|
|
if (!cpu_cache)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
init_arraycache(per_cpu_ptr(cpu_cache, cpu),
|
|
|
|
entries, batchcount);
|
|
|
|
}
|
|
|
|
|
|
|
|
return cpu_cache;
|
|
|
|
}
|
|
|
|
|
2016-08-02 21:03:33 +00:00
|
|
|
static int __ref setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
|
2006-03-22 08:08:11 +00:00
|
|
|
{
|
2012-07-06 20:25:11 +00:00
|
|
|
if (slab_state >= FULL)
|
2009-06-10 16:40:04 +00:00
|
|
|
return enable_cpucache(cachep, gfp);
|
2006-09-26 06:31:38 +00:00
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
|
|
|
|
if (!cachep->cpu_cache)
|
|
|
|
return 1;
|
|
|
|
|
2012-07-06 20:25:11 +00:00
|
|
|
if (slab_state == DOWN) {
|
2014-10-09 22:26:27 +00:00
|
|
|
/* Creation of first cache (kmem_cache). */
|
|
|
|
set_up_node(kmem_cache, CACHE_CACHE);
|
2012-11-28 16:23:09 +00:00
|
|
|
} else if (slab_state == PARTIAL) {
|
2014-10-09 22:26:27 +00:00
|
|
|
/* For kmem_cache_node */
|
|
|
|
set_up_node(cachep, SIZE_NODE);
|
2006-03-22 08:08:11 +00:00
|
|
|
} else {
|
2014-10-09 22:26:27 +00:00
|
|
|
int node;
|
2006-03-22 08:08:11 +00:00
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
for_each_online_node(node) {
|
|
|
|
cachep->node[node] = kmalloc_node(
|
|
|
|
sizeof(struct kmem_cache_node), gfp, node);
|
|
|
|
BUG_ON(!cachep->node[node]);
|
|
|
|
kmem_cache_node_init(cachep->node[node]);
|
2006-03-22 08:08:11 +00:00
|
|
|
}
|
|
|
|
}
|
2014-10-09 22:26:27 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep->node[numa_mem_id()]->next_reap =
|
2014-03-30 09:02:20 +00:00
|
|
|
jiffies + REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
2006-03-22 08:08:11 +00:00
|
|
|
|
|
|
|
cpu_cache_get(cachep)->avail = 0;
|
|
|
|
cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
|
|
|
|
cpu_cache_get(cachep)->batchcount = 1;
|
|
|
|
cpu_cache_get(cachep)->touched = 0;
|
|
|
|
cachep->batchcount = 1;
|
|
|
|
cachep->limit = BOOT_CPUCACHE_ENTRIES;
|
2006-09-26 06:31:38 +00:00
|
|
|
return 0;
|
2006-03-22 08:08:11 +00:00
|
|
|
}
|
|
|
|
|
2014-10-09 22:26:24 +00:00
|
|
|
unsigned long kmem_cache_flags(unsigned long object_size,
|
|
|
|
unsigned long flags, const char *name,
|
|
|
|
void (*ctor)(void *))
|
|
|
|
{
|
|
|
|
return flags;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct kmem_cache *
|
|
|
|
__kmem_cache_alias(const char *name, size_t size, size_t align,
|
|
|
|
unsigned long flags, void (*ctor)(void *))
|
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
|
|
|
|
cachep = find_mergeable(size, align, flags, name, ctor);
|
|
|
|
if (cachep) {
|
|
|
|
cachep->refcount++;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Adjust the object sizes so that we clear
|
|
|
|
* the complete object on kzalloc.
|
|
|
|
*/
|
|
|
|
cachep->object_size = max_t(int, cachep->object_size, size);
|
|
|
|
}
|
|
|
|
return cachep;
|
|
|
|
}
|
|
|
|
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
static bool set_objfreelist_slab_cache(struct kmem_cache *cachep,
|
|
|
|
size_t size, unsigned long flags)
|
|
|
|
{
|
|
|
|
size_t left;
|
|
|
|
|
|
|
|
cachep->num = 0;
|
|
|
|
|
|
|
|
if (cachep->ctor || flags & SLAB_DESTROY_BY_RCU)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
left = calculate_slab_order(cachep, size,
|
|
|
|
flags | CFLGS_OBJFREELIST_SLAB);
|
|
|
|
if (!cachep->num)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (cachep->num * sizeof(freelist_idx_t) > cachep->object_size)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
cachep->colour = left / cachep->colour_off;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:35 +00:00
|
|
|
static bool set_off_slab_cache(struct kmem_cache *cachep,
|
|
|
|
size_t size, unsigned long flags)
|
|
|
|
{
|
|
|
|
size_t left;
|
|
|
|
|
|
|
|
cachep->num = 0;
|
|
|
|
|
|
|
|
/*
|
2016-03-15 21:54:41 +00:00
|
|
|
* Always use on-slab management when SLAB_NOLEAKTRACE
|
|
|
|
* to avoid recursive calls into kmemleak.
|
2016-03-15 21:54:35 +00:00
|
|
|
*/
|
|
|
|
if (flags & SLAB_NOLEAKTRACE)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Size is large, assume best to place the slab management obj
|
|
|
|
* off-slab (should allow better packing of objs).
|
|
|
|
*/
|
|
|
|
left = calculate_slab_order(cachep, size, flags | CFLGS_OFF_SLAB);
|
|
|
|
if (!cachep->num)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the slab has been placed off-slab, and we have enough space then
|
|
|
|
* move it on-slab. This is at the expense of any extra colouring.
|
|
|
|
*/
|
|
|
|
if (left >= cachep->num * sizeof(freelist_idx_t))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
cachep->colour = left / cachep->colour_off;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool set_on_slab_cache(struct kmem_cache *cachep,
|
|
|
|
size_t size, unsigned long flags)
|
|
|
|
{
|
|
|
|
size_t left;
|
|
|
|
|
|
|
|
cachep->num = 0;
|
|
|
|
|
|
|
|
left = calculate_slab_order(cachep, size, flags);
|
|
|
|
if (!cachep->num)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
cachep->colour = left / cachep->colour_off;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
2012-07-06 20:25:10 +00:00
|
|
|
* __kmem_cache_create - Create a cache.
|
2012-11-07 01:10:10 +00:00
|
|
|
* @cachep: cache management descriptor
|
2005-04-16 22:20:36 +00:00
|
|
|
* @flags: SLAB flags
|
|
|
|
*
|
|
|
|
* Returns a ptr to the cache on success, NULL on failure.
|
|
|
|
* Cannot be called within a int, but can be interrupted.
|
2007-07-20 01:11:58 +00:00
|
|
|
* The @ctor is run when new pages are allocated by the cache.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* The flags are
|
|
|
|
*
|
|
|
|
* %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
|
|
|
|
* to catch references to uninitialised memory.
|
|
|
|
*
|
|
|
|
* %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
|
|
|
|
* for buffer overruns.
|
|
|
|
*
|
|
|
|
* %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
|
|
|
|
* cacheline. This can be beneficial if you're counting cycles as closely
|
|
|
|
* as davem.
|
|
|
|
*/
|
2012-09-05 00:20:34 +00:00
|
|
|
int
|
2012-09-04 23:18:33 +00:00
|
|
|
__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-09-25 23:05:20 +00:00
|
|
|
size_t ralign = BYTES_PER_WORD;
|
2009-06-10 16:40:04 +00:00
|
|
|
gfp_t gfp;
|
2012-09-05 00:20:34 +00:00
|
|
|
int err;
|
2012-09-04 23:18:33 +00:00
|
|
|
size_t size = cachep->size;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
#if FORCED_DEBUG
|
|
|
|
/*
|
|
|
|
* Enable redzoning and last user accounting, except for caches with
|
|
|
|
* large objects, if the increased size would increase the object size
|
|
|
|
* above the next power of two: caches with object sizes just above a
|
|
|
|
* power of two have a significant amount of internal fragmentation.
|
|
|
|
*/
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-05 01:26:44 +00:00
|
|
|
if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
|
|
|
|
2 * sizeof(unsigned long long)))
|
2006-01-08 09:00:37 +00:00
|
|
|
flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!(flags & SLAB_DESTROY_BY_RCU))
|
|
|
|
flags |= SLAB_POISON;
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Check that size is in terms of words. This is needed to avoid
|
2005-04-16 22:20:36 +00:00
|
|
|
* unaligned accesses for some archs when redzoning is used, and makes
|
|
|
|
* sure any on-slab bufctl's are also correctly aligned.
|
|
|
|
*/
|
2006-01-08 09:00:37 +00:00
|
|
|
if (size & (BYTES_PER_WORD - 1)) {
|
|
|
|
size += (BYTES_PER_WORD - 1);
|
|
|
|
size &= ~(BYTES_PER_WORD - 1);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-05 01:26:44 +00:00
|
|
|
if (flags & SLAB_RED_ZONE) {
|
|
|
|
ralign = REDZONE_ALIGN;
|
|
|
|
/* If redzoning, ensure that the second redzone is suitably
|
|
|
|
* aligned, by adjusting the object size accordingly. */
|
|
|
|
size += REDZONE_ALIGN - 1;
|
|
|
|
size &= ~(REDZONE_ALIGN - 1);
|
|
|
|
}
|
2006-09-26 06:31:25 +00:00
|
|
|
|
2006-12-07 04:32:11 +00:00
|
|
|
/* 3) caller mandated alignment */
|
2012-09-04 23:18:33 +00:00
|
|
|
if (ralign < cachep->align) {
|
|
|
|
ralign = cachep->align;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2011-02-14 15:46:21 +00:00
|
|
|
/* disable debug if necessary */
|
|
|
|
if (ralign > __alignof__(unsigned long long))
|
2006-12-07 04:32:11 +00:00
|
|
|
flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
2006-09-26 06:31:25 +00:00
|
|
|
* 4) Store it.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2012-09-04 23:18:33 +00:00
|
|
|
cachep->align = ralign;
|
2016-03-15 21:54:35 +00:00
|
|
|
cachep->colour_off = cache_line_size();
|
|
|
|
/* Offset must be a multiple of the alignment. */
|
|
|
|
if (cachep->colour_off < cachep->align)
|
|
|
|
cachep->colour_off = cachep->align;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-06-10 16:40:04 +00:00
|
|
|
if (slab_is_available())
|
|
|
|
gfp = GFP_KERNEL;
|
|
|
|
else
|
|
|
|
gfp = GFP_NOWAIT;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#if DEBUG
|
|
|
|
|
2006-09-26 06:31:25 +00:00
|
|
|
/*
|
|
|
|
* Both debugging options require word-alignment which is calculated
|
|
|
|
* into align above.
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
if (flags & SLAB_RED_ZONE) {
|
|
|
|
/* add space for red zone words */
|
2011-02-14 15:46:21 +00:00
|
|
|
cachep->obj_offset += sizeof(unsigned long long);
|
|
|
|
size += 2 * sizeof(unsigned long long);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
if (flags & SLAB_STORE_USER) {
|
2006-09-26 06:31:25 +00:00
|
|
|
/* user store requires one word storage behind the end of
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-05 01:26:44 +00:00
|
|
|
* the real object. But if the second red zone needs to be
|
|
|
|
* aligned to 64 bits, we must allow that much space.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-05 01:26:44 +00:00
|
|
|
if (flags & SLAB_RED_ZONE)
|
|
|
|
size += REDZONE_ALIGN;
|
|
|
|
else
|
|
|
|
size += BYTES_PER_WORD;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-03-15 21:54:33 +00:00
|
|
|
#endif
|
|
|
|
|
2016-03-25 21:21:59 +00:00
|
|
|
kasan_cache_create(cachep, &size, &flags);
|
|
|
|
|
2016-03-15 21:54:33 +00:00
|
|
|
size = ALIGN(size, cachep->align);
|
|
|
|
/*
|
|
|
|
* We should restrict the number of objects in a slab to implement
|
|
|
|
* byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
|
|
|
|
*/
|
|
|
|
if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
|
|
|
|
size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
|
|
|
|
|
|
|
|
#if DEBUG
|
mm/slab: fix unexpected index mapping result of kmalloc_size(INDEX_NODE+1)
Commit description is copied from the original post of this bug:
http://comments.gmane.org/gmane.linux.kernel.mm/135349
Kernels after v3.9 use kmalloc_size(INDEX_NODE + 1) to get the next
larger cache size than the size index INDEX_NODE mapping. In kernels
3.9 and earlier we used malloc_sizes[INDEX_L3 + 1].cs_size.
However, sometimes we can't get the right output we expected via
kmalloc_size(INDEX_NODE + 1), causing a BUG().
The mapping table in the latest kernel is like:
index = {0, 1, 2 , 3, 4, 5, 6, n}
size = {0, 96, 192, 8, 16, 32, 64, 2^n}
The mapping table before 3.10 is like this:
index = {0 , 1 , 2, 3, 4 , 5 , 6, n}
size = {32, 64, 96, 128, 192, 256, 512, 2^(n+3)}
The problem on my mips64 machine is as follows:
(1) When configured DEBUG_SLAB && DEBUG_PAGEALLOC && DEBUG_LOCK_ALLOC
&& DEBUG_SPINLOCK, the sizeof(struct kmem_cache_node) will be "150",
and the macro INDEX_NODE turns out to be "2": #define INDEX_NODE
kmalloc_index(sizeof(struct kmem_cache_node))
(2) Then the result of kmalloc_size(INDEX_NODE + 1) is 8.
(3) Then "if(size >= kmalloc_size(INDEX_NODE + 1)" will lead to "size
= PAGE_SIZE".
(4) Then "if ((size >= (PAGE_SIZE >> 3))" test will be satisfied and
"flags |= CFLGS_OFF_SLAB" will be covered.
(5) if (flags & CFLGS_OFF_SLAB)" test will be satisfied and will go to
"cachep->slabp_cache = kmalloc_slab(slab_size, 0u)", and the result
here may be NULL while kernel bootup.
(6) Finally,"BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));" causes the
BUG info as the following shows (may be only mips64 has this problem):
This patch fixes the problem of kmalloc_size(INDEX_NODE + 1) and removes
the BUG by adding 'size >= 256' check to guarantee that all necessary
small sized slabs are initialized regardless sequence of slab size in
mapping table.
Fixes: e33660165c90 ("slab: Use common kmalloc_index/kmalloc_size...")
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Liuhailong <liu.hailong6@zte.com.cn>
Acked-by: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-10-01 22:36:54 +00:00
|
|
|
/*
|
|
|
|
* To activate debug pagealloc, off-slab management is necessary
|
|
|
|
* requirement. In early phase of initialization, small sized slab
|
|
|
|
* doesn't get initialized so it would not be possible. So, we need
|
|
|
|
* to check size >= 256. It guarantees that all necessary small
|
|
|
|
* sized slab is initialized in current slab initialization sequence.
|
|
|
|
*/
|
2016-03-15 21:54:18 +00:00
|
|
|
if (debug_pagealloc_enabled() && (flags & SLAB_POISON) &&
|
2016-03-15 21:54:38 +00:00
|
|
|
size >= 256 && cachep->object_size > cache_line_size()) {
|
|
|
|
if (size < PAGE_SIZE || size % PAGE_SIZE == 0) {
|
|
|
|
size_t tmp_size = ALIGN(size, PAGE_SIZE);
|
|
|
|
|
|
|
|
if (set_off_slab_cache(cachep, tmp_size, flags)) {
|
|
|
|
flags |= CFLGS_OFF_SLAB;
|
|
|
|
cachep->obj_offset += tmp_size - size;
|
|
|
|
size = tmp_size;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (set_objfreelist_slab_cache(cachep, size, flags)) {
|
|
|
|
flags |= CFLGS_OBJFREELIST_SLAB;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:35 +00:00
|
|
|
if (set_off_slab_cache(cachep, size, flags)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
flags |= CFLGS_OFF_SLAB;
|
2016-03-15 21:54:35 +00:00
|
|
|
goto done;
|
2016-03-15 21:54:33 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-15 21:54:35 +00:00
|
|
|
if (set_on_slab_cache(cachep, size, flags))
|
|
|
|
goto done;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-15 21:54:35 +00:00
|
|
|
return -E2BIG;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-15 21:54:35 +00:00
|
|
|
done:
|
|
|
|
cachep->freelist_size = cachep->num * sizeof(freelist_idx_t);
|
2005-04-16 22:20:36 +00:00
|
|
|
cachep->flags = flags;
|
2013-10-24 01:07:44 +00:00
|
|
|
cachep->allocflags = __GFP_COMP;
|
2016-05-20 00:10:41 +00:00
|
|
|
if (flags & SLAB_CACHE_DMA)
|
2012-06-14 12:17:21 +00:00
|
|
|
cachep->allocflags |= GFP_DMA;
|
2012-06-13 15:24:57 +00:00
|
|
|
cachep->size = size;
|
2006-12-13 08:34:27 +00:00
|
|
|
cachep->reciprocal_buffer_size = reciprocal_value(size);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-15 21:54:21 +00:00
|
|
|
#if DEBUG
|
|
|
|
/*
|
|
|
|
* If we're going to use the generic kernel_map_pages()
|
|
|
|
* poisoning, then it's going to smash the contents of
|
|
|
|
* the redzone and userword anyhow, so switch them off.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_PAGE_POISONING) &&
|
|
|
|
(cachep->flags & SLAB_POISON) &&
|
|
|
|
is_debug_pagealloc_cache(cachep))
|
|
|
|
cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (OFF_SLAB(cachep)) {
|
2016-03-15 21:54:35 +00:00
|
|
|
cachep->freelist_cache =
|
|
|
|
kmalloc_slab(cachep->freelist_size, 0u);
|
2006-09-26 06:31:34 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-09-05 00:20:34 +00:00
|
|
|
err = setup_cpu_cache(cachep, gfp);
|
|
|
|
if (err) {
|
2016-02-17 21:11:37 +00:00
|
|
|
__kmem_cache_release(cachep);
|
2012-09-05 00:20:34 +00:00
|
|
|
return err;
|
2006-09-26 06:31:38 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-09-05 00:20:34 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
static void check_irq_off(void)
|
|
|
|
{
|
|
|
|
BUG_ON(!irqs_disabled());
|
|
|
|
}
|
|
|
|
|
|
|
|
static void check_irq_on(void)
|
|
|
|
{
|
|
|
|
BUG_ON(irqs_disabled());
|
|
|
|
}
|
|
|
|
|
2016-05-20 00:10:02 +00:00
|
|
|
static void check_mutex_acquired(void)
|
|
|
|
{
|
|
|
|
BUG_ON(!mutex_is_locked(&slab_mutex));
|
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void check_spinlock_acquired(struct kmem_cache *cachep)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
check_irq_off();
|
2014-08-06 23:04:11 +00:00
|
|
|
assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
}
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
check_irq_off();
|
2014-08-06 23:04:11 +00:00
|
|
|
assert_spin_locked(&get_node(cachep, node)->list_lock);
|
2005-09-09 20:03:32 +00:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#else
|
|
|
|
#define check_irq_off() do { } while(0)
|
|
|
|
#define check_irq_on() do { } while(0)
|
2016-05-20 00:10:02 +00:00
|
|
|
#define check_mutex_acquired() do { } while(0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define check_spinlock_acquired(x) do { } while(0)
|
2005-09-09 20:03:32 +00:00
|
|
|
#define check_spinlock_acquired_node(x, y) do { } while(0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2016-05-20 00:10:02 +00:00
|
|
|
static void drain_array_locked(struct kmem_cache *cachep, struct array_cache *ac,
|
|
|
|
int node, bool free_all, struct list_head *list)
|
|
|
|
{
|
|
|
|
int tofree;
|
|
|
|
|
|
|
|
if (!ac || !ac->avail)
|
|
|
|
return;
|
|
|
|
|
|
|
|
tofree = free_all ? ac->avail : (ac->limit + 4) / 5;
|
|
|
|
if (tofree > ac->avail)
|
|
|
|
tofree = (ac->avail + 1) / 2;
|
|
|
|
|
|
|
|
free_block(cachep, ac->entry, tofree, node, list);
|
|
|
|
ac->avail -= tofree;
|
|
|
|
memmove(ac->entry, &(ac->entry[tofree]), sizeof(void *) * ac->avail);
|
|
|
|
}
|
2006-03-22 08:09:06 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static void do_drain(void *arg)
|
|
|
|
{
|
2006-03-22 08:08:11 +00:00
|
|
|
struct kmem_cache *cachep = arg;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct array_cache *ac;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
int node = numa_mem_id();
|
2014-08-06 23:04:11 +00:00
|
|
|
struct kmem_cache_node *n;
|
2014-08-06 23:04:25 +00:00
|
|
|
LIST_HEAD(list);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
check_irq_off();
|
2006-02-01 11:05:49 +00:00
|
|
|
ac = cpu_cache_get(cachep);
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
spin_lock(&n->list_lock);
|
2014-08-06 23:04:25 +00:00
|
|
|
free_block(cachep, ac->entry, ac->avail, node, &list);
|
2014-08-06 23:04:11 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
2014-08-06 23:04:25 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
ac->avail = 0;
|
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void drain_cpu_caches(struct kmem_cache *cachep)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2005-09-09 20:03:32 +00:00
|
|
|
int node;
|
2016-05-20 00:10:02 +00:00
|
|
|
LIST_HEAD(list);
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2008-05-09 07:39:44 +00:00
|
|
|
on_each_cpu(do_drain, cachep, 1);
|
2005-04-16 22:20:36 +00:00
|
|
|
check_irq_on();
|
2014-08-06 23:04:11 +00:00
|
|
|
for_each_kmem_cache_node(cachep, node, n)
|
|
|
|
if (n->alien)
|
2013-01-10 19:14:19 +00:00
|
|
|
drain_alien_cache(cachep, n->alien);
|
2006-05-15 18:41:00 +00:00
|
|
|
|
2016-05-20 00:10:02 +00:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
drain_array_locked(cachep, n->shared, node, true, &list);
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
|
|
|
|
|
|
|
slabs_destroy(cachep, &list);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-06-30 08:55:45 +00:00
|
|
|
/*
|
|
|
|
* Remove slabs from the list of free slabs.
|
|
|
|
* Specify the number of slabs to drain in tofree.
|
|
|
|
*
|
|
|
|
* Returns the actual number of slabs released.
|
|
|
|
*/
|
|
|
|
static int drain_freelist(struct kmem_cache *cache,
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n, int tofree)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-06-30 08:55:45 +00:00
|
|
|
struct list_head *p;
|
|
|
|
int nr_freed;
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-06-30 08:55:45 +00:00
|
|
|
nr_freed = 0;
|
2013-01-10 19:14:19 +00:00
|
|
|
while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
p = n->slabs_free.prev;
|
|
|
|
if (p == &n->slabs_free) {
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
2006-06-30 08:55:45 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
page = list_entry(p, struct page, lru);
|
|
|
|
list_del(&page->lru);
|
2006-06-30 08:55:45 +00:00
|
|
|
/*
|
|
|
|
* Safe to drop the lock. The slab is no longer linked
|
|
|
|
* to the cache.
|
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
n->free_objects -= cache->num;
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
2013-10-24 01:07:49 +00:00
|
|
|
slab_destroy(cache, page);
|
2006-06-30 08:55:45 +00:00
|
|
|
nr_freed++;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2006-06-30 08:55:45 +00:00
|
|
|
out:
|
|
|
|
return nr_freed;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
slub: make dead caches discard free slabs immediately
To speed up further allocations SLUB may store empty slabs in per cpu/node
partial lists instead of freeing them immediately. This prevents per
memcg caches destruction, because kmem caches created for a memory cgroup
are only destroyed after the last page charged to the cgroup is freed.
To fix this issue, this patch resurrects approach first proposed in [1].
It forbids SLUB to cache empty slabs after the memory cgroup that the
cache belongs to was destroyed. It is achieved by setting kmem_cache's
cpu_partial and min_partial constants to 0 and tuning put_cpu_partial() so
that it would drop frozen empty slabs immediately if cpu_partial = 0.
The runtime overhead is minimal. From all the hot functions, we only
touch relatively cold put_cpu_partial(): we make it call
unfreeze_partials() after freezing a slab that belongs to an offline
memory cgroup. Since slab freezing exists to avoid moving slabs from/to a
partial list on free/alloc, and there can't be allocations from dead
caches, it shouldn't cause any overhead. We do have to disable preemption
for put_cpu_partial() to achieve that though.
The original patch was accepted well and even merged to the mm tree.
However, I decided to withdraw it due to changes happening to the memcg
core at that time. I had an idea of introducing per-memcg shrinkers for
kmem caches, but now, as memcg has finally settled down, I do not see it
as an option, because SLUB shrinker would be too costly to call since SLUB
does not keep free slabs on a separate list. Besides, we currently do not
even call per-memcg shrinkers for offline memcgs. Overall, it would
introduce much more complexity to both SLUB and memcg than this small
patch.
Regarding to SLAB, there's no problem with it, because it shrinks
per-cpu/node caches periodically. Thanks to list_lru reparenting, we no
longer keep entries for offline cgroups in per-memcg arrays (such as
memcg_cache_params->memcg_caches), so we do not have to bother if a
per-memcg cache will be shrunk a bit later than it could be.
[1] http://thread.gmane.org/gmane.linux.kernel.mm/118649/focus=118650
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-12 22:59:47 +00:00
|
|
|
int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2014-08-06 23:04:11 +00:00
|
|
|
int ret = 0;
|
|
|
|
int node;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2005-09-09 20:03:32 +00:00
|
|
|
|
|
|
|
drain_cpu_caches(cachep);
|
|
|
|
|
|
|
|
check_irq_on();
|
2014-08-06 23:04:11 +00:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2016-05-20 00:10:08 +00:00
|
|
|
drain_freelist(cachep, n, INT_MAX);
|
2006-06-30 08:55:45 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
ret += !list_empty(&n->slabs_full) ||
|
|
|
|
!list_empty(&n->slabs_partial);
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
return (ret ? 1 : 0);
|
|
|
|
}
|
|
|
|
|
2012-09-04 23:18:33 +00:00
|
|
|
int __kmem_cache_shutdown(struct kmem_cache *cachep)
|
2016-02-17 21:11:37 +00:00
|
|
|
{
|
|
|
|
return __kmem_cache_shrink(cachep, false);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __kmem_cache_release(struct kmem_cache *cachep)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-09-04 23:38:33 +00:00
|
|
|
int i;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-05-20 00:10:37 +00:00
|
|
|
cache_random_seq_destroy(cachep);
|
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
free_percpu(cachep->cpu_cache);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
/* NUMA: free the node structures */
|
2014-08-06 23:04:11 +00:00
|
|
|
for_each_kmem_cache_node(cachep, i, n) {
|
|
|
|
kfree(n->shared);
|
|
|
|
free_alien_cache(n->alien);
|
|
|
|
kfree(n);
|
|
|
|
cachep->node[i] = NULL;
|
2012-09-04 23:38:33 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-09-26 06:31:34 +00:00
|
|
|
/*
|
|
|
|
* Get the memory for a slab management obj.
|
2014-03-30 09:02:20 +00:00
|
|
|
*
|
|
|
|
* For a slab cache when the slab descriptor is off-slab, the
|
|
|
|
* slab descriptor can't come from the same cache which is being created,
|
|
|
|
* Because if it is the case, that means we defer the creation of
|
|
|
|
* the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
|
|
|
|
* And we eventually call down to __kmem_cache_create(), which
|
|
|
|
* in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
|
|
|
|
* This is a "chicken-and-egg" problem.
|
|
|
|
*
|
|
|
|
* So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
|
|
|
|
* which are all initialized during kmem_cache_init().
|
2006-09-26 06:31:34 +00:00
|
|
|
*/
|
2013-10-30 10:04:01 +00:00
|
|
|
static void *alloc_slabmgmt(struct kmem_cache *cachep,
|
2013-10-24 01:07:38 +00:00
|
|
|
struct page *page, int colour_off,
|
|
|
|
gfp_t local_flags, int nodeid)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-10-30 10:04:01 +00:00
|
|
|
void *freelist;
|
2013-10-24 01:07:38 +00:00
|
|
|
void *addr = page_address(page);
|
2006-01-08 09:00:37 +00:00
|
|
|
|
2016-03-15 21:54:30 +00:00
|
|
|
page->s_mem = addr + colour_off;
|
|
|
|
page->active = 0;
|
|
|
|
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (OBJFREELIST_SLAB(cachep))
|
|
|
|
freelist = NULL;
|
|
|
|
else if (OFF_SLAB(cachep)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Slab management obj is off-slab. */
|
2013-10-24 01:07:49 +00:00
|
|
|
freelist = kmem_cache_alloc_node(cachep->freelist_cache,
|
2008-11-26 08:01:31 +00:00
|
|
|
local_flags, nodeid);
|
2013-10-24 01:07:49 +00:00
|
|
|
if (!freelist)
|
2005-04-16 22:20:36 +00:00
|
|
|
return NULL;
|
|
|
|
} else {
|
2016-03-15 21:54:30 +00:00
|
|
|
/* We will use last bytes at the slab for freelist */
|
|
|
|
freelist = addr + (PAGE_SIZE << cachep->gfporder) -
|
|
|
|
cachep->freelist_size;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-03-15 21:54:30 +00:00
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
return freelist;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2014-04-18 07:24:09 +00:00
|
|
|
static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
slab: introduce byte sized index for the freelist of a slab
Currently, the freelist of a slab consist of unsigned int sized indexes.
Since most of slabs have less number of objects than 256, large sized
indexes is needless. For example, consider the minimum kmalloc slab. It's
object size is 32 byte and it would consist of one page, so 256 indexes
through byte sized index are enough to contain all possible indexes.
There can be some slabs whose object size is 8 byte. We cannot handle
this case with byte sized index, so we need to restrict minimum
object size. Since these slabs are not major, wasted memory from these
slabs would be negligible.
Some architectures' page size isn't 4096 bytes and rather larger than
4096 bytes (One example is 64KB page size on PPC or IA64) so that
byte sized index doesn't fit to them. In this case, we will use
two bytes sized index.
Below is some number for this patch.
* Before *
kmalloc-512 525 640 512 8 1 : tunables 54 27 0 : slabdata 80 80 0
kmalloc-256 210 210 256 15 1 : tunables 120 60 0 : slabdata 14 14 0
kmalloc-192 1016 1040 192 20 1 : tunables 120 60 0 : slabdata 52 52 0
kmalloc-96 560 620 128 31 1 : tunables 120 60 0 : slabdata 20 20 0
kmalloc-64 2148 2280 64 60 1 : tunables 120 60 0 : slabdata 38 38 0
kmalloc-128 647 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11360 11413 32 113 1 : tunables 120 60 0 : slabdata 101 101 0
kmem_cache 197 200 192 20 1 : tunables 120 60 0 : slabdata 10 10 0
* After *
kmalloc-512 521 648 512 8 1 : tunables 54 27 0 : slabdata 81 81 0
kmalloc-256 208 208 256 16 1 : tunables 120 60 0 : slabdata 13 13 0
kmalloc-192 1029 1029 192 21 1 : tunables 120 60 0 : slabdata 49 49 0
kmalloc-96 529 589 128 31 1 : tunables 120 60 0 : slabdata 19 19 0
kmalloc-64 2142 2142 64 63 1 : tunables 120 60 0 : slabdata 34 34 0
kmalloc-128 660 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11716 11780 32 124 1 : tunables 120 60 0 : slabdata 95 95 0
kmem_cache 197 210 192 21 1 : tunables 120 60 0 : slabdata 10 10 0
kmem_caches consisting of objects less than or equal to 256 byte have
one or more objects than before. In the case of kmalloc-32, we have 11 more
objects, so 352 bytes (11 * 32) are saved and this is roughly 9% saving of
memory. Of couse, this percentage decreases as the number of objects
in a slab decreases.
Here are the performance results on my 4 cpus machine.
* Before *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
229,945,138 cache-misses ( +- 0.23% )
11.627897174 seconds time elapsed ( +- 0.14% )
* After *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
218,640,472 cache-misses ( +- 0.42% )
11.504999837 seconds time elapsed ( +- 0.21% )
cache-misses are reduced by this patchset, roughly 5%.
And elapsed times are improved by 1%.
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2013-12-02 08:49:42 +00:00
|
|
|
return ((freelist_idx_t *)page->freelist)[idx];
|
2013-12-02 08:49:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_free_obj(struct page *page,
|
2014-04-18 07:24:09 +00:00
|
|
|
unsigned int idx, freelist_idx_t val)
|
2013-12-02 08:49:40 +00:00
|
|
|
{
|
slab: introduce byte sized index for the freelist of a slab
Currently, the freelist of a slab consist of unsigned int sized indexes.
Since most of slabs have less number of objects than 256, large sized
indexes is needless. For example, consider the minimum kmalloc slab. It's
object size is 32 byte and it would consist of one page, so 256 indexes
through byte sized index are enough to contain all possible indexes.
There can be some slabs whose object size is 8 byte. We cannot handle
this case with byte sized index, so we need to restrict minimum
object size. Since these slabs are not major, wasted memory from these
slabs would be negligible.
Some architectures' page size isn't 4096 bytes and rather larger than
4096 bytes (One example is 64KB page size on PPC or IA64) so that
byte sized index doesn't fit to them. In this case, we will use
two bytes sized index.
Below is some number for this patch.
* Before *
kmalloc-512 525 640 512 8 1 : tunables 54 27 0 : slabdata 80 80 0
kmalloc-256 210 210 256 15 1 : tunables 120 60 0 : slabdata 14 14 0
kmalloc-192 1016 1040 192 20 1 : tunables 120 60 0 : slabdata 52 52 0
kmalloc-96 560 620 128 31 1 : tunables 120 60 0 : slabdata 20 20 0
kmalloc-64 2148 2280 64 60 1 : tunables 120 60 0 : slabdata 38 38 0
kmalloc-128 647 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11360 11413 32 113 1 : tunables 120 60 0 : slabdata 101 101 0
kmem_cache 197 200 192 20 1 : tunables 120 60 0 : slabdata 10 10 0
* After *
kmalloc-512 521 648 512 8 1 : tunables 54 27 0 : slabdata 81 81 0
kmalloc-256 208 208 256 16 1 : tunables 120 60 0 : slabdata 13 13 0
kmalloc-192 1029 1029 192 21 1 : tunables 120 60 0 : slabdata 49 49 0
kmalloc-96 529 589 128 31 1 : tunables 120 60 0 : slabdata 19 19 0
kmalloc-64 2142 2142 64 63 1 : tunables 120 60 0 : slabdata 34 34 0
kmalloc-128 660 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11716 11780 32 124 1 : tunables 120 60 0 : slabdata 95 95 0
kmem_cache 197 210 192 21 1 : tunables 120 60 0 : slabdata 10 10 0
kmem_caches consisting of objects less than or equal to 256 byte have
one or more objects than before. In the case of kmalloc-32, we have 11 more
objects, so 352 bytes (11 * 32) are saved and this is roughly 9% saving of
memory. Of couse, this percentage decreases as the number of objects
in a slab decreases.
Here are the performance results on my 4 cpus machine.
* Before *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
229,945,138 cache-misses ( +- 0.23% )
11.627897174 seconds time elapsed ( +- 0.14% )
* After *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
218,640,472 cache-misses ( +- 0.42% )
11.504999837 seconds time elapsed ( +- 0.21% )
cache-misses are reduced by this patchset, roughly 5%.
And elapsed times are improved by 1%.
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2013-12-02 08:49:42 +00:00
|
|
|
((freelist_idx_t *)(page->freelist))[idx] = val;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:47 +00:00
|
|
|
static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2016-03-15 21:54:47 +00:00
|
|
|
#if DEBUG
|
2005-04-16 22:20:36 +00:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < cachep->num; i++) {
|
2013-10-24 01:07:49 +00:00
|
|
|
void *objp = index_to_obj(cachep, page, i);
|
2016-03-15 21:54:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
|
|
|
*dbg_userword(cachep, objp) = NULL;
|
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
|
|
|
*dbg_redzone1(cachep, objp) = RED_INACTIVE;
|
|
|
|
*dbg_redzone2(cachep, objp) = RED_INACTIVE;
|
|
|
|
}
|
|
|
|
/*
|
2006-03-22 08:08:11 +00:00
|
|
|
* Constructors are not allowed to allocate memory from the same
|
|
|
|
* cache which they are a constructor for. Otherwise, deadlock.
|
|
|
|
* They must also be threaded.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2016-03-25 21:21:59 +00:00
|
|
|
if (cachep->ctor && !(cachep->flags & SLAB_POISON)) {
|
|
|
|
kasan_unpoison_object_data(cachep,
|
|
|
|
objp + obj_offset(cachep));
|
2008-07-26 02:45:34 +00:00
|
|
|
cachep->ctor(objp + obj_offset(cachep));
|
2016-03-25 21:21:59 +00:00
|
|
|
kasan_poison_object_data(
|
|
|
|
cachep, objp + obj_offset(cachep));
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
|
|
|
if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
|
2016-03-17 21:19:47 +00:00
|
|
|
slab_error(cachep, "constructor overwrote the end of an object");
|
2005-04-16 22:20:36 +00:00
|
|
|
if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
|
2016-03-17 21:19:47 +00:00
|
|
|
slab_error(cachep, "constructor overwrote the start of an object");
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-03-15 21:54:21 +00:00
|
|
|
/* need to poison the objs? */
|
|
|
|
if (cachep->flags & SLAB_POISON) {
|
|
|
|
poison_obj(cachep, objp, POISON_FREE);
|
|
|
|
slab_kernel_map(cachep, objp, 0, 0);
|
|
|
|
}
|
2016-03-15 21:54:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
2016-03-15 21:54:47 +00:00
|
|
|
}
|
|
|
|
|
2016-05-20 00:10:37 +00:00
|
|
|
#ifdef CONFIG_SLAB_FREELIST_RANDOM
|
|
|
|
/* Hold information during a freelist initialization */
|
|
|
|
union freelist_init_state {
|
|
|
|
struct {
|
|
|
|
unsigned int pos;
|
2016-07-26 22:21:56 +00:00
|
|
|
unsigned int *list;
|
2016-05-20 00:10:37 +00:00
|
|
|
unsigned int count;
|
|
|
|
unsigned int rand;
|
|
|
|
};
|
|
|
|
struct rnd_state rnd_state;
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize the state based on the randomization methode available.
|
|
|
|
* return true if the pre-computed list is available, false otherwize.
|
|
|
|
*/
|
|
|
|
static bool freelist_state_initialize(union freelist_init_state *state,
|
|
|
|
struct kmem_cache *cachep,
|
|
|
|
unsigned int count)
|
|
|
|
{
|
|
|
|
bool ret;
|
|
|
|
unsigned int rand;
|
|
|
|
|
|
|
|
/* Use best entropy available to define a random shift */
|
2016-07-26 22:21:56 +00:00
|
|
|
rand = get_random_int();
|
2016-05-20 00:10:37 +00:00
|
|
|
|
|
|
|
/* Use a random state if the pre-computed list is not available */
|
|
|
|
if (!cachep->random_seq) {
|
|
|
|
prandom_seed_state(&state->rnd_state, rand);
|
|
|
|
ret = false;
|
|
|
|
} else {
|
|
|
|
state->list = cachep->random_seq;
|
|
|
|
state->count = count;
|
|
|
|
state->pos = 0;
|
|
|
|
state->rand = rand;
|
|
|
|
ret = true;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Get the next entry on the list and randomize it using a random shift */
|
|
|
|
static freelist_idx_t next_random_slot(union freelist_init_state *state)
|
|
|
|
{
|
|
|
|
return (state->list[state->pos++] + state->rand) % state->count;
|
|
|
|
}
|
|
|
|
|
2016-07-26 22:21:56 +00:00
|
|
|
/* Swap two freelist entries */
|
|
|
|
static void swap_free_obj(struct page *page, unsigned int a, unsigned int b)
|
|
|
|
{
|
|
|
|
swap(((freelist_idx_t *)page->freelist)[a],
|
|
|
|
((freelist_idx_t *)page->freelist)[b]);
|
|
|
|
}
|
|
|
|
|
2016-05-20 00:10:37 +00:00
|
|
|
/*
|
|
|
|
* Shuffle the freelist initialization state based on pre-computed lists.
|
|
|
|
* return true if the list was successfully shuffled, false otherwise.
|
|
|
|
*/
|
|
|
|
static bool shuffle_freelist(struct kmem_cache *cachep, struct page *page)
|
|
|
|
{
|
2016-07-26 22:21:56 +00:00
|
|
|
unsigned int objfreelist = 0, i, rand, count = cachep->num;
|
2016-05-20 00:10:37 +00:00
|
|
|
union freelist_init_state state;
|
|
|
|
bool precomputed;
|
|
|
|
|
|
|
|
if (count < 2)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
precomputed = freelist_state_initialize(&state, cachep, count);
|
|
|
|
|
|
|
|
/* Take a random entry as the objfreelist */
|
|
|
|
if (OBJFREELIST_SLAB(cachep)) {
|
|
|
|
if (!precomputed)
|
|
|
|
objfreelist = count - 1;
|
|
|
|
else
|
|
|
|
objfreelist = next_random_slot(&state);
|
|
|
|
page->freelist = index_to_obj(cachep, page, objfreelist) +
|
|
|
|
obj_offset(cachep);
|
|
|
|
count--;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* On early boot, generate the list dynamically.
|
|
|
|
* Later use a pre-computed list for speed.
|
|
|
|
*/
|
|
|
|
if (!precomputed) {
|
2016-07-26 22:21:56 +00:00
|
|
|
for (i = 0; i < count; i++)
|
|
|
|
set_free_obj(page, i, i);
|
|
|
|
|
|
|
|
/* Fisher-Yates shuffle */
|
|
|
|
for (i = count - 1; i > 0; i--) {
|
|
|
|
rand = prandom_u32_state(&state.rnd_state);
|
|
|
|
rand %= (i + 1);
|
|
|
|
swap_free_obj(page, i, rand);
|
|
|
|
}
|
2016-05-20 00:10:37 +00:00
|
|
|
} else {
|
|
|
|
for (i = 0; i < count; i++)
|
|
|
|
set_free_obj(page, i, next_random_slot(&state));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (OBJFREELIST_SLAB(cachep))
|
|
|
|
set_free_obj(page, cachep->num - 1, objfreelist);
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline bool shuffle_freelist(struct kmem_cache *cachep,
|
|
|
|
struct page *page)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_SLAB_FREELIST_RANDOM */
|
|
|
|
|
2016-03-15 21:54:47 +00:00
|
|
|
static void cache_init_objs(struct kmem_cache *cachep,
|
|
|
|
struct page *page)
|
|
|
|
{
|
|
|
|
int i;
|
2016-03-25 21:21:59 +00:00
|
|
|
void *objp;
|
2016-05-20 00:10:37 +00:00
|
|
|
bool shuffled;
|
2016-03-15 21:54:47 +00:00
|
|
|
|
|
|
|
cache_init_objs_debug(cachep, page);
|
|
|
|
|
2016-05-20 00:10:37 +00:00
|
|
|
/* Try to randomize the freelist if enabled */
|
|
|
|
shuffled = shuffle_freelist(cachep, page);
|
|
|
|
|
|
|
|
if (!shuffled && OBJFREELIST_SLAB(cachep)) {
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
page->freelist = index_to_obj(cachep, page, cachep->num - 1) +
|
|
|
|
obj_offset(cachep);
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:47 +00:00
|
|
|
for (i = 0; i < cachep->num; i++) {
|
2016-08-02 21:02:52 +00:00
|
|
|
objp = index_to_obj(cachep, page, i);
|
|
|
|
kasan_init_slab_obj(cachep, objp);
|
|
|
|
|
2016-03-15 21:54:47 +00:00
|
|
|
/* constructor could break poison info */
|
2016-03-25 21:21:59 +00:00
|
|
|
if (DEBUG == 0 && cachep->ctor) {
|
|
|
|
kasan_unpoison_object_data(cachep, objp);
|
|
|
|
cachep->ctor(objp);
|
|
|
|
kasan_poison_object_data(cachep, objp);
|
|
|
|
}
|
2016-03-15 21:54:47 +00:00
|
|
|
|
2016-05-20 00:10:37 +00:00
|
|
|
if (!shuffled)
|
|
|
|
set_free_obj(page, i, i);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:12 +00:00
|
|
|
static void *slab_get_obj(struct kmem_cache *cachep, struct page *page)
|
2006-02-01 11:05:47 +00:00
|
|
|
{
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-24 01:07:45 +00:00
|
|
|
void *objp;
|
2006-02-01 11:05:47 +00:00
|
|
|
|
2013-12-02 08:49:40 +00:00
|
|
|
objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
|
2013-10-24 01:07:49 +00:00
|
|
|
page->active++;
|
2006-02-01 11:05:47 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
#if DEBUG
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
|
|
|
set_store_user_dirty(cachep);
|
|
|
|
#endif
|
|
|
|
|
2006-02-01 11:05:47 +00:00
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:12 +00:00
|
|
|
static void slab_put_obj(struct kmem_cache *cachep,
|
|
|
|
struct page *page, void *objp)
|
2006-02-01 11:05:47 +00:00
|
|
|
{
|
2013-10-24 01:07:49 +00:00
|
|
|
unsigned int objnr = obj_to_index(cachep, page, objp);
|
2006-02-01 11:05:47 +00:00
|
|
|
#if DEBUG
|
2013-10-24 01:07:46 +00:00
|
|
|
unsigned int i;
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-24 01:07:45 +00:00
|
|
|
|
|
|
|
/* Verify double free bug */
|
2013-10-24 01:07:49 +00:00
|
|
|
for (i = page->active; i < cachep->num; i++) {
|
2013-12-02 08:49:40 +00:00
|
|
|
if (get_free_obj(page, i) == objnr) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("slab: double free detected in cache '%s', objp %p\n",
|
2016-03-17 21:19:47 +00:00
|
|
|
cachep->name, objp);
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-24 01:07:45 +00:00
|
|
|
BUG();
|
|
|
|
}
|
2006-02-01 11:05:47 +00:00
|
|
|
}
|
|
|
|
#endif
|
2013-10-24 01:07:49 +00:00
|
|
|
page->active--;
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (!page->freelist)
|
|
|
|
page->freelist = objp + obj_offset(cachep);
|
|
|
|
|
2013-12-02 08:49:40 +00:00
|
|
|
set_free_obj(page, page->active, objnr);
|
2006-02-01 11:05:47 +00:00
|
|
|
}
|
|
|
|
|
2006-06-23 09:03:07 +00:00
|
|
|
/*
|
|
|
|
* Map pages beginning at addr to the given cache and slab. This is required
|
|
|
|
* for the slab allocator to be able to lookup the cache and slab of a
|
2011-01-07 06:49:17 +00:00
|
|
|
* virtual address for kfree, ksize, and slab debugging.
|
2006-06-23 09:03:07 +00:00
|
|
|
*/
|
2013-10-24 01:07:49 +00:00
|
|
|
static void slab_map_pages(struct kmem_cache *cache, struct page *page,
|
2013-10-30 10:04:01 +00:00
|
|
|
void *freelist)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-10-24 01:07:44 +00:00
|
|
|
page->slab_cache = cache;
|
2013-10-24 01:07:49 +00:00
|
|
|
page->freelist = freelist;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Grow (by 1) the number of slabs within a cache. This is called by
|
|
|
|
* kmem_cache_alloc() when there are no active objs left in a cache.
|
|
|
|
*/
|
2016-05-20 00:10:26 +00:00
|
|
|
static struct page *cache_grow_begin(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags, int nodeid)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-10-30 10:04:01 +00:00
|
|
|
void *freelist;
|
2006-01-08 09:00:37 +00:00
|
|
|
size_t offset;
|
|
|
|
gfp_t local_flags;
|
2016-05-20 00:10:23 +00:00
|
|
|
int page_node;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2016-05-20 00:10:23 +00:00
|
|
|
struct page *page;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Be lazy and only check for valid flags here, keeping it out of the
|
|
|
|
* critical path in kmem_cache_alloc().
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2014-12-10 23:42:25 +00:00
|
|
|
if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
|
2016-07-26 22:22:02 +00:00
|
|
|
gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
|
2016-07-26 22:22:05 +00:00
|
|
|
flags &= ~GFP_SLAB_BUG_MASK;
|
|
|
|
pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
|
|
|
|
invalid_mask, &invalid_mask, flags, &flags);
|
|
|
|
dump_stack();
|
2014-12-10 23:42:25 +00:00
|
|
|
}
|
2007-10-16 08:25:41 +00:00
|
|
|
local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
check_irq_off();
|
2015-11-07 00:28:21 +00:00
|
|
|
if (gfpflags_allow_blocking(local_flags))
|
2005-04-16 22:20:36 +00:00
|
|
|
local_irq_enable();
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* Get mem for the objs. Attempt to allocate a physical page from
|
|
|
|
* 'nodeid'.
|
2005-09-09 20:03:32 +00:00
|
|
|
*/
|
2016-05-20 00:10:23 +00:00
|
|
|
page = kmem_getpages(cachep, local_flags, nodeid);
|
2013-10-24 01:07:38 +00:00
|
|
|
if (!page)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto failed;
|
|
|
|
|
2016-05-20 00:10:23 +00:00
|
|
|
page_node = page_to_nid(page);
|
|
|
|
n = get_node(cachep, page_node);
|
2016-05-20 00:10:20 +00:00
|
|
|
|
|
|
|
/* Get colour for the slab, and cal the next value. */
|
|
|
|
n->colour_next++;
|
|
|
|
if (n->colour_next >= cachep->colour)
|
|
|
|
n->colour_next = 0;
|
|
|
|
|
|
|
|
offset = n->colour_next;
|
|
|
|
if (offset >= cachep->colour)
|
|
|
|
offset = 0;
|
|
|
|
|
|
|
|
offset *= cachep->colour_off;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Get slab management. */
|
2013-10-24 01:07:49 +00:00
|
|
|
freelist = alloc_slabmgmt(cachep, page, offset,
|
2016-05-20 00:10:23 +00:00
|
|
|
local_flags & ~GFP_CONSTRAINT_MASK, page_node);
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (OFF_SLAB(cachep) && !freelist)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto opps1;
|
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
slab_map_pages(cachep, page, freelist);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-25 21:21:59 +00:00
|
|
|
kasan_poison_slab(page);
|
2013-10-24 01:07:49 +00:00
|
|
|
cache_init_objs(cachep, page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-11-07 00:28:21 +00:00
|
|
|
if (gfpflags_allow_blocking(local_flags))
|
2005-04-16 22:20:36 +00:00
|
|
|
local_irq_disable();
|
|
|
|
|
2016-05-20 00:10:26 +00:00
|
|
|
return page;
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
opps1:
|
2013-10-24 01:07:38 +00:00
|
|
|
kmem_freepages(cachep, page);
|
2006-03-22 08:08:11 +00:00
|
|
|
failed:
|
2015-11-07 00:28:21 +00:00
|
|
|
if (gfpflags_allow_blocking(local_flags))
|
2005-04-16 22:20:36 +00:00
|
|
|
local_irq_disable();
|
2016-05-20 00:10:26 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void cache_grow_end(struct kmem_cache *cachep, struct page *page)
|
|
|
|
{
|
|
|
|
struct kmem_cache_node *n;
|
|
|
|
void *list = NULL;
|
|
|
|
|
|
|
|
check_irq_off();
|
|
|
|
|
|
|
|
if (!page)
|
|
|
|
return;
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&page->lru);
|
|
|
|
n = get_node(cachep, page_to_nid(page));
|
|
|
|
|
|
|
|
spin_lock(&n->list_lock);
|
|
|
|
if (!page->active)
|
|
|
|
list_add_tail(&page->lru, &(n->slabs_free));
|
|
|
|
else
|
|
|
|
fixup_slab_list(cachep, n, page, &list);
|
|
|
|
STATS_INC_GROWN(cachep);
|
|
|
|
n->free_objects += cachep->num - page->active;
|
|
|
|
spin_unlock(&n->list_lock);
|
|
|
|
|
|
|
|
fixup_objfreelist_debug(cachep, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform extra freeing checks:
|
|
|
|
* - detect bad pointers.
|
|
|
|
* - POISON/RED_ZONE checking
|
|
|
|
*/
|
|
|
|
static void kfree_debugcheck(const void *objp)
|
|
|
|
{
|
|
|
|
if (!virt_addr_valid(objp)) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("kfree_debugcheck: out of range ptr %lxh\n",
|
2006-01-08 09:00:37 +00:00
|
|
|
(unsigned long)objp);
|
|
|
|
BUG();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-06-23 09:03:24 +00:00
|
|
|
static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
|
|
|
|
{
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 07:22:59 +00:00
|
|
|
unsigned long long redzone1, redzone2;
|
2006-06-23 09:03:24 +00:00
|
|
|
|
|
|
|
redzone1 = *dbg_redzone1(cache, obj);
|
|
|
|
redzone2 = *dbg_redzone2(cache, obj);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Redzone is ok.
|
|
|
|
*/
|
|
|
|
if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
|
|
|
|
slab_error(cache, "double free detected");
|
|
|
|
else
|
|
|
|
slab_error(cache, "memory outside object was overwritten");
|
|
|
|
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
|
|
|
|
obj, redzone1, redzone2);
|
2006-06-23 09:03:24 +00:00
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
|
2012-09-08 20:47:55 +00:00
|
|
|
unsigned long caller)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
unsigned int objnr;
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-11-29 19:05:13 +00:00
|
|
|
BUG_ON(virt_to_cache(objp) != cachep);
|
|
|
|
|
2006-02-01 11:05:42 +00:00
|
|
|
objp -= obj_offset(cachep);
|
2005-04-16 22:20:36 +00:00
|
|
|
kfree_debugcheck(objp);
|
2007-05-06 21:49:41 +00:00
|
|
|
page = virt_to_head_page(objp);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
2006-06-23 09:03:24 +00:00
|
|
|
verify_redzone_free(cachep, objp);
|
2005-04-16 22:20:36 +00:00
|
|
|
*dbg_redzone1(cachep, objp) = RED_INACTIVE;
|
|
|
|
*dbg_redzone2(cachep, objp) = RED_INACTIVE;
|
|
|
|
}
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
if (cachep->flags & SLAB_STORE_USER) {
|
|
|
|
set_store_user_dirty(cachep);
|
2012-09-08 20:47:55 +00:00
|
|
|
*dbg_userword(cachep, objp) = (void *)caller;
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
objnr = obj_to_index(cachep, page, objp);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
BUG_ON(objnr >= cachep->num);
|
2013-10-24 01:07:49 +00:00
|
|
|
BUG_ON(objp != index_to_obj(cachep, page, objnr));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_POISON) {
|
|
|
|
poison_obj(cachep, objp, POISON_FREE);
|
2016-03-15 21:54:21 +00:00
|
|
|
slab_kernel_map(cachep, objp, 0, caller);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
#define kfree_debugcheck(x) do { } while(0)
|
|
|
|
#define cache_free_debugcheck(x,objp,z) (objp)
|
|
|
|
#endif
|
|
|
|
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
|
|
|
|
void **list)
|
|
|
|
{
|
|
|
|
#if DEBUG
|
|
|
|
void *next = *list;
|
|
|
|
void *objp;
|
|
|
|
|
|
|
|
while (next) {
|
|
|
|
objp = next - obj_offset(cachep);
|
|
|
|
next = *(void **)next;
|
|
|
|
poison_obj(cachep, objp, POISON_FREE);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:44 +00:00
|
|
|
static inline void fixup_slab_list(struct kmem_cache *cachep,
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
struct kmem_cache_node *n, struct page *page,
|
|
|
|
void **list)
|
2016-03-15 21:54:44 +00:00
|
|
|
{
|
|
|
|
/* move slabp to correct slabp list: */
|
|
|
|
list_del(&page->lru);
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (page->active == cachep->num) {
|
2016-03-15 21:54:44 +00:00
|
|
|
list_add(&page->lru, &n->slabs_full);
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
if (OBJFREELIST_SLAB(cachep)) {
|
|
|
|
#if DEBUG
|
|
|
|
/* Poisoning will be done without holding the lock */
|
|
|
|
if (cachep->flags & SLAB_POISON) {
|
|
|
|
void **objp = page->freelist;
|
|
|
|
|
|
|
|
*objp = *list;
|
|
|
|
*list = objp;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
page->freelist = NULL;
|
|
|
|
}
|
|
|
|
} else
|
2016-03-15 21:54:44 +00:00
|
|
|
list_add(&page->lru, &n->slabs_partial);
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
/* Try to find non-pfmemalloc slab if needed */
|
|
|
|
static noinline struct page *get_valid_first_slab(struct kmem_cache_node *n,
|
|
|
|
struct page *page, bool pfmemalloc)
|
|
|
|
{
|
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (pfmemalloc)
|
|
|
|
return page;
|
|
|
|
|
|
|
|
if (!PageSlabPfmemalloc(page))
|
|
|
|
return page;
|
|
|
|
|
|
|
|
/* No need to keep pfmemalloc slab if we have enough free objects */
|
|
|
|
if (n->free_objects > n->free_limit) {
|
|
|
|
ClearPageSlabPfmemalloc(page);
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Move pfmemalloc slab to the end of list to speed up next search */
|
|
|
|
list_del(&page->lru);
|
|
|
|
if (!page->active)
|
|
|
|
list_add_tail(&page->lru, &n->slabs_free);
|
|
|
|
else
|
|
|
|
list_add_tail(&page->lru, &n->slabs_partial);
|
|
|
|
|
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru) {
|
|
|
|
if (!PageSlabPfmemalloc(page))
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(page, &n->slabs_free, lru) {
|
|
|
|
if (!PageSlabPfmemalloc(page))
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct page *get_first_slab(struct kmem_cache_node *n, bool pfmemalloc)
|
2016-01-14 23:18:02 +00:00
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
page = list_first_entry_or_null(&n->slabs_partial,
|
|
|
|
struct page, lru);
|
|
|
|
if (!page) {
|
|
|
|
n->free_touched = 1;
|
|
|
|
page = list_first_entry_or_null(&n->slabs_free,
|
|
|
|
struct page, lru);
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
if (sk_memalloc_socks())
|
|
|
|
return get_valid_first_slab(n, page, pfmemalloc);
|
|
|
|
|
2016-01-14 23:18:02 +00:00
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
static noinline void *cache_alloc_pfmemalloc(struct kmem_cache *cachep,
|
|
|
|
struct kmem_cache_node *n, gfp_t flags)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
void *obj;
|
|
|
|
void *list = NULL;
|
|
|
|
|
|
|
|
if (!gfp_pfmemalloc_allowed(flags))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
spin_lock(&n->list_lock);
|
|
|
|
page = get_first_slab(n, true);
|
|
|
|
if (!page) {
|
|
|
|
spin_unlock(&n->list_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
obj = slab_get_obj(cachep, page);
|
|
|
|
n->free_objects--;
|
|
|
|
|
|
|
|
fixup_slab_list(cachep, n, page, &list);
|
|
|
|
|
|
|
|
spin_unlock(&n->list_lock);
|
|
|
|
fixup_objfreelist_debug(cachep, &list);
|
|
|
|
|
|
|
|
return obj;
|
|
|
|
}
|
|
|
|
|
2016-05-20 00:10:29 +00:00
|
|
|
/*
|
|
|
|
* Slab list should be fixed up by fixup_slab_list() for existing slab
|
|
|
|
* or cache_grow_end() for new slab
|
|
|
|
*/
|
|
|
|
static __always_inline int alloc_block(struct kmem_cache *cachep,
|
|
|
|
struct array_cache *ac, struct page *page, int batchcount)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* There must be at least one object available for
|
|
|
|
* allocation.
|
|
|
|
*/
|
|
|
|
BUG_ON(page->active >= cachep->num);
|
|
|
|
|
|
|
|
while (page->active < cachep->num && batchcount--) {
|
|
|
|
STATS_INC_ALLOCED(cachep);
|
|
|
|
STATS_INC_ACTIVE(cachep);
|
|
|
|
STATS_SET_HIGH(cachep);
|
|
|
|
|
|
|
|
ac->entry[ac->avail++] = slab_get_obj(cachep, page);
|
|
|
|
}
|
|
|
|
|
|
|
|
return batchcount;
|
|
|
|
}
|
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int batchcount;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
mm/slab: lockless decision to grow cache
To check whether free objects exist or not precisely, we need to grab a
lock. But, accuracy isn't that important because race window would be
even small and if there is too much free object, cache reaper would reap
it. So, this patch makes the check for free object exisistence not to
hold a lock. This will reduce lock contention in heavily allocation
case.
Note that until now, n->shared can be freed during the processing by
writing slabinfo, but, with some trick in this patch, we can access it
freely within interrupt disabled period.
Below is the result of concurrent allocation/free in slab allocation
benchmark made by Christoph a long time ago. I make the output simpler.
The number shows cycle count during alloc/free respectively so less is
better.
* Before
Kmalloc N*alloc N*free(32): Average=248/966
Kmalloc N*alloc N*free(64): Average=261/949
Kmalloc N*alloc N*free(128): Average=314/1016
Kmalloc N*alloc N*free(256): Average=741/1061
Kmalloc N*alloc N*free(512): Average=1246/1152
Kmalloc N*alloc N*free(1024): Average=2437/1259
Kmalloc N*alloc N*free(2048): Average=4980/1800
Kmalloc N*alloc N*free(4096): Average=9000/2078
* After
Kmalloc N*alloc N*free(32): Average=344/792
Kmalloc N*alloc N*free(64): Average=347/882
Kmalloc N*alloc N*free(128): Average=390/959
Kmalloc N*alloc N*free(256): Average=393/1067
Kmalloc N*alloc N*free(512): Average=683/1229
Kmalloc N*alloc N*free(1024): Average=1295/1325
Kmalloc N*alloc N*free(2048): Average=2513/1664
Kmalloc N*alloc N*free(4096): Average=4742/2172
It shows that allocation performance decreases for the object size up to
128 and it may be due to extra checks in cache_alloc_refill(). But,
with considering improvement of free performance, net result looks the
same. Result for other size class looks very promising, roughly, 50%
performance improvement.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 00:10:31 +00:00
|
|
|
struct array_cache *ac, *shared;
|
2006-10-06 07:43:52 +00:00
|
|
|
int node;
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
void *list = NULL;
|
2016-05-20 00:10:26 +00:00
|
|
|
struct page *page;
|
2006-10-06 07:43:52 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
check_irq_off();
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
node = numa_mem_id();
|
2016-03-15 21:54:56 +00:00
|
|
|
|
2006-02-01 11:05:49 +00:00
|
|
|
ac = cpu_cache_get(cachep);
|
2005-04-16 22:20:36 +00:00
|
|
|
batchcount = ac->batchcount;
|
|
|
|
if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* If there was little recent activity on this cache, then
|
|
|
|
* perform only a partial refill. Otherwise we could generate
|
|
|
|
* refill bouncing.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
batchcount = BATCHREFILL_LIMIT;
|
|
|
|
}
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
BUG_ON(ac->avail > 0 || !n);
|
mm/slab: lockless decision to grow cache
To check whether free objects exist or not precisely, we need to grab a
lock. But, accuracy isn't that important because race window would be
even small and if there is too much free object, cache reaper would reap
it. So, this patch makes the check for free object exisistence not to
hold a lock. This will reduce lock contention in heavily allocation
case.
Note that until now, n->shared can be freed during the processing by
writing slabinfo, but, with some trick in this patch, we can access it
freely within interrupt disabled period.
Below is the result of concurrent allocation/free in slab allocation
benchmark made by Christoph a long time ago. I make the output simpler.
The number shows cycle count during alloc/free respectively so less is
better.
* Before
Kmalloc N*alloc N*free(32): Average=248/966
Kmalloc N*alloc N*free(64): Average=261/949
Kmalloc N*alloc N*free(128): Average=314/1016
Kmalloc N*alloc N*free(256): Average=741/1061
Kmalloc N*alloc N*free(512): Average=1246/1152
Kmalloc N*alloc N*free(1024): Average=2437/1259
Kmalloc N*alloc N*free(2048): Average=4980/1800
Kmalloc N*alloc N*free(4096): Average=9000/2078
* After
Kmalloc N*alloc N*free(32): Average=344/792
Kmalloc N*alloc N*free(64): Average=347/882
Kmalloc N*alloc N*free(128): Average=390/959
Kmalloc N*alloc N*free(256): Average=393/1067
Kmalloc N*alloc N*free(512): Average=683/1229
Kmalloc N*alloc N*free(1024): Average=1295/1325
Kmalloc N*alloc N*free(2048): Average=2513/1664
Kmalloc N*alloc N*free(4096): Average=4742/2172
It shows that allocation performance decreases for the object size up to
128 and it may be due to extra checks in cache_alloc_refill(). But,
with considering improvement of free performance, net result looks the
same. Result for other size class looks very promising, roughly, 50%
performance improvement.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 00:10:31 +00:00
|
|
|
shared = READ_ONCE(n->shared);
|
|
|
|
if (!n->free_objects && (!shared || !shared->avail))
|
|
|
|
goto direct_grow;
|
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock(&n->list_lock);
|
mm/slab: lockless decision to grow cache
To check whether free objects exist or not precisely, we need to grab a
lock. But, accuracy isn't that important because race window would be
even small and if there is too much free object, cache reaper would reap
it. So, this patch makes the check for free object exisistence not to
hold a lock. This will reduce lock contention in heavily allocation
case.
Note that until now, n->shared can be freed during the processing by
writing slabinfo, but, with some trick in this patch, we can access it
freely within interrupt disabled period.
Below is the result of concurrent allocation/free in slab allocation
benchmark made by Christoph a long time ago. I make the output simpler.
The number shows cycle count during alloc/free respectively so less is
better.
* Before
Kmalloc N*alloc N*free(32): Average=248/966
Kmalloc N*alloc N*free(64): Average=261/949
Kmalloc N*alloc N*free(128): Average=314/1016
Kmalloc N*alloc N*free(256): Average=741/1061
Kmalloc N*alloc N*free(512): Average=1246/1152
Kmalloc N*alloc N*free(1024): Average=2437/1259
Kmalloc N*alloc N*free(2048): Average=4980/1800
Kmalloc N*alloc N*free(4096): Average=9000/2078
* After
Kmalloc N*alloc N*free(32): Average=344/792
Kmalloc N*alloc N*free(64): Average=347/882
Kmalloc N*alloc N*free(128): Average=390/959
Kmalloc N*alloc N*free(256): Average=393/1067
Kmalloc N*alloc N*free(512): Average=683/1229
Kmalloc N*alloc N*free(1024): Average=1295/1325
Kmalloc N*alloc N*free(2048): Average=2513/1664
Kmalloc N*alloc N*free(4096): Average=4742/2172
It shows that allocation performance decreases for the object size up to
128 and it may be due to extra checks in cache_alloc_refill(). But,
with considering improvement of free performance, net result looks the
same. Result for other size class looks very promising, roughly, 50%
performance improvement.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 00:10:31 +00:00
|
|
|
shared = READ_ONCE(n->shared);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-25 11:06:44 +00:00
|
|
|
/* See if we can refill from the shared array */
|
mm/slab: lockless decision to grow cache
To check whether free objects exist or not precisely, we need to grab a
lock. But, accuracy isn't that important because race window would be
even small and if there is too much free object, cache reaper would reap
it. So, this patch makes the check for free object exisistence not to
hold a lock. This will reduce lock contention in heavily allocation
case.
Note that until now, n->shared can be freed during the processing by
writing slabinfo, but, with some trick in this patch, we can access it
freely within interrupt disabled period.
Below is the result of concurrent allocation/free in slab allocation
benchmark made by Christoph a long time ago. I make the output simpler.
The number shows cycle count during alloc/free respectively so less is
better.
* Before
Kmalloc N*alloc N*free(32): Average=248/966
Kmalloc N*alloc N*free(64): Average=261/949
Kmalloc N*alloc N*free(128): Average=314/1016
Kmalloc N*alloc N*free(256): Average=741/1061
Kmalloc N*alloc N*free(512): Average=1246/1152
Kmalloc N*alloc N*free(1024): Average=2437/1259
Kmalloc N*alloc N*free(2048): Average=4980/1800
Kmalloc N*alloc N*free(4096): Average=9000/2078
* After
Kmalloc N*alloc N*free(32): Average=344/792
Kmalloc N*alloc N*free(64): Average=347/882
Kmalloc N*alloc N*free(128): Average=390/959
Kmalloc N*alloc N*free(256): Average=393/1067
Kmalloc N*alloc N*free(512): Average=683/1229
Kmalloc N*alloc N*free(1024): Average=1295/1325
Kmalloc N*alloc N*free(2048): Average=2513/1664
Kmalloc N*alloc N*free(4096): Average=4742/2172
It shows that allocation performance decreases for the object size up to
128 and it may be due to extra checks in cache_alloc_refill(). But,
with considering improvement of free performance, net result looks the
same. Result for other size class looks very promising, roughly, 50%
performance improvement.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 00:10:31 +00:00
|
|
|
if (shared && transfer_objects(ac, shared, batchcount)) {
|
|
|
|
shared->touched = 1;
|
2006-03-25 11:06:44 +00:00
|
|
|
goto alloc_done;
|
2010-01-27 11:27:40 +00:00
|
|
|
}
|
2006-03-25 11:06:44 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
while (batchcount > 0) {
|
|
|
|
/* Get slab alloc is to come from. */
|
2016-03-15 21:54:56 +00:00
|
|
|
page = get_first_slab(n, false);
|
2016-01-14 23:18:02 +00:00
|
|
|
if (!page)
|
|
|
|
goto must_grow;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
check_spinlock_acquired(cachep);
|
2007-05-06 21:49:03 +00:00
|
|
|
|
2016-05-20 00:10:29 +00:00
|
|
|
batchcount = alloc_block(cachep, ac, page, batchcount);
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
fixup_slab_list(cachep, n, page, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
must_grow:
|
2013-01-10 19:14:19 +00:00
|
|
|
n->free_objects -= ac->avail;
|
2006-03-22 08:08:11 +00:00
|
|
|
alloc_done:
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
fixup_objfreelist_debug(cachep, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm/slab: lockless decision to grow cache
To check whether free objects exist or not precisely, we need to grab a
lock. But, accuracy isn't that important because race window would be
even small and if there is too much free object, cache reaper would reap
it. So, this patch makes the check for free object exisistence not to
hold a lock. This will reduce lock contention in heavily allocation
case.
Note that until now, n->shared can be freed during the processing by
writing slabinfo, but, with some trick in this patch, we can access it
freely within interrupt disabled period.
Below is the result of concurrent allocation/free in slab allocation
benchmark made by Christoph a long time ago. I make the output simpler.
The number shows cycle count during alloc/free respectively so less is
better.
* Before
Kmalloc N*alloc N*free(32): Average=248/966
Kmalloc N*alloc N*free(64): Average=261/949
Kmalloc N*alloc N*free(128): Average=314/1016
Kmalloc N*alloc N*free(256): Average=741/1061
Kmalloc N*alloc N*free(512): Average=1246/1152
Kmalloc N*alloc N*free(1024): Average=2437/1259
Kmalloc N*alloc N*free(2048): Average=4980/1800
Kmalloc N*alloc N*free(4096): Average=9000/2078
* After
Kmalloc N*alloc N*free(32): Average=344/792
Kmalloc N*alloc N*free(64): Average=347/882
Kmalloc N*alloc N*free(128): Average=390/959
Kmalloc N*alloc N*free(256): Average=393/1067
Kmalloc N*alloc N*free(512): Average=683/1229
Kmalloc N*alloc N*free(1024): Average=1295/1325
Kmalloc N*alloc N*free(2048): Average=2513/1664
Kmalloc N*alloc N*free(4096): Average=4742/2172
It shows that allocation performance decreases for the object size up to
128 and it may be due to extra checks in cache_alloc_refill(). But,
with considering improvement of free performance, net result looks the
same. Result for other size class looks very promising, roughly, 50%
performance improvement.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 00:10:31 +00:00
|
|
|
direct_grow:
|
2005-04-16 22:20:36 +00:00
|
|
|
if (unlikely(!ac->avail)) {
|
2016-03-15 21:54:56 +00:00
|
|
|
/* Check if we can use obj in pfmemalloc slab */
|
|
|
|
if (sk_memalloc_socks()) {
|
|
|
|
void *obj = cache_alloc_pfmemalloc(cachep, n, flags);
|
|
|
|
|
|
|
|
if (obj)
|
|
|
|
return obj;
|
|
|
|
}
|
|
|
|
|
2016-05-20 00:10:26 +00:00
|
|
|
page = cache_grow_begin(cachep, gfp_exact_node(flags), node);
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2016-05-20 00:10:26 +00:00
|
|
|
/*
|
|
|
|
* cache_grow_begin() can reenable interrupts,
|
|
|
|
* then ac could change.
|
|
|
|
*/
|
2006-02-01 11:05:49 +00:00
|
|
|
ac = cpu_cache_get(cachep);
|
2016-05-20 00:10:29 +00:00
|
|
|
if (!ac->avail && page)
|
|
|
|
alloc_block(cachep, ac, page, batchcount);
|
|
|
|
cache_grow_end(cachep, page);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
|
2016-05-20 00:10:29 +00:00
|
|
|
if (!ac->avail)
|
2005-04-16 22:20:36 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
ac->touched = 1;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
return ac->entry[--ac->avail];
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-11-07 00:28:21 +00:00
|
|
|
might_sleep_if(gfpflags_allow_blocking(flags));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
2006-03-22 08:08:11 +00:00
|
|
|
static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
|
2012-09-08 20:47:55 +00:00
|
|
|
gfp_t flags, void *objp, unsigned long caller)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-01-08 09:00:37 +00:00
|
|
|
if (!objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
return objp;
|
2006-01-08 09:00:37 +00:00
|
|
|
if (cachep->flags & SLAB_POISON) {
|
2005-04-16 22:20:36 +00:00
|
|
|
check_poison_obj(cachep, objp);
|
2016-03-15 21:54:21 +00:00
|
|
|
slab_kernel_map(cachep, objp, 1, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
poison_obj(cachep, objp, POISON_INUSE);
|
|
|
|
}
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
2012-09-08 20:47:55 +00:00
|
|
|
*dbg_userword(cachep, objp) = (void *)caller;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
2006-03-22 08:08:11 +00:00
|
|
|
if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
|
|
|
|
*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
|
2016-03-17 21:19:47 +00:00
|
|
|
slab_error(cachep, "double free, or memory outside object was overwritten");
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
|
|
|
|
objp, *dbg_redzone1(cachep, objp),
|
|
|
|
*dbg_redzone2(cachep, objp));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
*dbg_redzone1(cachep, objp) = RED_ACTIVE;
|
|
|
|
*dbg_redzone2(cachep, objp) = RED_ACTIVE;
|
|
|
|
}
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 20:22:06 +00:00
|
|
|
|
2006-02-01 11:05:42 +00:00
|
|
|
objp += obj_offset(cachep);
|
2007-05-06 21:50:17 +00:00
|
|
|
if (cachep->ctor && cachep->flags & SLAB_POISON)
|
2008-07-26 02:45:34 +00:00
|
|
|
cachep->ctor(objp);
|
2011-07-21 00:42:45 +00:00
|
|
|
if (ARCH_SLAB_MINALIGN &&
|
|
|
|
((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
|
2011-07-11 20:35:08 +00:00
|
|
|
objp, (int)ARCH_SLAB_MINALIGN);
|
2006-12-07 04:32:11 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
|
|
|
|
#endif
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-01-08 09:00:37 +00:00
|
|
|
void *objp;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct array_cache *ac;
|
|
|
|
|
2005-09-28 04:45:46 +00:00
|
|
|
check_irq_off();
|
2006-12-08 10:39:44 +00:00
|
|
|
|
2006-02-01 11:05:49 +00:00
|
|
|
ac = cpu_cache_get(cachep);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (likely(ac->avail)) {
|
|
|
|
ac->touched = 1;
|
2016-03-15 21:54:56 +00:00
|
|
|
objp = ac->entry[--ac->avail];
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
STATS_INC_ALLOCHIT(cachep);
|
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
|
|
|
|
STATS_INC_ALLOCMISS(cachep);
|
2016-03-15 21:54:56 +00:00
|
|
|
objp = cache_alloc_refill(cachep, flags);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
/*
|
|
|
|
* the 'ac' may be updated by cache_alloc_refill(),
|
|
|
|
* and kmemleak_erase() requires its correct value.
|
|
|
|
*/
|
|
|
|
ac = cpu_cache_get(cachep);
|
|
|
|
|
|
|
|
out:
|
2009-06-11 12:22:40 +00:00
|
|
|
/*
|
|
|
|
* To avoid a false negative, if an object that is in one of the
|
|
|
|
* per-CPU caches is leaked, we need to make sure kmemleak doesn't
|
|
|
|
* treat the array pointers as a reference to the object.
|
|
|
|
*/
|
2009-12-02 07:55:49 +00:00
|
|
|
if (objp)
|
|
|
|
kmemleak_erase(&ac->entry[ac->avail]);
|
2005-09-28 04:45:46 +00:00
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
2005-09-09 20:03:32 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
2006-03-24 11:16:08 +00:00
|
|
|
/*
|
2014-09-25 01:41:02 +00:00
|
|
|
* Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
|
2006-03-24 11:16:08 +00:00
|
|
|
*
|
|
|
|
* If we are in_interrupt, then process context, including cpusets and
|
|
|
|
* mempolicy, may not apply and should not be used for allocation policy.
|
|
|
|
*/
|
|
|
|
static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
|
|
|
|
{
|
|
|
|
int nid_alloc, nid_here;
|
|
|
|
|
2006-09-27 08:50:08 +00:00
|
|
|
if (in_interrupt() || (flags & __GFP_THISNODE))
|
2006-03-24 11:16:08 +00:00
|
|
|
return NULL;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
nid_alloc = nid_here = numa_mem_id();
|
2006-03-24 11:16:08 +00:00
|
|
|
if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
|
2010-05-26 21:42:49 +00:00
|
|
|
nid_alloc = cpuset_slab_spread_node();
|
2006-03-24 11:16:08 +00:00
|
|
|
else if (current->mempolicy)
|
2014-04-07 22:37:29 +00:00
|
|
|
nid_alloc = mempolicy_slab_node();
|
2006-03-24 11:16:08 +00:00
|
|
|
if (nid_alloc != nid_here)
|
2006-12-07 04:32:30 +00:00
|
|
|
return ____cache_alloc_node(cachep, flags, nid_alloc);
|
2006-03-24 11:16:08 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2006-09-27 08:50:08 +00:00
|
|
|
/*
|
|
|
|
* Fallback function if there was no memory available and no objects on a
|
2006-12-07 04:33:29 +00:00
|
|
|
* certain node and fall back is permitted. First we scan all the
|
2013-01-10 19:14:19 +00:00
|
|
|
* available node for available objects. If that fails then we
|
2006-12-07 04:33:29 +00:00
|
|
|
* perform an allocation without specifying a node. This allows the page
|
|
|
|
* allocator to do its reclaim / fallback magic. We then insert the
|
|
|
|
* slab into the proper nodelist and then allocate from it.
|
2006-09-27 08:50:08 +00:00
|
|
|
*/
|
2007-02-10 09:42:53 +00:00
|
|
|
static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
|
2006-09-27 08:50:08 +00:00
|
|
|
{
|
2007-02-10 09:42:53 +00:00
|
|
|
struct zonelist *zonelist;
|
2008-04-28 09:12:17 +00:00
|
|
|
struct zoneref *z;
|
2008-04-28 09:12:16 +00:00
|
|
|
struct zone *zone;
|
|
|
|
enum zone_type high_zoneidx = gfp_zone(flags);
|
2006-09-27 08:50:08 +00:00
|
|
|
void *obj = NULL;
|
2016-05-20 00:10:26 +00:00
|
|
|
struct page *page;
|
2006-12-07 04:33:29 +00:00
|
|
|
int nid;
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 23:34:11 +00:00
|
|
|
unsigned int cpuset_mems_cookie;
|
2007-02-10 09:42:53 +00:00
|
|
|
|
|
|
|
if (flags & __GFP_THISNODE)
|
|
|
|
return NULL;
|
|
|
|
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 23:34:11 +00:00
|
|
|
retry_cpuset:
|
2014-04-03 21:47:24 +00:00
|
|
|
cpuset_mems_cookie = read_mems_allowed_begin();
|
2014-04-07 22:37:29 +00:00
|
|
|
zonelist = node_zonelist(mempolicy_slab_node(), flags);
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 23:34:11 +00:00
|
|
|
|
2006-12-07 04:33:29 +00:00
|
|
|
retry:
|
|
|
|
/*
|
|
|
|
* Look through allowed nodes for objects available
|
|
|
|
* from existing per node queues.
|
|
|
|
*/
|
2008-04-28 09:12:16 +00:00
|
|
|
for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
|
|
|
|
nid = zone_to_nid(zone);
|
2006-10-21 17:24:16 +00:00
|
|
|
|
2014-12-13 00:58:25 +00:00
|
|
|
if (cpuset_zone_allowed(zone, flags) &&
|
2014-08-06 23:04:11 +00:00
|
|
|
get_node(cache, nid) &&
|
|
|
|
get_node(cache, nid)->free_objects) {
|
2006-12-07 04:33:29 +00:00
|
|
|
obj = ____cache_alloc_node(cache,
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 22:46:55 +00:00
|
|
|
gfp_exact_node(flags), nid);
|
2008-06-21 23:46:35 +00:00
|
|
|
if (obj)
|
|
|
|
break;
|
|
|
|
}
|
2006-12-07 04:33:29 +00:00
|
|
|
}
|
|
|
|
|
2007-05-06 21:50:17 +00:00
|
|
|
if (!obj) {
|
2006-12-07 04:33:29 +00:00
|
|
|
/*
|
|
|
|
* This allocation will be performed within the constraints
|
|
|
|
* of the current cpuset / memory policy requirements.
|
|
|
|
* We may trigger various forms of reclaim on the allowed
|
|
|
|
* set and go into memory reserves if necessary.
|
|
|
|
*/
|
2016-05-20 00:10:26 +00:00
|
|
|
page = cache_grow_begin(cache, flags, numa_mem_id());
|
|
|
|
cache_grow_end(cache, page);
|
|
|
|
if (page) {
|
|
|
|
nid = page_to_nid(page);
|
2016-05-20 00:10:23 +00:00
|
|
|
obj = ____cache_alloc_node(cache,
|
|
|
|
gfp_exact_node(flags), nid);
|
2013-10-24 01:07:38 +00:00
|
|
|
|
2006-12-07 04:33:29 +00:00
|
|
|
/*
|
2016-05-20 00:10:23 +00:00
|
|
|
* Another processor may allocate the objects in
|
|
|
|
* the slab since we are not holding any locks.
|
2006-12-07 04:33:29 +00:00
|
|
|
*/
|
2016-05-20 00:10:23 +00:00
|
|
|
if (!obj)
|
|
|
|
goto retry;
|
2006-12-07 04:33:29 +00:00
|
|
|
}
|
2006-10-21 17:24:16 +00:00
|
|
|
}
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 23:34:11 +00:00
|
|
|
|
2014-04-03 21:47:24 +00:00
|
|
|
if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 23:34:11 +00:00
|
|
|
goto retry_cpuset;
|
2006-09-27 08:50:08 +00:00
|
|
|
return obj;
|
|
|
|
}
|
|
|
|
|
2005-09-09 20:03:32 +00:00
|
|
|
/*
|
|
|
|
* A interface to enable slab creation on nodeid
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2006-12-07 04:32:30 +00:00
|
|
|
static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
|
2006-03-22 08:08:11 +00:00
|
|
|
int nodeid)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2016-05-20 00:10:29 +00:00
|
|
|
void *obj = NULL;
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
void *list = NULL;
|
2006-01-08 09:00:37 +00:00
|
|
|
|
slab: fix nodeid bounds check for non-contiguous node IDs
The bounds check for nodeid in ____cache_alloc_node gives false
positives on machines where the node IDs are not contiguous, leading to
a panic at boot time. For example, on a POWER8 machine the node IDs are
typically 0, 1, 16 and 17. This means that num_online_nodes() returns
4, so when ____cache_alloc_node is called with nodeid = 16 the VM_BUG_ON
triggers, like this:
kernel BUG at /home/paulus/kernel/kvm/mm/slab.c:3079!
Call Trace:
.____cache_alloc_node+0x5c/0x270 (unreliable)
.kmem_cache_alloc_node_trace+0xdc/0x360
.init_list+0x3c/0x128
.kmem_cache_init+0x1dc/0x258
.start_kernel+0x2a0/0x568
start_here_common+0x20/0xa8
To fix this, we instead compare the nodeid with MAX_NUMNODES, and
additionally make sure it isn't negative (since nodeid is an int). The
check is there mainly to protect the array dereference in the get_node()
call in the next line, and the array being dereferenced is of size
MAX_NUMNODES. If the nodeid is in range but invalid (for example if the
node is off-line), the BUG_ON in the next line will catch that.
Fixes: 14e50c6a9bc2 ("mm: slab: Verify the nodeid passed to ____cache_alloc_node")
Signed-off-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Reviewed-by: Pekka Enberg <penberg@kernel.org>
Acked-by: David Rientjes <rientjes@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-02 23:59:48 +00:00
|
|
|
VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, nodeid);
|
2013-01-10 19:14:19 +00:00
|
|
|
BUG_ON(!n);
|
2006-01-08 09:00:37 +00:00
|
|
|
|
2006-02-05 07:27:58 +00:00
|
|
|
check_irq_off();
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock(&n->list_lock);
|
2016-03-15 21:54:56 +00:00
|
|
|
page = get_first_slab(n, false);
|
2016-01-14 23:18:02 +00:00
|
|
|
if (!page)
|
|
|
|
goto must_grow;
|
2006-01-08 09:00:37 +00:00
|
|
|
|
|
|
|
check_spinlock_acquired_node(cachep, nodeid);
|
|
|
|
|
|
|
|
STATS_INC_NODEALLOCS(cachep);
|
|
|
|
STATS_INC_ACTIVE(cachep);
|
|
|
|
STATS_SET_HIGH(cachep);
|
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
BUG_ON(page->active == cachep->num);
|
2006-01-08 09:00:37 +00:00
|
|
|
|
2016-03-15 21:54:12 +00:00
|
|
|
obj = slab_get_obj(cachep, page);
|
2013-01-10 19:14:19 +00:00
|
|
|
n->free_objects--;
|
2006-01-08 09:00:37 +00:00
|
|
|
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
fixup_slab_list(cachep, n, page, &list);
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
mm/slab: introduce new slab management type, OBJFREELIST_SLAB
SLAB needs an array to manage freed objects in a slab. It is only used
if some objects are freed so we can use free object itself as this
array. This requires additional branch in somewhat critical lock path
to check if it is first freed object or not but that's all we need.
Benefits is that we can save extra memory usage and reduce some
computational overhead by allocating a management array when new slab is
created.
Code change is rather complex than what we can expect from the idea, in
order to handle debugging feature efficiently. If you want to see core
idea only, please remove '#if DEBUG' block in the patch.
Although this idea can apply to all caches whose size is larger than
management array size, it isn't applied to caches which have a
constructor. If such cache's object is used for management array,
constructor should be called for it before that object is returned to
user. I guess that overhead overwhelm benefit in that case so this idea
doesn't applied to them at least now.
For summary, from now on, slab management type is determined by
following logic.
1) if management array size is smaller than object size and no ctor, it
becomes OBJFREELIST_SLAB.
2) if management array size is smaller than leftover, it becomes
NORMAL_SLAB which uses leftover as a array.
3) if OFF_SLAB help to save memory than way 4), it becomes OFF_SLAB.
It allocate a management array from the other cache so memory waste
happens.
4) others become NORMAL_SLAB. It uses dedicated internal memory in a
slab as a management array so it causes memory waste.
In my system, without enabling CONFIG_DEBUG_SLAB, Almost caches become
OBJFREELIST_SLAB and NORMAL_SLAB (using leftover) which doesn't waste
memory. Following is the result of number of caches with specific slab
management type.
TOTAL = OBJFREELIST + NORMAL(leftover) + NORMAL + OFF
/Before/
126 = 0 + 60 + 25 + 41
/After/
126 = 97 + 12 + 15 + 2
Result shows that number of caches that doesn't waste memory increase
from 60 to 109.
I did some benchmarking and it looks that benefit are more than loss.
Kmalloc: Repeatedly allocate then free test
/Before/
[ 0.286809] 1. Kmalloc: Repeatedly allocate then free test
[ 1.143674] 100000 times kmalloc(32) -> 116 cycles kfree -> 78 cycles
[ 1.441726] 100000 times kmalloc(64) -> 121 cycles kfree -> 80 cycles
[ 1.815734] 100000 times kmalloc(128) -> 168 cycles kfree -> 85 cycles
[ 2.380709] 100000 times kmalloc(256) -> 287 cycles kfree -> 95 cycles
[ 3.101153] 100000 times kmalloc(512) -> 370 cycles kfree -> 117 cycles
[ 3.942432] 100000 times kmalloc(1024) -> 413 cycles kfree -> 156 cycles
[ 5.227396] 100000 times kmalloc(2048) -> 622 cycles kfree -> 248 cycles
[ 7.519793] 100000 times kmalloc(4096) -> 1102 cycles kfree -> 452 cycles
/After/
[ 1.205313] 100000 times kmalloc(32) -> 117 cycles kfree -> 78 cycles
[ 1.510526] 100000 times kmalloc(64) -> 124 cycles kfree -> 81 cycles
[ 1.827382] 100000 times kmalloc(128) -> 130 cycles kfree -> 84 cycles
[ 2.226073] 100000 times kmalloc(256) -> 177 cycles kfree -> 92 cycles
[ 2.814747] 100000 times kmalloc(512) -> 286 cycles kfree -> 112 cycles
[ 3.532952] 100000 times kmalloc(1024) -> 344 cycles kfree -> 141 cycles
[ 4.608777] 100000 times kmalloc(2048) -> 519 cycles kfree -> 210 cycles
[ 6.350105] 100000 times kmalloc(4096) -> 789 cycles kfree -> 391 cycles
In fact, I tested another idea implementing OBJFREELIST_SLAB with
extendable linked array through another freed object. It can remove
memory waste completely but it causes more computational overhead in
critical lock path and it seems that overhead outweigh benefit. So, this
patch doesn't include it.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:50 +00:00
|
|
|
fixup_objfreelist_debug(cachep, &list);
|
2016-05-20 00:10:29 +00:00
|
|
|
return obj;
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
must_grow:
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
2016-05-20 00:10:26 +00:00
|
|
|
page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid);
|
2016-05-20 00:10:29 +00:00
|
|
|
if (page) {
|
|
|
|
/* This slab isn't counted yet so don't update free_objects */
|
|
|
|
obj = slab_get_obj(cachep, page);
|
|
|
|
}
|
2016-05-20 00:10:26 +00:00
|
|
|
cache_grow_end(cachep, page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-05-20 00:10:29 +00:00
|
|
|
return obj ? obj : fallback_alloc(cachep, flags);
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
2007-02-10 09:42:53 +00:00
|
|
|
|
|
|
|
static __always_inline void *
|
2012-09-08 20:47:57 +00:00
|
|
|
slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
|
2012-09-08 20:47:55 +00:00
|
|
|
unsigned long caller)
|
2007-02-10 09:42:53 +00:00
|
|
|
{
|
|
|
|
unsigned long save_flags;
|
|
|
|
void *ptr;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
int slab_node = numa_mem_id();
|
2007-02-10 09:42:53 +00:00
|
|
|
|
2009-06-18 03:24:12 +00:00
|
|
|
flags &= gfp_allowed_mask;
|
2016-03-15 21:53:41 +00:00
|
|
|
cachep = slab_pre_alloc_hook(cachep, flags);
|
|
|
|
if (unlikely(!cachep))
|
2007-05-06 21:49:58 +00:00
|
|
|
return NULL;
|
|
|
|
|
2007-02-10 09:42:53 +00:00
|
|
|
cache_alloc_debugcheck_before(cachep, flags);
|
|
|
|
local_irq_save(save_flags);
|
|
|
|
|
2011-07-28 20:59:49 +00:00
|
|
|
if (nodeid == NUMA_NO_NODE)
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
nodeid = slab_node;
|
2007-02-10 09:42:53 +00:00
|
|
|
|
2014-08-06 23:04:11 +00:00
|
|
|
if (unlikely(!get_node(cachep, nodeid))) {
|
2007-02-10 09:42:53 +00:00
|
|
|
/* Node not bootstrapped yet */
|
|
|
|
ptr = fallback_alloc(cachep, flags);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
if (nodeid == slab_node) {
|
2007-02-10 09:42:53 +00:00
|
|
|
/*
|
|
|
|
* Use the locally cached objects if possible.
|
|
|
|
* However ____cache_alloc does not allow fallback
|
|
|
|
* to other nodes. It may fail while we still have
|
|
|
|
* objects on other nodes available.
|
|
|
|
*/
|
|
|
|
ptr = ____cache_alloc(cachep, flags);
|
|
|
|
if (ptr)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/* ___cache_alloc_node can fall back to other nodes */
|
|
|
|
ptr = ____cache_alloc_node(cachep, flags, nodeid);
|
|
|
|
out:
|
|
|
|
local_irq_restore(save_flags);
|
|
|
|
ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
|
|
|
|
|
2016-03-15 21:53:47 +00:00
|
|
|
if (unlikely(flags & __GFP_ZERO) && ptr)
|
|
|
|
memset(ptr, 0, cachep->object_size);
|
2007-07-17 11:03:23 +00:00
|
|
|
|
2016-03-15 21:53:47 +00:00
|
|
|
slab_post_alloc_hook(cachep, flags, 1, &ptr);
|
2007-02-10 09:42:53 +00:00
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline void *
|
|
|
|
__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
|
|
|
|
{
|
|
|
|
void *objp;
|
|
|
|
|
2014-09-25 01:41:02 +00:00
|
|
|
if (current->mempolicy || cpuset_do_slab_mem_spread()) {
|
2007-02-10 09:42:53 +00:00
|
|
|
objp = alternate_node_alloc(cache, flags);
|
|
|
|
if (objp)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
objp = ____cache_alloc(cache, flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We may just have run out of memory on the local node.
|
|
|
|
* ____cache_alloc_node() knows how to locate memory on other nodes
|
|
|
|
*/
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
if (!objp)
|
|
|
|
objp = ____cache_alloc_node(cache, flags, numa_mem_id());
|
2007-02-10 09:42:53 +00:00
|
|
|
|
|
|
|
out:
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
|
|
|
|
static __always_inline void *
|
|
|
|
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
|
|
|
{
|
|
|
|
return ____cache_alloc(cachep, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
|
|
|
|
static __always_inline void *
|
2012-09-08 20:47:57 +00:00
|
|
|
slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
|
2007-02-10 09:42:53 +00:00
|
|
|
{
|
|
|
|
unsigned long save_flags;
|
|
|
|
void *objp;
|
|
|
|
|
2009-06-18 03:24:12 +00:00
|
|
|
flags &= gfp_allowed_mask;
|
2016-03-15 21:53:41 +00:00
|
|
|
cachep = slab_pre_alloc_hook(cachep, flags);
|
|
|
|
if (unlikely(!cachep))
|
2007-05-06 21:49:58 +00:00
|
|
|
return NULL;
|
|
|
|
|
2007-02-10 09:42:53 +00:00
|
|
|
cache_alloc_debugcheck_before(cachep, flags);
|
|
|
|
local_irq_save(save_flags);
|
|
|
|
objp = __do_cache_alloc(cachep, flags);
|
|
|
|
local_irq_restore(save_flags);
|
|
|
|
objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
|
|
|
|
prefetchw(objp);
|
|
|
|
|
2016-03-15 21:53:47 +00:00
|
|
|
if (unlikely(flags & __GFP_ZERO) && objp)
|
|
|
|
memset(objp, 0, cachep->object_size);
|
2007-07-17 11:03:23 +00:00
|
|
|
|
2016-03-15 21:53:47 +00:00
|
|
|
slab_post_alloc_hook(cachep, flags, 1, &objp);
|
2007-02-10 09:42:53 +00:00
|
|
|
return objp;
|
|
|
|
}
|
2005-09-09 20:03:32 +00:00
|
|
|
|
|
|
|
/*
|
2014-03-30 09:02:20 +00:00
|
|
|
* Caller needs to acquire correct kmem_cache_node's list_lock
|
2014-08-06 23:04:25 +00:00
|
|
|
* @list: List of detached free slabs should be freed by caller
|
2005-09-09 20:03:32 +00:00
|
|
|
*/
|
2014-08-06 23:04:25 +00:00
|
|
|
static void free_block(struct kmem_cache *cachep, void **objpp,
|
|
|
|
int nr_objects, int node, struct list_head *list)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int i;
|
2014-08-06 23:04:22 +00:00
|
|
|
struct kmem_cache_node *n = get_node(cachep, node);
|
2016-05-20 00:10:17 +00:00
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
n->free_objects += nr_objects;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
for (i = 0; i < nr_objects; i++) {
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
void *objp;
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 23:43:58 +00:00
|
|
|
objp = objpp[i];
|
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
page = virt_to_head_page(objp);
|
|
|
|
list_del(&page->lru);
|
2005-09-23 04:44:02 +00:00
|
|
|
check_spinlock_acquired_node(cachep, node);
|
2016-03-15 21:54:12 +00:00
|
|
|
slab_put_obj(cachep, page, objp);
|
2005-04-16 22:20:36 +00:00
|
|
|
STATS_DEC_ACTIVE(cachep);
|
|
|
|
|
|
|
|
/* fixup slab chains */
|
2016-05-20 00:10:17 +00:00
|
|
|
if (page->active == 0)
|
|
|
|
list_add(&page->lru, &n->slabs_free);
|
|
|
|
else {
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Unconditionally move a slab to the end of the
|
|
|
|
* partial list on free - maximum time for the
|
|
|
|
* other objects to be freed, too.
|
|
|
|
*/
|
2013-10-24 01:07:49 +00:00
|
|
|
list_add_tail(&page->lru, &n->slabs_partial);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
2016-05-20 00:10:17 +00:00
|
|
|
|
|
|
|
while (n->free_objects > n->free_limit && !list_empty(&n->slabs_free)) {
|
|
|
|
n->free_objects -= cachep->num;
|
|
|
|
|
|
|
|
page = list_last_entry(&n->slabs_free, struct page, lru);
|
2016-07-26 22:22:11 +00:00
|
|
|
list_move(&page->lru, list);
|
2016-05-20 00:10:17 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-02-01 11:05:50 +00:00
|
|
|
static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int batchcount;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
int node = numa_mem_id();
|
2014-08-06 23:04:25 +00:00
|
|
|
LIST_HEAD(list);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
batchcount = ac->batchcount;
|
2016-03-15 21:54:12 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
check_irq_off();
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock(&n->list_lock);
|
|
|
|
if (n->shared) {
|
|
|
|
struct array_cache *shared_array = n->shared;
|
2006-01-08 09:00:37 +00:00
|
|
|
int max = shared_array->limit - shared_array->avail;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (max) {
|
|
|
|
if (batchcount > max)
|
|
|
|
batchcount = max;
|
2005-09-09 20:03:32 +00:00
|
|
|
memcpy(&(shared_array->entry[shared_array->avail]),
|
2006-01-08 09:00:37 +00:00
|
|
|
ac->entry, sizeof(void *) * batchcount);
|
2005-04-16 22:20:36 +00:00
|
|
|
shared_array->avail += batchcount;
|
|
|
|
goto free_done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-06 23:04:25 +00:00
|
|
|
free_block(cachep, ac->entry, batchcount, node, &list);
|
2006-03-22 08:08:11 +00:00
|
|
|
free_done:
|
2005-04-16 22:20:36 +00:00
|
|
|
#if STATS
|
|
|
|
{
|
|
|
|
int i = 0;
|
2016-01-14 23:17:59 +00:00
|
|
|
struct page *page;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-01-14 23:17:59 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_free, lru) {
|
2013-10-24 01:07:49 +00:00
|
|
|
BUG_ON(page->active);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
STATS_SET_FREEABLE(cachep, i);
|
|
|
|
}
|
|
|
|
#endif
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock(&n->list_lock);
|
2014-08-06 23:04:25 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
ac->avail -= batchcount;
|
2006-03-22 08:08:11 +00:00
|
|
|
memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2006-03-22 08:08:11 +00:00
|
|
|
* Release an obj back to its cache. If the obj has a constructed state, it must
|
|
|
|
* be in this state _before_ it is released. Called with disabled ints.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2011-06-02 07:16:42 +00:00
|
|
|
static inline void __cache_free(struct kmem_cache *cachep, void *objp,
|
2012-09-08 20:47:55 +00:00
|
|
|
unsigned long caller)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
mm: kasan: initial memory quarantine implementation
Quarantine isolates freed objects in a separate queue. The objects are
returned to the allocator later, which helps to detect use-after-free
errors.
When the object is freed, its state changes from KASAN_STATE_ALLOC to
KASAN_STATE_QUARANTINE. The object is poisoned and put into quarantine
instead of being returned to the allocator, therefore every subsequent
access to that object triggers a KASAN error, and the error handler is
able to say where the object has been allocated and deallocated.
When it's time for the object to leave quarantine, its state becomes
KASAN_STATE_FREE and it's returned to the allocator. From now on the
allocator may reuse it for another allocation. Before that happens,
it's still possible to detect a use-after free on that object (it
retains the allocation/deallocation stacks).
When the allocator reuses this object, the shadow is unpoisoned and old
allocation/deallocation stacks are wiped. Therefore a use of this
object, even an incorrect one, won't trigger ASan warning.
Without the quarantine, it's not guaranteed that the objects aren't
reused immediately, that's why the probability of catching a
use-after-free is lower than with quarantine in place.
Quarantine isolates freed objects in a separate queue. The objects are
returned to the allocator later, which helps to detect use-after-free
errors.
Freed objects are first added to per-cpu quarantine queues. When a
cache is destroyed or memory shrinking is requested, the objects are
moved into the global quarantine queue. Whenever a kmalloc call allows
memory reclaiming, the oldest objects are popped out of the global queue
until the total size of objects in quarantine is less than 3/4 of the
maximum quarantine size (which is a fraction of installed physical
memory).
As long as an object remains in the quarantine, KASAN is able to report
accesses to it, so the chance of reporting a use-after-free is
increased. Once the object leaves quarantine, the allocator may reuse
it, in which case the object is unpoisoned and KASAN can't detect
incorrect accesses to it.
Right now quarantine support is only enabled in SLAB allocator.
Unification of KASAN features in SLAB and SLUB will be done later.
This patch is based on the "mm: kasan: quarantine" patch originally
prepared by Dmitry Chernenkov. A number of improvements have been
suggested by Andrey Ryabinin.
[glider@google.com: v9]
Link: http://lkml.kernel.org/r/1462987130-144092-1-git-send-email-glider@google.com
Signed-off-by: Alexander Potapenko <glider@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Andrey Konovalov <adech.fo@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Konstantin Serebryany <kcc@google.com>
Cc: Dmitry Chernenkov <dmitryc@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 23:59:11 +00:00
|
|
|
/* Put the object into the quarantine, don't touch it for now. */
|
|
|
|
if (kasan_slab_free(cachep, objp))
|
|
|
|
return;
|
|
|
|
|
|
|
|
___cache_free(cachep, objp, caller);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm: kasan: initial memory quarantine implementation
Quarantine isolates freed objects in a separate queue. The objects are
returned to the allocator later, which helps to detect use-after-free
errors.
When the object is freed, its state changes from KASAN_STATE_ALLOC to
KASAN_STATE_QUARANTINE. The object is poisoned and put into quarantine
instead of being returned to the allocator, therefore every subsequent
access to that object triggers a KASAN error, and the error handler is
able to say where the object has been allocated and deallocated.
When it's time for the object to leave quarantine, its state becomes
KASAN_STATE_FREE and it's returned to the allocator. From now on the
allocator may reuse it for another allocation. Before that happens,
it's still possible to detect a use-after free on that object (it
retains the allocation/deallocation stacks).
When the allocator reuses this object, the shadow is unpoisoned and old
allocation/deallocation stacks are wiped. Therefore a use of this
object, even an incorrect one, won't trigger ASan warning.
Without the quarantine, it's not guaranteed that the objects aren't
reused immediately, that's why the probability of catching a
use-after-free is lower than with quarantine in place.
Quarantine isolates freed objects in a separate queue. The objects are
returned to the allocator later, which helps to detect use-after-free
errors.
Freed objects are first added to per-cpu quarantine queues. When a
cache is destroyed or memory shrinking is requested, the objects are
moved into the global quarantine queue. Whenever a kmalloc call allows
memory reclaiming, the oldest objects are popped out of the global queue
until the total size of objects in quarantine is less than 3/4 of the
maximum quarantine size (which is a fraction of installed physical
memory).
As long as an object remains in the quarantine, KASAN is able to report
accesses to it, so the chance of reporting a use-after-free is
increased. Once the object leaves quarantine, the allocator may reuse
it, in which case the object is unpoisoned and KASAN can't detect
incorrect accesses to it.
Right now quarantine support is only enabled in SLAB allocator.
Unification of KASAN features in SLAB and SLUB will be done later.
This patch is based on the "mm: kasan: quarantine" patch originally
prepared by Dmitry Chernenkov. A number of improvements have been
suggested by Andrey Ryabinin.
[glider@google.com: v9]
Link: http://lkml.kernel.org/r/1462987130-144092-1-git-send-email-glider@google.com
Signed-off-by: Alexander Potapenko <glider@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Andrey Konovalov <adech.fo@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Konstantin Serebryany <kcc@google.com>
Cc: Dmitry Chernenkov <dmitryc@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 23:59:11 +00:00
|
|
|
void ___cache_free(struct kmem_cache *cachep, void *objp,
|
|
|
|
unsigned long caller)
|
|
|
|
{
|
|
|
|
struct array_cache *ac = cpu_cache_get(cachep);
|
2016-03-25 21:21:59 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
check_irq_off();
|
2009-06-11 12:22:40 +00:00
|
|
|
kmemleak_free_recursive(objp, cachep->flags);
|
2011-06-02 07:16:42 +00:00
|
|
|
objp = cache_free_debugcheck(cachep, objp, caller);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-06-13 15:24:58 +00:00
|
|
|
kmemcheck_slab_free(cachep, objp, cachep->object_size);
|
2008-05-09 18:35:53 +00:00
|
|
|
|
2007-08-22 21:01:49 +00:00
|
|
|
/*
|
|
|
|
* Skip calling cache_free_alien() when the platform is not numa.
|
|
|
|
* This will avoid cache misses that happen while accessing slabp (which
|
|
|
|
* is per page memory reference) to get nodeid. Instead use a global
|
|
|
|
* variable to skip the call, which is mostly likely to be present in
|
|
|
|
* the cache.
|
|
|
|
*/
|
2009-06-16 22:32:16 +00:00
|
|
|
if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
|
2006-06-23 09:03:05 +00:00
|
|
|
return;
|
|
|
|
|
2014-10-09 22:26:04 +00:00
|
|
|
if (ac->avail < ac->limit) {
|
2005-04-16 22:20:36 +00:00
|
|
|
STATS_INC_FREEHIT(cachep);
|
|
|
|
} else {
|
|
|
|
STATS_INC_FREEMISS(cachep);
|
|
|
|
cache_flusharray(cachep, ac);
|
|
|
|
}
|
2011-08-26 16:26:17 +00:00
|
|
|
|
2016-03-15 21:54:56 +00:00
|
|
|
if (sk_memalloc_socks()) {
|
|
|
|
struct page *page = virt_to_head_page(objp);
|
|
|
|
|
|
|
|
if (unlikely(PageSlabPfmemalloc(page))) {
|
|
|
|
cache_free_pfmemalloc(cachep, page, objp);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ac->entry[ac->avail++] = objp;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kmem_cache_alloc - Allocate an object
|
|
|
|
* @cachep: The cache to allocate from.
|
|
|
|
* @flags: See kmalloc().
|
|
|
|
*
|
|
|
|
* Allocate an object from this cache. The flags are only relevant
|
|
|
|
* if the cache has no available objects.
|
|
|
|
*/
|
2006-02-01 11:05:50 +00:00
|
|
|
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-09-08 20:47:57 +00:00
|
|
|
void *ret = slab_alloc(cachep, flags, _RET_IP_);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
2016-03-25 21:22:02 +00:00
|
|
|
kasan_slab_alloc(cachep, ret, flags);
|
2009-03-23 13:12:24 +00:00
|
|
|
trace_kmem_cache_alloc(_RET_IP_, ret,
|
2012-06-13 15:24:58 +00:00
|
|
|
cachep->object_size, cachep->size, flags);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc);
|
|
|
|
|
2016-03-15 21:53:53 +00:00
|
|
|
static __always_inline void
|
|
|
|
cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
|
|
|
|
size_t size, void **p, unsigned long caller)
|
|
|
|
{
|
|
|
|
size_t i;
|
|
|
|
|
|
|
|
for (i = 0; i < size; i++)
|
|
|
|
p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
|
|
|
|
}
|
|
|
|
|
2015-11-20 23:57:58 +00:00
|
|
|
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
|
2016-03-15 21:53:50 +00:00
|
|
|
void **p)
|
2015-09-04 22:45:34 +00:00
|
|
|
{
|
2016-03-15 21:53:50 +00:00
|
|
|
size_t i;
|
|
|
|
|
|
|
|
s = slab_pre_alloc_hook(s, flags);
|
|
|
|
if (!s)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
cache_alloc_debugcheck_before(s, flags);
|
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
for (i = 0; i < size; i++) {
|
|
|
|
void *objp = __do_cache_alloc(s, flags);
|
|
|
|
|
|
|
|
if (unlikely(!objp))
|
|
|
|
goto error;
|
|
|
|
p[i] = objp;
|
|
|
|
}
|
|
|
|
local_irq_enable();
|
|
|
|
|
2016-03-15 21:53:53 +00:00
|
|
|
cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
|
|
|
|
|
2016-03-15 21:53:50 +00:00
|
|
|
/* Clear memory outside IRQ disabled section */
|
|
|
|
if (unlikely(flags & __GFP_ZERO))
|
|
|
|
for (i = 0; i < size; i++)
|
|
|
|
memset(p[i], 0, s->object_size);
|
|
|
|
|
|
|
|
slab_post_alloc_hook(s, flags, size, p);
|
|
|
|
/* FIXME: Trace call missing. Christoph would like a bulk variant */
|
|
|
|
return size;
|
|
|
|
error:
|
|
|
|
local_irq_enable();
|
2016-03-15 21:53:53 +00:00
|
|
|
cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
|
2016-03-15 21:53:50 +00:00
|
|
|
slab_post_alloc_hook(s, flags, i, p);
|
|
|
|
__kmem_cache_free_bulk(s, i, p);
|
|
|
|
return 0;
|
2015-09-04 22:45:34 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_bulk);
|
|
|
|
|
2009-12-11 07:45:30 +00:00
|
|
|
#ifdef CONFIG_TRACING
|
2010-11-24 21:23:34 +00:00
|
|
|
void *
|
2012-09-08 20:47:56 +00:00
|
|
|
kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
|
2008-08-10 17:14:05 +00:00
|
|
|
{
|
2010-11-24 21:23:34 +00:00
|
|
|
void *ret;
|
|
|
|
|
2012-09-08 20:47:57 +00:00
|
|
|
ret = slab_alloc(cachep, flags, _RET_IP_);
|
2010-11-24 21:23:34 +00:00
|
|
|
|
2016-03-25 21:22:02 +00:00
|
|
|
kasan_kmalloc(cachep, ret, size, flags);
|
2010-11-24 21:23:34 +00:00
|
|
|
trace_kmalloc(_RET_IP_, ret,
|
2012-09-08 20:47:52 +00:00
|
|
|
size, cachep->size, flags);
|
2010-11-24 21:23:34 +00:00
|
|
|
return ret;
|
2008-08-10 17:14:05 +00:00
|
|
|
}
|
2010-11-24 21:23:34 +00:00
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_trace);
|
2008-08-10 17:14:05 +00:00
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
2013-05-16 03:36:23 +00:00
|
|
|
/**
|
|
|
|
* kmem_cache_alloc_node - Allocate an object on the specified node
|
|
|
|
* @cachep: The cache to allocate from.
|
|
|
|
* @flags: See kmalloc().
|
|
|
|
* @nodeid: node number of the target node.
|
|
|
|
*
|
|
|
|
* Identical to kmem_cache_alloc but it will allocate memory on the given
|
|
|
|
* node, which can improve the performance for cpu bound structures.
|
|
|
|
*
|
|
|
|
* Fallback to other node is possible if __GFP_THISNODE is not set.
|
|
|
|
*/
|
2006-12-07 04:32:30 +00:00
|
|
|
void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
|
|
|
|
{
|
2012-09-08 20:47:57 +00:00
|
|
|
void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
2016-03-25 21:22:02 +00:00
|
|
|
kasan_slab_alloc(cachep, ret, flags);
|
2009-03-23 13:12:24 +00:00
|
|
|
trace_kmem_cache_alloc_node(_RET_IP_, ret,
|
2012-06-13 15:24:58 +00:00
|
|
|
cachep->object_size, cachep->size,
|
2009-03-23 13:12:24 +00:00
|
|
|
flags, nodeid);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
|
|
|
return ret;
|
2006-12-07 04:32:30 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_node);
|
|
|
|
|
2009-12-11 07:45:30 +00:00
|
|
|
#ifdef CONFIG_TRACING
|
2012-09-08 20:47:56 +00:00
|
|
|
void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
|
2010-11-24 21:23:34 +00:00
|
|
|
gfp_t flags,
|
2012-09-08 20:47:56 +00:00
|
|
|
int nodeid,
|
|
|
|
size_t size)
|
2008-08-10 17:14:05 +00:00
|
|
|
{
|
2010-11-24 21:23:34 +00:00
|
|
|
void *ret;
|
|
|
|
|
2012-09-25 11:07:08 +00:00
|
|
|
ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
|
2016-03-25 21:22:02 +00:00
|
|
|
|
|
|
|
kasan_kmalloc(cachep, ret, size, flags);
|
2010-11-24 21:23:34 +00:00
|
|
|
trace_kmalloc_node(_RET_IP_, ret,
|
2012-09-08 20:47:52 +00:00
|
|
|
size, cachep->size,
|
2010-11-24 21:23:34 +00:00
|
|
|
flags, nodeid);
|
|
|
|
return ret;
|
2008-08-10 17:14:05 +00:00
|
|
|
}
|
2010-11-24 21:23:34 +00:00
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
|
2008-08-10 17:14:05 +00:00
|
|
|
#endif
|
|
|
|
|
2006-12-07 04:32:30 +00:00
|
|
|
static __always_inline void *
|
2012-09-08 20:47:55 +00:00
|
|
|
__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
|
2005-05-01 15:58:38 +00:00
|
|
|
{
|
2006-02-01 11:05:50 +00:00
|
|
|
struct kmem_cache *cachep;
|
2016-03-25 21:21:59 +00:00
|
|
|
void *ret;
|
2005-05-01 15:58:38 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep = kmalloc_slab(size, flags);
|
2007-07-17 11:03:22 +00:00
|
|
|
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
|
|
|
|
return cachep;
|
2016-03-25 21:21:59 +00:00
|
|
|
ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
|
2016-03-25 21:22:02 +00:00
|
|
|
kasan_kmalloc(cachep, ret, size, flags);
|
2016-03-25 21:21:59 +00:00
|
|
|
|
|
|
|
return ret;
|
2005-05-01 15:58:38 +00:00
|
|
|
}
|
2006-12-07 04:32:30 +00:00
|
|
|
|
|
|
|
void *__kmalloc_node(size_t size, gfp_t flags, int node)
|
|
|
|
{
|
2012-09-08 20:47:55 +00:00
|
|
|
return __do_kmalloc_node(size, flags, node, _RET_IP_);
|
2006-12-07 04:32:30 +00:00
|
|
|
}
|
2006-09-26 06:31:36 +00:00
|
|
|
EXPORT_SYMBOL(__kmalloc_node);
|
2006-12-07 04:32:30 +00:00
|
|
|
|
|
|
|
void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
|
2008-08-19 17:43:25 +00:00
|
|
|
int node, unsigned long caller)
|
2006-12-07 04:32:30 +00:00
|
|
|
{
|
2012-09-08 20:47:55 +00:00
|
|
|
return __do_kmalloc_node(size, flags, node, caller);
|
2006-12-07 04:32:30 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__kmalloc_node_track_caller);
|
|
|
|
#endif /* CONFIG_NUMA */
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/**
|
2006-06-23 09:03:48 +00:00
|
|
|
* __do_kmalloc - allocate memory
|
2005-04-16 22:20:36 +00:00
|
|
|
* @size: how many bytes of memory are required.
|
2006-06-23 09:03:48 +00:00
|
|
|
* @flags: the type of memory to allocate (see kmalloc).
|
2006-03-22 08:08:14 +00:00
|
|
|
* @caller: function caller for debug tracking of the caller
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2006-02-01 11:05:52 +00:00
|
|
|
static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
|
2012-09-08 20:47:55 +00:00
|
|
|
unsigned long caller)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-02-01 11:05:50 +00:00
|
|
|
struct kmem_cache *cachep;
|
2008-08-10 17:14:05 +00:00
|
|
|
void *ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep = kmalloc_slab(size, flags);
|
2007-07-19 20:17:15 +00:00
|
|
|
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
|
|
|
|
return cachep;
|
2012-09-08 20:47:57 +00:00
|
|
|
ret = slab_alloc(cachep, flags, caller);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
2016-03-25 21:22:02 +00:00
|
|
|
kasan_kmalloc(cachep, ret, size, flags);
|
2012-09-08 20:47:55 +00:00
|
|
|
trace_kmalloc(caller, ret,
|
2012-06-13 15:24:57 +00:00
|
|
|
size, cachep->size, flags);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
|
|
|
return ret;
|
2006-02-01 11:05:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void *__kmalloc(size_t size, gfp_t flags)
|
|
|
|
{
|
2012-09-08 20:47:55 +00:00
|
|
|
return __do_kmalloc(size, flags, _RET_IP_);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__kmalloc);
|
|
|
|
|
2008-08-19 17:43:25 +00:00
|
|
|
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
|
2006-02-01 11:05:52 +00:00
|
|
|
{
|
2012-09-08 20:47:55 +00:00
|
|
|
return __do_kmalloc(size, flags, caller);
|
2006-02-01 11:05:52 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__kmalloc_track_caller);
|
2006-10-04 09:15:25 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
|
|
|
* kmem_cache_free - Deallocate an object
|
|
|
|
* @cachep: The cache the allocation was from.
|
|
|
|
* @objp: The previously allocated object.
|
|
|
|
*
|
|
|
|
* Free an object which was previously allocated from this
|
|
|
|
* cache.
|
|
|
|
*/
|
2006-02-01 11:05:50 +00:00
|
|
|
void kmem_cache_free(struct kmem_cache *cachep, void *objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
2012-12-18 22:22:46 +00:00
|
|
|
cachep = cache_from_obj(cachep, objp);
|
|
|
|
if (!cachep)
|
|
|
|
return;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
local_irq_save(flags);
|
2012-07-02 06:29:10 +00:00
|
|
|
debug_check_no_locks_freed(objp, cachep->object_size);
|
2008-04-30 07:55:01 +00:00
|
|
|
if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
|
2012-06-13 15:24:58 +00:00
|
|
|
debug_check_no_obj_freed(objp, cachep->object_size);
|
2012-09-08 20:47:55 +00:00
|
|
|
__cache_free(cachep, objp, _RET_IP_);
|
2005-04-16 22:20:36 +00:00
|
|
|
local_irq_restore(flags);
|
2008-08-10 17:14:05 +00:00
|
|
|
|
2009-03-23 13:12:24 +00:00
|
|
|
trace_kmem_cache_free(_RET_IP_, objp);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kmem_cache_free);
|
|
|
|
|
2016-03-15 21:53:56 +00:00
|
|
|
void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
|
|
|
|
{
|
|
|
|
struct kmem_cache *s;
|
|
|
|
size_t i;
|
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
for (i = 0; i < size; i++) {
|
|
|
|
void *objp = p[i];
|
|
|
|
|
2016-03-15 21:54:00 +00:00
|
|
|
if (!orig_s) /* called via kfree_bulk */
|
|
|
|
s = virt_to_cache(objp);
|
|
|
|
else
|
|
|
|
s = cache_from_obj(orig_s, objp);
|
2016-03-15 21:53:56 +00:00
|
|
|
|
|
|
|
debug_check_no_locks_freed(objp, s->object_size);
|
|
|
|
if (!(s->flags & SLAB_DEBUG_OBJECTS))
|
|
|
|
debug_check_no_obj_freed(objp, s->object_size);
|
|
|
|
|
|
|
|
__cache_free(s, objp, _RET_IP_);
|
|
|
|
}
|
|
|
|
local_irq_enable();
|
|
|
|
|
|
|
|
/* FIXME: add tracing */
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kmem_cache_free_bulk);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
|
|
|
* kfree - free previously allocated memory
|
|
|
|
* @objp: pointer returned by kmalloc.
|
|
|
|
*
|
2005-09-09 20:10:16 +00:00
|
|
|
* If @objp is NULL, no operation is performed.
|
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Don't free memory not originally allocated by kmalloc()
|
|
|
|
* or you will run into trouble.
|
|
|
|
*/
|
|
|
|
void kfree(const void *objp)
|
|
|
|
{
|
2006-02-01 11:05:50 +00:00
|
|
|
struct kmem_cache *c;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long flags;
|
|
|
|
|
2009-03-25 09:05:57 +00:00
|
|
|
trace_kfree(_RET_IP_, objp);
|
|
|
|
|
2007-07-17 11:03:22 +00:00
|
|
|
if (unlikely(ZERO_OR_NULL_PTR(objp)))
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
|
|
|
local_irq_save(flags);
|
|
|
|
kfree_debugcheck(objp);
|
2006-02-01 11:05:49 +00:00
|
|
|
c = virt_to_cache(objp);
|
2012-06-13 15:24:58 +00:00
|
|
|
debug_check_no_locks_freed(objp, c->object_size);
|
|
|
|
|
|
|
|
debug_check_no_obj_freed(objp, c->object_size);
|
2012-09-08 20:47:55 +00:00
|
|
|
__cache_free(c, (void *)objp, _RET_IP_);
|
2005-04-16 22:20:36 +00:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kfree);
|
|
|
|
|
2005-09-09 20:03:32 +00:00
|
|
|
/*
|
2013-01-10 19:14:19 +00:00
|
|
|
* This initializes kmem_cache_node or resizes various caches for all nodes.
|
2005-09-09 20:03:32 +00:00
|
|
|
*/
|
2016-05-20 00:10:14 +00:00
|
|
|
static int setup_kmem_cache_nodes(struct kmem_cache *cachep, gfp_t gfp)
|
2005-09-09 20:03:32 +00:00
|
|
|
{
|
2016-05-20 00:10:14 +00:00
|
|
|
int ret;
|
2005-09-09 20:03:32 +00:00
|
|
|
int node;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2008-01-24 13:49:54 +00:00
|
|
|
for_each_online_node(node) {
|
2016-05-20 00:10:14 +00:00
|
|
|
ret = setup_kmem_cache_node(cachep, node, gfp, true);
|
|
|
|
if (ret)
|
2005-09-09 20:03:32 +00:00
|
|
|
goto fail;
|
|
|
|
|
|
|
|
}
|
2016-05-20 00:10:14 +00:00
|
|
|
|
2006-03-25 11:06:46 +00:00
|
|
|
return 0;
|
2006-03-25 11:06:47 +00:00
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
fail:
|
2012-06-13 15:24:57 +00:00
|
|
|
if (!cachep->list.next) {
|
2006-03-25 11:06:47 +00:00
|
|
|
/* Cache is not active yet. Roll back what we did */
|
|
|
|
node--;
|
|
|
|
while (node >= 0) {
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
if (n) {
|
2013-01-10 19:14:19 +00:00
|
|
|
kfree(n->shared);
|
|
|
|
free_alien_cache(n->alien);
|
|
|
|
kfree(n);
|
2013-01-10 19:14:19 +00:00
|
|
|
cachep->node[node] = NULL;
|
2006-03-25 11:06:47 +00:00
|
|
|
}
|
|
|
|
node--;
|
|
|
|
}
|
|
|
|
}
|
2006-03-25 11:06:46 +00:00
|
|
|
return -ENOMEM;
|
2005-09-09 20:03:32 +00:00
|
|
|
}
|
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
/* Always called with the slab_mutex held */
|
2012-12-18 22:23:03 +00:00
|
|
|
static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
|
2009-06-10 16:40:04 +00:00
|
|
|
int batchcount, int shared, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:26:27 +00:00
|
|
|
struct array_cache __percpu *cpu_cache, *prev;
|
|
|
|
int cpu;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
|
|
|
|
if (!cpu_cache)
|
2006-09-26 06:31:47 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
prev = cachep->cpu_cache;
|
|
|
|
cachep->cpu_cache = cpu_cache;
|
|
|
|
kick_all_cpus_sync();
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
check_irq_on();
|
|
|
|
cachep->batchcount = batchcount;
|
|
|
|
cachep->limit = limit;
|
2005-09-09 20:03:32 +00:00
|
|
|
cachep->shared = shared;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
if (!prev)
|
2016-05-20 00:10:14 +00:00
|
|
|
goto setup_node;
|
2014-10-09 22:26:27 +00:00
|
|
|
|
|
|
|
for_each_online_cpu(cpu) {
|
2014-08-06 23:04:25 +00:00
|
|
|
LIST_HEAD(list);
|
2014-08-06 23:04:11 +00:00
|
|
|
int node;
|
|
|
|
struct kmem_cache_node *n;
|
2014-10-09 22:26:27 +00:00
|
|
|
struct array_cache *ac = per_cpu_ptr(prev, cpu);
|
2014-08-06 23:04:11 +00:00
|
|
|
|
2014-10-09 22:26:27 +00:00
|
|
|
node = cpu_to_mem(cpu);
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
spin_lock_irq(&n->list_lock);
|
2014-10-09 22:26:27 +00:00
|
|
|
free_block(cachep, ac->entry, ac->avail, node, &list);
|
2014-08-06 23:04:11 +00:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2014-08-06 23:04:25 +00:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2014-10-09 22:26:27 +00:00
|
|
|
free_percpu(prev);
|
|
|
|
|
2016-05-20 00:10:14 +00:00
|
|
|
setup_node:
|
|
|
|
return setup_kmem_cache_nodes(cachep, gfp);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-12-18 22:23:03 +00:00
|
|
|
static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
|
|
|
|
int batchcount, int shared, gfp_t gfp)
|
|
|
|
{
|
|
|
|
int ret;
|
2015-02-12 22:59:23 +00:00
|
|
|
struct kmem_cache *c;
|
2012-12-18 22:23:03 +00:00
|
|
|
|
|
|
|
ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
|
|
|
|
|
|
|
|
if (slab_state < FULL)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if ((ret < 0) || !is_root_cache(cachep))
|
|
|
|
return ret;
|
|
|
|
|
2015-02-12 22:59:23 +00:00
|
|
|
lockdep_assert_held(&slab_mutex);
|
|
|
|
for_each_memcg_cache(c, cachep) {
|
|
|
|
/* return value determined by the root cache only */
|
|
|
|
__do_tune_cpucache(c, limit, batchcount, shared, gfp);
|
2012-12-18 22:23:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
/* Called with slab_mutex held always */
|
2009-06-10 16:40:04 +00:00
|
|
|
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
int err;
|
2012-12-18 22:23:03 +00:00
|
|
|
int limit = 0;
|
|
|
|
int shared = 0;
|
|
|
|
int batchcount = 0;
|
|
|
|
|
2016-07-26 22:21:56 +00:00
|
|
|
err = cache_random_seq_create(cachep, cachep->num, gfp);
|
2016-05-20 00:10:37 +00:00
|
|
|
if (err)
|
|
|
|
goto end;
|
|
|
|
|
2012-12-18 22:23:03 +00:00
|
|
|
if (!is_root_cache(cachep)) {
|
|
|
|
struct kmem_cache *root = memcg_root_cache(cachep);
|
|
|
|
limit = root->limit;
|
|
|
|
shared = root->shared;
|
|
|
|
batchcount = root->batchcount;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-12-18 22:23:03 +00:00
|
|
|
if (limit && shared && batchcount)
|
|
|
|
goto skip_setup;
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* The head array serves three purposes:
|
2005-04-16 22:20:36 +00:00
|
|
|
* - create a LIFO ordering, i.e. return objects that are cache-warm
|
|
|
|
* - reduce the number of spinlock operations.
|
2006-03-22 08:08:11 +00:00
|
|
|
* - reduce the number of linked list operations on the slab and
|
2005-04-16 22:20:36 +00:00
|
|
|
* bufctl chains: array operations are cheaper.
|
|
|
|
* The numbers are guessed, we should auto-tune as described by
|
|
|
|
* Bonwick.
|
|
|
|
*/
|
2012-06-13 15:24:57 +00:00
|
|
|
if (cachep->size > 131072)
|
2005-04-16 22:20:36 +00:00
|
|
|
limit = 1;
|
2012-06-13 15:24:57 +00:00
|
|
|
else if (cachep->size > PAGE_SIZE)
|
2005-04-16 22:20:36 +00:00
|
|
|
limit = 8;
|
2012-06-13 15:24:57 +00:00
|
|
|
else if (cachep->size > 1024)
|
2005-04-16 22:20:36 +00:00
|
|
|
limit = 24;
|
2012-06-13 15:24:57 +00:00
|
|
|
else if (cachep->size > 256)
|
2005-04-16 22:20:36 +00:00
|
|
|
limit = 54;
|
|
|
|
else
|
|
|
|
limit = 120;
|
|
|
|
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* CPU bound tasks (e.g. network routing) can exhibit cpu bound
|
2005-04-16 22:20:36 +00:00
|
|
|
* allocation behaviour: Most allocs on one cpu, most free operations
|
|
|
|
* on another cpu. For these cases, an efficient object passing between
|
|
|
|
* cpus is necessary. This is provided by a shared array. The array
|
|
|
|
* replaces Bonwick's magazine layer.
|
|
|
|
* On uniprocessor, it's functionally equivalent (but less efficient)
|
|
|
|
* to a larger limit. Thus disabled by default.
|
|
|
|
*/
|
|
|
|
shared = 0;
|
2012-06-13 15:24:57 +00:00
|
|
|
if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
|
2005-04-16 22:20:36 +00:00
|
|
|
shared = 8;
|
|
|
|
|
|
|
|
#if DEBUG
|
2006-03-22 08:08:11 +00:00
|
|
|
/*
|
|
|
|
* With debugging enabled, large batchcount lead to excessively long
|
|
|
|
* periods with disabled local interrupts. Limit the batchcount
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
if (limit > 32)
|
|
|
|
limit = 32;
|
|
|
|
#endif
|
2012-12-18 22:23:03 +00:00
|
|
|
batchcount = (limit + 1) / 2;
|
|
|
|
skip_setup:
|
|
|
|
err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
|
2016-05-20 00:10:37 +00:00
|
|
|
end:
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err)
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("enable_cpucache failed for %s, error %d\n",
|
2006-01-08 09:00:37 +00:00
|
|
|
cachep->name, -err);
|
2006-09-26 06:31:38 +00:00
|
|
|
return err;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-03-22 08:09:07 +00:00
|
|
|
/*
|
2013-01-10 19:14:19 +00:00
|
|
|
* Drain an array if it contains any elements taking the node lock only if
|
|
|
|
* necessary. Note that the node listlock also protects the array_cache
|
2006-03-22 08:09:07 +00:00
|
|
|
* if drain_array() is used on the shared array.
|
2006-03-22 08:09:07 +00:00
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
|
2016-05-20 00:10:02 +00:00
|
|
|
struct array_cache *ac, int node)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-08-06 23:04:25 +00:00
|
|
|
LIST_HEAD(list);
|
2016-05-20 00:10:02 +00:00
|
|
|
|
|
|
|
/* ac from n->shared can be freed if we don't hold the slab_mutex. */
|
|
|
|
check_mutex_acquired();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-22 08:09:07 +00:00
|
|
|
if (!ac || !ac->avail)
|
|
|
|
return;
|
2016-05-20 00:10:02 +00:00
|
|
|
|
|
|
|
if (ac->touched) {
|
2005-04-16 22:20:36 +00:00
|
|
|
ac->touched = 0;
|
2016-05-20 00:10:02 +00:00
|
|
|
return;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-05-20 00:10:02 +00:00
|
|
|
|
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
drain_array_locked(cachep, ac, node, false, &list);
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
|
|
|
|
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cache_reap - Reclaim memory from caches.
|
2007-03-01 04:12:13 +00:00
|
|
|
* @w: work descriptor
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* Called from workqueue/eventd every few seconds.
|
|
|
|
* Purpose:
|
|
|
|
* - clear the per-cpu caches for this CPU.
|
|
|
|
* - return freeable pages to the main free memory pool.
|
|
|
|
*
|
2006-03-22 08:08:11 +00:00
|
|
|
* If we cannot acquire the cache chain mutex then just give up - we'll try
|
|
|
|
* again on the next iteration.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-02-10 09:42:55 +00:00
|
|
|
static void cache_reap(struct work_struct *w)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-06-23 09:03:17 +00:00
|
|
|
struct kmem_cache *searchp;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 21:45:03 +00:00
|
|
|
int node = numa_mem_id();
|
2009-04-02 23:56:54 +00:00
|
|
|
struct delayed_work *work = to_delayed_work(w);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
if (!mutex_trylock(&slab_mutex))
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Give up. Setup the next iteration. */
|
2007-02-10 09:42:55 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(searchp, &slab_caches, list) {
|
2005-04-16 22:20:36 +00:00
|
|
|
check_irq_on();
|
|
|
|
|
2006-03-22 08:09:05 +00:00
|
|
|
/*
|
2013-01-10 19:14:19 +00:00
|
|
|
* We only take the node lock if absolutely necessary and we
|
2006-03-22 08:09:05 +00:00
|
|
|
* have established with reasonable certainty that
|
|
|
|
* we can do some work if the lock was obtained.
|
|
|
|
*/
|
2014-08-06 23:04:11 +00:00
|
|
|
n = get_node(searchp, node);
|
2006-03-22 08:09:05 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
reap_alien(searchp, n);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-05-20 00:10:02 +00:00
|
|
|
drain_array(searchp, n, cpu_cache_get(searchp), node);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-22 08:09:05 +00:00
|
|
|
/*
|
|
|
|
* These are racy checks but it does not matter
|
|
|
|
* if we skip one check or scan twice.
|
|
|
|
*/
|
2013-01-10 19:14:19 +00:00
|
|
|
if (time_after(n->next_reap, jiffies))
|
2006-03-22 08:09:05 +00:00
|
|
|
goto next;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-03-30 09:02:20 +00:00
|
|
|
n->next_reap = jiffies + REAPTIMEOUT_NODE;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-05-20 00:10:02 +00:00
|
|
|
drain_array(searchp, n, n->shared, node);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
if (n->free_touched)
|
|
|
|
n->free_touched = 0;
|
2006-06-30 08:55:45 +00:00
|
|
|
else {
|
|
|
|
int freed;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
freed = drain_freelist(searchp, n, (n->free_limit +
|
2006-06-30 08:55:45 +00:00
|
|
|
5 * searchp->num - 1) / (5 * searchp->num));
|
|
|
|
STATS_ADD_REAPED(searchp, freed);
|
|
|
|
}
|
2006-03-22 08:09:05 +00:00
|
|
|
next:
|
2005-04-16 22:20:36 +00:00
|
|
|
cond_resched();
|
|
|
|
}
|
|
|
|
check_irq_on();
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_unlock(&slab_mutex);
|
2006-03-10 01:33:54 +00:00
|
|
|
next_reap_node();
|
2007-02-10 09:42:55 +00:00
|
|
|
out:
|
2006-03-22 08:08:11 +00:00
|
|
|
/* Set up the next iteration */
|
2014-03-30 09:02:20 +00:00
|
|
|
schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-01-02 21:04:48 +00:00
|
|
|
#ifdef CONFIG_SLABINFO
|
2012-10-19 14:20:27 +00:00
|
|
|
void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2006-01-08 09:00:37 +00:00
|
|
|
unsigned long active_objs;
|
|
|
|
unsigned long num_objs;
|
|
|
|
unsigned long active_slabs = 0;
|
|
|
|
unsigned long num_slabs, free_objects = 0, shared_avail = 0;
|
2005-09-09 20:03:32 +00:00
|
|
|
const char *name;
|
2005-04-16 22:20:36 +00:00
|
|
|
char *error = NULL;
|
2005-09-09 20:03:32 +00:00
|
|
|
int node;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
active_objs = 0;
|
|
|
|
num_slabs = 0;
|
2014-08-06 23:04:11 +00:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2006-02-05 07:27:58 +00:00
|
|
|
check_irq_on();
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru) {
|
|
|
|
if (page->active != cachep->num && !error)
|
2005-09-09 20:03:32 +00:00
|
|
|
error = "slabs_full accounting error";
|
|
|
|
active_objs += cachep->num;
|
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-24 01:07:49 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru) {
|
|
|
|
if (page->active == cachep->num && !error)
|
2013-10-24 01:07:48 +00:00
|
|
|
error = "slabs_partial accounting error";
|
2013-10-24 01:07:49 +00:00
|
|
|
if (!page->active && !error)
|
2013-10-24 01:07:48 +00:00
|
|
|
error = "slabs_partial accounting error";
|
2013-10-24 01:07:49 +00:00
|
|
|
active_objs += page->active;
|
2005-09-09 20:03:32 +00:00
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-24 01:07:49 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_free, lru) {
|
|
|
|
if (page->active && !error)
|
2013-10-24 01:07:48 +00:00
|
|
|
error = "slabs_free accounting error";
|
2005-09-09 20:03:32 +00:00
|
|
|
num_slabs++;
|
|
|
|
}
|
2013-01-10 19:14:19 +00:00
|
|
|
free_objects += n->free_objects;
|
|
|
|
if (n->shared)
|
|
|
|
shared_avail += n->shared->avail;
|
2005-09-09 20:03:32 +00:00
|
|
|
|
2013-01-10 19:14:19 +00:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2006-01-08 09:00:37 +00:00
|
|
|
num_slabs += active_slabs;
|
|
|
|
num_objs = num_slabs * cachep->num;
|
2005-09-09 20:03:32 +00:00
|
|
|
if (num_objs - active_objs != free_objects && !error)
|
2005-04-16 22:20:36 +00:00
|
|
|
error = "free_objects accounting error";
|
|
|
|
|
2006-01-08 09:00:37 +00:00
|
|
|
name = cachep->name;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (error)
|
2016-03-17 21:19:50 +00:00
|
|
|
pr_err("slab: cache %s error: %s\n", name, error);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-10-19 14:20:27 +00:00
|
|
|
sinfo->active_objs = active_objs;
|
|
|
|
sinfo->num_objs = num_objs;
|
|
|
|
sinfo->active_slabs = active_slabs;
|
|
|
|
sinfo->num_slabs = num_slabs;
|
|
|
|
sinfo->shared_avail = shared_avail;
|
|
|
|
sinfo->limit = cachep->limit;
|
|
|
|
sinfo->batchcount = cachep->batchcount;
|
|
|
|
sinfo->shared = cachep->shared;
|
|
|
|
sinfo->objects_per_slab = cachep->num;
|
|
|
|
sinfo->cache_order = cachep->gfporder;
|
|
|
|
}
|
|
|
|
|
|
|
|
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
|
|
|
|
{
|
2005-04-16 22:20:36 +00:00
|
|
|
#if STATS
|
2013-01-10 19:14:19 +00:00
|
|
|
{ /* node stats */
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long high = cachep->high_mark;
|
|
|
|
unsigned long allocs = cachep->num_allocations;
|
|
|
|
unsigned long grown = cachep->grown;
|
|
|
|
unsigned long reaped = cachep->reaped;
|
|
|
|
unsigned long errors = cachep->errors;
|
|
|
|
unsigned long max_freeable = cachep->max_freeable;
|
|
|
|
unsigned long node_allocs = cachep->node_allocs;
|
2005-09-09 20:03:32 +00:00
|
|
|
unsigned long node_frees = cachep->node_frees;
|
2006-04-11 05:52:54 +00:00
|
|
|
unsigned long overflows = cachep->node_overflow;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-17 21:19:47 +00:00
|
|
|
seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu %4lu",
|
2010-03-27 02:27:58 +00:00
|
|
|
allocs, high, grown,
|
|
|
|
reaped, errors, max_freeable, node_allocs,
|
|
|
|
node_frees, overflows);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
/* cpu stats */
|
|
|
|
{
|
|
|
|
unsigned long allochit = atomic_read(&cachep->allochit);
|
|
|
|
unsigned long allocmiss = atomic_read(&cachep->allocmiss);
|
|
|
|
unsigned long freehit = atomic_read(&cachep->freehit);
|
|
|
|
unsigned long freemiss = atomic_read(&cachep->freemiss);
|
|
|
|
|
|
|
|
seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
|
2006-01-08 09:00:37 +00:00
|
|
|
allochit, allocmiss, freehit, freemiss);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
#define MAX_SLABINFO_WRITE 128
|
|
|
|
/**
|
|
|
|
* slabinfo_write - Tuning for the slab allocator
|
|
|
|
* @file: unused
|
|
|
|
* @buffer: user buffer
|
|
|
|
* @count: data length
|
|
|
|
* @ppos: unused
|
|
|
|
*/
|
2012-10-19 14:20:25 +00:00
|
|
|
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
|
2006-01-08 09:00:37 +00:00
|
|
|
size_t count, loff_t *ppos)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-01-08 09:00:37 +00:00
|
|
|
char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
|
2005-04-16 22:20:36 +00:00
|
|
|
int limit, batchcount, shared, res;
|
2006-06-23 09:03:17 +00:00
|
|
|
struct kmem_cache *cachep;
|
2006-01-08 09:00:37 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (count > MAX_SLABINFO_WRITE)
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&kbuf, buffer, count))
|
|
|
|
return -EFAULT;
|
2006-01-08 09:00:37 +00:00
|
|
|
kbuf[MAX_SLABINFO_WRITE] = '\0';
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
tmp = strchr(kbuf, ' ');
|
|
|
|
if (!tmp)
|
|
|
|
return -EINVAL;
|
|
|
|
*tmp = '\0';
|
|
|
|
tmp++;
|
|
|
|
if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Find the cache in the chain of caches. */
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
res = -EINVAL;
|
2012-07-06 20:25:12 +00:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!strcmp(cachep->name, kbuf)) {
|
2006-03-22 08:08:11 +00:00
|
|
|
if (limit < 1 || batchcount < 1 ||
|
|
|
|
batchcount > limit || shared < 0) {
|
2005-09-09 20:03:32 +00:00
|
|
|
res = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2005-09-09 20:03:32 +00:00
|
|
|
res = do_tune_cpucache(cachep, limit,
|
2009-06-10 16:40:04 +00:00
|
|
|
batchcount, shared,
|
|
|
|
GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_unlock(&slab_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (res >= 0)
|
|
|
|
res = count;
|
|
|
|
return res;
|
|
|
|
}
|
2006-03-25 11:06:39 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB_LEAK
|
|
|
|
|
|
|
|
static inline int add_caller(unsigned long *n, unsigned long v)
|
|
|
|
{
|
|
|
|
unsigned long *p;
|
|
|
|
int l;
|
|
|
|
if (!v)
|
|
|
|
return 1;
|
|
|
|
l = n[1];
|
|
|
|
p = n + 2;
|
|
|
|
while (l) {
|
|
|
|
int i = l/2;
|
|
|
|
unsigned long *q = p + 2 * i;
|
|
|
|
if (*q == v) {
|
|
|
|
q[1]++;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
if (*q > v) {
|
|
|
|
l = i;
|
|
|
|
} else {
|
|
|
|
p = q + 2;
|
|
|
|
l -= i + 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (++n[1] == n[0])
|
|
|
|
return 0;
|
|
|
|
memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
|
|
|
|
p[0] = v;
|
|
|
|
p[1] = 1;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2013-10-24 01:07:49 +00:00
|
|
|
static void handle_slab(unsigned long *n, struct kmem_cache *c,
|
|
|
|
struct page *page)
|
2006-03-25 11:06:39 +00:00
|
|
|
{
|
|
|
|
void *p;
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
int i, j;
|
|
|
|
unsigned long v;
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-24 01:07:45 +00:00
|
|
|
|
2006-03-25 11:06:39 +00:00
|
|
|
if (n[0] == n[1])
|
|
|
|
return;
|
2013-10-24 01:07:49 +00:00
|
|
|
for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
bool active = true;
|
|
|
|
|
|
|
|
for (j = page->active; j < c->num; j++) {
|
|
|
|
if (get_free_obj(page, j) == i) {
|
|
|
|
active = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!active)
|
2006-03-25 11:06:39 +00:00
|
|
|
continue;
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-24 01:07:45 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
/*
|
|
|
|
* probe_kernel_read() is used for DEBUG_PAGEALLOC. page table
|
|
|
|
* mapping is established when actual object allocation and
|
|
|
|
* we could mistakenly access the unmapped object in the cpu
|
|
|
|
* cache.
|
|
|
|
*/
|
|
|
|
if (probe_kernel_read(&v, dbg_userword(c, p), sizeof(v)))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!add_caller(n, v))
|
2006-03-25 11:06:39 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void show_symbol(struct seq_file *m, unsigned long address)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_KALLSYMS
|
|
|
|
unsigned long offset, size;
|
2007-07-17 11:03:51 +00:00
|
|
|
char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
|
2006-03-25 11:06:39 +00:00
|
|
|
|
2007-05-08 07:28:47 +00:00
|
|
|
if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
|
2006-03-25 11:06:39 +00:00
|
|
|
seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
|
2007-05-08 07:28:47 +00:00
|
|
|
if (modname[0])
|
2006-03-25 11:06:39 +00:00
|
|
|
seq_printf(m, " [%s]", modname);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
seq_printf(m, "%p", (void *)address);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int leaks_show(struct seq_file *m, void *p)
|
|
|
|
{
|
2012-06-22 17:42:49 +00:00
|
|
|
struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
|
2013-10-24 01:07:49 +00:00
|
|
|
struct page *page;
|
2013-01-10 19:14:19 +00:00
|
|
|
struct kmem_cache_node *n;
|
2006-03-25 11:06:39 +00:00
|
|
|
const char *name;
|
2013-02-05 18:45:23 +00:00
|
|
|
unsigned long *x = m->private;
|
2006-03-25 11:06:39 +00:00
|
|
|
int node;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!(cachep->flags & SLAB_STORE_USER))
|
|
|
|
return 0;
|
|
|
|
if (!(cachep->flags & SLAB_RED_ZONE))
|
|
|
|
return 0;
|
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
/*
|
|
|
|
* Set store_user_clean and start to grab stored user information
|
|
|
|
* for all objects on this cache. If some alloc/free requests comes
|
|
|
|
* during the processing, information would be wrong so restart
|
|
|
|
* whole processing.
|
|
|
|
*/
|
|
|
|
do {
|
|
|
|
set_store_user_clean(cachep);
|
|
|
|
drain_cpu_caches(cachep);
|
|
|
|
|
|
|
|
x[1] = 0;
|
2006-03-25 11:06:39 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2006-03-25 11:06:39 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
check_irq_on();
|
|
|
|
spin_lock_irq(&n->list_lock);
|
2006-03-25 11:06:39 +00:00
|
|
|
|
mm/slab: alternative implementation for DEBUG_SLAB_LEAK
DEBUG_SLAB_LEAK is a debug option. It's current implementation requires
status buffer so we need more memory to use it. And, it cause
kmem_cache initialization step more complex.
To remove this extra memory usage and to simplify initialization step,
this patch implement this feature with another way.
When user requests to get slab object owner information, it marks that
getting information is started. And then, all free objects in caches
are flushed to corresponding slab page. Now, we can distinguish all
freed object so we can know all allocated objects, too. After
collecting slab object owner information on allocated objects, mark is
checked that there is no free during the processing. If true, we can be
sure that our information is correct so information is returned to user.
Although this way is rather complex, it has two important benefits
mentioned above. So, I think it is worth changing.
There is one drawback that it takes more time to get slab object owner
information but it is just a debug option so it doesn't matter at all.
To help review, this patch implements new way only. Following patch
will remove useless code.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 21:54:24 +00:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru)
|
|
|
|
handle_slab(x, cachep, page);
|
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru)
|
|
|
|
handle_slab(x, cachep, page);
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
|
|
|
}
|
|
|
|
} while (!is_store_user_clean(cachep));
|
2006-03-25 11:06:39 +00:00
|
|
|
|
|
|
|
name = cachep->name;
|
2013-02-05 18:45:23 +00:00
|
|
|
if (x[0] == x[1]) {
|
2006-03-25 11:06:39 +00:00
|
|
|
/* Increase the buffer size */
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_unlock(&slab_mutex);
|
2013-02-05 18:45:23 +00:00
|
|
|
m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
|
2006-03-25 11:06:39 +00:00
|
|
|
if (!m->private) {
|
|
|
|
/* Too bad, we are really out */
|
2013-02-05 18:45:23 +00:00
|
|
|
m->private = x;
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
2006-03-25 11:06:39 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2013-02-05 18:45:23 +00:00
|
|
|
*(unsigned long *)m->private = x[0] * 2;
|
|
|
|
kfree(x);
|
2012-07-06 20:25:12 +00:00
|
|
|
mutex_lock(&slab_mutex);
|
2006-03-25 11:06:39 +00:00
|
|
|
/* Now make sure this entry will be retried */
|
|
|
|
m->count = m->size;
|
|
|
|
return 0;
|
|
|
|
}
|
2013-02-05 18:45:23 +00:00
|
|
|
for (i = 0; i < x[1]; i++) {
|
|
|
|
seq_printf(m, "%s: %lu ", name, x[2*i+3]);
|
|
|
|
show_symbol(m, x[2*i+2]);
|
2006-03-25 11:06:39 +00:00
|
|
|
seq_putc(m, '\n');
|
|
|
|
}
|
2006-09-26 06:31:47 +00:00
|
|
|
|
2006-03-25 11:06:39 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-10-05 20:59:10 +00:00
|
|
|
static const struct seq_operations slabstats_op = {
|
2014-12-10 23:42:16 +00:00
|
|
|
.start = slab_start,
|
2013-07-08 00:08:28 +00:00
|
|
|
.next = slab_next,
|
|
|
|
.stop = slab_stop,
|
2006-03-25 11:06:39 +00:00
|
|
|
.show = leaks_show,
|
|
|
|
};
|
2008-10-05 20:59:10 +00:00
|
|
|
|
|
|
|
static int slabstats_open(struct inode *inode, struct file *file)
|
|
|
|
{
|
2014-10-09 22:28:03 +00:00
|
|
|
unsigned long *n;
|
|
|
|
|
|
|
|
n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
|
|
|
|
if (!n)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
*n = PAGE_SIZE / (2 * sizeof(unsigned long));
|
|
|
|
|
|
|
|
return 0;
|
2008-10-05 20:59:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct file_operations proc_slabstats_operations = {
|
|
|
|
.open = slabstats_open,
|
|
|
|
.read = seq_read,
|
|
|
|
.llseek = seq_lseek,
|
|
|
|
.release = seq_release_private,
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static int __init slab_proc_init(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB_LEAK
|
|
|
|
proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
|
2006-03-25 11:06:39 +00:00
|
|
|
#endif
|
2008-10-05 20:59:10 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
module_init(slab_proc_init);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2016-06-23 22:20:59 +00:00
|
|
|
#ifdef CONFIG_HARDENED_USERCOPY
|
|
|
|
/*
|
|
|
|
* Rejects objects that are incorrectly sized.
|
|
|
|
*
|
|
|
|
* Returns NULL if check passes, otherwise const char * to name of cache
|
|
|
|
* to indicate an error.
|
|
|
|
*/
|
|
|
|
const char *__check_heap_object(const void *ptr, unsigned long n,
|
|
|
|
struct page *page)
|
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
unsigned int objnr;
|
|
|
|
unsigned long offset;
|
|
|
|
|
|
|
|
/* Find and validate object. */
|
|
|
|
cachep = page->slab_cache;
|
|
|
|
objnr = obj_to_index(cachep, page, (void *)ptr);
|
|
|
|
BUG_ON(objnr >= cachep->num);
|
|
|
|
|
|
|
|
/* Find offset within object. */
|
|
|
|
offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
|
|
|
|
|
|
|
|
/* Allow address range falling entirely within object size. */
|
|
|
|
if (offset <= cachep->object_size && n <= cachep->object_size - offset)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return cachep->name;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_HARDENED_USERCOPY */
|
|
|
|
|
2005-09-03 22:55:07 +00:00
|
|
|
/**
|
|
|
|
* ksize - get the actual amount of memory allocated for a given object
|
|
|
|
* @objp: Pointer to the object
|
|
|
|
*
|
|
|
|
* kmalloc may internally round up allocations and return more memory
|
|
|
|
* than requested. ksize() can be used to determine the actual amount of
|
|
|
|
* memory allocated. The caller may use this additional memory, even though
|
|
|
|
* a smaller amount of memory was initially specified with the kmalloc call.
|
|
|
|
* The caller must guarantee that objp points to a valid object previously
|
|
|
|
* allocated with either kmalloc() or kmem_cache_alloc(). The object
|
|
|
|
* must not be freed during the duration of the call.
|
|
|
|
*/
|
2007-05-06 21:48:40 +00:00
|
|
|
size_t ksize(const void *objp)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2016-03-25 21:21:59 +00:00
|
|
|
size_t size;
|
|
|
|
|
2007-10-16 08:24:46 +00:00
|
|
|
BUG_ON(!objp);
|
|
|
|
if (unlikely(objp == ZERO_SIZE_PTR))
|
2005-09-03 22:55:07 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-03-25 21:21:59 +00:00
|
|
|
size = virt_to_cache(objp)->object_size;
|
|
|
|
/* We assume that ksize callers could use the whole allocated area,
|
|
|
|
* so we need to unpoison this area.
|
|
|
|
*/
|
2016-05-20 23:59:14 +00:00
|
|
|
kasan_unpoison_shadow(objp, size);
|
2016-03-25 21:21:59 +00:00
|
|
|
|
|
|
|
return size;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-02-10 13:21:44 +00:00
|
|
|
EXPORT_SYMBOL(ksize);
|