2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* linux/mm/swapfile.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
|
|
|
|
* Swap reorganised 29.12.95, Stephen Tweedie
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <linux/mman.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/kernel_stat.h>
|
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/vmalloc.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/namei.h>
|
2011-06-27 23:18:02 +00:00
|
|
|
#include <linux/shmem_fs.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/blkdev.h>
|
2009-01-06 22:39:54 +00:00
|
|
|
#include <linux/random.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/init.h>
|
ksm: let shared pages be swappable
Initial implementation for swapping out KSM's shared pages: add
page_referenced_ksm() and try_to_unmap_ksm(), which rmap.c calls when
faced with a PageKsm page.
Most of what's needed can be got from the rmap_items listed from the
stable_node of the ksm page, without discovering the actual vma: so in
this patch just fake up a struct vma for page_referenced_one() or
try_to_unmap_one(), then refine that in the next patch.
Add VM_NONLINEAR to ksm_madvise()'s list of exclusions: it has always been
implicit there (being only set with VM_SHARED, already excluded), but
let's make it explicit, to help justify the lack of nonlinear unmap.
Rely on the page lock to protect against concurrent modifications to that
page's node of the stable tree.
The awkward part is not swapout but swapin: do_swap_page() and
page_add_anon_rmap() now have to allow for new possibilities - perhaps a
ksm page still in swapcache, perhaps a swapcache page associated with one
location in one anon_vma now needed for another location or anon_vma.
(And the vma might even be no longer VM_MERGEABLE when that happens.)
ksm_might_need_to_copy() checks for that case, and supplies a duplicate
page when necessary, simply leaving it to a subsequent pass of ksmd to
rediscover the identity and merge them back into one ksm page.
Disappointingly primitive: but the alternative would have to accumulate
unswappable info about the swapped out ksm pages, limiting swappability.
Remove page_add_ksm_rmap(): page_add_anon_rmap() now has to allow for the
particular case it was handling, so just use it instead.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Chris Wright <chrisw@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:59:24 +00:00
|
|
|
#include <linux/ksm.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/rmap.h>
|
|
|
|
#include <linux/security.h>
|
|
|
|
#include <linux/backing-dev.h>
|
2006-01-19 01:42:33 +00:00
|
|
|
#include <linux/mutex.h>
|
2006-01-11 20:17:46 +00:00
|
|
|
#include <linux/capability.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/syscalls.h>
|
2008-02-07 08:13:53 +00:00
|
|
|
#include <linux/memcontrol.h>
|
2010-10-26 21:22:06 +00:00
|
|
|
#include <linux/poll.h>
|
2011-05-25 00:11:40 +00:00
|
|
|
#include <linux/oom.h>
|
2012-04-09 23:08:06 +00:00
|
|
|
#include <linux/frontswap.h>
|
|
|
|
#include <linux/swapfile.h>
|
2012-07-31 23:44:47 +00:00
|
|
|
#include <linux/export.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <linux/swapops.h>
|
2009-01-08 02:07:58 +00:00
|
|
|
#include <linux/page_cgroup.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
|
|
|
|
unsigned char);
|
|
|
|
static void free_swap_count_continuations(struct swap_info_struct *);
|
2009-12-15 01:58:49 +00:00
|
|
|
static sector_t map_swap_entry(swp_entry_t, struct block_device**);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
|
2012-04-09 23:08:06 +00:00
|
|
|
DEFINE_SPINLOCK(swap_lock);
|
2008-07-26 02:46:24 +00:00
|
|
|
static unsigned int nr_swapfiles;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_t nr_swap_pages;
|
|
|
|
/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
|
2005-04-16 22:20:36 +00:00
|
|
|
long total_swap_pages;
|
mm: fix ever-decreasing swap priority
Vegard Nossum has noticed the ever-decreasing negative priority in a
swapon /swapoff loop, which eventually would misprioritize when int wraps
positive. Not worth spending much code on, but probably better fixed.
It's easy to handle the swapping on and off of just one area, but there's
not much point if a pair or more still misbehave. To handle the general
case, swapoff should compact negative priorities, keeping them always from
-1 to -MAX_SWAPFILES. That's a change, but should cause no regression,
since these negative (unspecified) priorities are disjoint from the the
positive specified priorities 0 to 32767.
One small functional difference, which seems appropriate: when swapoff
fails to free all swap from a negative priority area, that area is now
reinserted at lowest priority, rather than at its original priority.
In moving down swapon's setting of priority, I notice that an area is
visible to /proc/swaps when it has swap_map set, yet that was being set
before all the visible fields were properly filled in: corrected.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reported-by: Vegard Nossum <vegard.nossum@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-24 04:28:23 +00:00
|
|
|
static int least_priority;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
static atomic_t highest_priority_index = ATOMIC_INIT(-1);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
static const char Bad_file[] = "Bad swap file entry ";
|
|
|
|
static const char Unused_file[] = "Unused swap file entry ";
|
|
|
|
static const char Bad_offset[] = "Bad swap offset entry ";
|
|
|
|
static const char Unused_offset[] = "Unused swap offset entry ";
|
|
|
|
|
2012-04-09 23:08:06 +00:00
|
|
|
struct swap_list_t swap_list = {-1, -1};
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-04-09 23:08:06 +00:00
|
|
|
struct swap_info_struct *swap_info[MAX_SWAPFILES];
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-01-19 01:42:33 +00:00
|
|
|
static DEFINE_MUTEX(swapon_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-10-26 21:22:06 +00:00
|
|
|
static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
|
|
|
|
/* Activity counter to indicate that a swapon or swapoff has occurred */
|
|
|
|
static atomic_t proc_poll_event = ATOMIC_INIT(0);
|
|
|
|
|
2009-12-15 01:58:45 +00:00
|
|
|
static inline unsigned char swap_count(unsigned char ent)
|
2009-06-16 22:32:53 +00:00
|
|
|
{
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
return ent & ~SWAP_HAS_CACHE; /* may include SWAP_HAS_CONT flag */
|
2009-06-16 22:32:53 +00:00
|
|
|
}
|
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
/* returns 1 if swap entry is freed */
|
2009-06-16 22:32:54 +00:00
|
|
|
static int
|
|
|
|
__try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
|
|
|
|
{
|
2009-12-15 01:58:41 +00:00
|
|
|
swp_entry_t entry = swp_entry(si->type, offset);
|
2009-06-16 22:32:54 +00:00
|
|
|
struct page *page;
|
|
|
|
int ret = 0;
|
|
|
|
|
2013-02-23 00:34:37 +00:00
|
|
|
page = find_get_page(swap_address_space(entry), entry.val);
|
2009-06-16 22:32:54 +00:00
|
|
|
if (!page)
|
|
|
|
return 0;
|
|
|
|
/*
|
|
|
|
* This function is called from scan_swap_map() and it's called
|
|
|
|
* by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
|
|
|
|
* We have to use trylock for avoiding deadlock. This is a special
|
|
|
|
* case and you should use try_to_free_swap() with explicit lock_page()
|
|
|
|
* in usual operations.
|
|
|
|
*/
|
|
|
|
if (trylock_page(page)) {
|
|
|
|
ret = try_to_free_swap(page);
|
|
|
|
unlock_page(page);
|
|
|
|
}
|
|
|
|
page_cache_release(page);
|
|
|
|
return ret;
|
|
|
|
}
|
2009-06-16 22:32:53 +00:00
|
|
|
|
2009-01-06 22:39:51 +00:00
|
|
|
/*
|
|
|
|
* swapon tell device that all the old swap contents can be discarded,
|
|
|
|
* to allow the swap device to optimize its wear-levelling.
|
|
|
|
*/
|
|
|
|
static int discard_swap(struct swap_info_struct *si)
|
|
|
|
{
|
|
|
|
struct swap_extent *se;
|
2009-12-15 01:58:42 +00:00
|
|
|
sector_t start_block;
|
|
|
|
sector_t nr_blocks;
|
2009-01-06 22:39:51 +00:00
|
|
|
int err = 0;
|
|
|
|
|
2009-12-15 01:58:42 +00:00
|
|
|
/* Do not discard the swap header page! */
|
|
|
|
se = &si->first_swap_extent;
|
|
|
|
start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
|
|
|
|
nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
|
|
|
|
if (nr_blocks) {
|
|
|
|
err = blkdev_issue_discard(si->bdev, start_block,
|
2010-09-16 18:51:46 +00:00
|
|
|
nr_blocks, GFP_KERNEL, 0);
|
2009-12-15 01:58:42 +00:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
cond_resched();
|
|
|
|
}
|
2009-01-06 22:39:51 +00:00
|
|
|
|
2009-12-15 01:58:42 +00:00
|
|
|
list_for_each_entry(se, &si->first_swap_extent.list, list) {
|
|
|
|
start_block = se->start_block << (PAGE_SHIFT - 9);
|
|
|
|
nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
|
2009-01-06 22:39:51 +00:00
|
|
|
|
|
|
|
err = blkdev_issue_discard(si->bdev, start_block,
|
2010-09-16 18:51:46 +00:00
|
|
|
nr_blocks, GFP_KERNEL, 0);
|
2009-01-06 22:39:51 +00:00
|
|
|
if (err)
|
|
|
|
break;
|
|
|
|
|
|
|
|
cond_resched();
|
|
|
|
}
|
|
|
|
return err; /* That will often be -EOPNOTSUPP */
|
|
|
|
}
|
|
|
|
|
2009-01-06 22:39:53 +00:00
|
|
|
/*
|
|
|
|
* swap allocation tell device that a cluster of swap can now be discarded,
|
|
|
|
* to allow the swap device to optimize its wear-levelling.
|
|
|
|
*/
|
|
|
|
static void discard_swap_cluster(struct swap_info_struct *si,
|
|
|
|
pgoff_t start_page, pgoff_t nr_pages)
|
|
|
|
{
|
|
|
|
struct swap_extent *se = si->curr_swap_extent;
|
|
|
|
int found_extent = 0;
|
|
|
|
|
|
|
|
while (nr_pages) {
|
|
|
|
struct list_head *lh;
|
|
|
|
|
|
|
|
if (se->start_page <= start_page &&
|
|
|
|
start_page < se->start_page + se->nr_pages) {
|
|
|
|
pgoff_t offset = start_page - se->start_page;
|
|
|
|
sector_t start_block = se->start_block + offset;
|
2009-01-06 22:39:56 +00:00
|
|
|
sector_t nr_blocks = se->nr_pages - offset;
|
2009-01-06 22:39:53 +00:00
|
|
|
|
|
|
|
if (nr_blocks > nr_pages)
|
|
|
|
nr_blocks = nr_pages;
|
|
|
|
start_page += nr_blocks;
|
|
|
|
nr_pages -= nr_blocks;
|
|
|
|
|
|
|
|
if (!found_extent++)
|
|
|
|
si->curr_swap_extent = se;
|
|
|
|
|
|
|
|
start_block <<= PAGE_SHIFT - 9;
|
|
|
|
nr_blocks <<= PAGE_SHIFT - 9;
|
|
|
|
if (blkdev_issue_discard(si->bdev, start_block,
|
2010-09-16 18:51:46 +00:00
|
|
|
nr_blocks, GFP_NOIO, 0))
|
2009-01-06 22:39:53 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
lh = se->list.next;
|
|
|
|
se = list_entry(lh, struct swap_extent, list);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int wait_for_discard(void *word)
|
|
|
|
{
|
|
|
|
schedule();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-09-03 22:54:40 +00:00
|
|
|
#define SWAPFILE_CLUSTER 256
|
|
|
|
#define LATENCY_LIMIT 256
|
|
|
|
|
2011-03-22 23:33:38 +00:00
|
|
|
static unsigned long scan_swap_map(struct swap_info_struct *si,
|
|
|
|
unsigned char usage)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-01-06 22:39:50 +00:00
|
|
|
unsigned long offset;
|
2009-01-06 22:39:55 +00:00
|
|
|
unsigned long scan_base;
|
2009-01-06 22:39:53 +00:00
|
|
|
unsigned long last_in_cluster = 0;
|
2005-09-03 22:54:40 +00:00
|
|
|
int latency_ration = LATENCY_LIMIT;
|
2009-01-06 22:39:53 +00:00
|
|
|
int found_free_cluster = 0;
|
2005-09-03 22:54:38 +00:00
|
|
|
|
2009-01-06 22:39:48 +00:00
|
|
|
/*
|
2005-09-03 22:54:38 +00:00
|
|
|
* We try to cluster swap pages by allocating them sequentially
|
|
|
|
* in swap. Once we've allocated SWAPFILE_CLUSTER pages this
|
|
|
|
* way, however, we resort to first-free allocation, starting
|
|
|
|
* a new cluster. This prevents us from scattering swap pages
|
|
|
|
* all over the entire swap partition, so that we reduce
|
|
|
|
* overall disk seek times between swap pages. -- sct
|
|
|
|
* But we do now try to find an empty cluster. -Andrea
|
2009-01-06 22:39:55 +00:00
|
|
|
* And we let swap pages go all over an SSD partition. Hugh
|
2005-09-03 22:54:38 +00:00
|
|
|
*/
|
|
|
|
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
si->flags += SWP_SCANNING;
|
2009-01-06 22:39:55 +00:00
|
|
|
scan_base = offset = si->cluster_next;
|
2009-01-06 22:39:50 +00:00
|
|
|
|
|
|
|
if (unlikely(!si->cluster_nr--)) {
|
|
|
|
if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
|
|
|
|
si->cluster_nr = SWAPFILE_CLUSTER - 1;
|
|
|
|
goto checks;
|
|
|
|
}
|
2009-01-06 22:39:53 +00:00
|
|
|
if (si->flags & SWP_DISCARDABLE) {
|
|
|
|
/*
|
|
|
|
* Start range check on racing allocations, in case
|
|
|
|
* they overlap the cluster we eventually decide on
|
|
|
|
* (we scan without swap_lock to allow preemption).
|
|
|
|
* It's hardly conceivable that cluster_nr could be
|
|
|
|
* wrapped during our scan, but don't depend on it.
|
|
|
|
*/
|
|
|
|
if (si->lowest_alloc)
|
|
|
|
goto checks;
|
|
|
|
si->lowest_alloc = si->max;
|
|
|
|
si->highest_alloc = 0;
|
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2005-09-03 22:54:38 +00:00
|
|
|
|
2009-01-06 22:39:55 +00:00
|
|
|
/*
|
|
|
|
* If seek is expensive, start searching for new cluster from
|
|
|
|
* start of partition, to minimize the span of allocated swap.
|
|
|
|
* But if seek is cheap, search from our current position, so
|
|
|
|
* that swap is allocated from all over the partition: if the
|
|
|
|
* Flash Translation Layer only remaps within limited zones,
|
|
|
|
* we don't want to wear out the first zone too quickly.
|
|
|
|
*/
|
|
|
|
if (!(si->flags & SWP_SOLIDSTATE))
|
|
|
|
scan_base = offset = si->lowest_bit;
|
2005-09-03 22:54:38 +00:00
|
|
|
last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
|
|
|
|
|
|
|
|
/* Locate the first empty (unaligned) cluster */
|
|
|
|
for (; last_in_cluster <= si->highest_bit; offset++) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (si->swap_map[offset])
|
2005-09-03 22:54:38 +00:00
|
|
|
last_in_cluster = offset + SWAPFILE_CLUSTER;
|
|
|
|
else if (offset == last_in_cluster) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-01-06 22:39:50 +00:00
|
|
|
offset -= SWAPFILE_CLUSTER - 1;
|
|
|
|
si->cluster_next = offset;
|
|
|
|
si->cluster_nr = SWAPFILE_CLUSTER - 1;
|
2009-01-06 22:39:53 +00:00
|
|
|
found_free_cluster = 1;
|
2009-01-06 22:39:50 +00:00
|
|
|
goto checks;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-09-03 22:54:40 +00:00
|
|
|
if (unlikely(--latency_ration < 0)) {
|
|
|
|
cond_resched();
|
|
|
|
latency_ration = LATENCY_LIMIT;
|
|
|
|
}
|
2005-09-03 22:54:38 +00:00
|
|
|
}
|
2009-01-06 22:39:50 +00:00
|
|
|
|
|
|
|
offset = si->lowest_bit;
|
2009-01-06 22:39:55 +00:00
|
|
|
last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
|
|
|
|
|
|
|
|
/* Locate the first empty (unaligned) cluster */
|
|
|
|
for (; last_in_cluster < scan_base; offset++) {
|
|
|
|
if (si->swap_map[offset])
|
|
|
|
last_in_cluster = offset + SWAPFILE_CLUSTER;
|
|
|
|
else if (offset == last_in_cluster) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-01-06 22:39:55 +00:00
|
|
|
offset -= SWAPFILE_CLUSTER - 1;
|
|
|
|
si->cluster_next = offset;
|
|
|
|
si->cluster_nr = SWAPFILE_CLUSTER - 1;
|
|
|
|
found_free_cluster = 1;
|
|
|
|
goto checks;
|
|
|
|
}
|
|
|
|
if (unlikely(--latency_ration < 0)) {
|
|
|
|
cond_resched();
|
|
|
|
latency_ration = LATENCY_LIMIT;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
offset = scan_base;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-01-06 22:39:50 +00:00
|
|
|
si->cluster_nr = SWAPFILE_CLUSTER - 1;
|
2009-01-06 22:39:53 +00:00
|
|
|
si->lowest_alloc = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-09-03 22:54:38 +00:00
|
|
|
|
2009-01-06 22:39:50 +00:00
|
|
|
checks:
|
|
|
|
if (!(si->flags & SWP_WRITEOK))
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
goto no_page;
|
2005-09-03 22:54:38 +00:00
|
|
|
if (!si->highest_bit)
|
|
|
|
goto no_page;
|
2009-01-06 22:39:50 +00:00
|
|
|
if (offset > si->highest_bit)
|
2009-01-06 22:39:55 +00:00
|
|
|
scan_base = offset = si->lowest_bit;
|
2009-06-16 22:32:54 +00:00
|
|
|
|
2010-09-09 23:38:09 +00:00
|
|
|
/* reuse swap entry of cache-only swap if not busy. */
|
|
|
|
if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
|
2009-06-16 22:32:54 +00:00
|
|
|
int swap_was_freed;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2009-06-16 22:32:54 +00:00
|
|
|
swap_was_freed = __try_to_reclaim_swap(si, offset);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-06-16 22:32:54 +00:00
|
|
|
/* entry was freed successfully, try to use this again */
|
|
|
|
if (swap_was_freed)
|
|
|
|
goto checks;
|
|
|
|
goto scan; /* check next one */
|
|
|
|
}
|
|
|
|
|
2009-01-06 22:39:50 +00:00
|
|
|
if (si->swap_map[offset])
|
|
|
|
goto scan;
|
|
|
|
|
|
|
|
if (offset == si->lowest_bit)
|
|
|
|
si->lowest_bit++;
|
|
|
|
if (offset == si->highest_bit)
|
|
|
|
si->highest_bit--;
|
|
|
|
si->inuse_pages++;
|
|
|
|
if (si->inuse_pages == si->pages) {
|
|
|
|
si->lowest_bit = si->max;
|
|
|
|
si->highest_bit = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-12-15 01:58:44 +00:00
|
|
|
si->swap_map[offset] = usage;
|
2009-01-06 22:39:50 +00:00
|
|
|
si->cluster_next = offset + 1;
|
|
|
|
si->flags -= SWP_SCANNING;
|
2009-01-06 22:39:53 +00:00
|
|
|
|
|
|
|
if (si->lowest_alloc) {
|
|
|
|
/*
|
|
|
|
* Only set when SWP_DISCARDABLE, and there's a scan
|
|
|
|
* for a free cluster in progress or just completed.
|
|
|
|
*/
|
|
|
|
if (found_free_cluster) {
|
|
|
|
/*
|
|
|
|
* To optimize wear-levelling, discard the
|
|
|
|
* old data of the cluster, taking care not to
|
|
|
|
* discard any of its pages that have already
|
|
|
|
* been allocated by racing tasks (offset has
|
|
|
|
* already stepped over any at the beginning).
|
|
|
|
*/
|
|
|
|
if (offset < si->highest_alloc &&
|
|
|
|
si->lowest_alloc <= last_in_cluster)
|
|
|
|
last_in_cluster = si->lowest_alloc - 1;
|
|
|
|
si->flags |= SWP_DISCARDING;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2009-01-06 22:39:53 +00:00
|
|
|
|
|
|
|
if (offset < last_in_cluster)
|
|
|
|
discard_swap_cluster(si, offset,
|
|
|
|
last_in_cluster - offset + 1);
|
|
|
|
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-01-06 22:39:53 +00:00
|
|
|
si->lowest_alloc = 0;
|
|
|
|
si->flags &= ~SWP_DISCARDING;
|
|
|
|
|
|
|
|
smp_mb(); /* wake_up_bit advises this */
|
|
|
|
wake_up_bit(&si->flags, ilog2(SWP_DISCARDING));
|
|
|
|
|
|
|
|
} else if (si->flags & SWP_DISCARDING) {
|
|
|
|
/*
|
|
|
|
* Delay using pages allocated by racing tasks
|
|
|
|
* until the whole discard has been issued. We
|
|
|
|
* could defer that delay until swap_writepage,
|
|
|
|
* but it's easier to keep this self-contained.
|
|
|
|
*/
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2009-01-06 22:39:53 +00:00
|
|
|
wait_on_bit(&si->flags, ilog2(SWP_DISCARDING),
|
|
|
|
wait_for_discard, TASK_UNINTERRUPTIBLE);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-01-06 22:39:53 +00:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Note pages allocated by racing tasks while
|
|
|
|
* scan for a free cluster is in progress, so
|
|
|
|
* that its final discard can exclude them.
|
|
|
|
*/
|
|
|
|
if (offset < si->lowest_alloc)
|
|
|
|
si->lowest_alloc = offset;
|
|
|
|
if (offset > si->highest_alloc)
|
|
|
|
si->highest_alloc = offset;
|
|
|
|
}
|
|
|
|
}
|
2009-01-06 22:39:50 +00:00
|
|
|
return offset;
|
2005-09-03 22:54:38 +00:00
|
|
|
|
2009-01-06 22:39:50 +00:00
|
|
|
scan:
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2005-09-03 22:54:38 +00:00
|
|
|
while (++offset <= si->highest_bit) {
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
if (!si->swap_map[offset]) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
goto checks;
|
|
|
|
}
|
2009-06-16 22:32:54 +00:00
|
|
|
if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-06-16 22:32:54 +00:00
|
|
|
goto checks;
|
|
|
|
}
|
2005-09-03 22:54:40 +00:00
|
|
|
if (unlikely(--latency_ration < 0)) {
|
|
|
|
cond_resched();
|
|
|
|
latency_ration = LATENCY_LIMIT;
|
|
|
|
}
|
2005-09-03 22:54:38 +00:00
|
|
|
}
|
2009-01-06 22:39:55 +00:00
|
|
|
offset = si->lowest_bit;
|
|
|
|
while (++offset < scan_base) {
|
|
|
|
if (!si->swap_map[offset]) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-01-06 22:39:55 +00:00
|
|
|
goto checks;
|
|
|
|
}
|
2009-06-16 22:32:54 +00:00
|
|
|
if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2009-06-16 22:32:54 +00:00
|
|
|
goto checks;
|
|
|
|
}
|
2009-01-06 22:39:55 +00:00
|
|
|
if (unlikely(--latency_ration < 0)) {
|
|
|
|
cond_resched();
|
|
|
|
latency_ration = LATENCY_LIMIT;
|
|
|
|
}
|
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2005-09-03 22:54:38 +00:00
|
|
|
|
|
|
|
no_page:
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
si->flags -= SWP_SCANNING;
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
swp_entry_t get_swap_page(void)
|
|
|
|
{
|
2005-09-03 22:54:37 +00:00
|
|
|
struct swap_info_struct *si;
|
|
|
|
pgoff_t offset;
|
|
|
|
int type, next;
|
|
|
|
int wrapped = 0;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
int hp_index;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
if (atomic_long_read(&nr_swap_pages) <= 0)
|
2005-09-03 22:54:37 +00:00
|
|
|
goto noswap;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_dec(&nr_swap_pages);
|
2005-09-03 22:54:37 +00:00
|
|
|
|
|
|
|
for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
hp_index = atomic_xchg(&highest_priority_index, -1);
|
|
|
|
/*
|
|
|
|
* highest_priority_index records current highest priority swap
|
|
|
|
* type which just frees swap entries. If its priority is
|
|
|
|
* higher than that of swap_list.next swap type, we use it. It
|
|
|
|
* isn't protected by swap_lock, so it can be an invalid value
|
|
|
|
* if the corresponding swap type is swapoff. We double check
|
|
|
|
* the flags here. It's even possible the swap type is swapoff
|
|
|
|
* and swapon again and its priority is changed. In such rare
|
|
|
|
* case, low prority swap type might be used, but eventually
|
|
|
|
* high priority swap will be used after several rounds of
|
|
|
|
* swap.
|
|
|
|
*/
|
|
|
|
if (hp_index != -1 && hp_index != type &&
|
|
|
|
swap_info[type]->prio < swap_info[hp_index]->prio &&
|
|
|
|
(swap_info[hp_index]->flags & SWP_WRITEOK)) {
|
|
|
|
type = hp_index;
|
|
|
|
swap_list.next = type;
|
|
|
|
}
|
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
si = swap_info[type];
|
2005-09-03 22:54:37 +00:00
|
|
|
next = si->next;
|
|
|
|
if (next < 0 ||
|
2009-12-15 01:58:41 +00:00
|
|
|
(!wrapped && si->prio != swap_info[next]->prio)) {
|
2005-09-03 22:54:37 +00:00
|
|
|
next = swap_list.head;
|
|
|
|
wrapped++;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-09-03 22:54:37 +00:00
|
|
|
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
|
|
|
if (!si->highest_bit) {
|
|
|
|
spin_unlock(&si->lock);
|
2005-09-03 22:54:37 +00:00
|
|
|
continue;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
}
|
|
|
|
if (!(si->flags & SWP_WRITEOK)) {
|
|
|
|
spin_unlock(&si->lock);
|
2005-09-03 22:54:37 +00:00
|
|
|
continue;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
}
|
2005-09-03 22:54:37 +00:00
|
|
|
|
|
|
|
swap_list.next = next;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
|
|
|
|
spin_unlock(&swap_lock);
|
2009-06-16 22:32:53 +00:00
|
|
|
/* This is called for allocating swap entry for cache */
|
2009-12-15 01:58:44 +00:00
|
|
|
offset = scan_swap_map(si, SWAP_HAS_CACHE);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
|
|
|
if (offset)
|
2005-09-03 22:54:37 +00:00
|
|
|
return swp_entry(type, offset);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&swap_lock);
|
2005-09-03 22:54:37 +00:00
|
|
|
next = swap_list.next;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-09-03 22:54:37 +00:00
|
|
|
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_inc(&nr_swap_pages);
|
2005-09-03 22:54:37 +00:00
|
|
|
noswap:
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-09-03 22:54:37 +00:00
|
|
|
return (swp_entry_t) {0};
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2010-09-09 23:38:07 +00:00
|
|
|
/* The only caller of this function is now susupend routine */
|
|
|
|
swp_entry_t get_swap_page_of_type(int type)
|
|
|
|
{
|
|
|
|
struct swap_info_struct *si;
|
|
|
|
pgoff_t offset;
|
|
|
|
|
|
|
|
si = swap_info[type];
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&si->lock);
|
2010-09-09 23:38:07 +00:00
|
|
|
if (si && (si->flags & SWP_WRITEOK)) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_dec(&nr_swap_pages);
|
2010-09-09 23:38:07 +00:00
|
|
|
/* This is called for allocating swap entry, not cache */
|
|
|
|
offset = scan_swap_map(si, 1);
|
|
|
|
if (offset) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2010-09-09 23:38:07 +00:00
|
|
|
return swp_entry(type, offset);
|
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_inc(&nr_swap_pages);
|
2010-09-09 23:38:07 +00:00
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
2010-09-09 23:38:07 +00:00
|
|
|
return (swp_entry_t) {0};
|
|
|
|
}
|
|
|
|
|
2009-12-15 01:58:43 +00:00
|
|
|
static struct swap_info_struct *swap_info_get(swp_entry_t entry)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:43 +00:00
|
|
|
struct swap_info_struct *p;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long offset, type;
|
|
|
|
|
|
|
|
if (!entry.val)
|
|
|
|
goto out;
|
|
|
|
type = swp_type(entry);
|
|
|
|
if (type >= nr_swapfiles)
|
|
|
|
goto bad_nofile;
|
2009-12-15 01:58:41 +00:00
|
|
|
p = swap_info[type];
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!(p->flags & SWP_USED))
|
|
|
|
goto bad_device;
|
|
|
|
offset = swp_offset(entry);
|
|
|
|
if (offset >= p->max)
|
|
|
|
goto bad_offset;
|
|
|
|
if (!p->swap_map[offset])
|
|
|
|
goto bad_free;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
return p;
|
|
|
|
|
|
|
|
bad_free:
|
|
|
|
printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val);
|
|
|
|
goto out;
|
|
|
|
bad_offset:
|
|
|
|
printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val);
|
|
|
|
goto out;
|
|
|
|
bad_device:
|
|
|
|
printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val);
|
|
|
|
goto out;
|
|
|
|
bad_nofile:
|
|
|
|
printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val);
|
|
|
|
out:
|
|
|
|
return NULL;
|
2009-01-06 22:39:48 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
/*
|
|
|
|
* This swap type frees swap entry, check if it is the highest priority swap
|
|
|
|
* type which just frees swap entry. get_swap_page() uses
|
|
|
|
* highest_priority_index to search highest priority swap type. The
|
|
|
|
* swap_info_struct.lock can't protect us if there are multiple swap types
|
|
|
|
* active, so we use atomic_cmpxchg.
|
|
|
|
*/
|
|
|
|
static void set_highest_priority_index(int type)
|
|
|
|
{
|
|
|
|
int old_hp_index, new_hp_index;
|
|
|
|
|
|
|
|
do {
|
|
|
|
old_hp_index = atomic_read(&highest_priority_index);
|
|
|
|
if (old_hp_index != -1 &&
|
|
|
|
swap_info[old_hp_index]->prio >= swap_info[type]->prio)
|
|
|
|
break;
|
|
|
|
new_hp_index = type;
|
|
|
|
} while (atomic_cmpxchg(&highest_priority_index,
|
|
|
|
old_hp_index, new_hp_index) != old_hp_index);
|
|
|
|
}
|
|
|
|
|
2009-12-15 01:58:45 +00:00
|
|
|
static unsigned char swap_entry_free(struct swap_info_struct *p,
|
|
|
|
swp_entry_t entry, unsigned char usage)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:44 +00:00
|
|
|
unsigned long offset = swp_offset(entry);
|
2009-12-15 01:58:45 +00:00
|
|
|
unsigned char count;
|
|
|
|
unsigned char has_cache;
|
2009-06-16 22:32:53 +00:00
|
|
|
|
2009-12-15 01:58:44 +00:00
|
|
|
count = p->swap_map[offset];
|
|
|
|
has_cache = count & SWAP_HAS_CACHE;
|
|
|
|
count &= ~SWAP_HAS_CACHE;
|
2009-06-16 22:32:53 +00:00
|
|
|
|
2009-12-15 01:58:44 +00:00
|
|
|
if (usage == SWAP_HAS_CACHE) {
|
2009-06-16 22:32:53 +00:00
|
|
|
VM_BUG_ON(!has_cache);
|
2009-12-15 01:58:44 +00:00
|
|
|
has_cache = 0;
|
2009-12-15 01:58:47 +00:00
|
|
|
} else if (count == SWAP_MAP_SHMEM) {
|
|
|
|
/*
|
|
|
|
* Or we could insist on shmem.c using a special
|
|
|
|
* swap_shmem_free() and free_shmem_swap_and_cache()...
|
|
|
|
*/
|
|
|
|
count = 0;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
|
|
|
|
if (count == COUNT_CONTINUED) {
|
|
|
|
if (swap_count_continued(p, offset, count))
|
|
|
|
count = SWAP_MAP_MAX | COUNT_CONTINUED;
|
|
|
|
else
|
|
|
|
count = SWAP_MAP_MAX;
|
|
|
|
} else
|
|
|
|
count--;
|
|
|
|
}
|
2009-12-15 01:58:44 +00:00
|
|
|
|
|
|
|
if (!count)
|
|
|
|
mem_cgroup_uncharge_swap(entry);
|
|
|
|
|
|
|
|
usage = count | has_cache;
|
|
|
|
p->swap_map[offset] = usage;
|
2009-06-16 22:32:53 +00:00
|
|
|
|
|
|
|
/* free if no reference */
|
2009-12-15 01:58:44 +00:00
|
|
|
if (!usage) {
|
2009-06-16 22:32:53 +00:00
|
|
|
if (offset < p->lowest_bit)
|
|
|
|
p->lowest_bit = offset;
|
|
|
|
if (offset > p->highest_bit)
|
|
|
|
p->highest_bit = offset;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
set_highest_priority_index(p->type);
|
|
|
|
atomic_long_inc(&nr_swap_pages);
|
2009-06-16 22:32:53 +00:00
|
|
|
p->inuse_pages--;
|
2012-04-09 23:08:06 +00:00
|
|
|
frontswap_invalidate_page(p->type, offset);
|
2012-07-31 23:45:20 +00:00
|
|
|
if (p->flags & SWP_BLKDEV) {
|
|
|
|
struct gendisk *disk = p->bdev->bd_disk;
|
|
|
|
if (disk->fops->swap_slot_free_notify)
|
|
|
|
disk->fops->swap_slot_free_notify(p->bdev,
|
|
|
|
offset);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-12-15 01:58:44 +00:00
|
|
|
|
|
|
|
return usage;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Caller has made sure that the swapdevice corresponding to entry
|
|
|
|
* is still around or has not been recycled.
|
|
|
|
*/
|
|
|
|
void swap_free(swp_entry_t entry)
|
|
|
|
{
|
2009-12-15 01:58:43 +00:00
|
|
|
struct swap_info_struct *p;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
p = swap_info_get(entry);
|
|
|
|
if (p) {
|
2009-12-15 01:58:44 +00:00
|
|
|
swap_entry_free(p, entry, 1);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-06-16 22:32:52 +00:00
|
|
|
/*
|
|
|
|
* Called after dropping swapcache to decrease refcnt to swap entries.
|
|
|
|
*/
|
|
|
|
void swapcache_free(swp_entry_t entry, struct page *page)
|
|
|
|
{
|
2009-06-16 22:32:53 +00:00
|
|
|
struct swap_info_struct *p;
|
2009-12-15 01:58:45 +00:00
|
|
|
unsigned char count;
|
2009-06-16 22:32:53 +00:00
|
|
|
|
|
|
|
p = swap_info_get(entry);
|
|
|
|
if (p) {
|
2009-12-15 01:58:44 +00:00
|
|
|
count = swap_entry_free(p, entry, SWAP_HAS_CACHE);
|
|
|
|
if (page)
|
|
|
|
mem_cgroup_uncharge_swapcache(page, entry, count != 0);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2009-06-16 22:32:53 +00:00
|
|
|
}
|
2009-06-16 22:32:52 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
* How many references to page are currently swapped out?
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
* This does not give an exact answer when swap count is continued,
|
|
|
|
* but does include the high COUNT_CONTINUED flag to allow for that.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
shmem: replace page if mapping excludes its zone
The GMA500 GPU driver uses GEM shmem objects, but with a new twist: the
backing RAM has to be below 4GB. Not a problem while the boards
supported only 4GB: but now Intel's D2700MUD boards support 8GB, and
their GMA3600 is managed by the GMA500 driver.
shmem/tmpfs has never pretended to support hardware restrictions on the
backing memory, but it might have appeared to do so before v3.1, and
even now it works fine until a page is swapped out then back in. When
read_cache_page_gfp() supplied a freshly allocated page for copy, that
compensated for whatever choice might have been made by earlier swapin
readahead; but swapoff was likely to destroy the illusion.
We'd like to continue to support GMA500, so now add a new
shmem_should_replace_page() check on the zone when about to move a page
from swapcache to filecache (in swapin and swapoff cases), with
shmem_replace_page() to allocate and substitute a suitable page (given
gma500/gem.c's mapping_set_gfp_mask GFP_KERNEL | __GFP_DMA32).
This does involve a minor extension to mem_cgroup_replace_page_cache()
(the page may or may not have already been charged); and I've removed a
comment and call to mem_cgroup_uncharge_cache_page(), which in fact is
always a no-op while PageSwapCache.
Also removed optimization of an unlikely path in shmem_getpage_gfp(),
now that we need to check PageSwapCache more carefully (a racing caller
might already have made the copy). And at one point shmem_unuse_inode()
needs to use the hitherto private page_swapcount(), to guard against
racing with inode eviction.
It would make sense to extend shmem_should_replace_page(), to cover
cpuset and NUMA mempolicy restrictions too, but set that aside for now:
needs a cleanup of shmem mempolicy handling, and more testing, and ought
to handle swap faults in do_swap_page() as well as shmem.
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: Christoph Hellwig <hch@infradead.org>
Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Alan Cox <alan@lxorguk.ukuu.org.uk>
Cc: Stephane Marchesin <marcheu@chromium.org>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Dave Airlie <airlied@gmail.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Cc: Rob Clark <rob.clark@linaro.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-05-29 22:06:38 +00:00
|
|
|
int page_swapcount(struct page *page)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
int count = 0;
|
|
|
|
struct swap_info_struct *p;
|
2005-04-16 22:20:36 +00:00
|
|
|
swp_entry_t entry;
|
|
|
|
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:40 +00:00
|
|
|
entry.val = page_private(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
p = swap_info_get(entry);
|
|
|
|
if (p) {
|
2009-06-16 22:32:53 +00:00
|
|
|
count = swap_count(p->swap_map[swp_offset(entry)]);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
return count;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-01-06 22:39:34 +00:00
|
|
|
* We can write to an anon page without COW if there are no other references
|
|
|
|
* to it. And as a side-effect, free up its swap: because the old content
|
|
|
|
* on disk will never be read, and seeking back there to write new content
|
|
|
|
* later would only waste time away from clustering.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-01-06 22:39:34 +00:00
|
|
|
int reuse_swap_page(struct page *page)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
int count;
|
|
|
|
|
2009-01-06 22:39:25 +00:00
|
|
|
VM_BUG_ON(!PageLocked(page));
|
ksm: let shared pages be swappable
Initial implementation for swapping out KSM's shared pages: add
page_referenced_ksm() and try_to_unmap_ksm(), which rmap.c calls when
faced with a PageKsm page.
Most of what's needed can be got from the rmap_items listed from the
stable_node of the ksm page, without discovering the actual vma: so in
this patch just fake up a struct vma for page_referenced_one() or
try_to_unmap_one(), then refine that in the next patch.
Add VM_NONLINEAR to ksm_madvise()'s list of exclusions: it has always been
implicit there (being only set with VM_SHARED, already excluded), but
let's make it explicit, to help justify the lack of nonlinear unmap.
Rely on the page lock to protect against concurrent modifications to that
page's node of the stable tree.
The awkward part is not swapout but swapin: do_swap_page() and
page_add_anon_rmap() now have to allow for new possibilities - perhaps a
ksm page still in swapcache, perhaps a swapcache page associated with one
location in one anon_vma now needed for another location or anon_vma.
(And the vma might even be no longer VM_MERGEABLE when that happens.)
ksm_might_need_to_copy() checks for that case, and supplies a duplicate
page when necessary, simply leaving it to a subsequent pass of ksmd to
rediscover the identity and merge them back into one ksm page.
Disappointingly primitive: but the alternative would have to accumulate
unswappable info about the swapped out ksm pages, limiting swappability.
Remove page_add_ksm_rmap(): page_add_anon_rmap() now has to allow for the
particular case it was handling, so just use it instead.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Chris Wright <chrisw@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:59:24 +00:00
|
|
|
if (unlikely(PageKsm(page)))
|
|
|
|
return 0;
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
count = page_mapcount(page);
|
2009-01-06 22:39:34 +00:00
|
|
|
if (count <= 1 && PageSwapCache(page)) {
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
count += page_swapcount(page);
|
2009-01-06 22:39:34 +00:00
|
|
|
if (count == 1 && !PageWriteback(page)) {
|
|
|
|
delete_from_swap_cache(page);
|
|
|
|
SetPageDirty(page);
|
|
|
|
}
|
|
|
|
}
|
ksm: let shared pages be swappable
Initial implementation for swapping out KSM's shared pages: add
page_referenced_ksm() and try_to_unmap_ksm(), which rmap.c calls when
faced with a PageKsm page.
Most of what's needed can be got from the rmap_items listed from the
stable_node of the ksm page, without discovering the actual vma: so in
this patch just fake up a struct vma for page_referenced_one() or
try_to_unmap_one(), then refine that in the next patch.
Add VM_NONLINEAR to ksm_madvise()'s list of exclusions: it has always been
implicit there (being only set with VM_SHARED, already excluded), but
let's make it explicit, to help justify the lack of nonlinear unmap.
Rely on the page lock to protect against concurrent modifications to that
page's node of the stable tree.
The awkward part is not swapout but swapin: do_swap_page() and
page_add_anon_rmap() now have to allow for new possibilities - perhaps a
ksm page still in swapcache, perhaps a swapcache page associated with one
location in one anon_vma now needed for another location or anon_vma.
(And the vma might even be no longer VM_MERGEABLE when that happens.)
ksm_might_need_to_copy() checks for that case, and supplies a duplicate
page when necessary, simply leaving it to a subsequent pass of ksmd to
rediscover the identity and merge them back into one ksm page.
Disappointingly primitive: but the alternative would have to accumulate
unswappable info about the swapped out ksm pages, limiting swappability.
Remove page_add_ksm_rmap(): page_add_anon_rmap() now has to allow for the
particular case it was handling, so just use it instead.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Chris Wright <chrisw@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:59:24 +00:00
|
|
|
return count <= 1;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-01-06 22:39:36 +00:00
|
|
|
* If swap is getting full, or if there are no more mappings of this page,
|
|
|
|
* then try_to_free_swap is called to free its swap space.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-01-06 22:39:36 +00:00
|
|
|
int try_to_free_swap(struct page *page)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-01-06 22:39:25 +00:00
|
|
|
VM_BUG_ON(!PageLocked(page));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (!PageSwapCache(page))
|
|
|
|
return 0;
|
|
|
|
if (PageWriteback(page))
|
|
|
|
return 0;
|
2009-01-06 22:39:36 +00:00
|
|
|
if (page_swapcount(page))
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
2010-09-09 23:38:09 +00:00
|
|
|
/*
|
|
|
|
* Once hibernation has begun to create its image of memory,
|
|
|
|
* there's a danger that one of the calls to try_to_free_swap()
|
|
|
|
* - most probably a call from __try_to_reclaim_swap() while
|
|
|
|
* hibernation is allocating its own swap pages for the image,
|
|
|
|
* but conceivably even a call from memory reclaim - will free
|
|
|
|
* the swap from a page which has already been recorded in the
|
|
|
|
* image as a clean swapcache page, and then reuse its swap for
|
|
|
|
* another page of the image. On waking from hibernation, the
|
|
|
|
* original page might be freed under memory pressure, then
|
|
|
|
* later read back in from swap, now with the wrong data.
|
|
|
|
*
|
2012-01-10 23:07:15 +00:00
|
|
|
* Hibration suspends storage while it is writing the image
|
|
|
|
* to disk so check that here.
|
2010-09-09 23:38:09 +00:00
|
|
|
*/
|
2012-01-10 23:07:15 +00:00
|
|
|
if (pm_suspended_storage())
|
2010-09-09 23:38:09 +00:00
|
|
|
return 0;
|
|
|
|
|
2009-01-06 22:39:36 +00:00
|
|
|
delete_from_swap_cache(page);
|
|
|
|
SetPageDirty(page);
|
|
|
|
return 1;
|
2008-10-19 03:26:23 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Free the swap entry like above, but also try to
|
|
|
|
* free the page cache entry if it is the last user.
|
|
|
|
*/
|
2009-01-06 22:40:10 +00:00
|
|
|
int free_swap_and_cache(swp_entry_t entry)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-01-06 22:40:10 +00:00
|
|
|
struct swap_info_struct *p;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct page *page = NULL;
|
|
|
|
|
2009-09-16 09:50:05 +00:00
|
|
|
if (non_swap_entry(entry))
|
2009-01-06 22:40:10 +00:00
|
|
|
return 1;
|
[PATCH] Swapless page migration: add R/W migration entries
Implement read/write migration ptes
We take the upper two swapfiles for the two types of migration ptes and define
a series of macros in swapops.h.
The VM is modified to handle the migration entries. migration entries can
only be encountered when the page they are pointing to is locked. This limits
the number of places one has to fix. We also check in copy_pte_range and in
mprotect_pte_range() for migration ptes.
We check for migration ptes in do_swap_cache and call a function that will
then wait on the page lock. This allows us to effectively stop all accesses
to apge.
Migration entries are created by try_to_unmap if called for migration and
removed by local functions in migrate.c
From: Hugh Dickins <hugh@veritas.com>
Several times while testing swapless page migration (I've no NUMA, just
hacking it up to migrate recklessly while running load), I've hit the
BUG_ON(!PageLocked(p)) in migration_entry_to_page.
This comes from an orphaned migration entry, unrelated to the current
correctly locked migration, but hit by remove_anon_migration_ptes as it
checks an address in each vma of the anon_vma list.
Such an orphan may be left behind if an earlier migration raced with fork:
copy_one_pte can duplicate a migration entry from parent to child, after
remove_anon_migration_ptes has checked the child vma, but before it has
removed it from the parent vma. (If the process were later to fault on this
orphaned entry, it would hit the same BUG from migration_entry_wait.)
This could be fixed by locking anon_vma in copy_one_pte, but we'd rather
not. There's no such problem with file pages, because vma_prio_tree_add
adds child vma after parent vma, and the page table locking at each end is
enough to serialize. Follow that example with anon_vma: add new vmas to the
tail instead of the head.
(There's no corresponding problem when inserting migration entries,
because a missed pte will leave the page count and mapcount high, which is
allowed for. And there's no corresponding problem when migrating via swap,
because a leftover swap entry will be correctly faulted. But the swapless
method has no refcounting of its entries.)
From: Ingo Molnar <mingo@elte.hu>
pte_unmap_unlock() takes the pte pointer as an argument.
From: Hugh Dickins <hugh@veritas.com>
Several times while testing swapless page migration, gcc has tried to exec
a pointer instead of a string: smells like COW mappings are not being
properly write-protected on fork.
The protection in copy_one_pte looks very convincing, until at last you
realize that the second arg to make_migration_entry is a boolean "write",
and SWP_MIGRATION_READ is 30.
Anyway, it's better done like in change_pte_range, using
is_write_migration_entry and make_migration_entry_read.
From: Hugh Dickins <hugh@veritas.com>
Remove unnecessary obfuscation from sys_swapon's range check on swap type,
which blew up causing memory corruption once swapless migration made
MAX_SWAPFILES no longer 2 ^ MAX_SWAPFILES_SHIFT.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Christoph Lameter <clameter@engr.sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
From: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 09:03:35 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
p = swap_info_get(entry);
|
|
|
|
if (p) {
|
2009-12-15 01:58:44 +00:00
|
|
|
if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) {
|
2013-02-23 00:34:37 +00:00
|
|
|
page = find_get_page(swap_address_space(entry),
|
|
|
|
entry.val);
|
2008-10-19 03:26:59 +00:00
|
|
|
if (page && !trylock_page(page)) {
|
2006-03-31 10:29:56 +00:00
|
|
|
page_cache_release(page);
|
|
|
|
page = NULL;
|
|
|
|
}
|
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
if (page) {
|
2009-01-06 22:39:36 +00:00
|
|
|
/*
|
|
|
|
* Not mapped elsewhere, or swap space full? Free it!
|
|
|
|
* Also recheck PageSwapCache now page is locked (above).
|
|
|
|
*/
|
2006-03-31 10:29:56 +00:00
|
|
|
if (PageSwapCache(page) && !PageWriteback(page) &&
|
2009-01-06 22:39:36 +00:00
|
|
|
(!page_mapped(page) || vm_swap_full())) {
|
2005-04-16 22:20:36 +00:00
|
|
|
delete_from_swap_cache(page);
|
|
|
|
SetPageDirty(page);
|
|
|
|
}
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
}
|
2009-01-06 22:40:10 +00:00
|
|
|
return p != NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2007-07-29 21:24:36 +00:00
|
|
|
#ifdef CONFIG_HIBERNATION
|
2006-03-23 10:59:59 +00:00
|
|
|
/*
|
2006-12-07 04:34:07 +00:00
|
|
|
* Find the swap type that corresponds to given device (if any).
|
2006-03-23 10:59:59 +00:00
|
|
|
*
|
2006-12-07 04:34:07 +00:00
|
|
|
* @offset - number of the PAGE_SIZE-sized block of the device, starting
|
|
|
|
* from 0, in which the swap header is expected to be located.
|
|
|
|
*
|
|
|
|
* This is needed for the suspend to disk (aka swsusp).
|
2006-03-23 10:59:59 +00:00
|
|
|
*/
|
2007-01-06 00:36:28 +00:00
|
|
|
int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
|
2006-03-23 10:59:59 +00:00
|
|
|
{
|
2006-12-07 04:34:07 +00:00
|
|
|
struct block_device *bdev = NULL;
|
2009-12-15 01:58:41 +00:00
|
|
|
int type;
|
2006-03-23 10:59:59 +00:00
|
|
|
|
2006-12-07 04:34:07 +00:00
|
|
|
if (device)
|
|
|
|
bdev = bdget(device);
|
|
|
|
|
2006-03-23 10:59:59 +00:00
|
|
|
spin_lock(&swap_lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
for (type = 0; type < nr_swapfiles; type++) {
|
|
|
|
struct swap_info_struct *sis = swap_info[type];
|
2006-03-23 10:59:59 +00:00
|
|
|
|
2006-12-07 04:34:07 +00:00
|
|
|
if (!(sis->flags & SWP_WRITEOK))
|
2006-03-23 10:59:59 +00:00
|
|
|
continue;
|
2006-08-27 08:23:25 +00:00
|
|
|
|
2006-12-07 04:34:07 +00:00
|
|
|
if (!bdev) {
|
2007-01-06 00:36:28 +00:00
|
|
|
if (bdev_p)
|
2009-07-29 19:07:55 +00:00
|
|
|
*bdev_p = bdgrab(sis->bdev);
|
2007-01-06 00:36:28 +00:00
|
|
|
|
2006-03-23 11:00:03 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
return type;
|
2006-03-23 11:00:03 +00:00
|
|
|
}
|
2006-12-07 04:34:07 +00:00
|
|
|
if (bdev == sis->bdev) {
|
2009-12-15 01:58:42 +00:00
|
|
|
struct swap_extent *se = &sis->first_swap_extent;
|
2006-12-07 04:34:07 +00:00
|
|
|
|
|
|
|
if (se->start_block == offset) {
|
2007-01-06 00:36:28 +00:00
|
|
|
if (bdev_p)
|
2009-07-29 19:07:55 +00:00
|
|
|
*bdev_p = bdgrab(sis->bdev);
|
2007-01-06 00:36:28 +00:00
|
|
|
|
2006-12-07 04:34:07 +00:00
|
|
|
spin_unlock(&swap_lock);
|
|
|
|
bdput(bdev);
|
2009-12-15 01:58:41 +00:00
|
|
|
return type;
|
2006-12-07 04:34:07 +00:00
|
|
|
}
|
2006-03-23 10:59:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
spin_unlock(&swap_lock);
|
2006-12-07 04:34:07 +00:00
|
|
|
if (bdev)
|
|
|
|
bdput(bdev);
|
|
|
|
|
2006-03-23 10:59:59 +00:00
|
|
|
return -ENODEV;
|
|
|
|
}
|
|
|
|
|
2009-12-15 01:58:43 +00:00
|
|
|
/*
|
|
|
|
* Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
|
|
|
|
* corresponding to given index in swap_info (swap type).
|
|
|
|
*/
|
|
|
|
sector_t swapdev_block(int type, pgoff_t offset)
|
|
|
|
{
|
|
|
|
struct block_device *bdev;
|
|
|
|
|
|
|
|
if ((unsigned int)type >= nr_swapfiles)
|
|
|
|
return 0;
|
|
|
|
if (!(swap_info[type]->flags & SWP_WRITEOK))
|
|
|
|
return 0;
|
2009-12-15 01:58:49 +00:00
|
|
|
return map_swap_entry(swp_entry(type, offset), &bdev);
|
2009-12-15 01:58:43 +00:00
|
|
|
}
|
|
|
|
|
2006-03-23 10:59:59 +00:00
|
|
|
/*
|
|
|
|
* Return either the total number of swap pages of given type, or the number
|
|
|
|
* of free pages of that type (depending on @free)
|
|
|
|
*
|
|
|
|
* This is needed for software suspend
|
|
|
|
*/
|
|
|
|
unsigned int count_swap_pages(int type, int free)
|
|
|
|
{
|
|
|
|
unsigned int n = 0;
|
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
|
|
|
if ((unsigned int)type < nr_swapfiles) {
|
|
|
|
struct swap_info_struct *sis = swap_info[type];
|
|
|
|
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&sis->lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
if (sis->flags & SWP_WRITEOK) {
|
|
|
|
n = sis->pages;
|
2006-03-23 10:59:59 +00:00
|
|
|
if (free)
|
2009-12-15 01:58:41 +00:00
|
|
|
n -= sis->inuse_pages;
|
2006-03-23 10:59:59 +00:00
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&sis->lock);
|
2006-03-23 10:59:59 +00:00
|
|
|
}
|
2009-12-15 01:58:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2006-03-23 10:59:59 +00:00
|
|
|
return n;
|
|
|
|
}
|
2009-12-15 01:58:43 +00:00
|
|
|
#endif /* CONFIG_HIBERNATION */
|
2006-03-23 10:59:59 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2005-10-30 01:15:55 +00:00
|
|
|
* No need to decide whether this PTE shares the swap entry with others,
|
|
|
|
* just let do_wp_page work it out if a write is requested later - to
|
|
|
|
* force COW, vm_page_prot omits write permission from any private vma.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2008-02-07 08:14:04 +00:00
|
|
|
static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long addr, swp_entry_t entry, struct page *page)
|
|
|
|
{
|
2013-02-23 00:36:09 +00:00
|
|
|
struct page *swapcache;
|
2012-01-13 01:18:32 +00:00
|
|
|
struct mem_cgroup *memcg;
|
2008-02-07 08:14:04 +00:00
|
|
|
spinlock_t *ptl;
|
|
|
|
pte_t *pte;
|
|
|
|
int ret = 1;
|
|
|
|
|
2013-02-23 00:36:09 +00:00
|
|
|
swapcache = page;
|
|
|
|
page = ksm_might_need_to_copy(page, vma, addr);
|
|
|
|
if (unlikely(!page))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2012-01-13 01:18:32 +00:00
|
|
|
if (mem_cgroup_try_charge_swapin(vma->vm_mm, page,
|
|
|
|
GFP_KERNEL, &memcg)) {
|
2008-02-07 08:14:04 +00:00
|
|
|
ret = -ENOMEM;
|
2009-01-29 22:25:13 +00:00
|
|
|
goto out_nolock;
|
|
|
|
}
|
2008-02-07 08:14:04 +00:00
|
|
|
|
|
|
|
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
|
|
|
|
if (unlikely(!pte_same(*pte, swp_entry_to_pte(entry)))) {
|
2012-07-31 23:45:28 +00:00
|
|
|
mem_cgroup_cancel_charge_swapin(memcg);
|
2008-02-07 08:14:04 +00:00
|
|
|
ret = 0;
|
|
|
|
goto out;
|
|
|
|
}
|
2008-02-07 08:13:53 +00:00
|
|
|
|
2010-03-05 21:41:42 +00:00
|
|
|
dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
|
2010-03-05 21:41:39 +00:00
|
|
|
inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
|
2005-04-16 22:20:36 +00:00
|
|
|
get_page(page);
|
|
|
|
set_pte_at(vma->vm_mm, addr, pte,
|
|
|
|
pte_mkold(mk_pte(page, vma->vm_page_prot)));
|
2013-02-23 00:36:09 +00:00
|
|
|
if (page == swapcache)
|
|
|
|
page_add_anon_rmap(page, vma, addr);
|
|
|
|
else /* ksm created a completely new copy */
|
|
|
|
page_add_new_anon_rmap(page, vma, addr);
|
2012-01-13 01:18:32 +00:00
|
|
|
mem_cgroup_commit_charge_swapin(page, memcg);
|
2005-04-16 22:20:36 +00:00
|
|
|
swap_free(entry);
|
|
|
|
/*
|
|
|
|
* Move the page to the active list so it is not
|
|
|
|
* immediately swapped out again after swapon.
|
|
|
|
*/
|
|
|
|
activate_page(page);
|
2008-02-07 08:14:04 +00:00
|
|
|
out:
|
|
|
|
pte_unmap_unlock(pte, ptl);
|
2009-01-29 22:25:13 +00:00
|
|
|
out_nolock:
|
2013-02-23 00:36:09 +00:00
|
|
|
if (page != swapcache) {
|
|
|
|
unlock_page(page);
|
|
|
|
put_page(page);
|
|
|
|
}
|
2008-02-07 08:14:04 +00:00
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
|
|
|
|
unsigned long addr, unsigned long end,
|
|
|
|
swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
pte_t swp_pte = swp_entry_to_pte(entry);
|
2005-10-30 01:16:27 +00:00
|
|
|
pte_t *pte;
|
2008-02-07 08:13:53 +00:00
|
|
|
int ret = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-02-07 08:14:04 +00:00
|
|
|
/*
|
|
|
|
* We don't actually need pte lock while scanning for swp_pte: since
|
|
|
|
* we hold page lock and mmap_sem, swp_pte cannot be inserted into the
|
|
|
|
* page table while we're scanning; though it could get zapped, and on
|
|
|
|
* some architectures (e.g. x86_32 with PAE) we might catch a glimpse
|
|
|
|
* of unmatched parts which look like swp_pte, so unuse_pte must
|
|
|
|
* recheck under pte lock. Scanning without pte lock lets it be
|
|
|
|
* preemptible whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE.
|
|
|
|
*/
|
|
|
|
pte = pte_offset_map(pmd, addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
do {
|
|
|
|
/*
|
|
|
|
* swapoff spends a _lot_ of time in this loop!
|
|
|
|
* Test inline before going to call unuse_pte.
|
|
|
|
*/
|
|
|
|
if (unlikely(pte_same(*pte, swp_pte))) {
|
2008-02-07 08:14:04 +00:00
|
|
|
pte_unmap(pte);
|
|
|
|
ret = unuse_pte(vma, pmd, addr, entry, page);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
pte = pte_offset_map(pmd, addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
2008-02-07 08:14:04 +00:00
|
|
|
pte_unmap(pte - 1);
|
|
|
|
out:
|
2008-02-07 08:13:53 +00:00
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
|
|
|
|
unsigned long addr, unsigned long end,
|
|
|
|
swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
unsigned long next;
|
2008-02-07 08:13:53 +00:00
|
|
|
int ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
pmd = pmd_offset(pud, addr);
|
|
|
|
do {
|
|
|
|
next = pmd_addr_end(addr, end);
|
mm: thp: fix pmd_bad() triggering in code paths holding mmap_sem read mode
In some cases it may happen that pmd_none_or_clear_bad() is called with
the mmap_sem hold in read mode. In those cases the huge page faults can
allocate hugepmds under pmd_none_or_clear_bad() and that can trigger a
false positive from pmd_bad() that will not like to see a pmd
materializing as trans huge.
It's not khugepaged causing the problem, khugepaged holds the mmap_sem
in write mode (and all those sites must hold the mmap_sem in read mode
to prevent pagetables to go away from under them, during code review it
seems vm86 mode on 32bit kernels requires that too unless it's
restricted to 1 thread per process or UP builds). The race is only with
the huge pagefaults that can convert a pmd_none() into a
pmd_trans_huge().
Effectively all these pmd_none_or_clear_bad() sites running with
mmap_sem in read mode are somewhat speculative with the page faults, and
the result is always undefined when they run simultaneously. This is
probably why it wasn't common to run into this. For example if the
madvise(MADV_DONTNEED) runs zap_page_range() shortly before the page
fault, the hugepage will not be zapped, if the page fault runs first it
will be zapped.
Altering pmd_bad() not to error out if it finds hugepmds won't be enough
to fix this, because zap_pmd_range would then proceed to call
zap_pte_range (which would be incorrect if the pmd become a
pmd_trans_huge()).
The simplest way to fix this is to read the pmd in the local stack
(regardless of what we read, no need of actual CPU barriers, only
compiler barrier needed), and be sure it is not changing under the code
that computes its value. Even if the real pmd is changing under the
value we hold on the stack, we don't care. If we actually end up in
zap_pte_range it means the pmd was not none already and it was not huge,
and it can't become huge from under us (khugepaged locking explained
above).
All we need is to enforce that there is no way anymore that in a code
path like below, pmd_trans_huge can be false, but pmd_none_or_clear_bad
can run into a hugepmd. The overhead of a barrier() is just a compiler
tweak and should not be measurable (I only added it for THP builds). I
don't exclude different compiler versions may have prevented the race
too by caching the value of *pmd on the stack (that hasn't been
verified, but it wouldn't be impossible considering
pmd_none_or_clear_bad, pmd_bad, pmd_trans_huge, pmd_none are all inlines
and there's no external function called in between pmd_trans_huge and
pmd_none_or_clear_bad).
if (pmd_trans_huge(*pmd)) {
if (next-addr != HPAGE_PMD_SIZE) {
VM_BUG_ON(!rwsem_is_locked(&tlb->mm->mmap_sem));
split_huge_page_pmd(vma->vm_mm, pmd);
} else if (zap_huge_pmd(tlb, vma, pmd, addr))
continue;
/* fall through */
}
if (pmd_none_or_clear_bad(pmd))
Because this race condition could be exercised without special
privileges this was reported in CVE-2012-1179.
The race was identified and fully explained by Ulrich who debugged it.
I'm quoting his accurate explanation below, for reference.
====== start quote =======
mapcount 0 page_mapcount 1
kernel BUG at mm/huge_memory.c:1384!
At some point prior to the panic, a "bad pmd ..." message similar to the
following is logged on the console:
mm/memory.c:145: bad pmd ffff8800376e1f98(80000000314000e7).
The "bad pmd ..." message is logged by pmd_clear_bad() before it clears
the page's PMD table entry.
143 void pmd_clear_bad(pmd_t *pmd)
144 {
-> 145 pmd_ERROR(*pmd);
146 pmd_clear(pmd);
147 }
After the PMD table entry has been cleared, there is an inconsistency
between the actual number of PMD table entries that are mapping the page
and the page's map count (_mapcount field in struct page). When the page
is subsequently reclaimed, __split_huge_page() detects this inconsistency.
1381 if (mapcount != page_mapcount(page))
1382 printk(KERN_ERR "mapcount %d page_mapcount %d\n",
1383 mapcount, page_mapcount(page));
-> 1384 BUG_ON(mapcount != page_mapcount(page));
The root cause of the problem is a race of two threads in a multithreaded
process. Thread B incurs a page fault on a virtual address that has never
been accessed (PMD entry is zero) while Thread A is executing an madvise()
system call on a virtual address within the same 2 MB (huge page) range.
virtual address space
.---------------------.
| |
| |
.-|---------------------|
| | |
| | |<-- B(fault)
| | |
2 MB | |/////////////////////|-.
huge < |/////////////////////| > A(range)
page | |/////////////////////|-'
| | |
| | |
'-|---------------------|
| |
| |
'---------------------'
- Thread A is executing an madvise(..., MADV_DONTNEED) system call
on the virtual address range "A(range)" shown in the picture.
sys_madvise
// Acquire the semaphore in shared mode.
down_read(¤t->mm->mmap_sem)
...
madvise_vma
switch (behavior)
case MADV_DONTNEED:
madvise_dontneed
zap_page_range
unmap_vmas
unmap_page_range
zap_pud_range
zap_pmd_range
//
// Assume that this huge page has never been accessed.
// I.e. content of the PMD entry is zero (not mapped).
//
if (pmd_trans_huge(*pmd)) {
// We don't get here due to the above assumption.
}
//
// Assume that Thread B incurred a page fault and
.---------> // sneaks in here as shown below.
| //
| if (pmd_none_or_clear_bad(pmd))
| {
| if (unlikely(pmd_bad(*pmd)))
| pmd_clear_bad
| {
| pmd_ERROR
| // Log "bad pmd ..." message here.
| pmd_clear
| // Clear the page's PMD entry.
| // Thread B incremented the map count
| // in page_add_new_anon_rmap(), but
| // now the page is no longer mapped
| // by a PMD entry (-> inconsistency).
| }
| }
|
v
- Thread B is handling a page fault on virtual address "B(fault)" shown
in the picture.
...
do_page_fault
__do_page_fault
// Acquire the semaphore in shared mode.
down_read_trylock(&mm->mmap_sem)
...
handle_mm_fault
if (pmd_none(*pmd) && transparent_hugepage_enabled(vma))
// We get here due to the above assumption (PMD entry is zero).
do_huge_pmd_anonymous_page
alloc_hugepage_vma
// Allocate a new transparent huge page here.
...
__do_huge_pmd_anonymous_page
...
spin_lock(&mm->page_table_lock)
...
page_add_new_anon_rmap
// Here we increment the page's map count (starts at -1).
atomic_set(&page->_mapcount, 0)
set_pmd_at
// Here we set the page's PMD entry which will be cleared
// when Thread A calls pmd_clear_bad().
...
spin_unlock(&mm->page_table_lock)
The mmap_sem does not prevent the race because both threads are acquiring
it in shared mode (down_read). Thread B holds the page_table_lock while
the page's map count and PMD table entry are updated. However, Thread A
does not synchronize on that lock.
====== end quote =======
[akpm@linux-foundation.org: checkpatch fixes]
Reported-by: Ulrich Obergfell <uobergfe@redhat.com>
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Jones <davej@redhat.com>
Acked-by: Larry Woodman <lwoodman@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: <stable@vger.kernel.org> [2.6.38+]
Cc: Mark Salter <msalter@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 23:33:42 +00:00
|
|
|
if (pmd_none_or_trans_huge_or_clear_bad(pmd))
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
2008-02-07 08:13:53 +00:00
|
|
|
ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
|
|
|
|
unsigned long addr, unsigned long end,
|
|
|
|
swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
pud_t *pud;
|
|
|
|
unsigned long next;
|
2008-02-07 08:13:53 +00:00
|
|
|
int ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
pud = pud_offset(pgd, addr);
|
|
|
|
do {
|
|
|
|
next = pud_addr_end(addr, end);
|
|
|
|
if (pud_none_or_clear_bad(pud))
|
|
|
|
continue;
|
2008-02-07 08:13:53 +00:00
|
|
|
ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
} while (pud++, addr = next, addr != end);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int unuse_vma(struct vm_area_struct *vma,
|
|
|
|
swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
unsigned long addr, end, next;
|
2008-02-07 08:13:53 +00:00
|
|
|
int ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-12-15 01:58:57 +00:00
|
|
|
if (page_anon_vma(page)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
addr = page_address_in_vma(page, vma);
|
|
|
|
if (addr == -EFAULT)
|
|
|
|
return 0;
|
|
|
|
else
|
|
|
|
end = addr + PAGE_SIZE;
|
|
|
|
} else {
|
|
|
|
addr = vma->vm_start;
|
|
|
|
end = vma->vm_end;
|
|
|
|
}
|
|
|
|
|
|
|
|
pgd = pgd_offset(vma->vm_mm, addr);
|
|
|
|
do {
|
|
|
|
next = pgd_addr_end(addr, end);
|
|
|
|
if (pgd_none_or_clear_bad(pgd))
|
|
|
|
continue;
|
2008-02-07 08:13:53 +00:00
|
|
|
ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int unuse_mm(struct mm_struct *mm,
|
|
|
|
swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
struct vm_area_struct *vma;
|
2008-02-07 08:13:53 +00:00
|
|
|
int ret = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (!down_read_trylock(&mm->mmap_sem)) {
|
|
|
|
/*
|
2008-07-30 05:33:41 +00:00
|
|
|
* Activate page so shrink_inactive_list is unlikely to unmap
|
|
|
|
* its ptes while lock is dropped, so swapoff can make progress.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
[PATCH] can_share_swap_page: use page_mapcount
Remember that ironic get_user_pages race? when the raised page_count on a
page swapped out led do_wp_page to decide that it had to copy on write, so
substituted a different page into userspace. 2.6.7 onwards have Andrea's
solution, where try_to_unmap_one backs out if it finds page_count raised.
Which works, but is unsatisfying (rmap.c has no other page_count heuristics),
and was found a few months ago to hang an intensive page migration test. A
year ago I was hesitant to engage page_mapcount, now it seems the right fix.
So remove the page_count hack from try_to_unmap_one; and use activate_page in
unuse_mm when dropping lock, to replace its secondary effect of helping
swapoff to make progress in that case.
Simplify can_share_swap_page (now called only on anonymous pages) to check
page_mapcount + page_swapcount == 1: still needs the page lock to stabilize
their (pessimistic) sum, but does not need swapper_space.tree_lock for that.
In do_swap_page, move swap_free and unlock_page below page_add_anon_rmap, to
keep sum on the high side, and correct when can_share_swap_page called.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-22 00:15:12 +00:00
|
|
|
activate_page(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
unlock_page(page);
|
|
|
|
down_read(&mm->mmap_sem);
|
|
|
|
lock_page(page);
|
|
|
|
}
|
|
|
|
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
2008-02-07 08:13:53 +00:00
|
|
|
if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
up_read(&mm->mmap_sem);
|
2008-02-07 08:13:53 +00:00
|
|
|
return (ret < 0)? ret: 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2012-04-09 23:08:06 +00:00
|
|
|
* Scan swap_map (or frontswap_map if frontswap parameter is true)
|
|
|
|
* from current position to next entry still in use.
|
2005-04-16 22:20:36 +00:00
|
|
|
* Recycle to start on reaching the end, returning 0 when empty.
|
|
|
|
*/
|
2005-09-03 22:54:35 +00:00
|
|
|
static unsigned int find_next_to_unuse(struct swap_info_struct *si,
|
2012-04-09 23:08:06 +00:00
|
|
|
unsigned int prev, bool frontswap)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2005-09-03 22:54:35 +00:00
|
|
|
unsigned int max = si->max;
|
|
|
|
unsigned int i = prev;
|
2009-12-15 01:58:45 +00:00
|
|
|
unsigned char count;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
* No need for swap_lock here: we're just looking
|
2005-04-16 22:20:36 +00:00
|
|
|
* for whether an entry is in use, not modifying it; false
|
|
|
|
* hits are okay, and sys_swapoff() has already prevented new
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
* allocations from this area (while holding swap_lock).
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
for (;;) {
|
|
|
|
if (++i >= max) {
|
|
|
|
if (!prev) {
|
|
|
|
i = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* No entries in use at top of swap_map,
|
|
|
|
* loop back to start and recheck there.
|
|
|
|
*/
|
|
|
|
max = prev + 1;
|
|
|
|
prev = 0;
|
|
|
|
i = 1;
|
|
|
|
}
|
2012-04-09 23:08:06 +00:00
|
|
|
if (frontswap) {
|
|
|
|
if (frontswap_test(si, i))
|
|
|
|
break;
|
|
|
|
else
|
|
|
|
continue;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
count = si->swap_map[i];
|
2009-06-16 22:32:53 +00:00
|
|
|
if (count && swap_count(count) != SWAP_MAP_BAD)
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We completely avoid races by reading each swap page in advance,
|
|
|
|
* and then search for the process using it. All the necessary
|
|
|
|
* page table adjustments can then be made atomically.
|
2012-04-09 23:08:06 +00:00
|
|
|
*
|
|
|
|
* if the boolean frontswap is true, only unuse pages_to_unuse pages;
|
|
|
|
* pages_to_unuse==0 means all pages; ignored if frontswap is false
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2012-04-09 23:08:06 +00:00
|
|
|
int try_to_unuse(unsigned int type, bool frontswap,
|
|
|
|
unsigned long pages_to_unuse)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:41 +00:00
|
|
|
struct swap_info_struct *si = swap_info[type];
|
2005-04-16 22:20:36 +00:00
|
|
|
struct mm_struct *start_mm;
|
2009-12-15 01:58:45 +00:00
|
|
|
unsigned char *swap_map;
|
|
|
|
unsigned char swcount;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct page *page;
|
|
|
|
swp_entry_t entry;
|
2005-09-03 22:54:35 +00:00
|
|
|
unsigned int i = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
int retval = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When searching mms for an entry, a good strategy is to
|
|
|
|
* start at the first mm we freed the previous entry from
|
|
|
|
* (though actually we don't notice whether we or coincidence
|
|
|
|
* freed the entry). Initialize this start_mm with a hold.
|
|
|
|
*
|
|
|
|
* A simpler strategy would be to start at the last mm we
|
|
|
|
* freed the previous entry from; but that would take less
|
|
|
|
* advantage of mmlist ordering, which clusters forked mms
|
|
|
|
* together, child after parent. If we race with dup_mmap(), we
|
|
|
|
* prefer to resolve parent before child, lest we miss entries
|
|
|
|
* duplicated after we scanned child: using last mm would invert
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
* that.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
start_mm = &init_mm;
|
|
|
|
atomic_inc(&init_mm.mm_users);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Keep on scanning until all entries have gone. Usually,
|
|
|
|
* one pass through swap_map is enough, but not necessarily:
|
|
|
|
* there are races when an instance of an entry might be missed.
|
|
|
|
*/
|
2012-04-09 23:08:06 +00:00
|
|
|
while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (signal_pending(current)) {
|
|
|
|
retval = -EINTR;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2009-01-06 22:39:48 +00:00
|
|
|
/*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Get a page for the entry, using the existing swap
|
|
|
|
* cache page if there is one. Otherwise, get a clean
|
2009-01-06 22:39:48 +00:00
|
|
|
* page and read the swap into it.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
swap_map = &si->swap_map[i];
|
|
|
|
entry = swp_entry(type, i);
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 06:28:42 +00:00
|
|
|
page = read_swap_cache_async(entry,
|
|
|
|
GFP_HIGHUSER_MOVABLE, NULL, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!page) {
|
|
|
|
/*
|
|
|
|
* Either swap_duplicate() failed because entry
|
|
|
|
* has been freed independently, and will not be
|
|
|
|
* reused since sys_swapoff() already disabled
|
|
|
|
* allocation from here, or alloc_page() failed.
|
|
|
|
*/
|
|
|
|
if (!*swap_map)
|
|
|
|
continue;
|
|
|
|
retval = -ENOMEM;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't hold on to start_mm if it looks like exiting.
|
|
|
|
*/
|
|
|
|
if (atomic_read(&start_mm->mm_users) == 1) {
|
|
|
|
mmput(start_mm);
|
|
|
|
start_mm = &init_mm;
|
|
|
|
atomic_inc(&init_mm.mm_users);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Wait for and lock page. When do_swap_page races with
|
|
|
|
* try_to_unuse, do_swap_page can handle the fault much
|
|
|
|
* faster than try_to_unuse can locate the entry. This
|
|
|
|
* apparently redundant "wait_on_page_locked" lets try_to_unuse
|
|
|
|
* defer to do_swap_page in such a case - in some tests,
|
|
|
|
* do_swap_page and try_to_unuse repeatedly compete.
|
|
|
|
*/
|
|
|
|
wait_on_page_locked(page);
|
|
|
|
wait_on_page_writeback(page);
|
|
|
|
lock_page(page);
|
|
|
|
wait_on_page_writeback(page);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove all references to entry.
|
|
|
|
*/
|
|
|
|
swcount = *swap_map;
|
2009-12-15 01:58:47 +00:00
|
|
|
if (swap_count(swcount) == SWAP_MAP_SHMEM) {
|
|
|
|
retval = shmem_unuse(entry, page);
|
|
|
|
/* page has already been unlocked and released */
|
|
|
|
if (retval < 0)
|
|
|
|
break;
|
|
|
|
continue;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-12-15 01:58:47 +00:00
|
|
|
if (swap_count(swcount) && start_mm != &init_mm)
|
|
|
|
retval = unuse_mm(start_mm, entry, page);
|
|
|
|
|
2009-06-16 22:32:53 +00:00
|
|
|
if (swap_count(*swap_map)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
int set_start_mm = (*swap_map >= swcount);
|
|
|
|
struct list_head *p = &start_mm->mmlist;
|
|
|
|
struct mm_struct *new_start_mm = start_mm;
|
|
|
|
struct mm_struct *prev_mm = start_mm;
|
|
|
|
struct mm_struct *mm;
|
|
|
|
|
|
|
|
atomic_inc(&new_start_mm->mm_users);
|
|
|
|
atomic_inc(&prev_mm->mm_users);
|
|
|
|
spin_lock(&mmlist_lock);
|
2009-12-15 01:58:47 +00:00
|
|
|
while (swap_count(*swap_map) && !retval &&
|
2005-04-16 22:20:36 +00:00
|
|
|
(p = p->next) != &start_mm->mmlist) {
|
|
|
|
mm = list_entry(p, struct mm_struct, mmlist);
|
2006-06-23 09:03:44 +00:00
|
|
|
if (!atomic_inc_not_zero(&mm->mm_users))
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
spin_unlock(&mmlist_lock);
|
|
|
|
mmput(prev_mm);
|
|
|
|
prev_mm = mm;
|
|
|
|
|
|
|
|
cond_resched();
|
|
|
|
|
|
|
|
swcount = *swap_map;
|
2009-06-16 22:32:53 +00:00
|
|
|
if (!swap_count(swcount)) /* any usage ? */
|
2005-04-16 22:20:36 +00:00
|
|
|
;
|
2009-12-15 01:58:47 +00:00
|
|
|
else if (mm == &init_mm)
|
2005-04-16 22:20:36 +00:00
|
|
|
set_start_mm = 1;
|
2009-12-15 01:58:47 +00:00
|
|
|
else
|
2005-04-16 22:20:36 +00:00
|
|
|
retval = unuse_mm(mm, entry, page);
|
2009-06-16 22:32:53 +00:00
|
|
|
|
2009-11-02 16:50:33 +00:00
|
|
|
if (set_start_mm && *swap_map < swcount) {
|
2005-04-16 22:20:36 +00:00
|
|
|
mmput(new_start_mm);
|
|
|
|
atomic_inc(&mm->mm_users);
|
|
|
|
new_start_mm = mm;
|
|
|
|
set_start_mm = 0;
|
|
|
|
}
|
|
|
|
spin_lock(&mmlist_lock);
|
|
|
|
}
|
|
|
|
spin_unlock(&mmlist_lock);
|
|
|
|
mmput(prev_mm);
|
|
|
|
mmput(start_mm);
|
|
|
|
start_mm = new_start_mm;
|
|
|
|
}
|
|
|
|
if (retval) {
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If a reference remains (rare), we would like to leave
|
|
|
|
* the page in the swap cache; but try_to_unmap could
|
|
|
|
* then re-duplicate the entry once we drop page lock,
|
|
|
|
* so we might loop indefinitely; also, that page could
|
|
|
|
* not be swapped out to other storage meanwhile. So:
|
|
|
|
* delete from cache even if there's another reference,
|
|
|
|
* after ensuring that the data has been saved to disk -
|
|
|
|
* since if the reference remains (rarer), it will be
|
|
|
|
* read from disk into another page. Splitting into two
|
|
|
|
* pages would be incorrect if swap supported "shared
|
|
|
|
* private" pages, but they are handled by tmpfs files.
|
ksm: let shared pages be swappable
Initial implementation for swapping out KSM's shared pages: add
page_referenced_ksm() and try_to_unmap_ksm(), which rmap.c calls when
faced with a PageKsm page.
Most of what's needed can be got from the rmap_items listed from the
stable_node of the ksm page, without discovering the actual vma: so in
this patch just fake up a struct vma for page_referenced_one() or
try_to_unmap_one(), then refine that in the next patch.
Add VM_NONLINEAR to ksm_madvise()'s list of exclusions: it has always been
implicit there (being only set with VM_SHARED, already excluded), but
let's make it explicit, to help justify the lack of nonlinear unmap.
Rely on the page lock to protect against concurrent modifications to that
page's node of the stable tree.
The awkward part is not swapout but swapin: do_swap_page() and
page_add_anon_rmap() now have to allow for new possibilities - perhaps a
ksm page still in swapcache, perhaps a swapcache page associated with one
location in one anon_vma now needed for another location or anon_vma.
(And the vma might even be no longer VM_MERGEABLE when that happens.)
ksm_might_need_to_copy() checks for that case, and supplies a duplicate
page when necessary, simply leaving it to a subsequent pass of ksmd to
rediscover the identity and merge them back into one ksm page.
Disappointingly primitive: but the alternative would have to accumulate
unswappable info about the swapped out ksm pages, limiting swappability.
Remove page_add_ksm_rmap(): page_add_anon_rmap() now has to allow for the
particular case it was handling, so just use it instead.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Chris Wright <chrisw@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:59:24 +00:00
|
|
|
*
|
|
|
|
* Given how unuse_vma() targets one particular offset
|
|
|
|
* in an anon_vma, once the anon_vma has been determined,
|
|
|
|
* this splitting happens to be just what is needed to
|
|
|
|
* handle where KSM pages have been swapped out: re-reading
|
|
|
|
* is unnecessarily slow, but we can fix that later on.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-06-16 22:32:53 +00:00
|
|
|
if (swap_count(*swap_map) &&
|
|
|
|
PageDirty(page) && PageSwapCache(page)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
struct writeback_control wbc = {
|
|
|
|
.sync_mode = WB_SYNC_NONE,
|
|
|
|
};
|
|
|
|
|
|
|
|
swap_writepage(page, &wbc);
|
|
|
|
lock_page(page);
|
|
|
|
wait_on_page_writeback(page);
|
|
|
|
}
|
2009-01-06 22:39:37 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* It is conceivable that a racing task removed this page from
|
|
|
|
* swap cache just before we acquired the page lock at the top,
|
|
|
|
* or while we dropped it in unuse_mm(). The page might even
|
|
|
|
* be back in swap cache on another swap area: that we must not
|
|
|
|
* delete, since it may not have been written out to swap yet.
|
|
|
|
*/
|
|
|
|
if (PageSwapCache(page) &&
|
|
|
|
likely(page_private(page) == entry.val))
|
2008-02-05 06:28:53 +00:00
|
|
|
delete_from_swap_cache(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* So we could skip searching mms once swap count went
|
|
|
|
* to 1, we did not mark any present ptes as dirty: must
|
2007-07-16 06:38:09 +00:00
|
|
|
* mark page dirty so shrink_page_list will preserve it.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
SetPageDirty(page);
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure that we aren't completely killing
|
|
|
|
* interactive performance.
|
|
|
|
*/
|
|
|
|
cond_resched();
|
2012-04-09 23:08:06 +00:00
|
|
|
if (frontswap && pages_to_unuse > 0) {
|
|
|
|
if (!--pages_to_unuse)
|
|
|
|
break;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
mmput(start_mm);
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
* After a successful try_to_unuse, if no swap is now in use, we know
|
|
|
|
* we can empty the mmlist. swap_lock must be held on entry and exit.
|
|
|
|
* Note that mmlist_lock nests inside swap_lock, and an mm must be
|
2005-04-16 22:20:36 +00:00
|
|
|
* added to the mmlist just after page_duplicate - before would be racy.
|
|
|
|
*/
|
|
|
|
static void drain_mmlist(void)
|
|
|
|
{
|
|
|
|
struct list_head *p, *next;
|
2009-12-15 01:58:41 +00:00
|
|
|
unsigned int type;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
for (type = 0; type < nr_swapfiles; type++)
|
|
|
|
if (swap_info[type]->inuse_pages)
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
|
|
|
spin_lock(&mmlist_lock);
|
|
|
|
list_for_each_safe(p, next, &init_mm.mmlist)
|
|
|
|
list_del_init(p);
|
|
|
|
spin_unlock(&mmlist_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Use this swapdev's extent info to locate the (PAGE_SIZE) block which
|
2009-12-15 01:58:49 +00:00
|
|
|
* corresponds to page offset for the specified swap entry.
|
|
|
|
* Note that the type of this function is sector_t, but it returns page offset
|
|
|
|
* into the bdev, not sector offset.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-12-15 01:58:49 +00:00
|
|
|
static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:40 +00:00
|
|
|
struct swap_info_struct *sis;
|
|
|
|
struct swap_extent *start_se;
|
|
|
|
struct swap_extent *se;
|
|
|
|
pgoff_t offset;
|
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
sis = swap_info[swp_type(entry)];
|
2009-12-15 01:58:40 +00:00
|
|
|
*bdev = sis->bdev;
|
|
|
|
|
|
|
|
offset = swp_offset(entry);
|
|
|
|
start_se = sis->curr_swap_extent;
|
|
|
|
se = start_se;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
for ( ; ; ) {
|
|
|
|
struct list_head *lh;
|
|
|
|
|
|
|
|
if (se->start_page <= offset &&
|
|
|
|
offset < (se->start_page + se->nr_pages)) {
|
|
|
|
return se->start_block + (offset - se->start_page);
|
|
|
|
}
|
2005-09-03 22:54:34 +00:00
|
|
|
lh = se->list.next;
|
2005-04-16 22:20:36 +00:00
|
|
|
se = list_entry(lh, struct swap_extent, list);
|
|
|
|
sis->curr_swap_extent = se;
|
|
|
|
BUG_ON(se == start_se); /* It *must* be present */
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-12-15 01:58:49 +00:00
|
|
|
/*
|
|
|
|
* Returns the page offset into bdev for the specified page's swap entry.
|
|
|
|
*/
|
|
|
|
sector_t map_swap_page(struct page *page, struct block_device **bdev)
|
|
|
|
{
|
|
|
|
swp_entry_t entry;
|
|
|
|
entry.val = page_private(page);
|
|
|
|
return map_swap_entry(entry, bdev);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Free all of a swapdev's extent information
|
|
|
|
*/
|
|
|
|
static void destroy_swap_extents(struct swap_info_struct *sis)
|
|
|
|
{
|
2009-12-15 01:58:42 +00:00
|
|
|
while (!list_empty(&sis->first_swap_extent.list)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
struct swap_extent *se;
|
|
|
|
|
2009-12-15 01:58:42 +00:00
|
|
|
se = list_entry(sis->first_swap_extent.list.next,
|
2005-04-16 22:20:36 +00:00
|
|
|
struct swap_extent, list);
|
|
|
|
list_del(&se->list);
|
|
|
|
kfree(se);
|
|
|
|
}
|
2012-07-31 23:44:55 +00:00
|
|
|
|
|
|
|
if (sis->flags & SWP_FILE) {
|
|
|
|
struct file *swap_file = sis->swap_file;
|
|
|
|
struct address_space *mapping = swap_file->f_mapping;
|
|
|
|
|
|
|
|
sis->flags &= ~SWP_FILE;
|
|
|
|
mapping->a_ops->swap_deactivate(swap_file);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add a block range (and the corresponding page range) into this swapdev's
|
2005-09-03 22:54:34 +00:00
|
|
|
* extent list. The extent list is kept sorted in page order.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2005-09-03 22:54:34 +00:00
|
|
|
* This function rather assumes that it is called in ascending page order.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2012-07-31 23:44:57 +00:00
|
|
|
int
|
2005-04-16 22:20:36 +00:00
|
|
|
add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
|
|
|
|
unsigned long nr_pages, sector_t start_block)
|
|
|
|
{
|
|
|
|
struct swap_extent *se;
|
|
|
|
struct swap_extent *new_se;
|
|
|
|
struct list_head *lh;
|
|
|
|
|
2009-12-15 01:58:42 +00:00
|
|
|
if (start_page == 0) {
|
|
|
|
se = &sis->first_swap_extent;
|
|
|
|
sis->curr_swap_extent = se;
|
|
|
|
se->start_page = 0;
|
|
|
|
se->nr_pages = nr_pages;
|
|
|
|
se->start_block = start_block;
|
|
|
|
return 1;
|
|
|
|
} else {
|
|
|
|
lh = sis->first_swap_extent.list.prev; /* Highest extent */
|
2005-04-16 22:20:36 +00:00
|
|
|
se = list_entry(lh, struct swap_extent, list);
|
2005-09-03 22:54:34 +00:00
|
|
|
BUG_ON(se->start_page + se->nr_pages != start_page);
|
|
|
|
if (se->start_block + se->nr_pages == start_block) {
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Merge it */
|
|
|
|
se->nr_pages += nr_pages;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No merge. Insert a new extent, preserving ordering.
|
|
|
|
*/
|
|
|
|
new_se = kmalloc(sizeof(*se), GFP_KERNEL);
|
|
|
|
if (new_se == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
new_se->start_page = start_page;
|
|
|
|
new_se->nr_pages = nr_pages;
|
|
|
|
new_se->start_block = start_block;
|
|
|
|
|
2009-12-15 01:58:42 +00:00
|
|
|
list_add_tail(&new_se->list, &sis->first_swap_extent.list);
|
2005-09-03 22:54:34 +00:00
|
|
|
return 1;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A `swap extent' is a simple thing which maps a contiguous range of pages
|
|
|
|
* onto a contiguous range of disk blocks. An ordered list of swap extents
|
|
|
|
* is built at swapon time and is then used at swap_writepage/swap_readpage
|
|
|
|
* time for locating where on disk a page belongs.
|
|
|
|
*
|
|
|
|
* If the swapfile is an S_ISBLK block device, a single extent is installed.
|
|
|
|
* This is done so that the main operating code can treat S_ISBLK and S_ISREG
|
|
|
|
* swap files identically.
|
|
|
|
*
|
|
|
|
* Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
|
|
|
|
* extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
|
|
|
|
* swapfiles are handled *identically* after swapon time.
|
|
|
|
*
|
|
|
|
* For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
|
|
|
|
* and will parse them into an ordered extent list, in PAGE_SIZE chunks. If
|
|
|
|
* some stray blocks are found which do not fall within the PAGE_SIZE alignment
|
|
|
|
* requirements, they are simply tossed out - we will never use those blocks
|
|
|
|
* for swapping.
|
|
|
|
*
|
2005-09-03 22:54:31 +00:00
|
|
|
* For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This
|
2005-04-16 22:20:36 +00:00
|
|
|
* prevents root from shooting her foot off by ftruncating an in-use swapfile,
|
|
|
|
* which will scribble on the fs.
|
|
|
|
*
|
|
|
|
* The amount of disk space which a single swap extent represents varies.
|
|
|
|
* Typically it is in the 1-4 megabyte range. So we can have hundreds of
|
|
|
|
* extents in the list. To avoid much list walking, we cache the previous
|
|
|
|
* search location in `curr_swap_extent', and start new searches from there.
|
|
|
|
* This is extremely effective. The average number of iterations in
|
|
|
|
* map_swap_page() has been measured at about 0.3 per page. - akpm.
|
|
|
|
*/
|
2005-09-03 22:54:34 +00:00
|
|
|
static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-07-31 23:44:55 +00:00
|
|
|
struct file *swap_file = sis->swap_file;
|
|
|
|
struct address_space *mapping = swap_file->f_mapping;
|
|
|
|
struct inode *inode = mapping->host;
|
2005-04-16 22:20:36 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (S_ISBLK(inode->i_mode)) {
|
|
|
|
ret = add_swap_extent(sis, 0, sis->max, 0);
|
2005-09-03 22:54:34 +00:00
|
|
|
*span = sis->pages;
|
2012-07-31 23:44:57 +00:00
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-07-31 23:44:55 +00:00
|
|
|
if (mapping->a_ops->swap_activate) {
|
2012-07-31 23:44:57 +00:00
|
|
|
ret = mapping->a_ops->swap_activate(sis, swap_file, span);
|
2012-07-31 23:44:55 +00:00
|
|
|
if (!ret) {
|
|
|
|
sis->flags |= SWP_FILE;
|
|
|
|
ret = add_swap_extent(sis, 0, sis->max, 0);
|
|
|
|
*span = sis->pages;
|
|
|
|
}
|
2012-07-31 23:44:57 +00:00
|
|
|
return ret;
|
2012-07-31 23:44:55 +00:00
|
|
|
}
|
|
|
|
|
2012-07-31 23:44:57 +00:00
|
|
|
return generic_swapfile_activate(sis, swap_file, span);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-12-12 00:01:13 +00:00
|
|
|
static void _enable_swap_info(struct swap_info_struct *p, int prio,
|
2013-04-30 22:26:54 +00:00
|
|
|
unsigned char *swap_map)
|
2011-03-22 23:33:37 +00:00
|
|
|
{
|
|
|
|
int i, prev;
|
|
|
|
|
|
|
|
if (prio >= 0)
|
|
|
|
p->prio = prio;
|
|
|
|
else
|
|
|
|
p->prio = --least_priority;
|
|
|
|
p->swap_map = swap_map;
|
|
|
|
p->flags |= SWP_WRITEOK;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_add(p->pages, &nr_swap_pages);
|
2011-03-22 23:33:37 +00:00
|
|
|
total_swap_pages += p->pages;
|
|
|
|
|
|
|
|
/* insert swap space into swap_list: */
|
|
|
|
prev = -1;
|
|
|
|
for (i = swap_list.head; i >= 0; i = swap_info[i]->next) {
|
|
|
|
if (p->prio >= swap_info[i]->prio)
|
|
|
|
break;
|
|
|
|
prev = i;
|
|
|
|
}
|
|
|
|
p->next = i;
|
|
|
|
if (prev < 0)
|
|
|
|
swap_list.head = swap_list.next = p->type;
|
|
|
|
else
|
|
|
|
swap_info[prev]->next = p->type;
|
2012-12-12 00:01:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void enable_swap_info(struct swap_info_struct *p, int prio,
|
|
|
|
unsigned char *swap_map,
|
|
|
|
unsigned long *frontswap_map)
|
|
|
|
{
|
2013-04-30 22:26:54 +00:00
|
|
|
frontswap_init(p->type, frontswap_map);
|
2012-12-12 00:01:13 +00:00
|
|
|
spin_lock(&swap_lock);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
2013-04-30 22:26:54 +00:00
|
|
|
_enable_swap_info(p, prio, swap_map);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2012-12-12 00:01:13 +00:00
|
|
|
spin_unlock(&swap_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void reinsert_swap_info(struct swap_info_struct *p)
|
|
|
|
{
|
|
|
|
spin_lock(&swap_lock);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
2013-04-30 22:26:54 +00:00
|
|
|
_enable_swap_info(p, p->prio, p->swap_map);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2011-03-22 23:33:37 +00:00
|
|
|
spin_unlock(&swap_lock);
|
|
|
|
}
|
|
|
|
|
2009-01-14 13:14:28 +00:00
|
|
|
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:43 +00:00
|
|
|
struct swap_info_struct *p = NULL;
|
2009-12-15 01:58:45 +00:00
|
|
|
unsigned char *swap_map;
|
2013-04-30 22:26:54 +00:00
|
|
|
unsigned long *frontswap_map;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct file *swap_file, *victim;
|
|
|
|
struct address_space *mapping;
|
|
|
|
struct inode *inode;
|
2012-10-10 19:25:28 +00:00
|
|
|
struct filename *pathname;
|
2005-04-16 22:20:36 +00:00
|
|
|
int i, type, prev;
|
|
|
|
int err;
|
2009-01-06 22:39:48 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!capable(CAP_SYS_ADMIN))
|
|
|
|
return -EPERM;
|
|
|
|
|
2012-02-13 03:58:52 +00:00
|
|
|
BUG_ON(!current->mm);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
pathname = getname(specialfile);
|
|
|
|
if (IS_ERR(pathname))
|
2012-11-16 22:14:55 +00:00
|
|
|
return PTR_ERR(pathname);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-10-10 20:43:10 +00:00
|
|
|
victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
err = PTR_ERR(victim);
|
|
|
|
if (IS_ERR(victim))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
mapping = victim->f_mapping;
|
|
|
|
prev = -1;
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
for (type = swap_list.head; type >= 0; type = swap_info[type]->next) {
|
|
|
|
p = swap_info[type];
|
2009-01-06 22:39:48 +00:00
|
|
|
if (p->flags & SWP_WRITEOK) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (p->swap_file->f_mapping == mapping)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
prev = type;
|
|
|
|
}
|
|
|
|
if (type < 0) {
|
|
|
|
err = -EINVAL;
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_dput;
|
|
|
|
}
|
2012-02-13 03:58:52 +00:00
|
|
|
if (!security_vm_enough_memory_mm(current->mm, p->pages))
|
2005-04-16 22:20:36 +00:00
|
|
|
vm_unacct_memory(p->pages);
|
|
|
|
else {
|
|
|
|
err = -ENOMEM;
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_dput;
|
|
|
|
}
|
2009-12-15 01:58:41 +00:00
|
|
|
if (prev < 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
swap_list.head = p->next;
|
2009-12-15 01:58:41 +00:00
|
|
|
else
|
|
|
|
swap_info[prev]->next = p->next;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (type == swap_list.next) {
|
|
|
|
/* just pick something that's safe... */
|
|
|
|
swap_list.next = swap_list.head;
|
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
mm: fix ever-decreasing swap priority
Vegard Nossum has noticed the ever-decreasing negative priority in a
swapon /swapoff loop, which eventually would misprioritize when int wraps
positive. Not worth spending much code on, but probably better fixed.
It's easy to handle the swapping on and off of just one area, but there's
not much point if a pair or more still misbehave. To handle the general
case, swapoff should compact negative priorities, keeping them always from
-1 to -MAX_SWAPFILES. That's a change, but should cause no regression,
since these negative (unspecified) priorities are disjoint from the the
positive specified priorities 0 to 32767.
One small functional difference, which seems appropriate: when swapoff
fails to free all swap from a negative priority area, that area is now
reinserted at lowest priority, rather than at its original priority.
In moving down swapon's setting of priority, I notice that an area is
visible to /proc/swaps when it has swap_map set, yet that was being set
before all the visible fields were properly filled in: corrected.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reported-by: Vegard Nossum <vegard.nossum@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-24 04:28:23 +00:00
|
|
|
if (p->prio < 0) {
|
2009-12-15 01:58:41 +00:00
|
|
|
for (i = p->next; i >= 0; i = swap_info[i]->next)
|
|
|
|
swap_info[i]->prio = p->prio--;
|
mm: fix ever-decreasing swap priority
Vegard Nossum has noticed the ever-decreasing negative priority in a
swapon /swapoff loop, which eventually would misprioritize when int wraps
positive. Not worth spending much code on, but probably better fixed.
It's easy to handle the swapping on and off of just one area, but there's
not much point if a pair or more still misbehave. To handle the general
case, swapoff should compact negative priorities, keeping them always from
-1 to -MAX_SWAPFILES. That's a change, but should cause no regression,
since these negative (unspecified) priorities are disjoint from the the
positive specified priorities 0 to 32767.
One small functional difference, which seems appropriate: when swapoff
fails to free all swap from a negative priority area, that area is now
reinserted at lowest priority, rather than at its original priority.
In moving down swapon's setting of priority, I notice that an area is
visible to /proc/swaps when it has swap_map set, yet that was being set
before all the visible fields were properly filled in: corrected.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reported-by: Vegard Nossum <vegard.nossum@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-24 04:28:23 +00:00
|
|
|
least_priority++;
|
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
atomic_long_sub(p->pages, &nr_swap_pages);
|
2005-04-16 22:20:36 +00:00
|
|
|
total_swap_pages -= p->pages;
|
|
|
|
p->flags &= ~SWP_WRITEOK;
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-09-03 22:54:37 +00:00
|
|
|
|
2012-12-12 00:02:56 +00:00
|
|
|
set_current_oom_origin();
|
2012-04-09 23:08:06 +00:00
|
|
|
err = try_to_unuse(type, false, 0); /* force all pages to be unused */
|
2012-12-12 00:02:56 +00:00
|
|
|
clear_current_oom_origin();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (err) {
|
|
|
|
/* re-insert swap space back into swap_list */
|
2012-12-12 00:01:13 +00:00
|
|
|
reinsert_swap_info(p);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out_dput;
|
|
|
|
}
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
destroy_swap_extents(p);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
if (p->flags & SWP_CONTINUED)
|
|
|
|
free_swap_count_continuations(p);
|
|
|
|
|
2006-01-19 01:42:33 +00:00
|
|
|
mutex_lock(&swapon_mutex);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
drain_mmlist();
|
|
|
|
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
/* wait for anyone still in scan_swap_map */
|
|
|
|
p->highest_bit = 0; /* cuts scans short */
|
|
|
|
while (p->flags >= SWP_SCANNING) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-09-10 07:27:25 +00:00
|
|
|
schedule_timeout_uninterruptible(1);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
[PATCH] swap: scan_swap_map drop swap_device_lock
get_swap_page has often shown up on latency traces, doing lengthy scans while
holding two spinlocks. swap_list_lock is already dropped, now scan_swap_map
drop swap_device_lock before scanning the swap_map.
While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating an
entry, check it's still free after retaking the lock. Avoid dropping the lock
in the expected common path. No barriers beyond the locks, just let the
cookie crumble; highest_bit limit is volatile, but benign.
Guard against swapoff: must check SWP_WRITEOK before allocating, must raise
SWP_SCANNING reference count while in scan_swap_map, swapoff wait for that to
fall - just use schedule_timeout, we don't want to burden scan_swap_map
itself, and it's very unlikely that anyone can really still be in
scan_swap_map once swapoff gets this far.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:39 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
swap_file = p->swap_file;
|
|
|
|
p->swap_file = NULL;
|
|
|
|
p->max = 0;
|
|
|
|
swap_map = p->swap_map;
|
|
|
|
p->swap_map = NULL;
|
|
|
|
p->flags = 0;
|
2013-04-30 22:26:54 +00:00
|
|
|
frontswap_map = frontswap_map_get(p);
|
|
|
|
frontswap_map_set(p, NULL);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2013-04-30 22:26:54 +00:00
|
|
|
frontswap_invalidate_area(type);
|
2006-01-19 01:42:33 +00:00
|
|
|
mutex_unlock(&swapon_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
vfree(swap_map);
|
2013-04-30 22:26:54 +00:00
|
|
|
vfree(frontswap_map);
|
2009-01-08 02:07:58 +00:00
|
|
|
/* Destroy swap account informatin */
|
|
|
|
swap_cgroup_swapoff(type);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
inode = mapping->host;
|
|
|
|
if (S_ISBLK(inode->i_mode)) {
|
|
|
|
struct block_device *bdev = I_BDEV(inode);
|
|
|
|
set_blocksize(bdev, p->old_block_size);
|
block: make blkdev_get/put() handle exclusive access
Over time, block layer has accumulated a set of APIs dealing with bdev
open, close, claim and release.
* blkdev_get/put() are the primary open and close functions.
* bd_claim/release() deal with exclusive open.
* open/close_bdev_exclusive() are combination of open and claim and
the other way around, respectively.
* bd_link/unlink_disk_holder() to create and remove holder/slave
symlinks.
* open_by_devnum() wraps bdget() + blkdev_get().
The interface is a bit confusing and the decoupling of open and claim
makes it impossible to properly guarantee exclusive access as
in-kernel open + claim sequence can disturb the existing exclusive
open even before the block layer knows the current open if for another
exclusive access. Reorganize the interface such that,
* blkdev_get() is extended to include exclusive access management.
@holder argument is added and, if is @FMODE_EXCL specified, it will
gain exclusive access atomically w.r.t. other exclusive accesses.
* blkdev_put() is similarly extended. It now takes @mode argument and
if @FMODE_EXCL is set, it releases an exclusive access. Also, when
the last exclusive claim is released, the holder/slave symlinks are
removed automatically.
* bd_claim/release() and close_bdev_exclusive() are no longer
necessary and either made static or removed.
* bd_link_disk_holder() remains the same but bd_unlink_disk_holder()
is no longer necessary and removed.
* open_bdev_exclusive() becomes a simple wrapper around lookup_bdev()
and blkdev_get(). It also has an unexpected extra bdev_read_only()
test which probably should be moved into blkdev_get().
* open_by_devnum() is modified to take @holder argument and pass it to
blkdev_get().
Most of bdev open/close operations are unified into blkdev_get/put()
and most exclusive accesses are tested atomically at the open time (as
it should). This cleans up code and removes some, both valid and
invalid, but unnecessary all the same, corner cases.
open_bdev_exclusive() and open_by_devnum() can use further cleanup -
rename to blkdev_get_by_path() and blkdev_get_by_devt() and drop
special features. Well, let's leave them for another day.
Most conversions are straight-forward. drbd conversion is a bit more
involved as there was some reordering, but the logic should stay the
same.
Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Neil Brown <neilb@suse.de>
Acked-by: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>
Acked-by: Mike Snitzer <snitzer@redhat.com>
Acked-by: Philipp Reisner <philipp.reisner@linbit.com>
Cc: Peter Osterlund <petero2@telia.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andreas Dilger <adilger.kernel@dilger.ca>
Cc: "Theodore Ts'o" <tytso@mit.edu>
Cc: Mark Fasheh <mfasheh@suse.com>
Cc: Joel Becker <joel.becker@oracle.com>
Cc: Alex Elder <aelder@sgi.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: dm-devel@redhat.com
Cc: drbd-dev@lists.linbit.com
Cc: Leo Chen <leochen@broadcom.com>
Cc: Scott Branden <sbranden@broadcom.com>
Cc: Chris Mason <chris.mason@oracle.com>
Cc: Steven Whitehouse <swhiteho@redhat.com>
Cc: Dave Kleikamp <shaggy@linux.vnet.ibm.com>
Cc: Joern Engel <joern@logfs.org>
Cc: reiserfs-devel@vger.kernel.org
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
2010-11-13 10:55:17 +00:00
|
|
|
blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2006-01-09 23:59:24 +00:00
|
|
|
mutex_lock(&inode->i_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
inode->i_flags &= ~S_SWAPFILE;
|
2006-01-09 23:59:24 +00:00
|
|
|
mutex_unlock(&inode->i_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
filp_close(swap_file, NULL);
|
|
|
|
err = 0;
|
2010-10-26 21:22:06 +00:00
|
|
|
atomic_inc(&proc_poll_event);
|
|
|
|
wake_up_interruptible(&proc_poll_wait);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
out_dput:
|
|
|
|
filp_close(victim, NULL);
|
|
|
|
out:
|
2012-11-16 22:14:55 +00:00
|
|
|
putname(pathname);
|
2005-04-16 22:20:36 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_PROC_FS
|
2010-10-26 21:22:06 +00:00
|
|
|
static unsigned swaps_poll(struct file *file, poll_table *wait)
|
|
|
|
{
|
2011-07-12 18:48:39 +00:00
|
|
|
struct seq_file *seq = file->private_data;
|
2010-10-26 21:22:06 +00:00
|
|
|
|
|
|
|
poll_wait(file, &proc_poll_wait, wait);
|
|
|
|
|
2011-07-12 18:48:39 +00:00
|
|
|
if (seq->poll_event != atomic_read(&proc_poll_event)) {
|
|
|
|
seq->poll_event = atomic_read(&proc_poll_event);
|
2010-10-26 21:22:06 +00:00
|
|
|
return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
|
|
|
|
}
|
|
|
|
|
|
|
|
return POLLIN | POLLRDNORM;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* iterator */
|
|
|
|
static void *swap_start(struct seq_file *swap, loff_t *pos)
|
|
|
|
{
|
2009-12-15 01:58:41 +00:00
|
|
|
struct swap_info_struct *si;
|
|
|
|
int type;
|
2005-04-16 22:20:36 +00:00
|
|
|
loff_t l = *pos;
|
|
|
|
|
2006-01-19 01:42:33 +00:00
|
|
|
mutex_lock(&swapon_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-12-07 04:32:28 +00:00
|
|
|
if (!l)
|
|
|
|
return SEQ_START_TOKEN;
|
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
for (type = 0; type < nr_swapfiles; type++) {
|
|
|
|
smp_rmb(); /* read nr_swapfiles before swap_info[type] */
|
|
|
|
si = swap_info[type];
|
|
|
|
if (!(si->flags & SWP_USED) || !si->swap_map)
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
2006-12-07 04:32:28 +00:00
|
|
|
if (!--l)
|
2009-12-15 01:58:41 +00:00
|
|
|
return si;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
|
|
|
|
{
|
2009-12-15 01:58:41 +00:00
|
|
|
struct swap_info_struct *si = v;
|
|
|
|
int type;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-12-07 04:32:28 +00:00
|
|
|
if (v == SEQ_START_TOKEN)
|
2009-12-15 01:58:41 +00:00
|
|
|
type = 0;
|
|
|
|
else
|
|
|
|
type = si->type + 1;
|
2006-12-07 04:32:28 +00:00
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
for (; type < nr_swapfiles; type++) {
|
|
|
|
smp_rmb(); /* read nr_swapfiles before swap_info[type] */
|
|
|
|
si = swap_info[type];
|
|
|
|
if (!(si->flags & SWP_USED) || !si->swap_map)
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
|
|
|
++*pos;
|
2009-12-15 01:58:41 +00:00
|
|
|
return si;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void swap_stop(struct seq_file *swap, void *v)
|
|
|
|
{
|
2006-01-19 01:42:33 +00:00
|
|
|
mutex_unlock(&swapon_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int swap_show(struct seq_file *swap, void *v)
|
|
|
|
{
|
2009-12-15 01:58:41 +00:00
|
|
|
struct swap_info_struct *si = v;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct file *file;
|
|
|
|
int len;
|
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
if (si == SEQ_START_TOKEN) {
|
2006-12-07 04:32:28 +00:00
|
|
|
seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-12-15 01:58:41 +00:00
|
|
|
file = si->swap_file;
|
2008-02-15 03:38:43 +00:00
|
|
|
len = seq_path(swap, &file->f_path, " \t\n\\");
|
2005-09-03 22:54:35 +00:00
|
|
|
seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
|
2009-01-06 22:39:48 +00:00
|
|
|
len < 40 ? 40 - len : 1, " ",
|
2013-01-23 22:07:38 +00:00
|
|
|
S_ISBLK(file_inode(file)->i_mode) ?
|
2005-04-16 22:20:36 +00:00
|
|
|
"partition" : "file\t",
|
2009-12-15 01:58:41 +00:00
|
|
|
si->pages << (PAGE_SHIFT - 10),
|
|
|
|
si->inuse_pages << (PAGE_SHIFT - 10),
|
|
|
|
si->prio);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-12-07 04:40:36 +00:00
|
|
|
static const struct seq_operations swaps_op = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.start = swap_start,
|
|
|
|
.next = swap_next,
|
|
|
|
.stop = swap_stop,
|
|
|
|
.show = swap_show
|
|
|
|
};
|
|
|
|
|
|
|
|
static int swaps_open(struct inode *inode, struct file *file)
|
|
|
|
{
|
2011-07-12 18:48:39 +00:00
|
|
|
struct seq_file *seq;
|
2010-10-26 21:22:06 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = seq_open(file, &swaps_op);
|
2011-07-12 18:48:39 +00:00
|
|
|
if (ret)
|
2010-10-26 21:22:06 +00:00
|
|
|
return ret;
|
|
|
|
|
2011-07-12 18:48:39 +00:00
|
|
|
seq = file->private_data;
|
|
|
|
seq->poll_event = atomic_read(&proc_poll_event);
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-12-07 04:40:36 +00:00
|
|
|
static const struct file_operations proc_swaps_operations = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.open = swaps_open,
|
|
|
|
.read = seq_read,
|
|
|
|
.llseek = seq_lseek,
|
|
|
|
.release = seq_release,
|
2010-10-26 21:22:06 +00:00
|
|
|
.poll = swaps_poll,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static int __init procswaps_init(void)
|
|
|
|
{
|
2008-04-29 08:02:13 +00:00
|
|
|
proc_create("swaps", 0, NULL, &proc_swaps_operations);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
__initcall(procswaps_init);
|
|
|
|
#endif /* CONFIG_PROC_FS */
|
|
|
|
|
2008-12-16 11:35:24 +00:00
|
|
|
#ifdef MAX_SWAPFILES_CHECK
|
|
|
|
static int __init max_swapfiles_check(void)
|
|
|
|
{
|
|
|
|
MAX_SWAPFILES_CHECK();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
late_initcall(max_swapfiles_check);
|
|
|
|
#endif
|
|
|
|
|
2011-03-22 23:33:17 +00:00
|
|
|
static struct swap_info_struct *alloc_swap_info(void)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:43 +00:00
|
|
|
struct swap_info_struct *p;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned int type;
|
2009-12-15 01:58:41 +00:00
|
|
|
|
|
|
|
p = kzalloc(sizeof(*p), GFP_KERNEL);
|
|
|
|
if (!p)
|
2011-03-22 23:33:17 +00:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2009-12-15 01:58:41 +00:00
|
|
|
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
for (type = 0; type < nr_swapfiles; type++) {
|
|
|
|
if (!(swap_info[type]->flags & SWP_USED))
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
2009-12-15 01:58:41 +00:00
|
|
|
}
|
[PATCH] Swapless page migration: add R/W migration entries
Implement read/write migration ptes
We take the upper two swapfiles for the two types of migration ptes and define
a series of macros in swapops.h.
The VM is modified to handle the migration entries. migration entries can
only be encountered when the page they are pointing to is locked. This limits
the number of places one has to fix. We also check in copy_pte_range and in
mprotect_pte_range() for migration ptes.
We check for migration ptes in do_swap_cache and call a function that will
then wait on the page lock. This allows us to effectively stop all accesses
to apge.
Migration entries are created by try_to_unmap if called for migration and
removed by local functions in migrate.c
From: Hugh Dickins <hugh@veritas.com>
Several times while testing swapless page migration (I've no NUMA, just
hacking it up to migrate recklessly while running load), I've hit the
BUG_ON(!PageLocked(p)) in migration_entry_to_page.
This comes from an orphaned migration entry, unrelated to the current
correctly locked migration, but hit by remove_anon_migration_ptes as it
checks an address in each vma of the anon_vma list.
Such an orphan may be left behind if an earlier migration raced with fork:
copy_one_pte can duplicate a migration entry from parent to child, after
remove_anon_migration_ptes has checked the child vma, but before it has
removed it from the parent vma. (If the process were later to fault on this
orphaned entry, it would hit the same BUG from migration_entry_wait.)
This could be fixed by locking anon_vma in copy_one_pte, but we'd rather
not. There's no such problem with file pages, because vma_prio_tree_add
adds child vma after parent vma, and the page table locking at each end is
enough to serialize. Follow that example with anon_vma: add new vmas to the
tail instead of the head.
(There's no corresponding problem when inserting migration entries,
because a missed pte will leave the page count and mapcount high, which is
allowed for. And there's no corresponding problem when migrating via swap,
because a leftover swap entry will be correctly faulted. But the swapless
method has no refcounting of its entries.)
From: Ingo Molnar <mingo@elte.hu>
pte_unmap_unlock() takes the pte pointer as an argument.
From: Hugh Dickins <hugh@veritas.com>
Several times while testing swapless page migration, gcc has tried to exec
a pointer instead of a string: smells like COW mappings are not being
properly write-protected on fork.
The protection in copy_one_pte looks very convincing, until at last you
realize that the second arg to make_migration_entry is a boolean "write",
and SWP_MIGRATION_READ is 30.
Anyway, it's better done like in change_pte_range, using
is_write_migration_entry and make_migration_entry_read.
From: Hugh Dickins <hugh@veritas.com>
Remove unnecessary obfuscation from sys_swapon's range check on swap type,
which blew up causing memory corruption once swapless migration made
MAX_SWAPFILES no longer 2 ^ MAX_SWAPFILES_SHIFT.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Christoph Lameter <clameter@engr.sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
From: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 09:03:35 +00:00
|
|
|
if (type >= MAX_SWAPFILES) {
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
kfree(p);
|
2011-03-22 23:33:19 +00:00
|
|
|
return ERR_PTR(-EPERM);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-12-15 01:58:41 +00:00
|
|
|
if (type >= nr_swapfiles) {
|
|
|
|
p->type = type;
|
|
|
|
swap_info[type] = p;
|
|
|
|
/*
|
|
|
|
* Write swap_info[type] before nr_swapfiles, in case a
|
|
|
|
* racing procfs swap_start() or swap_next() is reading them.
|
|
|
|
* (We never shrink nr_swapfiles, we never free this entry.)
|
|
|
|
*/
|
|
|
|
smp_wmb();
|
|
|
|
nr_swapfiles++;
|
|
|
|
} else {
|
|
|
|
kfree(p);
|
|
|
|
p = swap_info[type];
|
|
|
|
/*
|
|
|
|
* Do not memset this entry: a racing procfs swap_next()
|
|
|
|
* would be relying on p->type to remain valid.
|
|
|
|
*/
|
|
|
|
}
|
2009-12-15 01:58:42 +00:00
|
|
|
INIT_LIST_HEAD(&p->first_swap_extent.list);
|
2005-04-16 22:20:36 +00:00
|
|
|
p->flags = SWP_USED;
|
|
|
|
p->next = -1;
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock_init(&p->lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
|
2011-03-22 23:33:17 +00:00
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2011-03-22 23:33:26 +00:00
|
|
|
static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
|
|
|
|
if (S_ISBLK(inode->i_mode)) {
|
|
|
|
p->bdev = bdgrab(I_BDEV(inode));
|
|
|
|
error = blkdev_get(p->bdev,
|
|
|
|
FMODE_READ | FMODE_WRITE | FMODE_EXCL,
|
|
|
|
sys_swapon);
|
|
|
|
if (error < 0) {
|
|
|
|
p->bdev = NULL;
|
2011-03-22 23:33:27 +00:00
|
|
|
return -EINVAL;
|
2011-03-22 23:33:26 +00:00
|
|
|
}
|
|
|
|
p->old_block_size = block_size(p->bdev);
|
|
|
|
error = set_blocksize(p->bdev, PAGE_SIZE);
|
|
|
|
if (error < 0)
|
2011-03-22 23:33:27 +00:00
|
|
|
return error;
|
2011-03-22 23:33:26 +00:00
|
|
|
p->flags |= SWP_BLKDEV;
|
|
|
|
} else if (S_ISREG(inode->i_mode)) {
|
|
|
|
p->bdev = inode->i_sb->s_bdev;
|
|
|
|
mutex_lock(&inode->i_mutex);
|
2011-03-22 23:33:27 +00:00
|
|
|
if (IS_SWAPFILE(inode))
|
|
|
|
return -EBUSY;
|
|
|
|
} else
|
|
|
|
return -EINVAL;
|
2011-03-22 23:33:26 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-03-22 23:33:29 +00:00
|
|
|
static unsigned long read_swap_header(struct swap_info_struct *p,
|
|
|
|
union swap_header *swap_header,
|
|
|
|
struct inode *inode)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
unsigned long maxpages;
|
|
|
|
unsigned long swapfilepages;
|
|
|
|
|
|
|
|
if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
|
|
|
|
printk(KERN_ERR "Unable to find swap-space signature\n");
|
2011-03-22 23:33:30 +00:00
|
|
|
return 0;
|
2011-03-22 23:33:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* swap partition endianess hack... */
|
|
|
|
if (swab32(swap_header->info.version) == 1) {
|
|
|
|
swab32s(&swap_header->info.version);
|
|
|
|
swab32s(&swap_header->info.last_page);
|
|
|
|
swab32s(&swap_header->info.nr_badpages);
|
|
|
|
for (i = 0; i < swap_header->info.nr_badpages; i++)
|
|
|
|
swab32s(&swap_header->info.badpages[i]);
|
|
|
|
}
|
|
|
|
/* Check the swap header's sub-version */
|
|
|
|
if (swap_header->info.version != 1) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"Unable to handle swap header version %d\n",
|
|
|
|
swap_header->info.version);
|
2011-03-22 23:33:30 +00:00
|
|
|
return 0;
|
2011-03-22 23:33:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
p->lowest_bit = 1;
|
|
|
|
p->cluster_next = 1;
|
|
|
|
p->cluster_nr = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Find out how many pages are allowed for a single swap
|
swap: fix shmem swapping when more than 8 areas
Minchan Kim reports that when a system has many swap areas, and tmpfs
swaps out to the ninth or more, shmem_getpage_gfp()'s attempts to read
back the page cannot locate it, and the read fails with -ENOMEM.
Whoops. Yes, I blindly followed read_swap_header()'s pte_to_swp_entry(
swp_entry_to_pte()) technique for determining maximum usable swap
offset, without stopping to realize that that actually depends upon the
pte swap encoding shifting swap offset to the higher bits and truncating
it there. Whereas our radix_tree swap encoding leaves offset in the
lower bits: it's swap "type" (that is, index of swap area) that was
truncated.
Fix it by reducing the SWP_TYPE_SHIFT() in swapops.h, and removing the
broken radix_to_swp_entry(swp_to_radix_entry()) from read_swap_header().
This does not reduce the usable size of a swap area any further, it
leaves it as claimed when making the original commit: no change from 3.0
on x86_64, nor on i386 without PAE; but 3.0's 512GB is reduced to 128GB
per swapfile on i386 with PAE. It's not a change I would have risked
five years ago, but with x86_64 supported for ten years, I believe it's
appropriate now.
Hmm, and what if some architecture implements its swap pte with offset
encoded below type? That would equally break the maximum usable swap
offset check. Happily, they all follow the same tradition of encoding
offset above type, but I'll prepare a check on that for next.
Reported-and-Reviewed-and-Tested-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org [3.1, 3.2, 3.3, 3.4]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-16 00:55:50 +00:00
|
|
|
* device. There are two limiting factors: 1) the number
|
mm: let swap use exceptional entries
If swap entries are to be stored along with struct page pointers in a
radix tree, they need to be distinguished as exceptional entries.
Most of the handling of swap entries in radix tree will be contained in
shmem.c, but a few functions in filemap.c's common code need to check
for their appearance: find_get_page(), find_lock_page(),
find_get_pages() and find_get_pages_contig().
So as not to slow their fast paths, tuck those checks inside the
existing checks for unlikely radix_tree_deref_slot(); except for
find_lock_page(), where it is an added test. And make it a BUG in
find_get_pages_tag(), which is not applied to tmpfs files.
A part of the reason for eliminating shmem_readpage() earlier, was to
minimize the places where common code would need to allow for swap
entries.
The swp_entry_t known to swapfile.c must be massaged into a slightly
different form when stored in the radix tree, just as it gets massaged
into a pte_t when stored in page tables.
In an i386 kernel this limits its information (type and page offset) to
30 bits: given 32 "types" of swapfile and 4kB pagesize, that's a maximum
swapfile size of 128GB. Which is less than the 512GB we previously
allowed with X86_PAE (where the swap entry can occupy the entire upper
32 bits of a pte_t), but not a new limitation on 32-bit without PAE; and
there's not a new limitation on 64-bit (where swap filesize is already
limited to 16TB by a 32-bit page offset). Thirty areas of 128GB is
probably still enough swap for a 64GB 32-bit machine.
Provide swp_to_radix_entry() and radix_to_swp_entry() conversions, and
enforce filesize limit in read_swap_header(), just as for ptes.
Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-03 23:21:19 +00:00
|
|
|
* of bits for the swap offset in the swp_entry_t type, and
|
|
|
|
* 2) the number of bits in the swap pte as defined by the
|
swap: fix shmem swapping when more than 8 areas
Minchan Kim reports that when a system has many swap areas, and tmpfs
swaps out to the ninth or more, shmem_getpage_gfp()'s attempts to read
back the page cannot locate it, and the read fails with -ENOMEM.
Whoops. Yes, I blindly followed read_swap_header()'s pte_to_swp_entry(
swp_entry_to_pte()) technique for determining maximum usable swap
offset, without stopping to realize that that actually depends upon the
pte swap encoding shifting swap offset to the higher bits and truncating
it there. Whereas our radix_tree swap encoding leaves offset in the
lower bits: it's swap "type" (that is, index of swap area) that was
truncated.
Fix it by reducing the SWP_TYPE_SHIFT() in swapops.h, and removing the
broken radix_to_swp_entry(swp_to_radix_entry()) from read_swap_header().
This does not reduce the usable size of a swap area any further, it
leaves it as claimed when making the original commit: no change from 3.0
on x86_64, nor on i386 without PAE; but 3.0's 512GB is reduced to 128GB
per swapfile on i386 with PAE. It's not a change I would have risked
five years ago, but with x86_64 supported for ten years, I believe it's
appropriate now.
Hmm, and what if some architecture implements its swap pte with offset
encoded below type? That would equally break the maximum usable swap
offset check. Happily, they all follow the same tradition of encoding
offset above type, but I'll prepare a check on that for next.
Reported-and-Reviewed-and-Tested-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org [3.1, 3.2, 3.3, 3.4]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-16 00:55:50 +00:00
|
|
|
* different architectures. In order to find the
|
mm: let swap use exceptional entries
If swap entries are to be stored along with struct page pointers in a
radix tree, they need to be distinguished as exceptional entries.
Most of the handling of swap entries in radix tree will be contained in
shmem.c, but a few functions in filemap.c's common code need to check
for their appearance: find_get_page(), find_lock_page(),
find_get_pages() and find_get_pages_contig().
So as not to slow their fast paths, tuck those checks inside the
existing checks for unlikely radix_tree_deref_slot(); except for
find_lock_page(), where it is an added test. And make it a BUG in
find_get_pages_tag(), which is not applied to tmpfs files.
A part of the reason for eliminating shmem_readpage() earlier, was to
minimize the places where common code would need to allow for swap
entries.
The swp_entry_t known to swapfile.c must be massaged into a slightly
different form when stored in the radix tree, just as it gets massaged
into a pte_t when stored in page tables.
In an i386 kernel this limits its information (type and page offset) to
30 bits: given 32 "types" of swapfile and 4kB pagesize, that's a maximum
swapfile size of 128GB. Which is less than the 512GB we previously
allowed with X86_PAE (where the swap entry can occupy the entire upper
32 bits of a pte_t), but not a new limitation on 32-bit without PAE; and
there's not a new limitation on 64-bit (where swap filesize is already
limited to 16TB by a 32-bit page offset). Thirty areas of 128GB is
probably still enough swap for a 64GB 32-bit machine.
Provide swp_to_radix_entry() and radix_to_swp_entry() conversions, and
enforce filesize limit in read_swap_header(), just as for ptes.
Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-03 23:21:19 +00:00
|
|
|
* largest possible bit mask, a swap entry with swap type 0
|
2011-03-22 23:33:29 +00:00
|
|
|
* and swap offset ~0UL is created, encoded to a swap pte,
|
mm: let swap use exceptional entries
If swap entries are to be stored along with struct page pointers in a
radix tree, they need to be distinguished as exceptional entries.
Most of the handling of swap entries in radix tree will be contained in
shmem.c, but a few functions in filemap.c's common code need to check
for their appearance: find_get_page(), find_lock_page(),
find_get_pages() and find_get_pages_contig().
So as not to slow their fast paths, tuck those checks inside the
existing checks for unlikely radix_tree_deref_slot(); except for
find_lock_page(), where it is an added test. And make it a BUG in
find_get_pages_tag(), which is not applied to tmpfs files.
A part of the reason for eliminating shmem_readpage() earlier, was to
minimize the places where common code would need to allow for swap
entries.
The swp_entry_t known to swapfile.c must be massaged into a slightly
different form when stored in the radix tree, just as it gets massaged
into a pte_t when stored in page tables.
In an i386 kernel this limits its information (type and page offset) to
30 bits: given 32 "types" of swapfile and 4kB pagesize, that's a maximum
swapfile size of 128GB. Which is less than the 512GB we previously
allowed with X86_PAE (where the swap entry can occupy the entire upper
32 bits of a pte_t), but not a new limitation on 32-bit without PAE; and
there's not a new limitation on 64-bit (where swap filesize is already
limited to 16TB by a 32-bit page offset). Thirty areas of 128GB is
probably still enough swap for a 64GB 32-bit machine.
Provide swp_to_radix_entry() and radix_to_swp_entry() conversions, and
enforce filesize limit in read_swap_header(), just as for ptes.
Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-03 23:21:19 +00:00
|
|
|
* decoded to a swp_entry_t again, and finally the swap
|
2011-03-22 23:33:29 +00:00
|
|
|
* offset is extracted. This will mask all the bits from
|
|
|
|
* the initial ~0UL mask that can't be encoded in either
|
|
|
|
* the swp_entry_t or the architecture definition of a
|
swap: fix shmem swapping when more than 8 areas
Minchan Kim reports that when a system has many swap areas, and tmpfs
swaps out to the ninth or more, shmem_getpage_gfp()'s attempts to read
back the page cannot locate it, and the read fails with -ENOMEM.
Whoops. Yes, I blindly followed read_swap_header()'s pte_to_swp_entry(
swp_entry_to_pte()) technique for determining maximum usable swap
offset, without stopping to realize that that actually depends upon the
pte swap encoding shifting swap offset to the higher bits and truncating
it there. Whereas our radix_tree swap encoding leaves offset in the
lower bits: it's swap "type" (that is, index of swap area) that was
truncated.
Fix it by reducing the SWP_TYPE_SHIFT() in swapops.h, and removing the
broken radix_to_swp_entry(swp_to_radix_entry()) from read_swap_header().
This does not reduce the usable size of a swap area any further, it
leaves it as claimed when making the original commit: no change from 3.0
on x86_64, nor on i386 without PAE; but 3.0's 512GB is reduced to 128GB
per swapfile on i386 with PAE. It's not a change I would have risked
five years ago, but with x86_64 supported for ten years, I believe it's
appropriate now.
Hmm, and what if some architecture implements its swap pte with offset
encoded below type? That would equally break the maximum usable swap
offset check. Happily, they all follow the same tradition of encoding
offset above type, but I'll prepare a check on that for next.
Reported-and-Reviewed-and-Tested-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org [3.1, 3.2, 3.3, 3.4]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-16 00:55:50 +00:00
|
|
|
* swap pte.
|
2011-03-22 23:33:29 +00:00
|
|
|
*/
|
|
|
|
maxpages = swp_offset(pte_to_swp_entry(
|
swap: fix shmem swapping when more than 8 areas
Minchan Kim reports that when a system has many swap areas, and tmpfs
swaps out to the ninth or more, shmem_getpage_gfp()'s attempts to read
back the page cannot locate it, and the read fails with -ENOMEM.
Whoops. Yes, I blindly followed read_swap_header()'s pte_to_swp_entry(
swp_entry_to_pte()) technique for determining maximum usable swap
offset, without stopping to realize that that actually depends upon the
pte swap encoding shifting swap offset to the higher bits and truncating
it there. Whereas our radix_tree swap encoding leaves offset in the
lower bits: it's swap "type" (that is, index of swap area) that was
truncated.
Fix it by reducing the SWP_TYPE_SHIFT() in swapops.h, and removing the
broken radix_to_swp_entry(swp_to_radix_entry()) from read_swap_header().
This does not reduce the usable size of a swap area any further, it
leaves it as claimed when making the original commit: no change from 3.0
on x86_64, nor on i386 without PAE; but 3.0's 512GB is reduced to 128GB
per swapfile on i386 with PAE. It's not a change I would have risked
five years ago, but with x86_64 supported for ten years, I believe it's
appropriate now.
Hmm, and what if some architecture implements its swap pte with offset
encoded below type? That would equally break the maximum usable swap
offset check. Happily, they all follow the same tradition of encoding
offset above type, but I'll prepare a check on that for next.
Reported-and-Reviewed-and-Tested-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org [3.1, 3.2, 3.3, 3.4]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-16 00:55:50 +00:00
|
|
|
swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
|
2011-03-22 23:33:29 +00:00
|
|
|
if (maxpages > swap_header->info.last_page) {
|
|
|
|
maxpages = swap_header->info.last_page + 1;
|
|
|
|
/* p->max is an unsigned int: don't overflow it */
|
|
|
|
if ((unsigned int)maxpages == 0)
|
|
|
|
maxpages = UINT_MAX;
|
|
|
|
}
|
|
|
|
p->highest_bit = maxpages - 1;
|
|
|
|
|
|
|
|
if (!maxpages)
|
2011-03-22 23:33:30 +00:00
|
|
|
return 0;
|
2011-03-22 23:33:29 +00:00
|
|
|
swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
|
|
|
|
if (swapfilepages && maxpages > swapfilepages) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"Swap area shorter than signature indicates\n");
|
2011-03-22 23:33:30 +00:00
|
|
|
return 0;
|
2011-03-22 23:33:29 +00:00
|
|
|
}
|
|
|
|
if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
|
2011-03-22 23:33:30 +00:00
|
|
|
return 0;
|
2011-03-22 23:33:29 +00:00
|
|
|
if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
|
2011-03-22 23:33:30 +00:00
|
|
|
return 0;
|
2011-03-22 23:33:29 +00:00
|
|
|
|
|
|
|
return maxpages;
|
|
|
|
}
|
|
|
|
|
2011-03-22 23:33:32 +00:00
|
|
|
static int setup_swap_map_and_extents(struct swap_info_struct *p,
|
|
|
|
union swap_header *swap_header,
|
|
|
|
unsigned char *swap_map,
|
|
|
|
unsigned long maxpages,
|
|
|
|
sector_t *span)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
unsigned int nr_good_pages;
|
|
|
|
int nr_extents;
|
|
|
|
|
|
|
|
nr_good_pages = maxpages - 1; /* omit header page */
|
|
|
|
|
|
|
|
for (i = 0; i < swap_header->info.nr_badpages; i++) {
|
|
|
|
unsigned int page_nr = swap_header->info.badpages[i];
|
2011-03-22 23:33:33 +00:00
|
|
|
if (page_nr == 0 || page_nr > swap_header->info.last_page)
|
|
|
|
return -EINVAL;
|
2011-03-22 23:33:32 +00:00
|
|
|
if (page_nr < maxpages) {
|
|
|
|
swap_map[page_nr] = SWAP_MAP_BAD;
|
|
|
|
nr_good_pages--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (nr_good_pages) {
|
|
|
|
swap_map[0] = SWAP_MAP_BAD;
|
|
|
|
p->max = maxpages;
|
|
|
|
p->pages = nr_good_pages;
|
|
|
|
nr_extents = setup_swap_extents(p, span);
|
2011-03-22 23:33:33 +00:00
|
|
|
if (nr_extents < 0)
|
|
|
|
return nr_extents;
|
2011-03-22 23:33:32 +00:00
|
|
|
nr_good_pages = p->pages;
|
|
|
|
}
|
|
|
|
if (!nr_good_pages) {
|
|
|
|
printk(KERN_WARNING "Empty swap-file\n");
|
2011-03-22 23:33:33 +00:00
|
|
|
return -EINVAL;
|
2011-03-22 23:33:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nr_extents;
|
|
|
|
}
|
|
|
|
|
2011-03-22 23:33:17 +00:00
|
|
|
SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
|
|
|
|
{
|
|
|
|
struct swap_info_struct *p;
|
2012-10-10 19:25:28 +00:00
|
|
|
struct filename *name;
|
2011-03-22 23:33:17 +00:00
|
|
|
struct file *swap_file = NULL;
|
|
|
|
struct address_space *mapping;
|
2011-03-22 23:33:37 +00:00
|
|
|
int i;
|
|
|
|
int prio;
|
2011-03-22 23:33:17 +00:00
|
|
|
int error;
|
|
|
|
union swap_header *swap_header;
|
2011-03-22 23:33:32 +00:00
|
|
|
int nr_extents;
|
2011-03-22 23:33:17 +00:00
|
|
|
sector_t span;
|
|
|
|
unsigned long maxpages;
|
|
|
|
unsigned char *swap_map = NULL;
|
2012-04-09 23:08:06 +00:00
|
|
|
unsigned long *frontswap_map = NULL;
|
2011-03-22 23:33:17 +00:00
|
|
|
struct page *page = NULL;
|
|
|
|
struct inode *inode = NULL;
|
|
|
|
|
2012-03-28 21:42:42 +00:00
|
|
|
if (swap_flags & ~SWAP_FLAGS_VALID)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2011-03-22 23:33:17 +00:00
|
|
|
if (!capable(CAP_SYS_ADMIN))
|
|
|
|
return -EPERM;
|
|
|
|
|
|
|
|
p = alloc_swap_info();
|
2011-03-22 23:33:18 +00:00
|
|
|
if (IS_ERR(p))
|
|
|
|
return PTR_ERR(p);
|
2011-03-22 23:33:17 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
name = getname(specialfile);
|
|
|
|
if (IS_ERR(name)) {
|
2011-03-22 23:33:22 +00:00
|
|
|
error = PTR_ERR(name);
|
2005-04-16 22:20:36 +00:00
|
|
|
name = NULL;
|
2011-03-22 23:33:25 +00:00
|
|
|
goto bad_swap;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2012-10-10 20:43:10 +00:00
|
|
|
swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (IS_ERR(swap_file)) {
|
2011-03-22 23:33:22 +00:00
|
|
|
error = PTR_ERR(swap_file);
|
2005-04-16 22:20:36 +00:00
|
|
|
swap_file = NULL;
|
2011-03-22 23:33:25 +00:00
|
|
|
goto bad_swap;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
p->swap_file = swap_file;
|
|
|
|
mapping = swap_file->f_mapping;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_swapfiles; i++) {
|
2009-12-15 01:58:41 +00:00
|
|
|
struct swap_info_struct *q = swap_info[i];
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-03-22 23:33:16 +00:00
|
|
|
if (q == p || !q->swap_file)
|
2005-04-16 22:20:36 +00:00
|
|
|
continue;
|
2011-03-22 23:33:22 +00:00
|
|
|
if (mapping == q->swap_file->f_mapping) {
|
|
|
|
error = -EBUSY;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto bad_swap;
|
2011-03-22 23:33:22 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2011-03-23 02:03:13 +00:00
|
|
|
inode = mapping->host;
|
|
|
|
/* If S_ISREG(inode->i_mode) will do mutex_lock(&inode->i_mutex); */
|
2011-03-22 23:33:26 +00:00
|
|
|
error = claim_swapfile(p, inode);
|
|
|
|
if (unlikely(error))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto bad_swap;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Read the swap header.
|
|
|
|
*/
|
|
|
|
if (!mapping->a_ops->readpage) {
|
|
|
|
error = -EINVAL;
|
|
|
|
goto bad_swap;
|
|
|
|
}
|
2006-06-23 09:05:08 +00:00
|
|
|
page = read_mapping_page(mapping, 0, swap_file);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (IS_ERR(page)) {
|
|
|
|
error = PTR_ERR(page);
|
|
|
|
goto bad_swap;
|
|
|
|
}
|
2009-01-06 22:39:49 +00:00
|
|
|
swap_header = kmap(page);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-03-22 23:33:29 +00:00
|
|
|
maxpages = read_swap_header(p, swap_header, inode);
|
|
|
|
if (unlikely(!maxpages)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
error = -EINVAL;
|
|
|
|
goto bad_swap;
|
|
|
|
}
|
2009-01-06 22:39:48 +00:00
|
|
|
|
2009-01-06 22:39:49 +00:00
|
|
|
/* OK, set up the swap map and apply the bad block list */
|
2011-03-22 23:33:14 +00:00
|
|
|
swap_map = vzalloc(maxpages);
|
2009-01-06 22:39:49 +00:00
|
|
|
if (!swap_map) {
|
|
|
|
error = -ENOMEM;
|
|
|
|
goto bad_swap;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-03-22 23:33:31 +00:00
|
|
|
error = swap_cgroup_swapon(p->type, maxpages);
|
|
|
|
if (error)
|
|
|
|
goto bad_swap;
|
|
|
|
|
2011-03-22 23:33:32 +00:00
|
|
|
nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
|
|
|
|
maxpages, &span);
|
|
|
|
if (unlikely(nr_extents < 0)) {
|
|
|
|
error = nr_extents;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto bad_swap;
|
|
|
|
}
|
2012-04-09 23:08:06 +00:00
|
|
|
/* frontswap enabled? set up bit-per-page map for frontswap */
|
|
|
|
if (frontswap_enabled)
|
2013-06-12 21:05:08 +00:00
|
|
|
frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-09-30 08:53:48 +00:00
|
|
|
if (p->bdev) {
|
|
|
|
if (blk_queue_nonrot(bdev_get_queue(p->bdev))) {
|
|
|
|
p->flags |= SWP_SOLIDSTATE;
|
2013-04-29 23:21:29 +00:00
|
|
|
p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
|
2009-09-30 08:53:48 +00:00
|
|
|
}
|
2012-03-21 23:34:17 +00:00
|
|
|
if ((swap_flags & SWAP_FLAG_DISCARD) && discard_swap(p) == 0)
|
2009-09-30 08:53:48 +00:00
|
|
|
p->flags |= SWP_DISCARDABLE;
|
2009-01-06 22:39:54 +00:00
|
|
|
}
|
2009-01-06 22:39:51 +00:00
|
|
|
|
2006-01-19 01:42:33 +00:00
|
|
|
mutex_lock(&swapon_mutex);
|
2011-03-22 23:33:37 +00:00
|
|
|
prio = -1;
|
mm: fix ever-decreasing swap priority
Vegard Nossum has noticed the ever-decreasing negative priority in a
swapon /swapoff loop, which eventually would misprioritize when int wraps
positive. Not worth spending much code on, but probably better fixed.
It's easy to handle the swapping on and off of just one area, but there's
not much point if a pair or more still misbehave. To handle the general
case, swapoff should compact negative priorities, keeping them always from
-1 to -MAX_SWAPFILES. That's a change, but should cause no regression,
since these negative (unspecified) priorities are disjoint from the the
positive specified priorities 0 to 32767.
One small functional difference, which seems appropriate: when swapoff
fails to free all swap from a negative priority area, that area is now
reinserted at lowest priority, rather than at its original priority.
In moving down swapon's setting of priority, I notice that an area is
visible to /proc/swaps when it has swap_map set, yet that was being set
before all the visible fields were properly filled in: corrected.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reported-by: Vegard Nossum <vegard.nossum@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-24 04:28:23 +00:00
|
|
|
if (swap_flags & SWAP_FLAG_PREFER)
|
2011-03-22 23:33:37 +00:00
|
|
|
prio =
|
mm: fix ever-decreasing swap priority
Vegard Nossum has noticed the ever-decreasing negative priority in a
swapon /swapoff loop, which eventually would misprioritize when int wraps
positive. Not worth spending much code on, but probably better fixed.
It's easy to handle the swapping on and off of just one area, but there's
not much point if a pair or more still misbehave. To handle the general
case, swapoff should compact negative priorities, keeping them always from
-1 to -MAX_SWAPFILES. That's a change, but should cause no regression,
since these negative (unspecified) priorities are disjoint from the the
positive specified priorities 0 to 32767.
One small functional difference, which seems appropriate: when swapoff
fails to free all swap from a negative priority area, that area is now
reinserted at lowest priority, rather than at its original priority.
In moving down swapon's setting of priority, I notice that an area is
visible to /proc/swaps when it has swap_map set, yet that was being set
before all the visible fields were properly filled in: corrected.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reported-by: Vegard Nossum <vegard.nossum@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-24 04:28:23 +00:00
|
|
|
(swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
|
2012-04-09 23:08:06 +00:00
|
|
|
enable_swap_info(p, prio, swap_map, frontswap_map);
|
2011-03-22 23:33:35 +00:00
|
|
|
|
|
|
|
printk(KERN_INFO "Adding %uk swap on %s. "
|
2012-04-09 23:08:06 +00:00
|
|
|
"Priority:%d extents:%d across:%lluk %s%s%s\n",
|
2012-10-10 19:25:28 +00:00
|
|
|
p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
|
2011-03-22 23:33:35 +00:00
|
|
|
nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
|
|
|
|
(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
|
2012-04-09 23:08:06 +00:00
|
|
|
(p->flags & SWP_DISCARDABLE) ? "D" : "",
|
|
|
|
(frontswap_map) ? "FS" : "");
|
2011-03-22 23:33:35 +00:00
|
|
|
|
2006-01-19 01:42:33 +00:00
|
|
|
mutex_unlock(&swapon_mutex);
|
2010-10-26 21:22:06 +00:00
|
|
|
atomic_inc(&proc_poll_event);
|
|
|
|
wake_up_interruptible(&proc_poll_wait);
|
|
|
|
|
2011-03-22 23:33:24 +00:00
|
|
|
if (S_ISREG(inode->i_mode))
|
|
|
|
inode->i_flags |= S_SWAPFILE;
|
2005-04-16 22:20:36 +00:00
|
|
|
error = 0;
|
|
|
|
goto out;
|
|
|
|
bad_swap:
|
2011-03-22 23:33:25 +00:00
|
|
|
if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
|
2011-03-22 23:33:23 +00:00
|
|
|
set_blocksize(p->bdev, p->old_block_size);
|
|
|
|
blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-09-03 22:54:33 +00:00
|
|
|
destroy_swap_extents(p);
|
2011-03-22 23:33:16 +00:00
|
|
|
swap_cgroup_swapoff(p->type);
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
p->swap_file = NULL;
|
|
|
|
p->flags = 0;
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
vfree(swap_map);
|
2011-03-22 23:30:08 +00:00
|
|
|
if (swap_file) {
|
2011-03-23 02:03:13 +00:00
|
|
|
if (inode && S_ISREG(inode->i_mode)) {
|
2011-03-22 23:30:08 +00:00
|
|
|
mutex_unlock(&inode->i_mutex);
|
2011-03-23 02:03:13 +00:00
|
|
|
inode = NULL;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
filp_close(swap_file, NULL);
|
2011-03-22 23:30:08 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
|
|
|
if (page && !IS_ERR(page)) {
|
|
|
|
kunmap(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
}
|
|
|
|
if (name)
|
|
|
|
putname(name);
|
2011-03-22 23:33:24 +00:00
|
|
|
if (inode && S_ISREG(inode->i_mode))
|
2006-01-09 23:59:24 +00:00
|
|
|
mutex_unlock(&inode->i_mutex);
|
2005-04-16 22:20:36 +00:00
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
void si_swapinfo(struct sysinfo *val)
|
|
|
|
{
|
2009-12-15 01:58:41 +00:00
|
|
|
unsigned int type;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long nr_to_be_unused = 0;
|
|
|
|
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_lock(&swap_lock);
|
2009-12-15 01:58:41 +00:00
|
|
|
for (type = 0; type < nr_swapfiles; type++) {
|
|
|
|
struct swap_info_struct *si = swap_info[type];
|
|
|
|
|
|
|
|
if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
|
|
|
|
nr_to_be_unused += si->inuse_pages;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
|
2005-04-16 22:20:36 +00:00
|
|
|
val->totalswap = total_swap_pages + nr_to_be_unused;
|
[PATCH] swap: swap_lock replace list+device
The idea of a swap_device_lock per device, and a swap_list_lock over them all,
is appealing; but in practice almost every holder of swap_device_lock must
already hold swap_list_lock, which defeats the purpose of the split.
The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate does
demand attention. However, with the hold time in get_swap_pages so much
reduced, I've not yet found a load and set of swap device priorities to show
even swap_duplicate benefitting from the split. Certainly the split is mere
overhead in the common case of a single swap device.
So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).
If someone can show a regression in swap_duplicate, then probably we should
add a hashlock for the swap_map entries alone (shorts being anatomic), so as
to help the case of the single swap device too.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-03 22:54:41 +00:00
|
|
|
spin_unlock(&swap_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Verify that a swap entry is valid and increment its swap map count.
|
|
|
|
*
|
2009-06-16 22:32:53 +00:00
|
|
|
* Returns error code in following case.
|
|
|
|
* - success -> 0
|
|
|
|
* - swp_entry is invalid -> EINVAL
|
|
|
|
* - swp_entry is migration entry -> EINVAL
|
|
|
|
* - swap-cache reference is requested but there is already one. -> EEXIST
|
|
|
|
* - swap-cache reference is requested but the entry is not used. -> ENOENT
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
* - swap-mapped reference requested but needs continued swap count. -> ENOMEM
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-12-15 01:58:45 +00:00
|
|
|
static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-15 01:58:43 +00:00
|
|
|
struct swap_info_struct *p;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long offset, type;
|
2009-12-15 01:58:45 +00:00
|
|
|
unsigned char count;
|
|
|
|
unsigned char has_cache;
|
2009-12-15 01:58:44 +00:00
|
|
|
int err = -EINVAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-09-16 09:50:05 +00:00
|
|
|
if (non_swap_entry(entry))
|
2009-12-15 01:58:44 +00:00
|
|
|
goto out;
|
[PATCH] Swapless page migration: add R/W migration entries
Implement read/write migration ptes
We take the upper two swapfiles for the two types of migration ptes and define
a series of macros in swapops.h.
The VM is modified to handle the migration entries. migration entries can
only be encountered when the page they are pointing to is locked. This limits
the number of places one has to fix. We also check in copy_pte_range and in
mprotect_pte_range() for migration ptes.
We check for migration ptes in do_swap_cache and call a function that will
then wait on the page lock. This allows us to effectively stop all accesses
to apge.
Migration entries are created by try_to_unmap if called for migration and
removed by local functions in migrate.c
From: Hugh Dickins <hugh@veritas.com>
Several times while testing swapless page migration (I've no NUMA, just
hacking it up to migrate recklessly while running load), I've hit the
BUG_ON(!PageLocked(p)) in migration_entry_to_page.
This comes from an orphaned migration entry, unrelated to the current
correctly locked migration, but hit by remove_anon_migration_ptes as it
checks an address in each vma of the anon_vma list.
Such an orphan may be left behind if an earlier migration raced with fork:
copy_one_pte can duplicate a migration entry from parent to child, after
remove_anon_migration_ptes has checked the child vma, but before it has
removed it from the parent vma. (If the process were later to fault on this
orphaned entry, it would hit the same BUG from migration_entry_wait.)
This could be fixed by locking anon_vma in copy_one_pte, but we'd rather
not. There's no such problem with file pages, because vma_prio_tree_add
adds child vma after parent vma, and the page table locking at each end is
enough to serialize. Follow that example with anon_vma: add new vmas to the
tail instead of the head.
(There's no corresponding problem when inserting migration entries,
because a missed pte will leave the page count and mapcount high, which is
allowed for. And there's no corresponding problem when migrating via swap,
because a leftover swap entry will be correctly faulted. But the swapless
method has no refcounting of its entries.)
From: Ingo Molnar <mingo@elte.hu>
pte_unmap_unlock() takes the pte pointer as an argument.
From: Hugh Dickins <hugh@veritas.com>
Several times while testing swapless page migration, gcc has tried to exec
a pointer instead of a string: smells like COW mappings are not being
properly write-protected on fork.
The protection in copy_one_pte looks very convincing, until at last you
realize that the second arg to make_migration_entry is a boolean "write",
and SWP_MIGRATION_READ is 30.
Anyway, it's better done like in change_pte_range, using
is_write_migration_entry and make_migration_entry_read.
From: Hugh Dickins <hugh@veritas.com>
Remove unnecessary obfuscation from sys_swapon's range check on swap type,
which blew up causing memory corruption once swapless migration made
MAX_SWAPFILES no longer 2 ^ MAX_SWAPFILES_SHIFT.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Christoph Lameter <clameter@engr.sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
From: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 09:03:35 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
type = swp_type(entry);
|
|
|
|
if (type >= nr_swapfiles)
|
|
|
|
goto bad_file;
|
2009-12-15 01:58:41 +00:00
|
|
|
p = swap_info[type];
|
2005-04-16 22:20:36 +00:00
|
|
|
offset = swp_offset(entry);
|
|
|
|
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_lock(&p->lock);
|
2009-06-16 22:32:53 +00:00
|
|
|
if (unlikely(offset >= p->max))
|
|
|
|
goto unlock_out;
|
|
|
|
|
2009-12-15 01:58:44 +00:00
|
|
|
count = p->swap_map[offset];
|
|
|
|
has_cache = count & SWAP_HAS_CACHE;
|
|
|
|
count &= ~SWAP_HAS_CACHE;
|
|
|
|
err = 0;
|
2009-06-16 22:32:53 +00:00
|
|
|
|
2009-12-15 01:58:44 +00:00
|
|
|
if (usage == SWAP_HAS_CACHE) {
|
2009-06-16 22:32:53 +00:00
|
|
|
|
|
|
|
/* set SWAP_HAS_CACHE if there is no cache and entry is used */
|
2009-12-15 01:58:44 +00:00
|
|
|
if (!has_cache && count)
|
|
|
|
has_cache = SWAP_HAS_CACHE;
|
|
|
|
else if (has_cache) /* someone else added cache */
|
|
|
|
err = -EEXIST;
|
|
|
|
else /* no users remaining */
|
|
|
|
err = -ENOENT;
|
2009-06-16 22:32:53 +00:00
|
|
|
|
|
|
|
} else if (count || has_cache) {
|
2009-12-15 01:58:44 +00:00
|
|
|
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
|
|
|
|
count += usage;
|
|
|
|
else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
|
2009-12-15 01:58:44 +00:00
|
|
|
err = -EINVAL;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
else if (swap_count_continued(p, offset, count))
|
|
|
|
count = COUNT_CONTINUED;
|
|
|
|
else
|
|
|
|
err = -ENOMEM;
|
2009-06-16 22:32:53 +00:00
|
|
|
} else
|
2009-12-15 01:58:44 +00:00
|
|
|
err = -ENOENT; /* unused swap entry */
|
|
|
|
|
|
|
|
p->swap_map[offset] = count | has_cache;
|
|
|
|
|
2009-06-16 22:32:53 +00:00
|
|
|
unlock_out:
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&p->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
2009-12-15 01:58:44 +00:00
|
|
|
return err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
bad_file:
|
|
|
|
printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val);
|
|
|
|
goto out;
|
|
|
|
}
|
2009-12-15 01:58:44 +00:00
|
|
|
|
2009-12-15 01:58:47 +00:00
|
|
|
/*
|
|
|
|
* Help swapoff by noting that swap entry belongs to shmem/tmpfs
|
|
|
|
* (in which case its reference count is never incremented).
|
|
|
|
*/
|
|
|
|
void swap_shmem_alloc(swp_entry_t entry)
|
|
|
|
{
|
|
|
|
__swap_duplicate(entry, SWAP_MAP_SHMEM);
|
|
|
|
}
|
|
|
|
|
2009-06-16 22:32:53 +00:00
|
|
|
/*
|
2010-03-05 21:42:25 +00:00
|
|
|
* Increase reference count of swap entry by 1.
|
|
|
|
* Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
|
|
|
|
* but could not be atomically allocated. Returns 0, just as if it succeeded,
|
|
|
|
* if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
|
|
|
|
* might occur if a page table entry has got corrupted.
|
2009-06-16 22:32:53 +00:00
|
|
|
*/
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
int swap_duplicate(swp_entry_t entry)
|
2009-06-16 22:32:53 +00:00
|
|
|
{
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
|
|
|
|
err = add_swap_count_continuation(entry, GFP_ATOMIC);
|
|
|
|
return err;
|
2009-06-16 22:32:53 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-06-16 22:32:52 +00:00
|
|
|
/*
|
2009-06-16 22:32:53 +00:00
|
|
|
* @entry: swap entry for which we allocate swap cache.
|
|
|
|
*
|
2009-12-15 01:58:43 +00:00
|
|
|
* Called when allocating swap cache for existing swap entry,
|
2009-06-16 22:32:53 +00:00
|
|
|
* This can return error codes. Returns 0 at success.
|
|
|
|
* -EBUSY means there is a swap cache.
|
|
|
|
* Note: return code is different from swap_duplicate().
|
2009-06-16 22:32:52 +00:00
|
|
|
*/
|
|
|
|
int swapcache_prepare(swp_entry_t entry)
|
|
|
|
{
|
2009-12-15 01:58:44 +00:00
|
|
|
return __swap_duplicate(entry, SWAP_HAS_CACHE);
|
2009-06-16 22:32:52 +00:00
|
|
|
}
|
|
|
|
|
2012-07-31 23:44:47 +00:00
|
|
|
struct swap_info_struct *page_swap_info(struct page *page)
|
|
|
|
{
|
|
|
|
swp_entry_t swap = { .val = page_private(page) };
|
|
|
|
BUG_ON(!PageSwapCache(page));
|
|
|
|
return swap_info[swp_type(swap)];
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* out-of-line __page_file_ methods to avoid include hell.
|
|
|
|
*/
|
|
|
|
struct address_space *__page_file_mapping(struct page *page)
|
|
|
|
{
|
|
|
|
VM_BUG_ON(!PageSwapCache(page));
|
|
|
|
return page_swap_info(page)->swap_file->f_mapping;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__page_file_mapping);
|
|
|
|
|
|
|
|
pgoff_t __page_file_index(struct page *page)
|
|
|
|
{
|
|
|
|
swp_entry_t swap = { .val = page_private(page) };
|
|
|
|
VM_BUG_ON(!PageSwapCache(page));
|
|
|
|
return swp_offset(swap);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__page_file_index);
|
|
|
|
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
/*
|
|
|
|
* add_swap_count_continuation - called when a swap count is duplicated
|
|
|
|
* beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
|
|
|
|
* page of the original vmalloc'ed swap_map, to hold the continuation count
|
|
|
|
* (for that entry and for its neighbouring PAGE_SIZE swap entries). Called
|
|
|
|
* again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
|
|
|
|
*
|
|
|
|
* These continuation pages are seldom referenced: the common paths all work
|
|
|
|
* on the original swap_map, only referring to a continuation page when the
|
|
|
|
* low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
|
|
|
|
*
|
|
|
|
* add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
|
|
|
|
* page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
|
|
|
|
* can be called after dropping locks.
|
|
|
|
*/
|
|
|
|
int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
|
|
|
|
{
|
|
|
|
struct swap_info_struct *si;
|
|
|
|
struct page *head;
|
|
|
|
struct page *page;
|
|
|
|
struct page *list_page;
|
|
|
|
pgoff_t offset;
|
|
|
|
unsigned char count;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When debugging, it's easier to use __GFP_ZERO here; but it's better
|
|
|
|
* for latency not to zero a page while GFP_ATOMIC and holding locks.
|
|
|
|
*/
|
|
|
|
page = alloc_page(gfp_mask | __GFP_HIGHMEM);
|
|
|
|
|
|
|
|
si = swap_info_get(entry);
|
|
|
|
if (!si) {
|
|
|
|
/*
|
|
|
|
* An acceptable race has occurred since the failing
|
|
|
|
* __swap_duplicate(): the swap entry has been freed,
|
|
|
|
* perhaps even the whole swap_map cleared for swapoff.
|
|
|
|
*/
|
|
|
|
goto outer;
|
|
|
|
}
|
|
|
|
|
|
|
|
offset = swp_offset(entry);
|
|
|
|
count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
|
|
|
|
|
|
|
|
if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
|
|
|
|
/*
|
|
|
|
* The higher the swap count, the more likely it is that tasks
|
|
|
|
* will race to add swap count continuation: we need to avoid
|
|
|
|
* over-provisioning.
|
|
|
|
*/
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!page) {
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are fortunate that although vmalloc_to_page uses pte_offset_map,
|
|
|
|
* no architecture is using highmem pages for kernel pagetables: so it
|
|
|
|
* will not corrupt the GFP_ATOMIC caller's atomic pagetable kmaps.
|
|
|
|
*/
|
|
|
|
head = vmalloc_to_page(si->swap_map + offset);
|
|
|
|
offset &= ~PAGE_MASK;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Page allocation does not initialize the page's lru field,
|
|
|
|
* but it does always reset its private field.
|
|
|
|
*/
|
|
|
|
if (!page_private(head)) {
|
|
|
|
BUG_ON(count & COUNT_CONTINUED);
|
|
|
|
INIT_LIST_HEAD(&head->lru);
|
|
|
|
set_page_private(head, SWP_CONTINUED);
|
|
|
|
si->flags |= SWP_CONTINUED;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(list_page, &head->lru, lru) {
|
|
|
|
unsigned char *map;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the previous map said no continuation, but we've found
|
|
|
|
* a continuation page, free our allocation and use this one.
|
|
|
|
*/
|
|
|
|
if (!(count & COUNT_CONTINUED))
|
|
|
|
goto out;
|
|
|
|
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(list_page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
count = *map;
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this continuation count now has some space in it,
|
|
|
|
* free our allocation and use this one.
|
|
|
|
*/
|
|
|
|
if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_add_tail(&page->lru, &head->lru);
|
|
|
|
page = NULL; /* now it's attached, don't free it */
|
|
|
|
out:
|
swap: add per-partition lock for swapfile
swap_lock is heavily contended when I test swap to 3 fast SSD (even
slightly slower than swap to 2 such SSD). The main contention comes
from swap_info_get(). This patch tries to fix the gap with adding a new
per-partition lock.
Global data like nr_swapfiles, total_swap_pages, least_priority and
swap_list are still protected by swap_lock.
nr_swap_pages is an atomic now, it can be changed without swap_lock. In
theory, it's possible get_swap_page() finds no swap pages but actually
there are free swap pages. But sounds not a big problem.
Accessing partition specific data (like scan_swap_map and so on) is only
protected by swap_info_struct.lock.
Changing swap_info_struct.flags need hold swap_lock and
swap_info_struct.lock, because scan_scan_map() will check it. read the
flags is ok with either the locks hold.
If both swap_lock and swap_info_struct.lock must be hold, we always hold
the former first to avoid deadlock.
swap_entry_free() can change swap_list. To delete that code, we add a
new highest_priority_index. Whenever get_swap_page() is called, we
check it. If it's valid, we use it.
It's a pity get_swap_page() still holds swap_lock(). But in practice,
swap_lock() isn't heavily contended in my test with this patch (or I can
say there are other much more heavier bottlenecks like TLB flush). And
BTW, looks get_swap_page() doesn't really need the lock. We never free
swap_info[] and we check SWAP_WRITEOK flag. The only risk without the
lock is we could swapout to some low priority swap, but we can quickly
recover after several rounds of swap, so sounds not a big deal to me.
But I'd prefer to fix this if it's a real problem.
"swap: make each swap partition have one address_space" improved the
swapout speed from 1.7G/s to 2G/s. This patch further improves the
speed to 2.3G/s, so around 15% improvement. It's a multi-process test,
so TLB flush isn't the biggest bottleneck before the patches.
[arnd@arndb.de: fix it for nommu]
[hughd@google.com: add missing unlock]
[minchan@kernel.org: get rid of lockdep whinge on sys_swapon]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Seth Jennings <sjenning@linux.vnet.ibm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Dan Magenheimer <dan.magenheimer@oracle.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-23 00:34:38 +00:00
|
|
|
spin_unlock(&si->lock);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
outer:
|
|
|
|
if (page)
|
|
|
|
__free_page(page);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* swap_count_continued - when the original swap_map count is incremented
|
|
|
|
* from SWAP_MAP_MAX, check if there is already a continuation page to carry
|
|
|
|
* into, carry if so, or else fail until a new continuation page is allocated;
|
|
|
|
* when the original swap_map count is decremented from 0 with continuation,
|
|
|
|
* borrow from the continuation and report whether it still holds more.
|
|
|
|
* Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
|
|
|
|
*/
|
|
|
|
static bool swap_count_continued(struct swap_info_struct *si,
|
|
|
|
pgoff_t offset, unsigned char count)
|
|
|
|
{
|
|
|
|
struct page *head;
|
|
|
|
struct page *page;
|
|
|
|
unsigned char *map;
|
|
|
|
|
|
|
|
head = vmalloc_to_page(si->swap_map + offset);
|
|
|
|
if (page_private(head) != SWP_CONTINUED) {
|
|
|
|
BUG_ON(count & COUNT_CONTINUED);
|
|
|
|
return false; /* need to add count continuation */
|
|
|
|
}
|
|
|
|
|
|
|
|
offset &= ~PAGE_MASK;
|
|
|
|
page = list_entry(head->lru.next, struct page, lru);
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
|
|
|
|
if (count == SWAP_MAP_MAX) /* initial increment from swap_map */
|
|
|
|
goto init_map; /* jump over SWAP_CONT_MAX checks */
|
|
|
|
|
|
|
|
if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
|
|
|
|
/*
|
|
|
|
* Think of how you add 1 to 999
|
|
|
|
*/
|
|
|
|
while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.next, struct page, lru);
|
|
|
|
BUG_ON(page == head);
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
}
|
|
|
|
if (*map == SWAP_CONT_MAX) {
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.next, struct page, lru);
|
|
|
|
if (page == head)
|
|
|
|
return false; /* add count continuation */
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
init_map: *map = 0; /* we didn't zero the page */
|
|
|
|
}
|
|
|
|
*map += 1;
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.prev, struct page, lru);
|
|
|
|
while (page != head) {
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
*map = COUNT_CONTINUED;
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.prev, struct page, lru);
|
|
|
|
}
|
|
|
|
return true; /* incremented */
|
|
|
|
|
|
|
|
} else { /* decrementing */
|
|
|
|
/*
|
|
|
|
* Think of how you subtract 1 from 1000
|
|
|
|
*/
|
|
|
|
BUG_ON(count != COUNT_CONTINUED);
|
|
|
|
while (*map == COUNT_CONTINUED) {
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.next, struct page, lru);
|
|
|
|
BUG_ON(page == head);
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
}
|
|
|
|
BUG_ON(*map == 0);
|
|
|
|
*map -= 1;
|
|
|
|
if (*map == 0)
|
|
|
|
count = 0;
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.prev, struct page, lru);
|
|
|
|
while (page != head) {
|
2011-11-25 15:14:39 +00:00
|
|
|
map = kmap_atomic(page) + offset;
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
*map = SWAP_CONT_MAX | count;
|
|
|
|
count = COUNT_CONTINUED;
|
2011-11-25 15:14:39 +00:00
|
|
|
kunmap_atomic(map);
|
swap_info: swap count continuations
Swap is duplicated (reference count incremented by one) whenever the same
swap page is inserted into another mm (when forking finds a swap entry in
place of a pte, or when reclaim unmaps a pte to insert the swap entry).
swap_info_struct's vmalloc'ed swap_map is the array of these reference
counts: but what happens when the unsigned short (or unsigned char since
the preceding patch) is full? (and its high bit is kept for a cache flag)
We then lose track of it, never freeing, leaving it in use until swapoff:
at which point we _hope_ that a single pass will have found all instances,
assume there are no more, and will lose user data if we're wrong.
Swapping of KSM pages has not yet been enabled; but it is implemented,
and makes it very easy for a user to overflow the maximum swap count:
possible with ordinary process pages, but unlikely, even when pid_max
has been raised from PID_MAX_DEFAULT.
This patch implements swap count continuations: when the count overflows,
a continuation page is allocated and linked to the original vmalloc'ed
map page, and this used to hold the continuation counts for that entry
and its neighbours. These continuation pages are seldom referenced:
the common paths all work on the original swap_map, only referring to
a continuation page when the low "digit" of a count is incremented or
decremented through SWAP_MAP_MAX.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:58:46 +00:00
|
|
|
page = list_entry(page->lru.prev, struct page, lru);
|
|
|
|
}
|
|
|
|
return count == COUNT_CONTINUED;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* free_swap_count_continuations - swapoff free all the continuation pages
|
|
|
|
* appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
|
|
|
|
*/
|
|
|
|
static void free_swap_count_continuations(struct swap_info_struct *si)
|
|
|
|
{
|
|
|
|
pgoff_t offset;
|
|
|
|
|
|
|
|
for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
|
|
|
|
struct page *head;
|
|
|
|
head = vmalloc_to_page(si->swap_map + offset);
|
|
|
|
if (page_private(head)) {
|
|
|
|
struct list_head *this, *next;
|
|
|
|
list_for_each_safe(this, next, &head->lru) {
|
|
|
|
struct page *page;
|
|
|
|
page = list_entry(this, struct page, lru);
|
|
|
|
list_del(this);
|
|
|
|
__free_page(page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|