mirror of
https://github.com/torvalds/linux.git
synced 2024-11-27 22:51:35 +00:00
[PATCH] Dynamic hugepage addresses for ppc64
Paulus, I think this is now a reasonable candidate for the post-2.6.13 queue. Relax address restrictions for hugepages on ppc64 Presently, 64-bit applications on ppc64 may only use hugepages in the address region from 1-1.5T. Furthermore, if hugepages are enabled in the kernel config, they may only use hugepages and never normal pages in this area. This patch relaxes this restriction, allowing any address to be used with hugepages, but with a 1TB granularity. That is if you map a hugepage anywhere in the region 1TB-2TB, that entire area will be reserved exclusively for hugepages for the remainder of the process's lifetime. This works analagously to hugepages in 32-bit applications, where hugepages can be mapped anywhere, but with 256MB (mmu segment) granularity. This patch applies on top of the four level pagetable patch (http://patchwork.ozlabs.org/linuxppc64/patch?id=1936). Signed-off-by: David Gibson <dwg@au1.ibm.com> Signed-off-by: Paul Mackerras <paulus@samba.org>
This commit is contained in:
David Gibson
Paul Mackerras
parent
9a5573e378
commit
c594adad56
@ -94,7 +94,8 @@ int main(void)
|
||||
DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));
|
||||
DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
DEFINE(PACAHTLBSEGS, offsetof(struct paca_struct, context.htlb_segs));
|
||||
DEFINE(PACALOWHTLBAREAS, offsetof(struct paca_struct, context.low_htlb_areas));
|
||||
DEFINE(PACAHIGHHTLBAREAS, offsetof(struct paca_struct, context.high_htlb_areas));
|
||||
#endif /* CONFIG_HUGETLB_PAGE */
|
||||
DEFINE(PACADEFAULTDECR, offsetof(struct paca_struct, default_decr));
|
||||
DEFINE(PACA_EXGEN, offsetof(struct paca_struct, exgen));
|
||||
|
||||
@ -27,6 +27,9 @@
|
||||
|
||||
#include <linux/sysctl.h>
|
||||
|
||||
#define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
|
||||
#define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
|
||||
|
||||
/* Modelled after find_linux_pte() */
|
||||
pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
|
||||
{
|
||||
@ -129,15 +132,17 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void flush_segments(void *parm)
|
||||
static void flush_low_segments(void *parm)
|
||||
{
|
||||
u16 segs = (unsigned long) parm;
|
||||
u16 areas = (unsigned long) parm;
|
||||
unsigned long i;
|
||||
|
||||
asm volatile("isync" : : : "memory");
|
||||
|
||||
for (i = 0; i < 16; i++) {
|
||||
if (! (segs & (1U << i)))
|
||||
BUILD_BUG_ON((sizeof(areas)*8) != NUM_LOW_AREAS);
|
||||
|
||||
for (i = 0; i < NUM_LOW_AREAS; i++) {
|
||||
if (! (areas & (1U << i)))
|
||||
continue;
|
||||
asm volatile("slbie %0" : : "r" (i << SID_SHIFT));
|
||||
}
|
||||
@ -145,13 +150,33 @@ static void flush_segments(void *parm)
|
||||
asm volatile("isync" : : : "memory");
|
||||
}
|
||||
|
||||
static int prepare_low_seg_for_htlb(struct mm_struct *mm, unsigned long seg)
|
||||
static void flush_high_segments(void *parm)
|
||||
{
|
||||
unsigned long start = seg << SID_SHIFT;
|
||||
unsigned long end = (seg+1) << SID_SHIFT;
|
||||
u16 areas = (unsigned long) parm;
|
||||
unsigned long i, j;
|
||||
|
||||
asm volatile("isync" : : : "memory");
|
||||
|
||||
BUILD_BUG_ON((sizeof(areas)*8) != NUM_HIGH_AREAS);
|
||||
|
||||
for (i = 0; i < NUM_HIGH_AREAS; i++) {
|
||||
if (! (areas & (1U << i)))
|
||||
continue;
|
||||
for (j = 0; j < (1UL << (HTLB_AREA_SHIFT-SID_SHIFT)); j++)
|
||||
asm volatile("slbie %0"
|
||||
:: "r" ((i << HTLB_AREA_SHIFT) + (j << SID_SHIFT)));
|
||||
}
|
||||
|
||||
asm volatile("isync" : : : "memory");
|
||||
}
|
||||
|
||||
static int prepare_low_area_for_htlb(struct mm_struct *mm, unsigned long area)
|
||||
{
|
||||
unsigned long start = area << SID_SHIFT;
|
||||
unsigned long end = (area+1) << SID_SHIFT;
|
||||
struct vm_area_struct *vma;
|
||||
|
||||
BUG_ON(seg >= 16);
|
||||
BUG_ON(area >= NUM_LOW_AREAS);
|
||||
|
||||
/* Check no VMAs are in the region */
|
||||
vma = find_vma(mm, start);
|
||||
@ -161,20 +186,39 @@ static int prepare_low_seg_for_htlb(struct mm_struct *mm, unsigned long seg)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int open_low_hpage_segs(struct mm_struct *mm, u16 newsegs)
|
||||
static int prepare_high_area_for_htlb(struct mm_struct *mm, unsigned long area)
|
||||
{
|
||||
unsigned long start = area << HTLB_AREA_SHIFT;
|
||||
unsigned long end = (area+1) << HTLB_AREA_SHIFT;
|
||||
struct vm_area_struct *vma;
|
||||
|
||||
BUG_ON(area >= NUM_HIGH_AREAS);
|
||||
|
||||
/* Check no VMAs are in the region */
|
||||
vma = find_vma(mm, start);
|
||||
if (vma && (vma->vm_start < end))
|
||||
return -EBUSY;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int open_low_hpage_areas(struct mm_struct *mm, u16 newareas)
|
||||
{
|
||||
unsigned long i;
|
||||
|
||||
newsegs &= ~(mm->context.htlb_segs);
|
||||
if (! newsegs)
|
||||
BUILD_BUG_ON((sizeof(newareas)*8) != NUM_LOW_AREAS);
|
||||
BUILD_BUG_ON((sizeof(mm->context.low_htlb_areas)*8) != NUM_LOW_AREAS);
|
||||
|
||||
newareas &= ~(mm->context.low_htlb_areas);
|
||||
if (! newareas)
|
||||
return 0; /* The segments we want are already open */
|
||||
|
||||
for (i = 0; i < 16; i++)
|
||||
if ((1 << i) & newsegs)
|
||||
if (prepare_low_seg_for_htlb(mm, i) != 0)
|
||||
for (i = 0; i < NUM_LOW_AREAS; i++)
|
||||
if ((1 << i) & newareas)
|
||||
if (prepare_low_area_for_htlb(mm, i) != 0)
|
||||
return -EBUSY;
|
||||
|
||||
mm->context.htlb_segs |= newsegs;
|
||||
mm->context.low_htlb_areas |= newareas;
|
||||
|
||||
/* update the paca copy of the context struct */
|
||||
get_paca()->context = mm->context;
|
||||
@ -182,29 +226,63 @@ static int open_low_hpage_segs(struct mm_struct *mm, u16 newsegs)
|
||||
/* the context change must make it to memory before the flush,
|
||||
* so that further SLB misses do the right thing. */
|
||||
mb();
|
||||
on_each_cpu(flush_segments, (void *)(unsigned long)newsegs, 0, 1);
|
||||
on_each_cpu(flush_low_segments, (void *)(unsigned long)newareas, 0, 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int open_high_hpage_areas(struct mm_struct *mm, u16 newareas)
|
||||
{
|
||||
unsigned long i;
|
||||
|
||||
BUILD_BUG_ON((sizeof(newareas)*8) != NUM_HIGH_AREAS);
|
||||
BUILD_BUG_ON((sizeof(mm->context.high_htlb_areas)*8)
|
||||
!= NUM_HIGH_AREAS);
|
||||
|
||||
newareas &= ~(mm->context.high_htlb_areas);
|
||||
if (! newareas)
|
||||
return 0; /* The areas we want are already open */
|
||||
|
||||
for (i = 0; i < NUM_HIGH_AREAS; i++)
|
||||
if ((1 << i) & newareas)
|
||||
if (prepare_high_area_for_htlb(mm, i) != 0)
|
||||
return -EBUSY;
|
||||
|
||||
mm->context.high_htlb_areas |= newareas;
|
||||
|
||||
/* update the paca copy of the context struct */
|
||||
get_paca()->context = mm->context;
|
||||
|
||||
/* the context change must make it to memory before the flush,
|
||||
* so that further SLB misses do the right thing. */
|
||||
mb();
|
||||
on_each_cpu(flush_high_segments, (void *)(unsigned long)newareas, 0, 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int prepare_hugepage_range(unsigned long addr, unsigned long len)
|
||||
{
|
||||
if (within_hugepage_high_range(addr, len))
|
||||
return 0;
|
||||
else if ((addr < 0x100000000UL) && ((addr+len) < 0x100000000UL)) {
|
||||
int err;
|
||||
/* Yes, we need both tests, in case addr+len overflows
|
||||
* 64-bit arithmetic */
|
||||
err = open_low_hpage_segs(current->mm,
|
||||
int err;
|
||||
|
||||
if ( (addr+len) < addr )
|
||||
return -EINVAL;
|
||||
|
||||
if ((addr + len) < 0x100000000UL)
|
||||
err = open_low_hpage_areas(current->mm,
|
||||
LOW_ESID_MASK(addr, len));
|
||||
if (err)
|
||||
printk(KERN_DEBUG "prepare_hugepage_range(%lx, %lx)"
|
||||
" failed (segs: 0x%04hx)\n", addr, len,
|
||||
LOW_ESID_MASK(addr, len));
|
||||
else
|
||||
err = open_high_hpage_areas(current->mm,
|
||||
HTLB_AREA_MASK(addr, len));
|
||||
if (err) {
|
||||
printk(KERN_DEBUG "prepare_hugepage_range(%lx, %lx)"
|
||||
" failed (lowmask: 0x%04hx, highmask: 0x%04hx)\n",
|
||||
addr, len,
|
||||
LOW_ESID_MASK(addr, len), HTLB_AREA_MASK(addr, len));
|
||||
return err;
|
||||
}
|
||||
|
||||
return -EINVAL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct page *
|
||||
@ -276,8 +354,8 @@ full_search:
|
||||
vma = find_vma(mm, addr);
|
||||
continue;
|
||||
}
|
||||
if (touches_hugepage_high_range(addr, len)) {
|
||||
addr = TASK_HPAGE_END;
|
||||
if (touches_hugepage_high_range(mm, addr, len)) {
|
||||
addr = ALIGN(addr+1, 1UL<<HTLB_AREA_SHIFT);
|
||||
vma = find_vma(mm, addr);
|
||||
continue;
|
||||
}
|
||||
@ -356,8 +434,9 @@ hugepage_recheck:
|
||||
if (touches_hugepage_low_range(mm, addr, len)) {
|
||||
addr = (addr & ((~0) << SID_SHIFT)) - len;
|
||||
goto hugepage_recheck;
|
||||
} else if (touches_hugepage_high_range(addr, len)) {
|
||||
addr = TASK_HPAGE_BASE - len;
|
||||
} else if (touches_hugepage_high_range(mm, addr, len)) {
|
||||
addr = (addr & ((~0UL) << HTLB_AREA_SHIFT)) - len;
|
||||
goto hugepage_recheck;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -448,23 +527,28 @@ static unsigned long htlb_get_low_area(unsigned long len, u16 segmask)
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static unsigned long htlb_get_high_area(unsigned long len)
|
||||
static unsigned long htlb_get_high_area(unsigned long len, u16 areamask)
|
||||
{
|
||||
unsigned long addr = TASK_HPAGE_BASE;
|
||||
unsigned long addr = 0x100000000UL;
|
||||
struct vm_area_struct *vma;
|
||||
|
||||
vma = find_vma(current->mm, addr);
|
||||
for (vma = find_vma(current->mm, addr);
|
||||
addr + len <= TASK_HPAGE_END;
|
||||
vma = vma->vm_next) {
|
||||
while (addr + len <= TASK_SIZE_USER64) {
|
||||
BUG_ON(vma && (addr >= vma->vm_end)); /* invariant */
|
||||
BUG_ON(! within_hugepage_high_range(addr, len));
|
||||
|
||||
if (! __within_hugepage_high_range(addr, len, areamask)) {
|
||||
addr = ALIGN(addr+1, 1UL<<HTLB_AREA_SHIFT);
|
||||
vma = find_vma(current->mm, addr);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!vma || (addr + len) <= vma->vm_start)
|
||||
return addr;
|
||||
addr = ALIGN(vma->vm_end, HPAGE_SIZE);
|
||||
/* Because we're in a hugepage region, this alignment
|
||||
* should not skip us over any VMAs */
|
||||
/* Depending on segmask this might not be a confirmed
|
||||
* hugepage region, so the ALIGN could have skipped
|
||||
* some VMAs */
|
||||
vma = find_vma(current->mm, addr);
|
||||
}
|
||||
|
||||
return -ENOMEM;
|
||||
@ -474,6 +558,9 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
||||
unsigned long len, unsigned long pgoff,
|
||||
unsigned long flags)
|
||||
{
|
||||
int lastshift;
|
||||
u16 areamask, curareas;
|
||||
|
||||
if (len & ~HPAGE_MASK)
|
||||
return -EINVAL;
|
||||
|
||||
@ -481,31 +568,49 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
||||
return -EINVAL;
|
||||
|
||||
if (test_thread_flag(TIF_32BIT)) {
|
||||
int lastshift = 0;
|
||||
u16 segmask, cursegs = current->mm->context.htlb_segs;
|
||||
curareas = current->mm->context.low_htlb_areas;
|
||||
|
||||
/* First see if we can do the mapping in the existing
|
||||
* low hpage segments */
|
||||
addr = htlb_get_low_area(len, cursegs);
|
||||
* low areas */
|
||||
addr = htlb_get_low_area(len, curareas);
|
||||
if (addr != -ENOMEM)
|
||||
return addr;
|
||||
|
||||
for (segmask = LOW_ESID_MASK(0x100000000UL-len, len);
|
||||
! lastshift; segmask >>=1) {
|
||||
if (segmask & 1)
|
||||
lastshift = 0;
|
||||
for (areamask = LOW_ESID_MASK(0x100000000UL-len, len);
|
||||
! lastshift; areamask >>=1) {
|
||||
if (areamask & 1)
|
||||
lastshift = 1;
|
||||
|
||||
addr = htlb_get_low_area(len, cursegs | segmask);
|
||||
addr = htlb_get_low_area(len, curareas | areamask);
|
||||
if ((addr != -ENOMEM)
|
||||
&& open_low_hpage_segs(current->mm, segmask) == 0)
|
||||
&& open_low_hpage_areas(current->mm, areamask) == 0)
|
||||
return addr;
|
||||
}
|
||||
printk(KERN_DEBUG "hugetlb_get_unmapped_area() unable to open"
|
||||
" enough segments\n");
|
||||
return -ENOMEM;
|
||||
} else {
|
||||
return htlb_get_high_area(len);
|
||||
curareas = current->mm->context.high_htlb_areas;
|
||||
|
||||
/* First see if we can do the mapping in the existing
|
||||
* high areas */
|
||||
addr = htlb_get_high_area(len, curareas);
|
||||
if (addr != -ENOMEM)
|
||||
return addr;
|
||||
|
||||
lastshift = 0;
|
||||
for (areamask = HTLB_AREA_MASK(TASK_SIZE_USER64-len, len);
|
||||
! lastshift; areamask >>=1) {
|
||||
if (areamask & 1)
|
||||
lastshift = 1;
|
||||
|
||||
addr = htlb_get_high_area(len, curareas | areamask);
|
||||
if ((addr != -ENOMEM)
|
||||
&& open_high_hpage_areas(current->mm, areamask) == 0)
|
||||
return addr;
|
||||
}
|
||||
}
|
||||
printk(KERN_DEBUG "hugetlb_get_unmapped_area() unable to open"
|
||||
" enough areas\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
int hash_huge_page(struct mm_struct *mm, unsigned long access,
|
||||
|
||||
@ -89,28 +89,29 @@ END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
|
||||
b 9f
|
||||
|
||||
0: /* user address: proto-VSID = context<<15 | ESID */
|
||||
li r11,SLB_VSID_USER
|
||||
|
||||
srdi. r9,r3,USER_ESID_BITS
|
||||
bne- 8f /* invalid ea bits set */
|
||||
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
BEGIN_FTR_SECTION
|
||||
/* check against the hugepage ranges */
|
||||
cmpldi r3,(TASK_HPAGE_END>>SID_SHIFT)
|
||||
bge 6f /* >= TASK_HPAGE_END */
|
||||
cmpldi r3,(TASK_HPAGE_BASE>>SID_SHIFT)
|
||||
bge 5f /* TASK_HPAGE_BASE..TASK_HPAGE_END */
|
||||
cmpldi r3,16
|
||||
bge 6f /* 4GB..TASK_HPAGE_BASE */
|
||||
lhz r9,PACAHIGHHTLBAREAS(r13)
|
||||
srdi r11,r3,(HTLB_AREA_SHIFT-SID_SHIFT)
|
||||
srd r9,r9,r11
|
||||
andi. r9,r9,1
|
||||
bne 5f
|
||||
|
||||
lhz r9,PACAHTLBSEGS(r13)
|
||||
li r11,SLB_VSID_USER
|
||||
|
||||
cmpldi r3,16
|
||||
bge 6f
|
||||
|
||||
lhz r9,PACALOWHTLBAREAS(r13)
|
||||
srd r9,r9,r3
|
||||
andi. r9,r9,1
|
||||
|
||||
beq 6f
|
||||
|
||||
5: /* this is a hugepage user address */
|
||||
li r11,(SLB_VSID_USER|SLB_VSID_L)
|
||||
5: li r11,SLB_VSID_USER|SLB_VSID_L
|
||||
END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
|
||||
#endif /* CONFIG_HUGETLB_PAGE */
|
||||
|
||||
|
||||
@ -307,7 +307,7 @@ typedef unsigned long mm_context_id_t;
|
||||
typedef struct {
|
||||
mm_context_id_t id;
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
u16 htlb_segs; /* bitmask */
|
||||
u16 low_htlb_areas, high_htlb_areas;
|
||||
#endif
|
||||
} mm_context_t;
|
||||
|
||||
|
||||
@ -37,40 +37,45 @@
|
||||
|
||||
#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
|
||||
|
||||
/* For 64-bit processes the hugepage range is 1T-1.5T */
|
||||
#define TASK_HPAGE_BASE ASM_CONST(0x0000010000000000)
|
||||
#define TASK_HPAGE_END ASM_CONST(0x0000018000000000)
|
||||
#define HTLB_AREA_SHIFT 40
|
||||
#define HTLB_AREA_SIZE (1UL << HTLB_AREA_SHIFT)
|
||||
#define GET_HTLB_AREA(x) ((x) >> HTLB_AREA_SHIFT)
|
||||
|
||||
#define LOW_ESID_MASK(addr, len) (((1U << (GET_ESID(addr+len-1)+1)) \
|
||||
- (1U << GET_ESID(addr))) & 0xffff)
|
||||
#define HTLB_AREA_MASK(addr, len) (((1U << (GET_HTLB_AREA(addr+len-1)+1)) \
|
||||
- (1U << GET_HTLB_AREA(addr))) & 0xffff)
|
||||
|
||||
#define ARCH_HAS_HUGEPAGE_ONLY_RANGE
|
||||
#define ARCH_HAS_PREPARE_HUGEPAGE_RANGE
|
||||
#define ARCH_HAS_SETCLEAR_HUGE_PTE
|
||||
|
||||
#define touches_hugepage_low_range(mm, addr, len) \
|
||||
(LOW_ESID_MASK((addr), (len)) & mm->context.htlb_segs)
|
||||
#define touches_hugepage_high_range(addr, len) \
|
||||
(((addr) > (TASK_HPAGE_BASE-(len))) && ((addr) < TASK_HPAGE_END))
|
||||
(LOW_ESID_MASK((addr), (len)) & (mm)->context.low_htlb_areas)
|
||||
#define touches_hugepage_high_range(mm, addr, len) \
|
||||
(HTLB_AREA_MASK((addr), (len)) & (mm)->context.high_htlb_areas)
|
||||
|
||||
#define __within_hugepage_low_range(addr, len, segmask) \
|
||||
((LOW_ESID_MASK((addr), (len)) | (segmask)) == (segmask))
|
||||
#define within_hugepage_low_range(addr, len) \
|
||||
__within_hugepage_low_range((addr), (len), \
|
||||
current->mm->context.htlb_segs)
|
||||
#define within_hugepage_high_range(addr, len) (((addr) >= TASK_HPAGE_BASE) \
|
||||
&& ((addr)+(len) <= TASK_HPAGE_END) && ((addr)+(len) >= (addr)))
|
||||
current->mm->context.low_htlb_areas)
|
||||
#define __within_hugepage_high_range(addr, len, zonemask) \
|
||||
((HTLB_AREA_MASK((addr), (len)) | (zonemask)) == (zonemask))
|
||||
#define within_hugepage_high_range(addr, len) \
|
||||
__within_hugepage_high_range((addr), (len), \
|
||||
current->mm->context.high_htlb_areas)
|
||||
|
||||
#define is_hugepage_only_range(mm, addr, len) \
|
||||
(touches_hugepage_high_range((addr), (len)) || \
|
||||
(touches_hugepage_high_range((mm), (addr), (len)) || \
|
||||
touches_hugepage_low_range((mm), (addr), (len)))
|
||||
#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
|
||||
|
||||
#define in_hugepage_area(context, addr) \
|
||||
(cpu_has_feature(CPU_FTR_16M_PAGE) && \
|
||||
( (((addr) >= TASK_HPAGE_BASE) && ((addr) < TASK_HPAGE_END)) || \
|
||||
( ((1 << GET_HTLB_AREA(addr)) & (context).high_htlb_areas) || \
|
||||
( ((addr) < 0x100000000L) && \
|
||||
((1 << GET_ESID(addr)) & (context).htlb_segs) ) ) )
|
||||
((1 << GET_ESID(addr)) & (context).low_htlb_areas) ) ) )
|
||||
|
||||
#else /* !CONFIG_HUGETLB_PAGE */
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user