2008-10-23 05:26:29 +00:00
|
|
|
#ifndef _ASM_X86_PGALLOC_H
|
|
|
|
|
#define _ASM_X86_PGALLOC_H
|
2008-03-17 23:36:55 +00:00
|
|
|
|
|
|
|
|
#include <linux/threads.h>
|
|
|
|
|
#include <linux/mm.h> /* for struct page */
|
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
|
|
2008-06-25 04:19:12 +00:00
|
|
|
static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }
|
|
|
|
|
|
2008-03-17 23:36:56 +00:00
|
|
|
#ifdef CONFIG_PARAVIRT
|
|
|
|
|
#include <asm/paravirt.h>
|
|
|
|
|
#else
|
2008-06-25 04:19:12 +00:00
|
|
|
#define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm)
|
|
|
|
|
static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {}
|
2008-03-17 23:37:06 +00:00
|
|
|
static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {}
|
|
|
|
|
static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {}
|
|
|
|
|
static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
|
|
|
|
|
unsigned long start, unsigned long count) {}
|
|
|
|
|
static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {}
|
2017-03-30 08:07:28 +00:00
|
|
|
static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {}
|
2008-03-17 23:37:06 +00:00
|
|
|
static inline void paravirt_release_pte(unsigned long pfn) {}
|
|
|
|
|
static inline void paravirt_release_pmd(unsigned long pfn) {}
|
|
|
|
|
static inline void paravirt_release_pud(unsigned long pfn) {}
|
2017-03-30 08:07:28 +00:00
|
|
|
static inline void paravirt_release_p4d(unsigned long pfn) {}
|
2008-03-17 23:36:56 +00:00
|
|
|
#endif
|
|
|
|
|
|
2010-02-17 10:38:10 +00:00
|
|
|
/*
|
|
|
|
|
* Flags to use when allocating a user page table page.
|
|
|
|
|
*/
|
|
|
|
|
extern gfp_t __userpte_alloc_gfp;
|
|
|
|
|
|
2008-03-17 23:36:55 +00:00
|
|
|
/*
|
|
|
|
|
* Allocate and free page tables.
|
|
|
|
|
*/
|
|
|
|
|
extern pgd_t *pgd_alloc(struct mm_struct *);
|
|
|
|
|
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
|
|
|
|
|
|
|
|
|
|
extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long);
|
|
|
|
|
extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);
|
|
|
|
|
|
2008-03-17 23:36:57 +00:00
|
|
|
/* Should really implement gc for free page table pages. This could be
|
|
|
|
|
done with a reference count in struct page. */
|
|
|
|
|
|
|
|
|
|
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
|
|
|
|
|
{
|
|
|
|
|
BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
|
|
|
|
|
free_page((unsigned long)pte);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void pte_free(struct mm_struct *mm, struct page *pte)
|
|
|
|
|
{
|
2009-01-23 16:37:49 +00:00
|
|
|
pgtable_page_dtor(pte);
|
2008-03-17 23:36:57 +00:00
|
|
|
__free_page(pte);
|
|
|
|
|
}
|
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-22 05:44:28 +00:00
|
|
|
extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
|
|
|
|
|
|
|
|
|
|
static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
|
|
|
|
|
unsigned long address)
|
|
|
|
|
{
|
|
|
|
|
___pte_free_tlb(tlb, pte);
|
|
|
|
|
}
|
2008-03-17 23:36:57 +00:00
|
|
|
|
2008-03-17 23:36:58 +00:00
|
|
|
static inline void pmd_populate_kernel(struct mm_struct *mm,
|
|
|
|
|
pmd_t *pmd, pte_t *pte)
|
|
|
|
|
{
|
2008-03-17 23:37:01 +00:00
|
|
|
paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
|
2008-03-17 23:36:58 +00:00
|
|
|
set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
|
|
|
|
|
struct page *pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long pfn = page_to_pfn(pte);
|
|
|
|
|
|
2008-03-17 23:37:01 +00:00
|
|
|
paravirt_alloc_pte(mm, pfn);
|
2008-03-17 23:36:58 +00:00
|
|
|
set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define pmd_pgtable(pmd) pmd_page(pmd)
|
|
|
|
|
|
2015-04-14 22:46:14 +00:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 2
|
2008-03-17 23:36:58 +00:00
|
|
|
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
|
|
|
|
|
{
|
2013-11-14 22:31:10 +00:00
|
|
|
struct page *page;
|
2016-07-26 22:24:30 +00:00
|
|
|
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
|
|
|
|
|
|
|
|
|
|
if (mm == &init_mm)
|
|
|
|
|
gfp &= ~__GFP_ACCOUNT;
|
|
|
|
|
page = alloc_pages(gfp, 0);
|
2013-11-14 22:31:10 +00:00
|
|
|
if (!page)
|
|
|
|
|
return NULL;
|
|
|
|
|
if (!pgtable_pmd_page_ctor(page)) {
|
|
|
|
|
__free_pages(page, 0);
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
return (pmd_t *)page_address(page);
|
2008-03-17 23:36:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
|
|
|
|
|
{
|
|
|
|
|
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
|
2013-11-14 22:31:10 +00:00
|
|
|
pgtable_pmd_page_dtor(virt_to_page(pmd));
|
2008-03-17 23:36:58 +00:00
|
|
|
free_page((unsigned long)pmd);
|
|
|
|
|
}
|
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-22 05:44:28 +00:00
|
|
|
extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
|
|
|
|
|
|
|
|
|
|
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
|
tree-wide: fix comment/printk typos
"gadget", "through", "command", "maintain", "maintain", "controller", "address",
"between", "initiali[zs]e", "instead", "function", "select", "already",
"equal", "access", "management", "hierarchy", "registration", "interest",
"relative", "memory", "offset", "already",
Signed-off-by: Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2010-11-01 19:38:34 +00:00
|
|
|
unsigned long address)
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-22 05:44:28 +00:00
|
|
|
{
|
|
|
|
|
___pmd_free_tlb(tlb, pmd);
|
|
|
|
|
}
|
2008-03-17 23:36:58 +00:00
|
|
|
|
2008-03-17 23:36:59 +00:00
|
|
|
#ifdef CONFIG_X86_PAE
|
|
|
|
|
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
|
|
|
|
|
#else /* !CONFIG_X86_PAE */
|
|
|
|
|
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
|
|
|
|
|
{
|
2008-03-17 23:37:01 +00:00
|
|
|
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
|
2008-03-17 23:36:59 +00:00
|
|
|
set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
|
|
|
|
|
}
|
|
|
|
|
#endif /* CONFIG_X86_PAE */
|
|
|
|
|
|
2015-04-14 22:46:14 +00:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 3
|
2017-03-17 18:55:15 +00:00
|
|
|
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
|
2008-03-17 23:36:59 +00:00
|
|
|
{
|
2008-03-17 23:37:02 +00:00
|
|
|
paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
|
2017-03-17 18:55:15 +00:00
|
|
|
set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
|
2008-03-17 23:36:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
|
|
|
|
|
{
|
2016-07-26 22:24:30 +00:00
|
|
|
gfp_t gfp = GFP_KERNEL_ACCOUNT;
|
|
|
|
|
|
|
|
|
|
if (mm == &init_mm)
|
|
|
|
|
gfp &= ~__GFP_ACCOUNT;
|
|
|
|
|
return (pud_t *)get_zeroed_page(gfp);
|
2008-03-17 23:36:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
|
|
|
|
|
{
|
|
|
|
|
BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
|
|
|
|
|
free_page((unsigned long)pud);
|
|
|
|
|
}
|
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-22 05:44:28 +00:00
|
|
|
extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);
|
|
|
|
|
|
|
|
|
|
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
|
|
|
|
|
unsigned long address)
|
|
|
|
|
{
|
|
|
|
|
___pud_free_tlb(tlb, pud);
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-17 18:55:15 +00:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 4
|
|
|
|
|
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
|
|
|
|
|
{
|
|
|
|
|
paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
|
|
|
|
|
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
gfp_t gfp = GFP_KERNEL_ACCOUNT;
|
|
|
|
|
|
|
|
|
|
if (mm == &init_mm)
|
|
|
|
|
gfp &= ~__GFP_ACCOUNT;
|
|
|
|
|
return (p4d_t *)get_zeroed_page(gfp);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
|
|
|
|
|
{
|
|
|
|
|
BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
|
|
|
|
|
free_page((unsigned long)p4d);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
|
|
|
|
|
|
|
|
|
|
static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
|
|
|
|
|
unsigned long address)
|
|
|
|
|
{
|
|
|
|
|
___p4d_free_tlb(tlb, p4d);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 4 */
|
2015-04-14 22:46:14 +00:00
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 3 */
|
|
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
|
2008-03-17 23:36:55 +00:00
|
|
|
|
2008-10-23 05:26:29 +00:00
|
|
|
#endif /* _ASM_X86_PGALLOC_H */
|
