mirror of
https://github.com/torvalds/linux.git
synced 2024-11-26 06:02:05 +00:00
f0293cd1f4
When THP is enabled, 4K pages are collapsed into a single huge
page using the generic pmdp_collapse_flush() which will further
use flush_tlb_range() to shoot-down stale TLB entries. Unfortunately,
the generic pmdp_collapse_flush() only invalidates cached leaf PTEs
using address specific SFENCEs which results in repetitive (or
unpredictable) page faults on RISC-V implementations which cache
non-leaf PTEs.
Provide a RISC-V specific pmdp_collapse_flush() which ensures both
cached leaf and non-leaf PTEs are invalidated by using non-address
specific SFENCEs as recommended by the RISC-V privileged specification.
Fixes: e88b333142 ("riscv: mm: add THP support on 64-bit")
Signed-off-by: Mayuresh Chitale <mchitale@ventanamicro.com>
Link: https://lore.kernel.org/r/20230130074815.1694055-1-mchitale@ventanamicro.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
104 lines
2.1 KiB
C
104 lines
2.1 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
#include <asm/pgalloc.h>
|
|
#include <linux/gfp.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/pgtable.h>
|
|
|
|
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
|
|
int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
void p4d_clear_huge(p4d_t *p4d)
|
|
{
|
|
}
|
|
|
|
int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
|
|
{
|
|
pud_t new_pud = pfn_pud(__phys_to_pfn(phys), prot);
|
|
|
|
set_pud(pud, new_pud);
|
|
return 1;
|
|
}
|
|
|
|
int pud_clear_huge(pud_t *pud)
|
|
{
|
|
if (!pud_leaf(READ_ONCE(*pud)))
|
|
return 0;
|
|
pud_clear(pud);
|
|
return 1;
|
|
}
|
|
|
|
int pud_free_pmd_page(pud_t *pud, unsigned long addr)
|
|
{
|
|
pmd_t *pmd = pud_pgtable(*pud);
|
|
int i;
|
|
|
|
pud_clear(pud);
|
|
|
|
flush_tlb_kernel_range(addr, addr + PUD_SIZE);
|
|
|
|
for (i = 0; i < PTRS_PER_PMD; i++) {
|
|
if (!pmd_none(pmd[i])) {
|
|
pte_t *pte = (pte_t *)pmd_page_vaddr(pmd[i]);
|
|
|
|
pte_free_kernel(NULL, pte);
|
|
}
|
|
}
|
|
|
|
pmd_free(NULL, pmd);
|
|
|
|
return 1;
|
|
}
|
|
|
|
int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
|
|
{
|
|
pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), prot);
|
|
|
|
set_pmd(pmd, new_pmd);
|
|
return 1;
|
|
}
|
|
|
|
int pmd_clear_huge(pmd_t *pmd)
|
|
{
|
|
if (!pmd_leaf(READ_ONCE(*pmd)))
|
|
return 0;
|
|
pmd_clear(pmd);
|
|
return 1;
|
|
}
|
|
|
|
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
|
|
{
|
|
pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd);
|
|
|
|
pmd_clear(pmd);
|
|
|
|
flush_tlb_kernel_range(addr, addr + PMD_SIZE);
|
|
pte_free_kernel(NULL, pte);
|
|
return 1;
|
|
}
|
|
|
|
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
|
|
unsigned long address, pmd_t *pmdp)
|
|
{
|
|
pmd_t pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
|
|
|
|
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
|
VM_BUG_ON(pmd_trans_huge(*pmdp));
|
|
/*
|
|
* When leaf PTE entries (regular pages) are collapsed into a leaf
|
|
* PMD entry (huge page), a valid non-leaf PTE is converted into a
|
|
* valid leaf PTE at the level 1 page table. Since the sfence.vma
|
|
* forms that specify an address only apply to leaf PTEs, we need a
|
|
* global flush here. collapse_huge_page() assumes these flushes are
|
|
* eager, so just do the fence here.
|
|
*/
|
|
flush_tlb_mm(vma->vm_mm);
|
|
return pmd;
|
|
}
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|