leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tlb: mmu_gather: add tlb_flush_*_range APIs

Browse Source 
tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and
tlb->end, then set corresponding cleared_*.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20200625080314.230-5-yezhenyu2@huawei.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
This commit is contained in:
Peter Zijlstra (Intel) 2020-06-25 16:03:12 +08:00 committed by Catalin Marinas
parent e735b98a5f
commit 2631ed00b0
1 changed files with 40 additions and 15 deletions

55
include/asm-generic/tlb.h
View File

@ -512,6 +512,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
} }
#endif #endif
/*
* tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
* and set corresponding cleared_*.
*/
static inline void tlb_flush_pte_range(struct mmu_gather *tlb,
unsigned long address, unsigned long size)
{
__tlb_adjust_range(tlb, address, size);
tlb->cleared_ptes = 1;
}
static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,
unsigned long address, unsigned long size)
{
__tlb_adjust_range(tlb, address, size);
tlb->cleared_pmds = 1;
}
static inline void tlb_flush_pud_range(struct mmu_gather *tlb,
unsigned long address, unsigned long size)
{
__tlb_adjust_range(tlb, address, size);
tlb->cleared_puds = 1;
}
static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,
unsigned long address, unsigned long size)
{
__tlb_adjust_range(tlb, address, size);
tlb->cleared_p4ds = 1;
}
#ifndef __tlb_remove_tlb_entry #ifndef __tlb_remove_tlb_entry
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
#endif #endif
@ -525,19 +557,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
*/ */
#define tlb_remove_tlb_entry(tlb, ptep, address) \ #define tlb_remove_tlb_entry(tlb, ptep, address) \
do { \ do { \
__tlb_adjust_range(tlb, address, PAGE_SIZE); \ tlb_flush_pte_range(tlb, address, PAGE_SIZE); \
tlb->cleared_ptes = 1; \
__tlb_remove_tlb_entry(tlb, ptep, address); \ __tlb_remove_tlb_entry(tlb, ptep, address); \
} while (0) } while (0)
#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
do { \ do { \
unsigned long _sz = huge_page_size(h); \ unsigned long _sz = huge_page_size(h); \
__tlb_adjust_range(tlb, address, _sz); \
if (_sz == PMD_SIZE) \ if (_sz == PMD_SIZE) \
tlb->cleared_pmds = 1; \ tlb_flush_pmd_range(tlb, address, _sz); \
else if (_sz == PUD_SIZE) \ else if (_sz == PUD_SIZE) \
tlb->cleared_puds = 1; \ tlb_flush_pud_range(tlb, address, _sz); \
__tlb_remove_tlb_entry(tlb, ptep, address); \ __tlb_remove_tlb_entry(tlb, ptep, address); \
} while (0) } while (0)
@ -551,8 +581,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
do { \ do { \
__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \ tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \
tlb->cleared_pmds = 1; \
__tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \ __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
} while (0) } while (0)
@ -566,8 +595,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
#define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \
do { \ do { \
__tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \ tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \
tlb->cleared_puds = 1; \
__tlb_remove_pud_tlb_entry(tlb, pudp, address); \ __tlb_remove_pud_tlb_entry(tlb, pudp, address); \
} while (0) } while (0)
@ -592,9 +620,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
#ifndef pte_free_tlb #ifndef pte_free_tlb
#define pte_free_tlb(tlb, ptep, address) \ #define pte_free_tlb(tlb, ptep, address) \
do { \ do { \
__tlb_adjust_range(tlb, address, PAGE_SIZE); \ tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \
tlb->freed_tables = 1; \ tlb->freed_tables = 1; \
tlb->cleared_pmds = 1; \
__pte_free_tlb(tlb, ptep, address); \ __pte_free_tlb(tlb, ptep, address); \
} while (0) } while (0)
#endif #endif
@ -602,9 +629,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
#ifndef pmd_free_tlb #ifndef pmd_free_tlb
#define pmd_free_tlb(tlb, pmdp, address) \ #define pmd_free_tlb(tlb, pmdp, address) \
do { \ do { \
__tlb_adjust_range(tlb, address, PAGE_SIZE); \ tlb_flush_pud_range(tlb, address, PAGE_SIZE); \
tlb->freed_tables = 1; \ tlb->freed_tables = 1; \
tlb->cleared_puds = 1; \
__pmd_free_tlb(tlb, pmdp, address); \ __pmd_free_tlb(tlb, pmdp, address); \
} while (0) } while (0)
#endif #endif
@ -612,9 +638,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
#ifndef pud_free_tlb #ifndef pud_free_tlb
#define pud_free_tlb(tlb, pudp, address) \ #define pud_free_tlb(tlb, pudp, address) \
do { \ do { \
__tlb_adjust_range(tlb, address, PAGE_SIZE); \ tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \
tlb->freed_tables = 1; \ tlb->freed_tables = 1; \
tlb->cleared_p4ds = 1; \
__pud_free_tlb(tlb, pudp, address); \ __pud_free_tlb(tlb, pudp, address); \
} while (0) } while (0)
#endif #endif