leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s390/mm: uninline ptep_xxx functions from pgtable.h

Browse Source 
The code in the various ptep_xxx functions has grown quite large,
consolidate them to four out-of-line functions:
  ptep_xchg_direct to exchange a pte with another with immediate flushing
  ptep_xchg_lazy to exchange a pte with another in a batched update
  ptep_modify_prot_start to begin a protection flags update
  ptep_modify_prot_commit to commit a protection flags update

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
This commit is contained in:
Martin Schwidefsky 2016-03-08 11:08:09 +01:00
parent 988b86e69d
commit ebde765c0e
3 changed files with 324 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

386
arch/s390/include/asm/pgtable.h
View File

@ -680,69 +680,8 @@ static inline void pgste_set(pte_t *ptep, pgste_t pgste)
#endif #endif
} }
static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, bool pgste_test_and_clear_dirty(struct mm_struct *, unsigned long address);
struct mm_struct *mm) void ptep_ipte_notify(struct mm_struct *, unsigned long addr, pte_t *);
{
#ifdef CONFIG_PGSTE
unsigned long address, bits, skey;
if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
return pgste;
address = pte_val(pte) & PAGE_MASK;
skey = (unsigned long) page_get_storage_key(address);
bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
/* Transfer page changed & referenced bit to guest bits in pgste */
pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */
/* Copy page access key and fetch protection bit to pgste */
pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
#endif
return pgste;
}
static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
unsigned long address;
unsigned long nkey;
if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
return;
VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
address = pte_val(entry) & PAGE_MASK;
/*
* Set page access key and fetch protection bit from pgste.
* The guest C/R information is still in the PGSTE, set real
* key C/R to 0.
*/
nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
page_set_storage_key(address, nkey, 0);
#endif
}
static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
{
if ((pte_val(entry) & _PAGE_PRESENT) &&
(pte_val(entry) & _PAGE_WRITE) &&
!(pte_val(entry) & _PAGE_INVALID)) {
if (!MACHINE_HAS_ESOP) {
/*
* Without enhanced suppression-on-protection force
* the dirty bit on for all writable ptes.
*/
pte_val(entry) |= _PAGE_DIRTY;
pte_val(entry) &= ~_PAGE_PROTECT;
}
if (!(pte_val(entry) & _PAGE_PROTECT))
/* This pte allows write access, set user-dirty */
pgste_val(pgste) |= PGSTE_UC_BIT;
}
*ptep = entry;
return pgste;
}
/** /**
* struct gmap_struct - guest address space * struct gmap_struct - guest address space
@ -791,47 +730,11 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr);
int gmap_fault(struct gmap *, unsigned long gaddr, unsigned int fault_flags); int gmap_fault(struct gmap *, unsigned long gaddr, unsigned int fault_flags);
void gmap_discard(struct gmap *, unsigned long from, unsigned long to); void gmap_discard(struct gmap *, unsigned long from, unsigned long to);
void __gmap_zap(struct gmap *, unsigned long gaddr); void __gmap_zap(struct gmap *, unsigned long gaddr);
bool gmap_test_and_clear_dirty(unsigned long address, struct gmap *);
void gmap_register_ipte_notifier(struct gmap_notifier *); void gmap_register_ipte_notifier(struct gmap_notifier *);
void gmap_unregister_ipte_notifier(struct gmap_notifier *); void gmap_unregister_ipte_notifier(struct gmap_notifier *);
int gmap_ipte_notify(struct gmap *, unsigned long start, unsigned long len); int gmap_ipte_notify(struct gmap *, unsigned long start, unsigned long len);
void gmap_do_ipte_notify(struct mm_struct *, unsigned long addr, pte_t *);
static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, pgste_t pgste)
{
#ifdef CONFIG_PGSTE
if (pgste_val(pgste) & PGSTE_IN_BIT) {
pgste_val(pgste) &= ~PGSTE_IN_BIT;
gmap_do_ipte_notify(mm, addr, ptep);
}
#endif
return pgste;
}
/*
* Certain architectures need to do special things when PTEs
* within a page table are directly modified. Thus, the following
* hook is made available.
*/
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
pgste_t pgste;
if (mm_has_pgste(mm)) {
pgste = pgste_get_lock(ptep);
pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
pgste_set_key(ptep, pgste, entry, mm);
pgste = pgste_set_pte(ptep, pgste, entry);
pgste_set_unlock(ptep, pgste);
} else {
*ptep = entry;
}
}
/* /*
* query functions pte_write/pte_dirty/pte_young only work if * query functions pte_write/pte_dirty/pte_young only work if
@ -998,104 +901,6 @@ static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep)
} while (nr != 255); } while (nr != 255);
} }
static inline void ptep_flush_direct(struct mm_struct *mm,
unsigned long address, pte_t *ptep)
{
int active, count;
if (pte_val(*ptep) & _PAGE_INVALID)
return;
active = (mm == current->active_mm) ? 1 : 0;
count = atomic_add_return(0x10000, &mm->context.attach_count);
if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
__ptep_ipte_local(address, ptep);
else
__ptep_ipte(address, ptep);
atomic_sub(0x10000, &mm->context.attach_count);
}
static inline void ptep_flush_lazy(struct mm_struct *mm,
unsigned long address, pte_t *ptep)
{
int active, count;
if (pte_val(*ptep) & _PAGE_INVALID)
return;
active = (mm == current->active_mm) ? 1 : 0;
count = atomic_add_return(0x10000, &mm->context.attach_count);
if ((count & 0xffff) <= active) {
pte_val(*ptep) |= _PAGE_INVALID;
mm->context.flush_mm = 1;
} else
__ptep_ipte(address, ptep);
atomic_sub(0x10000, &mm->context.attach_count);
}
/*
* Get (and clear) the user dirty bit for a pte.
*/
static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep)
{
pgste_t pgste;
pte_t pte;
int dirty;
if (!mm_has_pgste(mm))
return 0;
pgste = pgste_get_lock(ptep);
dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
pgste_val(pgste) &= ~PGSTE_UC_BIT;
pte = *ptep;
if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
__ptep_ipte(addr, ptep);
if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
pte_val(pte) |= _PAGE_PROTECT;
else
pte_val(pte) |= _PAGE_INVALID;
*ptep = pte;
}
pgste_set_unlock(ptep, pgste);
return dirty;
}
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
pgste_t pgste;
pte_t pte;
int young;
if (mm_has_pgste(vma->vm_mm)) {
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste);
}
pte = *ptep;
ptep_flush_direct(vma->vm_mm, addr, ptep);
young = pte_young(pte);
if (mm_has_pgste(vma->vm_mm)) {
pgste = pgste_update_all(pte, pgste, vma->vm_mm);
pgste = pgste_set_pte(ptep, pgste, pte_mkold(pte));
pgste_set_unlock(ptep, pgste);
} else
*ptep = pte_mkold(pte);
return young;
}
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
return ptep_test_and_clear_young(vma, address, ptep);
}
/* /*
* This is hard to understand. ptep_get_and_clear and ptep_clear_flush * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
* both clear the TLB for the unmapped pte. The reason is that * both clear the TLB for the unmapped pte. The reason is that
@ -1109,91 +914,42 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
* have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
* is a nop. * is a nop.
*/ */
pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
pte_t pte = *ptep;
pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
return pte_young(pte);
}
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
return ptep_test_and_clear_young(vma, address, ptep);
}
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(struct mm_struct *mm, static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
unsigned long address, pte_t *ptep) unsigned long addr, pte_t *ptep)
{ {
pgste_t pgste; return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
pte_t pte;
if (mm_has_pgste(mm)) {
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(mm, address, ptep, pgste);
}
pte = *ptep;
ptep_flush_lazy(mm, address, ptep);
pte_val(*ptep) = _PAGE_INVALID;
if (mm_has_pgste(mm)) {
pgste = pgste_update_all(pte, pgste, mm);
pgste_set_unlock(ptep, pgste);
}
return pte;
} }
#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, pte_t ptep_modify_prot_start(struct mm_struct *, unsigned long, pte_t *);
unsigned long address, void ptep_modify_prot_commit(struct mm_struct *, unsigned long, pte_t *, pte_t);
pte_t *ptep)
{
pgste_t pgste;
pte_t pte;
if (mm_has_pgste(mm)) {
pgste = pgste_get_lock(ptep);
pgste_ipte_notify(mm, address, ptep, pgste);
}
pte = *ptep;
ptep_flush_lazy(mm, address, ptep);
if (mm_has_pgste(mm)) {
pgste = pgste_update_all(pte, pgste, mm);
pgste_set(ptep, pgste);
}
return pte;
}
static inline void ptep_modify_prot_commit(struct mm_struct *mm,
unsigned long address,
pte_t *ptep, pte_t pte)
{
pgste_t pgste;
if (mm_has_pgste(mm)) {
pgste = pgste_get(ptep);
pgste_set_key(ptep, pgste, pte, mm);
pgste = pgste_set_pte(ptep, pgste, pte);
pgste_set_unlock(ptep, pgste);
} else
*ptep = pte;
}
#define __HAVE_ARCH_PTEP_CLEAR_FLUSH #define __HAVE_ARCH_PTEP_CLEAR_FLUSH
static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep) unsigned long addr, pte_t *ptep)
{ {
pgste_t pgste; return ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
pte_t pte;
if (mm_has_pgste(vma->vm_mm)) {
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
}
pte = *ptep;
ptep_flush_direct(vma->vm_mm, address, ptep);
pte_val(*ptep) = _PAGE_INVALID;
if (mm_has_pgste(vma->vm_mm)) {
if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
_PGSTE_GPS_USAGE_UNUSED)
pte_val(pte) |= _PAGE_UNUSED;
pgste = pgste_update_all(pte, pgste, vma->vm_mm);
pgste_set_unlock(ptep, pgste);
}
return pte;
} }
/* /*
@ -1205,82 +961,54 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
*/ */
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
unsigned long address, unsigned long addr,
pte_t *ptep, int full) pte_t *ptep, int full)
{ {
pgste_t pgste; if (full) {
pte_t pte; pte_t pte = *ptep;
*ptep = __pte(_PAGE_INVALID);
if (!full && mm_has_pgste(mm)) { return pte;
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(mm, address, ptep, pgste);
} }
return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
pte = *ptep;
if (!full)
ptep_flush_lazy(mm, address, ptep);
pte_val(*ptep) = _PAGE_INVALID;
if (!full && mm_has_pgste(mm)) {
pgste = pgste_update_all(pte, pgste, mm);
pgste_set_unlock(ptep, pgste);
}
return pte;
} }
#define __HAVE_ARCH_PTEP_SET_WRPROTECT #define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline pte_t ptep_set_wrprotect(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm,
unsigned long address, pte_t *ptep) unsigned long addr, pte_t *ptep)
{ {
pgste_t pgste;
pte_t pte = *ptep; pte_t pte = *ptep;
if (pte_write(pte)) { if (pte_write(pte))
if (mm_has_pgste(mm)) { ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(mm, address, ptep, pgste);
}
ptep_flush_lazy(mm, address, ptep);
pte = pte_wrprotect(pte);
if (mm_has_pgste(mm)) {
pgste = pgste_set_pte(ptep, pgste, pte);
pgste_set_unlock(ptep, pgste);
} else
*ptep = pte;
}
return pte;
} }
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
static inline int ptep_set_access_flags(struct vm_area_struct *vma, static inline int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep, unsigned long addr, pte_t *ptep,
pte_t entry, int dirty) pte_t entry, int dirty)
{ {
pgste_t pgste; if (pte_same(*ptep, entry))
pte_t oldpte;
oldpte = *ptep;
if (pte_same(oldpte, entry))
return 0; return 0;
if (mm_has_pgste(vma->vm_mm)) { ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
}
ptep_flush_direct(vma->vm_mm, address, ptep);
if (mm_has_pgste(vma->vm_mm)) {
if (pte_val(oldpte) & _PAGE_INVALID)
pgste_set_key(ptep, pgste, entry, vma->vm_mm);
pgste = pgste_set_pte(ptep, pgste, entry);
pgste_set_unlock(ptep, pgste);
} else
*ptep = entry;
return 1; return 1;
} }
void set_pte_pgste_at(struct mm_struct *, unsigned long, pte_t *, pte_t);
/*
* Certain architectures need to do special things when PTEs
* within a page table are directly modified. Thus, the following
* hook is made available.
*/
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
if (mm_has_pgste(mm))
set_pte_pgste_at(mm, addr, ptep, entry);
else
*ptep = entry;
}
/* /*
* Conversion functions: convert a page and protection to a page entry, * Conversion functions: convert a page and protection to a page entry,
* and a page entry and page directory to the page they refer to. * and a page entry and page directory to the page they refer to.

2
arch/s390/kvm/kvm-s390.c
View File

@ -280,7 +280,7 @@ static void kvm_s390_sync_dirty_log(struct kvm *kvm,
for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
address = gfn_to_hva_memslot(memslot, cur_gfn); address = gfn_to_hva_memslot(memslot, cur_gfn);
if (gmap_test_and_clear_dirty(address, gmap)) if (pgste_test_and_clear_dirty(gmap->mm, address))
mark_page_dirty(kvm, cur_gfn); mark_page_dirty(kvm, cur_gfn);
} }
up_read(&gmap->mm->mmap_sem); up_read(&gmap->mm->mmap_sem);

295
arch/s390/mm/pgtable.c
View File

@ -772,7 +772,7 @@ int gmap_ipte_notify(struct gmap *gmap, unsigned long gaddr, unsigned long len)
EXPORT_SYMBOL_GPL(gmap_ipte_notify); EXPORT_SYMBOL_GPL(gmap_ipte_notify);
/** /**
* gmap_do_ipte_notify - call all invalidation callbacks for a specific pte. * ptep_ipte_notify - call all invalidation callbacks for a specific pte.
* @mm: pointer to the process mm_struct * @mm: pointer to the process mm_struct
* @addr: virtual address in the process address space * @addr: virtual address in the process address space
* @pte: pointer to the page table entry * @pte: pointer to the page table entry
@ -780,7 +780,7 @@ EXPORT_SYMBOL_GPL(gmap_ipte_notify);
* This function is assumed to be called with the page table lock held * This function is assumed to be called with the page table lock held
* for the pte to notify. * for the pte to notify.
*/ */
void gmap_do_ipte_notify(struct mm_struct *mm, unsigned long vmaddr, pte_t *pte) void ptep_ipte_notify(struct mm_struct *mm, unsigned long vmaddr, pte_t *pte)
{ {
unsigned long offset, gaddr; unsigned long offset, gaddr;
unsigned long *table; unsigned long *table;
@ -801,7 +801,7 @@ void gmap_do_ipte_notify(struct mm_struct *mm, unsigned long vmaddr, pte_t *pte)
} }
spin_unlock(&gmap_notifier_lock); spin_unlock(&gmap_notifier_lock);
} }
EXPORT_SYMBOL_GPL(gmap_do_ipte_notify); EXPORT_SYMBOL_GPL(ptep_ipte_notify);
int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
unsigned long key, bool nq) unsigned long key, bool nq)
@ -1158,6 +1158,266 @@ static inline void thp_split_mm(struct mm_struct *mm)
} }
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
static inline pte_t ptep_flush_direct(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
int active, count;
pte_t old;
old = *ptep;
if (unlikely(pte_val(old) & _PAGE_INVALID))
return old;
active = (mm == current->active_mm) ? 1 : 0;
count = atomic_add_return(0x10000, &mm->context.attach_count);
if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
__ptep_ipte_local(addr, ptep);
else
__ptep_ipte(addr, ptep);
atomic_sub(0x10000, &mm->context.attach_count);
return old;
}
static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
int active, count;
pte_t old;
old = *ptep;
if (unlikely(pte_val(old) & _PAGE_INVALID))
return old;
active = (mm == current->active_mm) ? 1 : 0;
count = atomic_add_return(0x10000, &mm->context.attach_count);
if ((count & 0xffff) <= active) {
pte_val(*ptep) |= _PAGE_INVALID;
mm->context.flush_mm = 1;
} else
__ptep_ipte(addr, ptep);
atomic_sub(0x10000, &mm->context.attach_count);
return old;
}
static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
unsigned long address, bits, skey;
if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
return pgste;
address = pte_val(pte) & PAGE_MASK;
skey = (unsigned long) page_get_storage_key(address);
bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
/* Transfer page changed & referenced bit to guest bits in pgste */
pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */
/* Copy page access key and fetch protection bit to pgste */
pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
#endif
return pgste;
}
static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
unsigned long address;
unsigned long nkey;
if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
return;
VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
address = pte_val(entry) & PAGE_MASK;
/*
* Set page access key and fetch protection bit from pgste.
* The guest C/R information is still in the PGSTE, set real
* key C/R to 0.
*/
nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
page_set_storage_key(address, nkey, 0);
#endif
}
static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
{
#ifdef CONFIG_PGSTE
if ((pte_val(entry) & _PAGE_PRESENT) &&
(pte_val(entry) & _PAGE_WRITE) &&
!(pte_val(entry) & _PAGE_INVALID)) {
if (!MACHINE_HAS_ESOP) {
/*
* Without enhanced suppression-on-protection force
* the dirty bit on for all writable ptes.
*/
pte_val(entry) |= _PAGE_DIRTY;
pte_val(entry) &= ~_PAGE_PROTECT;
}
if (!(pte_val(entry) & _PAGE_PROTECT))
/* This pte allows write access, set user-dirty */
pgste_val(pgste) |= PGSTE_UC_BIT;
}
#endif
*ptep = entry;
return pgste;
}
static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, pgste_t pgste)
{
#ifdef CONFIG_PGSTE
if (pgste_val(pgste) & PGSTE_IN_BIT) {
pgste_val(pgste) &= ~PGSTE_IN_BIT;
ptep_ipte_notify(mm, addr, ptep);
}
#endif
return pgste;
}
#ifdef CONFIG_PGSTE
/*
* Test and reset if a guest page is dirty
*/
bool pgste_test_and_clear_dirty(struct mm_struct *mm, unsigned long addr)
{
spinlock_t *ptl;
pgste_t pgste;
pte_t *ptep;
pte_t pte;
bool dirty;
ptep = get_locked_pte(mm, addr, &ptl);
if (unlikely(!ptep))
return false;
pgste = pgste_get_lock(ptep);
dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
pgste_val(pgste) &= ~PGSTE_UC_BIT;
pte = *ptep;
if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
__ptep_ipte(addr, ptep);
if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
pte_val(pte) |= _PAGE_PROTECT;
else
pte_val(pte) |= _PAGE_INVALID;
*ptep = pte;
}
pgste_set_unlock(ptep, pgste);
spin_unlock(ptl);
return dirty;
}
EXPORT_SYMBOL_GPL(pgste_test_and_clear_dirty);
void set_pte_pgste_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
pgste_t pgste;
/* the mm_has_pgste() check is done in set_pte_at() */
pgste = pgste_get_lock(ptep);
pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
pgste_set_key(ptep, pgste, entry, mm);
pgste = pgste_set_pte(ptep, pgste, entry);
pgste_set_unlock(ptep, pgste);
}
EXPORT_SYMBOL(set_pte_pgste_at);
#endif
static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
pgste_t pgste = __pgste(0);
if (mm_has_pgste(mm)) {
pgste = pgste_get_lock(ptep);
pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
}
return pgste;
}
static inline void ptep_xchg_commit(struct mm_struct *mm,
unsigned long addr, pte_t *ptep,
pgste_t pgste, pte_t old, pte_t new)
{
if (mm_has_pgste(mm)) {
if (pte_val(old) & _PAGE_INVALID)
pgste_set_key(ptep, pgste, new, mm);
if (pte_val(new) & _PAGE_INVALID) {
pgste = pgste_update_all(old, pgste, mm);
if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
_PGSTE_GPS_USAGE_UNUSED)
pte_val(old) |= _PAGE_UNUSED;
}
pgste = pgste_set_pte(ptep, pgste, new);
pgste_set_unlock(ptep, pgste);
} else {
*ptep = new;
}
}
pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t new)
{
pgste_t pgste;
pte_t old;
pgste = ptep_xchg_start(mm, addr, ptep);
old = ptep_flush_direct(mm, addr, ptep);
ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
return old;
}
EXPORT_SYMBOL(ptep_xchg_direct);
pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t new)
{
pgste_t pgste;
pte_t old;
pgste = ptep_xchg_start(mm, addr, ptep);
old = ptep_flush_lazy(mm, addr, ptep);
ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
return old;
}
EXPORT_SYMBOL(ptep_xchg_lazy);
pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
pgste_t pgste;
pte_t old;
pgste = ptep_xchg_start(mm, addr, ptep);
old = ptep_flush_lazy(mm, addr, ptep);
if (mm_has_pgste(mm)) {
pgste = pgste_update_all(old, pgste, mm);
pgste_set(ptep, pgste);
}
return old;
}
EXPORT_SYMBOL(ptep_modify_prot_start);
void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte)
{
pgste_t pgste;
if (mm_has_pgste(mm)) {
pgste = pgste_get(ptep);
pgste_set_key(ptep, pgste, pte, mm);
pgste = pgste_set_pte(ptep, pgste, pte);
pgste_set_unlock(ptep, pgste);
} else {
*ptep = pte;
}
}
EXPORT_SYMBOL(ptep_modify_prot_commit);
/* /*
* switch on pgstes for its userspace process (for kvm) * switch on pgstes for its userspace process (for kvm)
*/ */
@ -1190,17 +1450,15 @@ static int __s390_enable_skey(pte_t *pte, unsigned long addr,
unsigned long ptev; unsigned long ptev;
pgste_t pgste; pgste_t pgste;
pgste = pgste_get_lock(pte);
/* /*
* Remove all zero page mappings, * Remove all zero page mappings,
* after establishing a policy to forbid zero page mappings * after establishing a policy to forbid zero page mappings
* following faults for that page will get fresh anonymous pages * following faults for that page will get fresh anonymous pages
*/ */
if (is_zero_pfn(pte_pfn(*pte))) { if (is_zero_pfn(pte_pfn(*pte)))
ptep_flush_direct(walk->mm, addr, pte); ptep_xchg_direct(walk->mm, addr, pte, __pte(_PAGE_INVALID));
pte_val(*pte) = _PAGE_INVALID;
}
/* Clear storage key */ /* Clear storage key */
pgste = pgste_get_lock(pte);
pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT |
PGSTE_GR_BIT | PGSTE_GC_BIT); PGSTE_GR_BIT | PGSTE_GC_BIT);
ptev = pte_val(*pte); ptev = pte_val(*pte);
@ -1266,27 +1524,6 @@ void s390_reset_cmma(struct mm_struct *mm)
} }
EXPORT_SYMBOL_GPL(s390_reset_cmma); EXPORT_SYMBOL_GPL(s390_reset_cmma);
/*
* Test and reset if a guest page is dirty
*/
bool gmap_test_and_clear_dirty(unsigned long address, struct gmap *gmap)
{
pte_t *pte;
spinlock_t *ptl;
bool dirty = false;
pte = get_locked_pte(gmap->mm, address, &ptl);
if (unlikely(!pte))
return false;
if (ptep_test_and_clear_user_dirty(gmap->mm, address, pte))
dirty = true;
spin_unlock(ptl);
return dirty;
}
EXPORT_SYMBOL_GPL(gmap_test_and_clear_dirty);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp) pmd_t *pmdp)