forked from Minki/linux
5aec715d7d
Our current switch_mm implementation suffers from a number of problems:
(1) The ASID allocator relies on IPIs to synchronise the CPUs on a
rollover event
(2) Because of (1), we cannot allocate ASIDs with interrupts disabled
and therefore make use of a TIF_SWITCH_MM flag to postpone the
actual switch to finish_arch_post_lock_switch
(3) We run context switch with a reserved (invalid) TTBR0 value, even
though the ASID and pgd are updated atomically
(4) We take a global spinlock (cpu_asid_lock) during context-switch
(5) We use h/w broadcast TLB operations when they are not required
(e.g. in flush_context)
This patch addresses these problems by rewriting the ASID algorithm to
match the bitmap-based arch/arm/ implementation more closely. This in
turn allows us to remove much of the complications surrounding switch_mm,
including the ugly thread flag.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
228 lines
5.3 KiB
ArmAsm
228 lines
5.3 KiB
ArmAsm
/*
|
|
* Based on arch/arm/mm/proc.S
|
|
*
|
|
* Copyright (C) 2001 Deep Blue Solutions Ltd.
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
* Author: Catalin Marinas <catalin.marinas@arm.com>
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/linkage.h>
|
|
#include <asm/assembler.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/hwcap.h>
|
|
#include <asm/pgtable-hwdef.h>
|
|
#include <asm/pgtable.h>
|
|
|
|
#include "proc-macros.S"
|
|
|
|
#ifdef CONFIG_ARM64_64K_PAGES
|
|
#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
|
|
#else
|
|
#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
|
|
#endif
|
|
|
|
#define TCR_SMP_FLAGS TCR_SHARED
|
|
|
|
/* PTWs cacheable, inner/outer WBWA */
|
|
#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
|
|
|
|
#define MAIR(attr, mt) ((attr) << ((mt) * 8))
|
|
|
|
/*
|
|
* cpu_do_idle()
|
|
*
|
|
* Idle the processor (wait for interrupt).
|
|
*/
|
|
ENTRY(cpu_do_idle)
|
|
dsb sy // WFI may enter a low-power mode
|
|
wfi
|
|
ret
|
|
ENDPROC(cpu_do_idle)
|
|
|
|
#ifdef CONFIG_CPU_PM
|
|
/**
|
|
* cpu_do_suspend - save CPU registers context
|
|
*
|
|
* x0: virtual address of context pointer
|
|
*/
|
|
ENTRY(cpu_do_suspend)
|
|
mrs x2, tpidr_el0
|
|
mrs x3, tpidrro_el0
|
|
mrs x4, contextidr_el1
|
|
mrs x5, mair_el1
|
|
mrs x6, cpacr_el1
|
|
mrs x7, ttbr1_el1
|
|
mrs x8, tcr_el1
|
|
mrs x9, vbar_el1
|
|
mrs x10, mdscr_el1
|
|
mrs x11, oslsr_el1
|
|
mrs x12, sctlr_el1
|
|
stp x2, x3, [x0]
|
|
stp x4, x5, [x0, #16]
|
|
stp x6, x7, [x0, #32]
|
|
stp x8, x9, [x0, #48]
|
|
stp x10, x11, [x0, #64]
|
|
str x12, [x0, #80]
|
|
ret
|
|
ENDPROC(cpu_do_suspend)
|
|
|
|
/**
|
|
* cpu_do_resume - restore CPU register context
|
|
*
|
|
* x0: Physical address of context pointer
|
|
* x1: ttbr0_el1 to be restored
|
|
*
|
|
* Returns:
|
|
* sctlr_el1 value in x0
|
|
*/
|
|
ENTRY(cpu_do_resume)
|
|
/*
|
|
* Invalidate local tlb entries before turning on MMU
|
|
*/
|
|
tlbi vmalle1
|
|
ldp x2, x3, [x0]
|
|
ldp x4, x5, [x0, #16]
|
|
ldp x6, x7, [x0, #32]
|
|
ldp x8, x9, [x0, #48]
|
|
ldp x10, x11, [x0, #64]
|
|
ldr x12, [x0, #80]
|
|
msr tpidr_el0, x2
|
|
msr tpidrro_el0, x3
|
|
msr contextidr_el1, x4
|
|
msr mair_el1, x5
|
|
msr cpacr_el1, x6
|
|
msr ttbr0_el1, x1
|
|
msr ttbr1_el1, x7
|
|
tcr_set_idmap_t0sz x8, x7
|
|
msr tcr_el1, x8
|
|
msr vbar_el1, x9
|
|
msr mdscr_el1, x10
|
|
/*
|
|
* Restore oslsr_el1 by writing oslar_el1
|
|
*/
|
|
ubfx x11, x11, #1, #1
|
|
msr oslar_el1, x11
|
|
mov x0, x12
|
|
dsb nsh // Make sure local tlb invalidation completed
|
|
isb
|
|
ret
|
|
ENDPROC(cpu_do_resume)
|
|
#endif
|
|
|
|
/*
|
|
* cpu_do_switch_mm(pgd_phys, tsk)
|
|
*
|
|
* Set the translation table base pointer to be pgd_phys.
|
|
*
|
|
* - pgd_phys - physical address of new TTB
|
|
*/
|
|
ENTRY(cpu_do_switch_mm)
|
|
mmid x1, x1 // get mm->context.id
|
|
bfi x0, x1, #48, #16 // set the ASID
|
|
msr ttbr0_el1, x0 // set TTBR0
|
|
isb
|
|
ret
|
|
ENDPROC(cpu_do_switch_mm)
|
|
|
|
.section ".text.init", #alloc, #execinstr
|
|
|
|
/*
|
|
* __cpu_setup
|
|
*
|
|
* Initialise the processor for turning the MMU on. Return in x0 the
|
|
* value of the SCTLR_EL1 register.
|
|
*/
|
|
ENTRY(__cpu_setup)
|
|
tlbi vmalle1 // Invalidate local TLB
|
|
dsb nsh
|
|
|
|
mov x0, #3 << 20
|
|
msr cpacr_el1, x0 // Enable FP/ASIMD
|
|
mov x0, #1 << 12 // Reset mdscr_el1 and disable
|
|
msr mdscr_el1, x0 // access to the DCC from EL0
|
|
/*
|
|
* Memory region attributes for LPAE:
|
|
*
|
|
* n = AttrIndx[2:0]
|
|
* n MAIR
|
|
* DEVICE_nGnRnE 000 00000000
|
|
* DEVICE_nGnRE 001 00000100
|
|
* DEVICE_GRE 010 00001100
|
|
* NORMAL_NC 011 01000100
|
|
* NORMAL 100 11111111
|
|
*/
|
|
ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
|
|
MAIR(0x04, MT_DEVICE_nGnRE) | \
|
|
MAIR(0x0c, MT_DEVICE_GRE) | \
|
|
MAIR(0x44, MT_NORMAL_NC) | \
|
|
MAIR(0xff, MT_NORMAL)
|
|
msr mair_el1, x5
|
|
/*
|
|
* Prepare SCTLR
|
|
*/
|
|
adr x5, crval
|
|
ldp w5, w6, [x5]
|
|
mrs x0, sctlr_el1
|
|
bic x0, x0, x5 // clear bits
|
|
orr x0, x0, x6 // set bits
|
|
/*
|
|
* Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
|
|
* both user and kernel.
|
|
*/
|
|
ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
|
|
TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
|
|
tcr_set_idmap_t0sz x10, x9
|
|
|
|
/*
|
|
* Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
|
|
* TCR_EL1.
|
|
*/
|
|
mrs x9, ID_AA64MMFR0_EL1
|
|
bfi x10, x9, #32, #3
|
|
#ifdef CONFIG_ARM64_HW_AFDBM
|
|
/*
|
|
* Hardware update of the Access and Dirty bits.
|
|
*/
|
|
mrs x9, ID_AA64MMFR1_EL1
|
|
and x9, x9, #0xf
|
|
cbz x9, 2f
|
|
cmp x9, #2
|
|
b.lt 1f
|
|
orr x10, x10, #TCR_HD // hardware Dirty flag update
|
|
1: orr x10, x10, #TCR_HA // hardware Access flag update
|
|
2:
|
|
#endif /* CONFIG_ARM64_HW_AFDBM */
|
|
msr tcr_el1, x10
|
|
ret // return to head.S
|
|
ENDPROC(__cpu_setup)
|
|
|
|
/*
|
|
* We set the desired value explicitly, including those of the
|
|
* reserved bits. The values of bits EE & E0E were set early in
|
|
* el2_setup, which are left untouched below.
|
|
*
|
|
* n n T
|
|
* U E WT T UD US IHBS
|
|
* CE0 XWHW CZ ME TEEA S
|
|
* .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
|
|
* 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
|
|
* .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
|
|
*/
|
|
.type crval, #object
|
|
crval:
|
|
.word 0xfcffffff // clear
|
|
.word 0x34d5d91d // set
|