2012-12-10 16:29:28 +00:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2012,2013 - ARM Ltd
|
|
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
|
|
*
|
|
|
|
* Derived from arch/arm/include/asm/kvm_host.h:
|
|
|
|
* Copyright (C) 2012 - Virtual Open Systems and Columbia University
|
|
|
|
* Author: Christoffer Dall <c.dall@virtualopensystems.com>
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef __ARM64_KVM_HOST_H__
|
|
|
|
#define __ARM64_KVM_HOST_H__
|
|
|
|
|
2014-08-29 12:01:17 +00:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kvm_types.h>
|
arm64/cpufeature: don't use mutex in bringup path
Currently, cpus_set_cap() calls static_branch_enable_cpuslocked(), which
must take the jump_label mutex.
We call cpus_set_cap() in the secondary bringup path, from the idle
thread where interrupts are disabled. Taking a mutex in this path "is a
NONO" regardless of whether it's contended, and something we must avoid.
We didn't spot this until recently, as ___might_sleep() won't warn for
this case until all CPUs have been brought up.
This patch avoids taking the mutex in the secondary bringup path. The
poking of static keys is deferred until enable_cpu_capabilities(), which
runs in a suitable context on the boot CPU. To account for the static
keys being set later, cpus_have_const_cap() is updated to use another
static key to check whether the const cap keys have been initialised,
falling back to the caps bitmap until this is the case.
This means that users of cpus_have_const_cap() gain should only gain a
single additional NOP in the fast path once the const caps are
initialised, but should always see the current cap value.
The hyp code should never dereference the caps array, since the caps are
initialized before we run the module initcall to initialise hyp. A check
is added to the hyp init code to document this requirement.
This change will sidestep a number of issues when the upcoming hotplug
locking rework is merged.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Marc Zyniger <marc.zyngier@arm.com>
Reviewed-by: Suzuki Poulose <suzuki.poulose@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Sewior <bigeasy@linutronix.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-05-16 14:18:05 +00:00
|
|
|
#include <asm/cpufeature.h>
|
2018-01-15 19:39:00 +00:00
|
|
|
#include <asm/daifflags.h>
|
arm64/sve: KVM: Prevent guests from using SVE
Until KVM has full SVE support, guests must not be allowed to
execute SVE instructions.
This patch enables the necessary traps, and also ensures that the
traps are disabled again on exit from the guest so that the host
can still use SVE if it wants to.
On guest exit, high bits of the SVE Zn registers may have been
clobbered as a side-effect the execution of FPSIMD instructions in
the guest. The existing KVM host FPSIMD restore code is not
sufficient to restore these bits, so this patch explicitly marks
the CPU as not containing cached vector state for any task, thus
forcing a reload on the next return to userspace. This is an
interim measure, in advance of adding full SVE awareness to KVM.
This marking of cached vector state in the CPU as invalid is done
using __this_cpu_write(fpsimd_last_state, NULL) in fpsimd.c. Due
to the repeated use of this rather obscure operation, it makes
sense to factor it out as a separate helper with a clearer name.
This patch factors it out as fpsimd_flush_cpu_state(), and ports
all callers to use it.
As a side effect of this refactoring, a this_cpu_write() in
fpsimd_cpu_pm_notifier() is changed to __this_cpu_write(). This
should be fine, since cpu_pm_enter() is supposed to be called only
with interrupts disabled.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:16 +00:00
|
|
|
#include <asm/fpsimd.h>
|
2012-12-10 16:29:28 +00:00
|
|
|
#include <asm/kvm.h>
|
2015-01-29 13:19:45 +00:00
|
|
|
#include <asm/kvm_asm.h>
|
2012-12-10 16:29:28 +00:00
|
|
|
#include <asm/kvm_mmio.h>
|
2018-04-06 13:55:59 +00:00
|
|
|
#include <asm/thread_info.h>
|
2012-12-10 16:29:28 +00:00
|
|
|
|
2015-03-04 10:14:34 +00:00
|
|
|
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
|
|
|
|
|
2017-03-08 06:08:35 +00:00
|
|
|
#define KVM_USER_MEM_SLOTS 512
|
2015-09-18 10:34:53 +00:00
|
|
|
#define KVM_HALT_POLL_NS_DEFAULT 500000
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
#include <kvm/arm_vgic.h>
|
|
|
|
#include <kvm/arm_arch_timer.h>
|
2015-09-11 01:38:32 +00:00
|
|
|
#include <kvm/arm_pmu.h>
|
2012-12-10 16:29:28 +00:00
|
|
|
|
arm/arm64: KVM: Remove 'config KVM_ARM_MAX_VCPUS'
This patch removes config option of KVM_ARM_MAX_VCPUS,
and like other ARCHs, just choose the maximum allowed
value from hardware, and follows the reasons:
1) from distribution view, the option has to be
defined as the max allowed value because it need to
meet all kinds of virtulization applications and
need to support most of SoCs;
2) using a bigger value doesn't introduce extra memory
consumption, and the help text in Kconfig isn't accurate
because kvm_vpu structure isn't allocated until request
of creating VCPU is sent from QEMU;
3) the main effect is that the field of vcpus[] in 'struct kvm'
becomes a bit bigger(sizeof(void *) per vcpu) and need more cache
lines to hold the structure, but 'struct kvm' is one generic struct,
and it has worked well on other ARCHs already in this way. Also,
the world switch frequecy is often low, for example, it is ~2000
when running kernel building load in VM from APM xgene KVM host,
so the effect is very small, and the difference can't be observed
in my test at all.
Cc: Dann Frazier <dann.frazier@canonical.com>
Signed-off-by: Ming Lei <ming.lei@canonical.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2015-09-02 06:31:21 +00:00
|
|
|
#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
|
|
|
|
|
2016-01-11 14:46:15 +00:00
|
|
|
#define KVM_VCPU_MAX_FEATURES 4
|
2012-12-10 16:29:28 +00:00
|
|
|
|
2017-06-04 12:43:58 +00:00
|
|
|
#define KVM_REQ_SLEEP \
|
2017-06-04 12:43:51 +00:00
|
|
|
KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
|
2017-06-04 12:43:59 +00:00
|
|
|
#define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1)
|
2016-04-27 09:28:00 +00:00
|
|
|
|
2017-10-27 17:57:51 +00:00
|
|
|
DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
|
|
|
|
|
2014-08-26 14:13:20 +00:00
|
|
|
int __attribute_const__ kvm_target_cpu(void);
|
2012-12-10 16:29:28 +00:00
|
|
|
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
|
2018-10-12 16:12:48 +00:00
|
|
|
int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext);
|
arm64: kvm: Fix kvm teardown for systems using the extended idmap
If memory is located above 1<<VA_BITS, kvm adds an extra level to its page
tables, merging the runtime tables and boot tables that contain the idmap.
This lets us avoid the trampoline dance during initialisation.
This also means there is no trampoline page mapped, so
__cpu_reset_hyp_mode() can't call __kvm_hyp_reset() in this page. The good
news is the idmap is still mapped, so we don't need the trampoline page.
The bad news is we can't call it directly as the idmap is above
HYP_PAGE_OFFSET, so its address is masked by kvm_call_hyp.
Add a function __extended_idmap_trampoline which will branch into
__kvm_hyp_reset in the idmap, change kvm_hyp_reset_entry() to return
this address if __kvm_cpu_uses_extended_idmap(). In this case
__kvm_hyp_reset() will still switch to the boot tables (which are the
merged tables that were already in use), and branch into the idmap (where
it already was).
This fixes boot failures on these systems, where we fail to execute the
missing trampoline page when tearing down kvm in init_subsystems():
[ 2.508922] kvm [1]: 8-bit VMID
[ 2.512057] kvm [1]: Hyp mode initialized successfully
[ 2.517242] kvm [1]: interrupt-controller@e1140000 IRQ13
[ 2.522622] kvm [1]: timer IRQ3
[ 2.525783] Kernel panic - not syncing: HYP panic:
[ 2.525783] PS:200003c9 PC:0000007ffffff820 ESR:86000005
[ 2.525783] FAR:0000007ffffff820 HPFAR:00000000003ffff0 PAR:0000000000000000
[ 2.525783] VCPU: (null)
[ 2.525783]
[ 2.547667] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.6.0-rc5+ #1
[ 2.555137] Hardware name: Default string Default string/Default string, BIOS ROD0084E 09/03/2015
[ 2.563994] Call trace:
[ 2.566432] [<ffffff80080888d0>] dump_backtrace+0x0/0x240
[ 2.571818] [<ffffff8008088b24>] show_stack+0x14/0x20
[ 2.576858] [<ffffff80083423ac>] dump_stack+0x94/0xb8
[ 2.581899] [<ffffff8008152130>] panic+0x10c/0x250
[ 2.586677] [<ffffff8008152024>] panic+0x0/0x250
[ 2.591281] SMP: stopping secondary CPUs
[ 3.649692] SMP: failed to stop secondary CPUs 0-2,4-7
[ 3.654818] Kernel Offset: disabled
[ 3.658293] Memory Limit: none
[ 3.661337] ---[ end Kernel panic - not syncing: HYP panic:
[ 3.661337] PS:200003c9 PC:0000007ffffff820 ESR:86000005
[ 3.661337] FAR:0000007ffffff820 HPFAR:00000000003ffff0 PAR:0000000000000000
[ 3.661337] VCPU: (null)
[ 3.661337]
Reported-by: Will Deacon <will.deacon@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-04-29 17:27:03 +00:00
|
|
|
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
struct kvm_arch {
|
|
|
|
/* The VMID generation used for the virt. memory system */
|
|
|
|
u64 vmid_gen;
|
|
|
|
u32 vmid;
|
|
|
|
|
2018-09-26 16:32:43 +00:00
|
|
|
/* stage2 entry level table */
|
2012-12-10 16:29:28 +00:00
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
|
|
/* VTTBR value associated with above pgd and vmid */
|
|
|
|
u64 vttbr;
|
2018-09-26 16:32:43 +00:00
|
|
|
/* VTCR_EL2 value for this VM */
|
|
|
|
u64 vtcr;
|
2012-12-10 16:29:28 +00:00
|
|
|
|
2016-10-18 17:37:49 +00:00
|
|
|
/* The last vcpu id that ran on each physical CPU */
|
|
|
|
int __percpu *last_vcpu_ran;
|
|
|
|
|
2014-06-02 14:26:01 +00:00
|
|
|
/* The maximum number of vCPUs depends on the used GIC model */
|
|
|
|
int max_vcpus;
|
|
|
|
|
2012-12-10 16:29:28 +00:00
|
|
|
/* Interrupt controller */
|
|
|
|
struct vgic_dist vgic;
|
2018-01-21 16:42:56 +00:00
|
|
|
|
|
|
|
/* Mandated version of PSCI */
|
|
|
|
u32 psci_version;
|
2012-12-10 16:29:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#define KVM_NR_MEM_OBJS 40
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't want allocation failures within the mmu code, so we preallocate
|
|
|
|
* enough memory for a single page fault in a cache.
|
|
|
|
*/
|
|
|
|
struct kvm_mmu_memory_cache {
|
|
|
|
int nobjs;
|
|
|
|
void *objects[KVM_NR_MEM_OBJS];
|
|
|
|
};
|
|
|
|
|
|
|
|
struct kvm_vcpu_fault_info {
|
|
|
|
u32 esr_el2; /* Hyp Syndrom Register */
|
|
|
|
u64 far_el2; /* Hyp Fault Address Register */
|
|
|
|
u64 hpfar_el2; /* Hyp IPA Fault Address Register */
|
KVM: arm64: Handle RAS SErrors from EL2 on guest exit
We expect to have firmware-first handling of RAS SErrors, with errors
notified via an APEI method. For systems without firmware-first, add
some minimal handling to KVM.
There are two ways KVM can take an SError due to a guest, either may be a
RAS error: we exit the guest due to an SError routed to EL2 by HCR_EL2.AMO,
or we take an SError from EL2 when we unmask PSTATE.A from __guest_exit.
The current SError from EL2 code unmasks SError and tries to fence any
pending SError into a single instruction window. It then leaves SError
unmasked.
With the v8.2 RAS Extensions we may take an SError for a 'corrected'
error, but KVM is only able to handle SError from EL2 if they occur
during this single instruction window...
The RAS Extensions give us a new instruction to synchronise and
consume SErrors. The RAS Extensions document (ARM DDI0587),
'2.4.1 ESB and Unrecoverable errors' describes ESB as synchronising
SError interrupts generated by 'instructions, translation table walks,
hardware updates to the translation tables, and instruction fetches on
the same PE'. This makes ESB equivalent to KVMs existing
'dsb, mrs-daifclr, isb' sequence.
Use the alternatives to synchronise and consume any SError using ESB
instead of unmasking and taking the SError. Set ARM_EXIT_WITH_SERROR_BIT
in the exit_code so that we can restart the vcpu if it turns out this
SError has no impact on the vcpu.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-15 19:39:05 +00:00
|
|
|
u64 disr_el1; /* Deferred [SError] Status Register */
|
2012-12-10 16:29:28 +00:00
|
|
|
};
|
|
|
|
|
2015-10-25 19:57:11 +00:00
|
|
|
/*
|
|
|
|
* 0 is reserved as an invalid value.
|
|
|
|
* Order should be kept in sync with the save/restore code.
|
|
|
|
*/
|
|
|
|
enum vcpu_sysreg {
|
|
|
|
__INVALID_SYSREG__,
|
|
|
|
MPIDR_EL1, /* MultiProcessor Affinity Register */
|
|
|
|
CSSELR_EL1, /* Cache Size Selection Register */
|
|
|
|
SCTLR_EL1, /* System Control Register */
|
|
|
|
ACTLR_EL1, /* Auxiliary Control Register */
|
|
|
|
CPACR_EL1, /* Coprocessor Access Control */
|
|
|
|
TTBR0_EL1, /* Translation Table Base Register 0 */
|
|
|
|
TTBR1_EL1, /* Translation Table Base Register 1 */
|
|
|
|
TCR_EL1, /* Translation Control Register */
|
|
|
|
ESR_EL1, /* Exception Syndrome Register */
|
2016-02-24 17:52:41 +00:00
|
|
|
AFSR0_EL1, /* Auxiliary Fault Status Register 0 */
|
|
|
|
AFSR1_EL1, /* Auxiliary Fault Status Register 1 */
|
2015-10-25 19:57:11 +00:00
|
|
|
FAR_EL1, /* Fault Address Register */
|
|
|
|
MAIR_EL1, /* Memory Attribute Indirection Register */
|
|
|
|
VBAR_EL1, /* Vector Base Address Register */
|
|
|
|
CONTEXTIDR_EL1, /* Context ID Register */
|
|
|
|
TPIDR_EL0, /* Thread ID, User R/W */
|
|
|
|
TPIDRRO_EL0, /* Thread ID, User R/O */
|
|
|
|
TPIDR_EL1, /* Thread ID, Privileged */
|
|
|
|
AMAIR_EL1, /* Aux Memory Attribute Indirection Register */
|
|
|
|
CNTKCTL_EL1, /* Timer Control Register (EL1) */
|
|
|
|
PAR_EL1, /* Physical Address Register */
|
|
|
|
MDSCR_EL1, /* Monitor Debug System Control Register */
|
|
|
|
MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */
|
2018-01-15 19:39:02 +00:00
|
|
|
DISR_EL1, /* Deferred Interrupt Status Register */
|
2015-10-25 19:57:11 +00:00
|
|
|
|
2015-06-18 08:01:53 +00:00
|
|
|
/* Performance Monitors Registers */
|
|
|
|
PMCR_EL0, /* Control Register */
|
2015-08-31 09:20:22 +00:00
|
|
|
PMSELR_EL0, /* Event Counter Selection Register */
|
2015-12-08 07:29:06 +00:00
|
|
|
PMEVCNTR0_EL0, /* Event Counter Register (0-30) */
|
|
|
|
PMEVCNTR30_EL0 = PMEVCNTR0_EL0 + 30,
|
|
|
|
PMCCNTR_EL0, /* Cycle Counter Register */
|
2016-02-23 03:11:27 +00:00
|
|
|
PMEVTYPER0_EL0, /* Event Type Register (0-30) */
|
|
|
|
PMEVTYPER30_EL0 = PMEVTYPER0_EL0 + 30,
|
|
|
|
PMCCFILTR_EL0, /* Cycle Count Filter Register */
|
2015-09-08 04:26:13 +00:00
|
|
|
PMCNTENSET_EL0, /* Count Enable Set Register */
|
2015-09-08 06:40:20 +00:00
|
|
|
PMINTENSET_EL1, /* Interrupt Enable Set Register */
|
2015-09-08 07:03:26 +00:00
|
|
|
PMOVSSET_EL0, /* Overflow Flag Status Set Register */
|
2015-09-08 07:49:39 +00:00
|
|
|
PMSWINC_EL0, /* Software Increment Register */
|
2015-09-08 07:15:56 +00:00
|
|
|
PMUSERENR_EL0, /* User Enable Register */
|
2015-06-18 08:01:53 +00:00
|
|
|
|
2015-10-25 19:57:11 +00:00
|
|
|
/* 32bit specific registers. Keep them at the end of the range */
|
|
|
|
DACR32_EL2, /* Domain Access Control Register */
|
|
|
|
IFSR32_EL2, /* Instruction Fault Status Register */
|
|
|
|
FPEXC32_EL2, /* Floating-Point Exception Control Register */
|
|
|
|
DBGVCR32_EL2, /* Debug Vector Catch Register */
|
|
|
|
|
|
|
|
NR_SYS_REGS /* Nothing after this line! */
|
|
|
|
};
|
|
|
|
|
|
|
|
/* 32bit mapping */
|
|
|
|
#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */
|
|
|
|
#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */
|
|
|
|
#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */
|
|
|
|
#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */
|
|
|
|
#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */
|
|
|
|
#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */
|
|
|
|
#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */
|
|
|
|
#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */
|
|
|
|
#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */
|
|
|
|
#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */
|
|
|
|
#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */
|
|
|
|
#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */
|
|
|
|
#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */
|
|
|
|
#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */
|
|
|
|
#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */
|
|
|
|
#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */
|
|
|
|
#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */
|
|
|
|
#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */
|
|
|
|
#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */
|
|
|
|
#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */
|
|
|
|
#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */
|
|
|
|
#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */
|
|
|
|
#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */
|
|
|
|
#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */
|
|
|
|
#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */
|
|
|
|
#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */
|
|
|
|
#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */
|
|
|
|
#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
|
|
|
|
#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
|
|
|
|
|
|
|
|
#define cp14_DBGDSCRext (MDSCR_EL1 * 2)
|
|
|
|
#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2)
|
|
|
|
#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2)
|
|
|
|
#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1)
|
|
|
|
#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2)
|
|
|
|
#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2)
|
|
|
|
#define cp14_DBGDCCINT (MDCCINT_EL1 * 2)
|
|
|
|
|
|
|
|
#define NR_COPRO_REGS (NR_SYS_REGS * 2)
|
|
|
|
|
2012-12-10 16:29:28 +00:00
|
|
|
struct kvm_cpu_context {
|
|
|
|
struct kvm_regs gp_regs;
|
2013-02-06 19:17:50 +00:00
|
|
|
union {
|
|
|
|
u64 sys_regs[NR_SYS_REGS];
|
2014-04-24 09:27:13 +00:00
|
|
|
u32 copro[NR_COPRO_REGS];
|
2013-02-06 19:17:50 +00:00
|
|
|
};
|
2018-01-08 15:38:05 +00:00
|
|
|
|
|
|
|
struct kvm_vcpu *__hyp_running_vcpu;
|
2012-12-10 16:29:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
typedef struct kvm_cpu_context kvm_cpu_context_t;
|
|
|
|
|
|
|
|
struct kvm_vcpu_arch {
|
|
|
|
struct kvm_cpu_context ctxt;
|
|
|
|
|
|
|
|
/* HYP configuration */
|
|
|
|
u64 hcr_el2;
|
2015-07-07 16:29:56 +00:00
|
|
|
u32 mdcr_el2;
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
/* Exception Information */
|
|
|
|
struct kvm_vcpu_fault_info fault;
|
|
|
|
|
2018-05-29 12:11:16 +00:00
|
|
|
/* State of various workarounds, see kvm_asm.h for bit assignment */
|
|
|
|
u64 workaround_flags;
|
|
|
|
|
2018-05-08 13:47:23 +00:00
|
|
|
/* Miscellaneous vcpu state flags */
|
|
|
|
u64 flags;
|
2014-04-24 09:24:46 +00:00
|
|
|
|
2015-07-07 16:30:00 +00:00
|
|
|
/*
|
|
|
|
* We maintain more than a single set of debug registers to support
|
|
|
|
* debugging the guest from the host and to maintain separate host and
|
|
|
|
* guest state during world switches. vcpu_debug_state are the debug
|
|
|
|
* registers of the vcpu as the guest sees them. host_debug_state are
|
2015-07-07 16:30:02 +00:00
|
|
|
* the host registers which are saved and restored during
|
|
|
|
* world switches. external_debug_state contains the debug
|
|
|
|
* values we want to debug the guest. This is set via the
|
|
|
|
* KVM_SET_GUEST_DEBUG ioctl.
|
2015-07-07 16:30:00 +00:00
|
|
|
*
|
|
|
|
* debug_ptr points to the set of debug registers that should be loaded
|
|
|
|
* onto the hardware when running the guest.
|
|
|
|
*/
|
|
|
|
struct kvm_guest_debug_arch *debug_ptr;
|
|
|
|
struct kvm_guest_debug_arch vcpu_debug_state;
|
2015-07-07 16:30:02 +00:00
|
|
|
struct kvm_guest_debug_arch external_debug_state;
|
2015-07-07 16:30:00 +00:00
|
|
|
|
2012-12-10 16:29:28 +00:00
|
|
|
/* Pointer to host CPU context */
|
|
|
|
kvm_cpu_context_t *host_cpu_context;
|
2018-04-06 13:55:59 +00:00
|
|
|
|
|
|
|
struct thread_info *host_thread_info; /* hyp VA */
|
|
|
|
struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */
|
|
|
|
|
2016-09-22 10:35:43 +00:00
|
|
|
struct {
|
|
|
|
/* {Break,watch}point registers */
|
|
|
|
struct kvm_guest_debug_arch regs;
|
|
|
|
/* Statistical profiling extension */
|
|
|
|
u64 pmscr_el1;
|
|
|
|
} host_debug_state;
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
/* VGIC state */
|
|
|
|
struct vgic_cpu vgic_cpu;
|
|
|
|
struct arch_timer_cpu timer_cpu;
|
2015-09-11 01:38:32 +00:00
|
|
|
struct kvm_pmu pmu;
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Anything that is not used directly from assembly code goes
|
|
|
|
* here.
|
|
|
|
*/
|
|
|
|
|
2015-07-07 16:29:58 +00:00
|
|
|
/*
|
|
|
|
* Guest registers we preserve during guest debugging.
|
|
|
|
*
|
|
|
|
* These shadow registers are updated by the kvm_handle_sys_reg
|
|
|
|
* trap handler if the guest accesses or updates them while we
|
|
|
|
* are using guest debug.
|
|
|
|
*/
|
|
|
|
struct {
|
|
|
|
u32 mdscr_el1;
|
|
|
|
} guest_debug_preserved;
|
|
|
|
|
2015-09-25 21:41:14 +00:00
|
|
|
/* vcpu power-off state */
|
|
|
|
bool power_off;
|
2012-12-10 16:29:28 +00:00
|
|
|
|
2015-09-25 21:41:17 +00:00
|
|
|
/* Don't run the guest (internal implementation need) */
|
|
|
|
bool pause;
|
|
|
|
|
2012-12-10 16:29:28 +00:00
|
|
|
/* IO related fields */
|
|
|
|
struct kvm_decode mmio_decode;
|
|
|
|
|
|
|
|
/* Cache some mmu pages needed inside spinlock regions */
|
|
|
|
struct kvm_mmu_memory_cache mmu_page_cache;
|
|
|
|
|
|
|
|
/* Target CPU and feature flags */
|
2013-07-22 03:40:38 +00:00
|
|
|
int target;
|
2012-12-10 16:29:28 +00:00
|
|
|
DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
|
|
|
|
|
|
|
|
/* Detect first run of a vcpu */
|
|
|
|
bool has_run_once;
|
2018-01-15 19:39:01 +00:00
|
|
|
|
|
|
|
/* Virtual SError ESR to restore when HCR_EL2.VSE is set */
|
|
|
|
u64 vsesr_el2;
|
2017-12-23 20:53:48 +00:00
|
|
|
|
|
|
|
/* True when deferrable sysregs are loaded on the physical CPU,
|
|
|
|
* see kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs. */
|
|
|
|
bool sysregs_loaded_on_cpu;
|
2012-12-10 16:29:28 +00:00
|
|
|
};
|
|
|
|
|
2018-05-08 13:47:23 +00:00
|
|
|
/* vcpu_arch flags field values: */
|
|
|
|
#define KVM_ARM64_DEBUG_DIRTY (1 << 0)
|
2018-04-06 13:55:59 +00:00
|
|
|
#define KVM_ARM64_FP_ENABLED (1 << 1) /* guest FP regs loaded */
|
|
|
|
#define KVM_ARM64_FP_HOST (1 << 2) /* host FP regs loaded */
|
|
|
|
#define KVM_ARM64_HOST_SVE_IN_USE (1 << 3) /* backup for host TIF_SVE */
|
2018-06-15 15:47:25 +00:00
|
|
|
#define KVM_ARM64_HOST_SVE_ENABLED (1 << 4) /* SVE enabled for EL0 */
|
2018-05-08 13:47:23 +00:00
|
|
|
|
2012-12-10 16:29:28 +00:00
|
|
|
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
|
2016-03-16 14:38:53 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Only use __vcpu_sys_reg if you know you want the memory backed version of a
|
|
|
|
* register, and not the one most recently accessed by a running VCPU. For
|
|
|
|
* example, for userspace access or for system registers that are never context
|
|
|
|
* switched, but only emulated.
|
|
|
|
*/
|
|
|
|
#define __vcpu_sys_reg(v,r) ((v)->arch.ctxt.sys_regs[(r)])
|
|
|
|
|
2017-12-23 20:53:48 +00:00
|
|
|
u64 vcpu_read_sys_reg(struct kvm_vcpu *vcpu, int reg);
|
|
|
|
void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg);
|
2016-03-16 14:38:53 +00:00
|
|
|
|
2014-04-24 09:27:13 +00:00
|
|
|
/*
|
|
|
|
* CP14 and CP15 live in the same array, as they are backed by the
|
|
|
|
* same system registers.
|
|
|
|
*/
|
|
|
|
#define vcpu_cp14(v,r) ((v)->arch.ctxt.copro[(r)])
|
|
|
|
#define vcpu_cp15(v,r) ((v)->arch.ctxt.copro[(r)])
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
struct kvm_vm_stat {
|
2016-08-02 04:03:22 +00:00
|
|
|
ulong remote_tlb_flush;
|
2012-12-10 16:29:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct kvm_vcpu_stat {
|
2016-08-02 04:03:22 +00:00
|
|
|
u64 halt_successful_poll;
|
|
|
|
u64 halt_attempted_poll;
|
|
|
|
u64 halt_poll_invalid;
|
|
|
|
u64 halt_wakeup;
|
|
|
|
u64 hvc_exit_stat;
|
2015-11-26 10:09:43 +00:00
|
|
|
u64 wfe_exit_stat;
|
|
|
|
u64 wfi_exit_stat;
|
|
|
|
u64 mmio_exit_user;
|
|
|
|
u64 mmio_exit_kernel;
|
|
|
|
u64 exits;
|
2012-12-10 16:29:28 +00:00
|
|
|
};
|
|
|
|
|
2013-09-30 08:50:06 +00:00
|
|
|
int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
|
2012-12-10 16:29:28 +00:00
|
|
|
unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
|
|
|
|
int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
|
|
|
|
int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
|
|
|
|
int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
|
2018-07-19 15:24:24 +00:00
|
|
|
int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_vcpu_events *events);
|
2018-07-19 15:24:22 +00:00
|
|
|
|
2018-07-19 15:24:24 +00:00
|
|
|
int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_vcpu_events *events);
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
#define KVM_ARCH_WANT_MMU_NOTIFIER
|
|
|
|
int kvm_unmap_hva_range(struct kvm *kvm,
|
|
|
|
unsigned long start, unsigned long end);
|
|
|
|
void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
|
2015-03-12 18:16:51 +00:00
|
|
|
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
|
|
|
|
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
|
2014-08-26 14:13:21 +00:00
|
|
|
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
|
2016-04-27 09:28:00 +00:00
|
|
|
void kvm_arm_halt_guest(struct kvm *kvm);
|
|
|
|
void kvm_arm_resume_guest(struct kvm *kvm);
|
2012-12-10 16:29:28 +00:00
|
|
|
|
2016-02-16 12:52:39 +00:00
|
|
|
u64 __kvm_call_hyp(void *hypfn, ...);
|
2016-03-01 13:12:44 +00:00
|
|
|
#define kvm_call_hyp(f, ...) __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__)
|
|
|
|
|
2014-10-16 15:00:18 +00:00
|
|
|
void force_vm_exit(const cpumask_t *mask);
|
2015-01-15 23:58:59 +00:00
|
|
|
void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
|
|
|
|
int exception_index);
|
2018-01-15 19:39:04 +00:00
|
|
|
void handle_exit_early(struct kvm_vcpu *vcpu, struct kvm_run *run,
|
|
|
|
int exception_index);
|
2012-12-10 16:29:28 +00:00
|
|
|
|
|
|
|
int kvm_perf_init(void);
|
|
|
|
int kvm_perf_teardown(void);
|
|
|
|
|
2018-07-19 15:24:22 +00:00
|
|
|
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);
|
|
|
|
|
2014-06-02 13:37:13 +00:00
|
|
|
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
|
|
|
|
|
2017-10-08 15:01:56 +00:00
|
|
|
DECLARE_PER_CPU(kvm_cpu_context_t, kvm_host_cpu_state);
|
|
|
|
|
2018-08-08 15:10:54 +00:00
|
|
|
void __kvm_enable_ssbs(void);
|
|
|
|
|
2016-06-30 17:40:45 +00:00
|
|
|
static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
|
2012-12-17 17:07:52 +00:00
|
|
|
unsigned long hyp_stack_ptr,
|
|
|
|
unsigned long vector_ptr)
|
|
|
|
{
|
arm64: KVM: Cleanup tpidr_el2 init on non-VHE
When running on a non-VHE system, we initialize tpidr_el2 to
contain the per-CPU offset required to reach per-cpu variables.
Actually, we initialize it twice: the first time as part of the
EL2 initialization, by copying tpidr_el1 into its el2 counterpart,
and another time by calling into __kvm_set_tpidr_el2.
It turns out that the first part is wrong, as it includes the
distance between the kernel mapping and the linear mapping, while
EL2 only cares about the linear mapping. This was the last vestige
of the first per-cpu use of tpidr_el2 that came in with SDEI.
The only caller then was hyp_panic(), and its now using the
pc-relative get_host_ctxt() stuff, instead of kimage addresses
from the literal pool.
It is not a big deal, as we override it straight away, but it is
slightly confusing. In order to clear said confusion, let's
set this directly as part of the hyp-init code, and drop the
ad-hoc HYP helper.
Reviewed-by: James Morse <james.morse@arm.com>
Acked-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-07-10 12:20:47 +00:00
|
|
|
/*
|
|
|
|
* Calculate the raw per-cpu offset without a translation from the
|
|
|
|
* kernel's mapping to the linear mapping, and store it in tpidr_el2
|
|
|
|
* so that we can use adr_l to access per-cpu variables in EL2.
|
|
|
|
*/
|
|
|
|
u64 tpidr_el2 = ((u64)this_cpu_ptr(&kvm_host_cpu_state) -
|
|
|
|
(u64)kvm_ksym_ref(kvm_host_cpu_state));
|
2017-10-08 15:01:56 +00:00
|
|
|
|
2012-12-17 17:07:52 +00:00
|
|
|
/*
|
arm64/cpufeature: don't use mutex in bringup path
Currently, cpus_set_cap() calls static_branch_enable_cpuslocked(), which
must take the jump_label mutex.
We call cpus_set_cap() in the secondary bringup path, from the idle
thread where interrupts are disabled. Taking a mutex in this path "is a
NONO" regardless of whether it's contended, and something we must avoid.
We didn't spot this until recently, as ___might_sleep() won't warn for
this case until all CPUs have been brought up.
This patch avoids taking the mutex in the secondary bringup path. The
poking of static keys is deferred until enable_cpu_capabilities(), which
runs in a suitable context on the boot CPU. To account for the static
keys being set later, cpus_have_const_cap() is updated to use another
static key to check whether the const cap keys have been initialised,
falling back to the caps bitmap until this is the case.
This means that users of cpus_have_const_cap() gain should only gain a
single additional NOP in the fast path once the const caps are
initialised, but should always see the current cap value.
The hyp code should never dereference the caps array, since the caps are
initialized before we run the module initcall to initialise hyp. A check
is added to the hyp init code to document this requirement.
This change will sidestep a number of issues when the upcoming hotplug
locking rework is merged.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Marc Zyniger <marc.zyngier@arm.com>
Reviewed-by: Suzuki Poulose <suzuki.poulose@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Sewior <bigeasy@linutronix.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-05-16 14:18:05 +00:00
|
|
|
* Call initialization code, and switch to the full blown HYP code.
|
|
|
|
* If the cpucaps haven't been finalized yet, something has gone very
|
|
|
|
* wrong, and hyp will crash and burn when it uses any
|
|
|
|
* cpus_have_const_cap() wrapper.
|
2012-12-17 17:07:52 +00:00
|
|
|
*/
|
arm64/cpufeature: don't use mutex in bringup path
Currently, cpus_set_cap() calls static_branch_enable_cpuslocked(), which
must take the jump_label mutex.
We call cpus_set_cap() in the secondary bringup path, from the idle
thread where interrupts are disabled. Taking a mutex in this path "is a
NONO" regardless of whether it's contended, and something we must avoid.
We didn't spot this until recently, as ___might_sleep() won't warn for
this case until all CPUs have been brought up.
This patch avoids taking the mutex in the secondary bringup path. The
poking of static keys is deferred until enable_cpu_capabilities(), which
runs in a suitable context on the boot CPU. To account for the static
keys being set later, cpus_have_const_cap() is updated to use another
static key to check whether the const cap keys have been initialised,
falling back to the caps bitmap until this is the case.
This means that users of cpus_have_const_cap() gain should only gain a
single additional NOP in the fast path once the const caps are
initialised, but should always see the current cap value.
The hyp code should never dereference the caps array, since the caps are
initialized before we run the module initcall to initialise hyp. A check
is added to the hyp init code to document this requirement.
This change will sidestep a number of issues when the upcoming hotplug
locking rework is merged.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Marc Zyniger <marc.zyngier@arm.com>
Reviewed-by: Suzuki Poulose <suzuki.poulose@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Sewior <bigeasy@linutronix.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-05-16 14:18:05 +00:00
|
|
|
BUG_ON(!static_branch_likely(&arm64_const_caps_ready));
|
arm64: KVM: Cleanup tpidr_el2 init on non-VHE
When running on a non-VHE system, we initialize tpidr_el2 to
contain the per-CPU offset required to reach per-cpu variables.
Actually, we initialize it twice: the first time as part of the
EL2 initialization, by copying tpidr_el1 into its el2 counterpart,
and another time by calling into __kvm_set_tpidr_el2.
It turns out that the first part is wrong, as it includes the
distance between the kernel mapping and the linear mapping, while
EL2 only cares about the linear mapping. This was the last vestige
of the first per-cpu use of tpidr_el2 that came in with SDEI.
The only caller then was hyp_panic(), and its now using the
pc-relative get_host_ctxt() stuff, instead of kimage addresses
from the literal pool.
It is not a big deal, as we override it straight away, but it is
slightly confusing. In order to clear said confusion, let's
set this directly as part of the hyp-init code, and drop the
ad-hoc HYP helper.
Reviewed-by: James Morse <james.morse@arm.com>
Acked-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-07-10 12:20:47 +00:00
|
|
|
__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2);
|
2018-08-08 15:10:54 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Disabling SSBD on a non-VHE system requires us to enable SSBS
|
|
|
|
* at EL2.
|
|
|
|
*/
|
|
|
|
if (!has_vhe() && this_cpu_has_cap(ARM64_SSBS) &&
|
|
|
|
arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
|
|
|
|
kvm_call_hyp(__kvm_enable_ssbs);
|
|
|
|
}
|
2012-12-17 17:07:52 +00:00
|
|
|
}
|
arm64: kvm: allows kvm cpu hotplug
The current kvm implementation on arm64 does cpu-specific initialization
at system boot, and has no way to gracefully shutdown a core in terms of
kvm. This prevents kexec from rebooting the system at EL2.
This patch adds a cpu tear-down function and also puts an existing cpu-init
code into a separate function, kvm_arch_hardware_disable() and
kvm_arch_hardware_enable() respectively.
We don't need the arm64 specific cpu hotplug hook any more.
Since this patch modifies common code between arm and arm64, one stub
definition, __cpu_reset_hyp_mode(), is added on arm side to avoid
compilation errors.
Signed-off-by: AKASHI Takahiro <takahiro.akashi@linaro.org>
[Rebase, added separate VHE init/exit path, changed resets use of
kvm_call_hyp() to the __version, en/disabled hardware in init_subsystems(),
added icache maintenance to __kvm_hyp_reset() and removed lr restore, removed
guest-enter after teardown handling]
Signed-off-by: James Morse <james.morse@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-04-27 16:47:05 +00:00
|
|
|
|
2018-04-20 15:20:43 +00:00
|
|
|
static inline bool kvm_arch_check_sve_has_vhe(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The Arm architecture specifies that implementation of SVE
|
|
|
|
* requires VHE also to be implemented. The KVM code for arm64
|
|
|
|
* relies on this when SVE is present:
|
|
|
|
*/
|
|
|
|
if (system_supports_sve())
|
|
|
|
return has_vhe();
|
|
|
|
else
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2014-08-28 13:13:02 +00:00
|
|
|
static inline void kvm_arch_hardware_unsetup(void) {}
|
|
|
|
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
|
|
|
|
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
|
|
|
|
static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
|
2016-05-13 10:16:35 +00:00
|
|
|
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
|
2014-08-28 13:13:02 +00:00
|
|
|
|
2015-07-07 16:29:56 +00:00
|
|
|
void kvm_arm_init_debug(void);
|
|
|
|
void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_arm_clear_debug(struct kvm_vcpu *vcpu);
|
2015-07-07 16:30:00 +00:00
|
|
|
void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu);
|
2017-11-16 15:39:19 +00:00
|
|
|
bool kvm_arm_handle_step_debug(struct kvm_vcpu *vcpu, struct kvm_run *run);
|
2016-01-11 13:35:32 +00:00
|
|
|
int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_device_attr *attr);
|
|
|
|
int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_device_attr *attr);
|
|
|
|
int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_device_attr *attr);
|
2015-07-07 16:29:56 +00:00
|
|
|
|
2018-09-26 16:32:52 +00:00
|
|
|
static inline void __cpu_init_stage2(void) {}
|
2016-02-22 10:57:30 +00:00
|
|
|
|
2018-04-06 13:55:59 +00:00
|
|
|
/* Guest/host FPSIMD coordination helpers */
|
|
|
|
int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
|
|
#ifdef CONFIG_KVM /* Avoid conflicts with core headers if CONFIG_KVM=n */
|
|
|
|
static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
|
arm64/sve: KVM: Prevent guests from using SVE
Until KVM has full SVE support, guests must not be allowed to
execute SVE instructions.
This patch enables the necessary traps, and also ensures that the
traps are disabled again on exit from the guest so that the host
can still use SVE if it wants to.
On guest exit, high bits of the SVE Zn registers may have been
clobbered as a side-effect the execution of FPSIMD instructions in
the guest. The existing KVM host FPSIMD restore code is not
sufficient to restore these bits, so this patch explicitly marks
the CPU as not containing cached vector state for any task, thus
forcing a reload on the next return to userspace. This is an
interim measure, in advance of adding full SVE awareness to KVM.
This marking of cached vector state in the CPU as invalid is done
using __this_cpu_write(fpsimd_last_state, NULL) in fpsimd.c. Due
to the repeated use of this rather obscure operation, it makes
sense to factor it out as a separate helper with a clearer name.
This patch factors it out as fpsimd_flush_cpu_state(), and ports
all callers to use it.
As a side effect of this refactoring, a this_cpu_write() in
fpsimd_cpu_pm_notifier() is changed to __this_cpu_write(). This
should be fine, since cpu_pm_enter() is supposed to be called only
with interrupts disabled.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:16 +00:00
|
|
|
{
|
2018-04-06 13:55:59 +00:00
|
|
|
return kvm_arch_vcpu_run_map_fp(vcpu);
|
arm64/sve: KVM: Prevent guests from using SVE
Until KVM has full SVE support, guests must not be allowed to
execute SVE instructions.
This patch enables the necessary traps, and also ensures that the
traps are disabled again on exit from the guest so that the host
can still use SVE if it wants to.
On guest exit, high bits of the SVE Zn registers may have been
clobbered as a side-effect the execution of FPSIMD instructions in
the guest. The existing KVM host FPSIMD restore code is not
sufficient to restore these bits, so this patch explicitly marks
the CPU as not containing cached vector state for any task, thus
forcing a reload on the next return to userspace. This is an
interim measure, in advance of adding full SVE awareness to KVM.
This marking of cached vector state in the CPU as invalid is done
using __this_cpu_write(fpsimd_last_state, NULL) in fpsimd.c. Due
to the repeated use of this rather obscure operation, it makes
sense to factor it out as a separate helper with a clearer name.
This patch factors it out as fpsimd_flush_cpu_state(), and ports
all callers to use it.
As a side effect of this refactoring, a this_cpu_write() in
fpsimd_cpu_pm_notifier() is changed to __this_cpu_write(). This
should be fine, since cpu_pm_enter() is supposed to be called only
with interrupts disabled.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:16 +00:00
|
|
|
}
|
2018-04-06 13:55:59 +00:00
|
|
|
#endif
|
arm64/sve: KVM: Prevent guests from using SVE
Until KVM has full SVE support, guests must not be allowed to
execute SVE instructions.
This patch enables the necessary traps, and also ensures that the
traps are disabled again on exit from the guest so that the host
can still use SVE if it wants to.
On guest exit, high bits of the SVE Zn registers may have been
clobbered as a side-effect the execution of FPSIMD instructions in
the guest. The existing KVM host FPSIMD restore code is not
sufficient to restore these bits, so this patch explicitly marks
the CPU as not containing cached vector state for any task, thus
forcing a reload on the next return to userspace. This is an
interim measure, in advance of adding full SVE awareness to KVM.
This marking of cached vector state in the CPU as invalid is done
using __this_cpu_write(fpsimd_last_state, NULL) in fpsimd.c. Due
to the repeated use of this rather obscure operation, it makes
sense to factor it out as a separate helper with a clearer name.
This patch factors it out as fpsimd_flush_cpu_state(), and ports
all callers to use it.
As a side effect of this refactoring, a this_cpu_write() in
fpsimd_cpu_pm_notifier() is changed to __this_cpu_write(). This
should be fine, since cpu_pm_enter() is supposed to be called only
with interrupts disabled.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:16 +00:00
|
|
|
|
2018-01-15 19:39:00 +00:00
|
|
|
static inline void kvm_arm_vhe_guest_enter(void)
|
|
|
|
{
|
|
|
|
local_daif_mask();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void kvm_arm_vhe_guest_exit(void)
|
|
|
|
{
|
|
|
|
local_daif_restore(DAIF_PROCCTX_NOIRQ);
|
2017-10-03 12:02:12 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* When we exit from the guest we change a number of CPU configuration
|
|
|
|
* parameters, such as traps. Make sure these changes take effect
|
|
|
|
* before running the host or additional guests.
|
|
|
|
*/
|
|
|
|
isb();
|
2018-01-15 19:39:00 +00:00
|
|
|
}
|
2018-02-06 17:56:14 +00:00
|
|
|
|
|
|
|
static inline bool kvm_arm_harden_branch_predictor(void)
|
|
|
|
{
|
|
|
|
return cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR);
|
|
|
|
}
|
|
|
|
|
2018-05-29 12:11:18 +00:00
|
|
|
#define KVM_SSBD_UNKNOWN -1
|
|
|
|
#define KVM_SSBD_FORCE_DISABLE 0
|
|
|
|
#define KVM_SSBD_KERNEL 1
|
|
|
|
#define KVM_SSBD_FORCE_ENABLE 2
|
|
|
|
#define KVM_SSBD_MITIGATED 3
|
|
|
|
|
|
|
|
static inline int kvm_arm_have_ssbd(void)
|
|
|
|
{
|
|
|
|
switch (arm64_get_ssbd_state()) {
|
|
|
|
case ARM64_SSBD_FORCE_DISABLE:
|
|
|
|
return KVM_SSBD_FORCE_DISABLE;
|
|
|
|
case ARM64_SSBD_KERNEL:
|
|
|
|
return KVM_SSBD_KERNEL;
|
|
|
|
case ARM64_SSBD_FORCE_ENABLE:
|
|
|
|
return KVM_SSBD_FORCE_ENABLE;
|
|
|
|
case ARM64_SSBD_MITIGATED:
|
|
|
|
return KVM_SSBD_MITIGATED;
|
|
|
|
case ARM64_SSBD_UNKNOWN:
|
|
|
|
default:
|
|
|
|
return KVM_SSBD_UNKNOWN;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-10 08:21:18 +00:00
|
|
|
void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu);
|
|
|
|
|
2018-09-26 16:32:52 +00:00
|
|
|
void kvm_set_ipa_limit(void);
|
|
|
|
|
2018-05-15 11:37:37 +00:00
|
|
|
#define __KVM_HAVE_ARCH_VM_ALLOC
|
|
|
|
struct kvm *kvm_arch_alloc_vm(void);
|
|
|
|
void kvm_arch_free_vm(struct kvm *kvm);
|
|
|
|
|
2018-10-01 12:40:36 +00:00
|
|
|
int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type);
|
2018-09-26 16:32:42 +00:00
|
|
|
|
2012-12-10 16:29:28 +00:00
|
|
|
#endif /* __ARM64_KVM_HOST_H__ */
|