KVM: VMX: Move vCPU-run code to a proper assembly routine
As evidenced by the myriad patches leading up to this moment, using an inline asm blob for vCPU-run is nothing short of horrific. It's also been called "unholy", "an abomination" and likely a whole host of other names that would violate the Code of Conduct if recorded here and now. The code is relocated nearly verbatim, e.g. quotes, newlines, tabs and __stringify need to be dropped, but other than those cosmetic changes the only functional changees are to add the "call" and replace the final "jmp" with a "ret". Note that STACK_FRAME_NON_STANDARD is also dropped from __vmx_vcpu_run(). Suggested-by: Andi Kleen <ak@linux.intel.com> Suggested-by: Josh Poimboeuf <jpoimboe@redhat.com> Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
Sean Christopherson
Paolo Bonzini
parent
63c73aa07f
commit
5e0781df18
@ -1,6 +1,30 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/asm.h>
|
||||
#include <asm/bitsperlong.h>
|
||||
#include <asm/kvm_vcpu_regs.h>
|
||||
|
||||
#define WORD_SIZE (BITS_PER_LONG / 8)
|
||||
|
||||
#define VCPU_RAX __VCPU_REGS_RAX * WORD_SIZE
|
||||
#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE
|
||||
#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE
|
||||
#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE
|
||||
/* Intentionally omit RSP as it's context switched by hardware */
|
||||
#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE
|
||||
#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE
|
||||
#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE
|
||||
#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE
|
||||
#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE
|
||||
#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE
|
||||
#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE
|
||||
#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE
|
||||
#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE
|
||||
#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
|
||||
#endif
|
||||
|
||||
.text
|
||||
|
||||
@ -55,3 +79,123 @@ ENDPROC(vmx_vmenter)
|
||||
ENTRY(vmx_vmexit)
|
||||
ret
|
||||
ENDPROC(vmx_vmexit)
|
||||
|
||||
/**
|
||||
* ____vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
|
||||
* @vmx: struct vcpu_vmx *
|
||||
* @regs: unsigned long * (to guest registers)
|
||||
* %RBX: VMCS launched status (non-zero indicates already launched)
|
||||
*
|
||||
* Returns:
|
||||
* %RBX is 0 on VM-Exit, 1 on VM-Fail
|
||||
*/
|
||||
ENTRY(____vmx_vcpu_run)
|
||||
push %_ASM_BP
|
||||
mov %_ASM_SP, %_ASM_BP
|
||||
|
||||
/*
|
||||
* Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
|
||||
* @regs is needed after VM-Exit to save the guest's register values.
|
||||
*/
|
||||
push %_ASM_ARG2
|
||||
|
||||
/* Adjust RSP to account for the CALL to vmx_vmenter(). */
|
||||
lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
|
||||
call vmx_update_host_rsp
|
||||
|
||||
/* Load @regs to RCX. */
|
||||
mov (%_ASM_SP), %_ASM_CX
|
||||
|
||||
/* Check if vmlaunch or vmresume is needed */
|
||||
cmpb $0, %bl
|
||||
|
||||
/* Load guest registers. Don't clobber flags. */
|
||||
mov VCPU_RAX(%_ASM_CX), %_ASM_AX
|
||||
mov VCPU_RBX(%_ASM_CX), %_ASM_BX
|
||||
mov VCPU_RDX(%_ASM_CX), %_ASM_DX
|
||||
mov VCPU_RSI(%_ASM_CX), %_ASM_SI
|
||||
mov VCPU_RDI(%_ASM_CX), %_ASM_DI
|
||||
mov VCPU_RBP(%_ASM_CX), %_ASM_BP
|
||||
#ifdef CONFIG_X86_64
|
||||
mov VCPU_R8 (%_ASM_CX), %r8
|
||||
mov VCPU_R9 (%_ASM_CX), %r9
|
||||
mov VCPU_R10(%_ASM_CX), %r10
|
||||
mov VCPU_R11(%_ASM_CX), %r11
|
||||
mov VCPU_R12(%_ASM_CX), %r12
|
||||
mov VCPU_R13(%_ASM_CX), %r13
|
||||
mov VCPU_R14(%_ASM_CX), %r14
|
||||
mov VCPU_R15(%_ASM_CX), %r15
|
||||
#endif
|
||||
/* Load guest RCX. This kills the vmx_vcpu pointer! */
|
||||
mov VCPU_RCX(%_ASM_CX), %_ASM_CX
|
||||
|
||||
/* Enter guest mode */
|
||||
call vmx_vmenter
|
||||
|
||||
/* Jump on VM-Fail. */
|
||||
jbe 2f
|
||||
|
||||
/* Temporarily save guest's RCX. */
|
||||
push %_ASM_CX
|
||||
|
||||
/* Reload @regs to RCX. */
|
||||
mov WORD_SIZE(%_ASM_SP), %_ASM_CX
|
||||
|
||||
/* Save all guest registers, including RCX from the stack */
|
||||
mov %_ASM_AX, VCPU_RAX(%_ASM_CX)
|
||||
mov %_ASM_BX, VCPU_RBX(%_ASM_CX)
|
||||
__ASM_SIZE(pop) VCPU_RCX(%_ASM_CX)
|
||||
mov %_ASM_DX, VCPU_RDX(%_ASM_CX)
|
||||
mov %_ASM_SI, VCPU_RSI(%_ASM_CX)
|
||||
mov %_ASM_DI, VCPU_RDI(%_ASM_CX)
|
||||
mov %_ASM_BP, VCPU_RBP(%_ASM_CX)
|
||||
#ifdef CONFIG_X86_64
|
||||
mov %r8, VCPU_R8 (%_ASM_CX)
|
||||
mov %r9, VCPU_R9 (%_ASM_CX)
|
||||
mov %r10, VCPU_R10(%_ASM_CX)
|
||||
mov %r11, VCPU_R11(%_ASM_CX)
|
||||
mov %r12, VCPU_R12(%_ASM_CX)
|
||||
mov %r13, VCPU_R13(%_ASM_CX)
|
||||
mov %r14, VCPU_R14(%_ASM_CX)
|
||||
mov %r15, VCPU_R15(%_ASM_CX)
|
||||
#endif
|
||||
|
||||
/* Clear EBX to indicate VM-Exit (as opposed to VM-Fail). */
|
||||
xor %ebx, %ebx
|
||||
|
||||
/*
|
||||
* Clear all general purpose registers except RSP and RBX to prevent
|
||||
* speculative use of the guest's values, even those that are reloaded
|
||||
* via the stack. In theory, an L1 cache miss when restoring registers
|
||||
* could lead to speculative execution with the guest's values.
|
||||
* Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
|
||||
* free. RSP and RBX are exempt as RSP is restored by hardware during
|
||||
* VM-Exit and RBX is explicitly loaded with 0 or 1 to "return" VM-Fail.
|
||||
*/
|
||||
1:
|
||||
#ifdef CONFIG_X86_64
|
||||
xor %r8d, %r8d
|
||||
xor %r9d, %r9d
|
||||
xor %r10d, %r10d
|
||||
xor %r11d, %r11d
|
||||
xor %r12d, %r12d
|
||||
xor %r13d, %r13d
|
||||
xor %r14d, %r14d
|
||||
xor %r15d, %r15d
|
||||
#endif
|
||||
xor %eax, %eax
|
||||
xor %ecx, %ecx
|
||||
xor %edx, %edx
|
||||
xor %esi, %esi
|
||||
xor %edi, %edi
|
||||
xor %ebp, %ebp
|
||||
|
||||
/* "POP" @regs. */
|
||||
add $WORD_SIZE, %_ASM_SP
|
||||
pop %_ASM_BP
|
||||
ret
|
||||
|
||||
/* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */
|
||||
2: mov $1, %ebx
|
||||
jmp 1b
|
||||
ENDPROC(____vmx_vcpu_run)
|
||||
|
||||
@ -6371,33 +6371,6 @@ void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
#define WORD_SIZE 8
|
||||
#else
|
||||
#define WORD_SIZE 4
|
||||
#endif
|
||||
|
||||
#define _WORD_SIZE __stringify(WORD_SIZE)
|
||||
|
||||
#define VCPU_RAX __stringify(__VCPU_REGS_RAX * WORD_SIZE)
|
||||
#define VCPU_RCX __stringify(__VCPU_REGS_RCX * WORD_SIZE)
|
||||
#define VCPU_RDX __stringify(__VCPU_REGS_RDX * WORD_SIZE)
|
||||
#define VCPU_RBX __stringify(__VCPU_REGS_RBX * WORD_SIZE)
|
||||
/* Intentionally omit %RSP as it's context switched by hardware */
|
||||
#define VCPU_RBP __stringify(__VCPU_REGS_RBP * WORD_SIZE)
|
||||
#define VCPU_RSI __stringify(__VCPU_REGS_RSI * WORD_SIZE)
|
||||
#define VCPU_RDI __stringify(__VCPU_REGS_RDI * WORD_SIZE)
|
||||
#ifdef CONFIG_X86_64
|
||||
#define VCPU_R8 __stringify(__VCPU_REGS_R8 * WORD_SIZE)
|
||||
#define VCPU_R9 __stringify(__VCPU_REGS_R9 * WORD_SIZE)
|
||||
#define VCPU_R10 __stringify(__VCPU_REGS_R10 * WORD_SIZE)
|
||||
#define VCPU_R11 __stringify(__VCPU_REGS_R11 * WORD_SIZE)
|
||||
#define VCPU_R12 __stringify(__VCPU_REGS_R12 * WORD_SIZE)
|
||||
#define VCPU_R13 __stringify(__VCPU_REGS_R13 * WORD_SIZE)
|
||||
#define VCPU_R14 __stringify(__VCPU_REGS_R14 * WORD_SIZE)
|
||||
#define VCPU_R15 __stringify(__VCPU_REGS_R15 * WORD_SIZE)
|
||||
#endif
|
||||
|
||||
static void __vmx_vcpu_run(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx)
|
||||
{
|
||||
if (static_branch_unlikely(&vmx_l1d_should_flush))
|
||||
@ -6407,115 +6380,7 @@ static void __vmx_vcpu_run(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx)
|
||||
write_cr2(vcpu->arch.cr2);
|
||||
|
||||
asm(
|
||||
"push %%" _ASM_BP " \n\t"
|
||||
"mov %%" _ASM_SP ", %%" _ASM_BP " \n\t"
|
||||
|
||||
/*
|
||||
* Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
|
||||
* @regs is needed after VM-Exit to save the guest's register values.
|
||||
*/
|
||||
"push %%" _ASM_ARG2 " \n\t"
|
||||
|
||||
/* Adjust RSP to account for the CALL to vmx_vmenter(). */
|
||||
"lea -" _WORD_SIZE "(%%" _ASM_SP "), %%" _ASM_ARG2 " \n\t"
|
||||
"call vmx_update_host_rsp \n\t"
|
||||
|
||||
/* Load RCX with @regs. */
|
||||
"mov (%%" _ASM_SP "), %%" _ASM_CX " \n\t"
|
||||
|
||||
/* Check if vmlaunch or vmresume is needed */
|
||||
"cmpb $0, %%bl \n\t"
|
||||
|
||||
/* Load guest registers. Don't clobber flags. */
|
||||
"mov " VCPU_RAX "(%%" _ASM_CX "), %%" _ASM_AX " \n\t"
|
||||
"mov " VCPU_RBX "(%%" _ASM_CX "), %%" _ASM_BX " \n\t"
|
||||
"mov " VCPU_RDX "(%%" _ASM_CX "), %%" _ASM_DX " \n\t"
|
||||
"mov " VCPU_RSI "(%%" _ASM_CX "), %%" _ASM_SI " \n\t"
|
||||
"mov " VCPU_RDI "(%%" _ASM_CX "), %%" _ASM_DI " \n\t"
|
||||
"mov " VCPU_RBP "(%%" _ASM_CX "), %%" _ASM_BP " \n\t"
|
||||
#ifdef CONFIG_X86_64
|
||||
"mov " VCPU_R8 "(%%" _ASM_CX "), %%r8 \n\t"
|
||||
"mov " VCPU_R9 "(%%" _ASM_CX "), %%r9 \n\t"
|
||||
"mov " VCPU_R10 "(%%" _ASM_CX "), %%r10 \n\t"
|
||||
"mov " VCPU_R11 "(%%" _ASM_CX "), %%r11 \n\t"
|
||||
"mov " VCPU_R12 "(%%" _ASM_CX "), %%r12 \n\t"
|
||||
"mov " VCPU_R13 "(%%" _ASM_CX "), %%r13 \n\t"
|
||||
"mov " VCPU_R14 "(%%" _ASM_CX "), %%r14 \n\t"
|
||||
"mov " VCPU_R15 "(%%" _ASM_CX "), %%r15 \n\t"
|
||||
#endif
|
||||
/* Load guest RCX. This kills the vmx_vcpu pointer! */
|
||||
"mov " VCPU_RCX"(%%" _ASM_CX "), %%" _ASM_CX " \n\t"
|
||||
|
||||
/* Enter guest mode */
|
||||
"call vmx_vmenter\n\t"
|
||||
"jbe 2f \n\t"
|
||||
|
||||
/* Temporarily save guest's RCX. */
|
||||
"push %%" _ASM_CX " \n\t"
|
||||
|
||||
/* Reload RCX with @regs. */
|
||||
"mov " _WORD_SIZE "(%%" _ASM_SP "), %%" _ASM_CX " \n\t"
|
||||
|
||||
/* Save all guest registers, including RCX from the stack */
|
||||
"mov %%" _ASM_AX ", " VCPU_RAX "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%" _ASM_BX ", " VCPU_RBX "(%%" _ASM_CX ") \n\t"
|
||||
__ASM_SIZE(pop) " " VCPU_RCX "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%" _ASM_DX ", " VCPU_RDX "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%" _ASM_SI ", " VCPU_RSI "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%" _ASM_DI ", " VCPU_RDI "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%" _ASM_BP ", " VCPU_RBP "(%%" _ASM_CX ") \n\t"
|
||||
#ifdef CONFIG_X86_64
|
||||
"mov %%r8, " VCPU_R8 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r9, " VCPU_R9 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r10, " VCPU_R10 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r11, " VCPU_R11 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r12, " VCPU_R12 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r13, " VCPU_R13 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r14, " VCPU_R14 "(%%" _ASM_CX ") \n\t"
|
||||
"mov %%r15, " VCPU_R15 "(%%" _ASM_CX ") \n\t"
|
||||
#endif
|
||||
|
||||
/* Clear EBX to indicate VM-Exit (as opposed to VM-Fail). */
|
||||
"xor %%ebx, %%ebx \n\t"
|
||||
|
||||
/*
|
||||
* Clear all general purpose registers except RSP and RBX to prevent
|
||||
* speculative use of the guest's values, even those that are reloaded
|
||||
* via the stack. In theory, an L1 cache miss when restoring registers
|
||||
* could lead to speculative execution with the guest's values.
|
||||
* Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
|
||||
* free. RSP and RBX are exempt as RSP is restored by hardware during
|
||||
* VM-Exit and RBX is explicitly loaded with 0 or 1 to "return" VM-Fail.
|
||||
*/
|
||||
"1: \n\t"
|
||||
#ifdef CONFIG_X86_64
|
||||
"xor %%r8d, %%r8d \n\t"
|
||||
"xor %%r9d, %%r9d \n\t"
|
||||
"xor %%r10d, %%r10d \n\t"
|
||||
"xor %%r11d, %%r11d \n\t"
|
||||
"xor %%r12d, %%r12d \n\t"
|
||||
"xor %%r13d, %%r13d \n\t"
|
||||
"xor %%r14d, %%r14d \n\t"
|
||||
"xor %%r15d, %%r15d \n\t"
|
||||
#endif
|
||||
"xor %%eax, %%eax \n\t"
|
||||
"xor %%ecx, %%ecx \n\t"
|
||||
"xor %%edx, %%edx \n\t"
|
||||
"xor %%esi, %%esi \n\t"
|
||||
"xor %%edi, %%edi \n\t"
|
||||
"xor %%ebp, %%ebp \n\t"
|
||||
|
||||
/* "POP" the vcpu_vmx pointer. */
|
||||
"add $" _WORD_SIZE ", %%" _ASM_SP " \n\t"
|
||||
"pop %%" _ASM_BP " \n\t"
|
||||
"jmp 3f \n\t"
|
||||
|
||||
/* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */
|
||||
"2: \n\t"
|
||||
"mov $1, %%ebx \n\t"
|
||||
"jmp 1b \n\t"
|
||||
"3: \n\t"
|
||||
|
||||
"call ____vmx_vcpu_run \n\t"
|
||||
: ASM_CALL_CONSTRAINT, "=b"(vmx->fail),
|
||||
#ifdef CONFIG_X86_64
|
||||
"=D"((int){0}), "=S"((int){0})
|
||||
@ -6536,7 +6401,6 @@ static void __vmx_vcpu_run(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx)
|
||||
|
||||
vcpu->arch.cr2 = read_cr2();
|
||||
}
|
||||
STACK_FRAME_NON_STANDARD(__vmx_vcpu_run);
|
||||
|
||||
static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user