mirror of
https://github.com/torvalds/linux.git
synced 2024-12-15 23:51:46 +00:00
x86/asm/entry/64: Save R11 into pt_regs->flags on SYSCALL64 fastpath
Before this patch, R11 was saved in pt_regs->r11. Which looks natural, but requires messy shuffling to/from iret frame whenever ptrace or e.g. sys_iopl() wants to modify flags - because that's how this register is used by SYSCALL/SYSRET. This patch saves R11 in pt_regs->flags, and uses that value for the SYSRET64 instruction. Shuffling is eliminated. FIXUP/RESTORE_TOP_OF_STACK are simplified. stub_iopl is no longer needed: pt_regs->flags needs no fixing up. Testing shows that syscall fast path is ~54.3 ns before and after the patch (on 2.7 GHz Sandy Bridge CPU). Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com> Cc: Alexei Starovoitov <ast@plumgrid.com> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Borislav Petkov <bp@alien8.de> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Kees Cook <keescook@chromium.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Will Drewry <wad@chromium.org> Link: http://lkml.kernel.org/r/1425926364-9526-2-git-send-email-dvlasenk@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
3e1aa7cb59
commit
29722cd4ef
@ -95,9 +95,11 @@ For 32-bit we have the following conventions - kernel is built with
|
||||
CFI_ADJUST_CFA_OFFSET 15*8+\addskip
|
||||
.endm
|
||||
|
||||
.macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8plus=1
|
||||
.if \r8plus
|
||||
.macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1
|
||||
.if \r11
|
||||
movq_cfi r11, 6*8+\offset
|
||||
.endif
|
||||
.if \r8910
|
||||
movq_cfi r10, 7*8+\offset
|
||||
movq_cfi r9, 8*8+\offset
|
||||
movq_cfi r8, 9*8+\offset
|
||||
@ -113,16 +115,19 @@ For 32-bit we have the following conventions - kernel is built with
|
||||
movq_cfi rdi, 14*8+\offset
|
||||
.endm
|
||||
.macro SAVE_C_REGS offset=0
|
||||
SAVE_C_REGS_HELPER \offset, 1, 1, 1
|
||||
SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
|
||||
.endm
|
||||
.macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0
|
||||
SAVE_C_REGS_HELPER \offset, 0, 0, 1
|
||||
SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1
|
||||
.endm
|
||||
.macro SAVE_C_REGS_EXCEPT_R891011
|
||||
SAVE_C_REGS_HELPER 0, 1, 1, 0
|
||||
SAVE_C_REGS_HELPER 0, 1, 1, 0, 0
|
||||
.endm
|
||||
.macro SAVE_C_REGS_EXCEPT_RCX_R891011
|
||||
SAVE_C_REGS_HELPER 0, 1, 0, 0
|
||||
SAVE_C_REGS_HELPER 0, 1, 0, 0, 0
|
||||
.endm
|
||||
.macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11
|
||||
SAVE_C_REGS_HELPER 0, 0, 0, 1, 0
|
||||
.endm
|
||||
|
||||
.macro SAVE_EXTRA_REGS offset=0
|
||||
@ -179,6 +184,9 @@ For 32-bit we have the following conventions - kernel is built with
|
||||
.macro RESTORE_C_REGS_EXCEPT_R11
|
||||
RESTORE_C_REGS_HELPER 1,1,0,1,1
|
||||
.endm
|
||||
.macro RESTORE_C_REGS_EXCEPT_RCX_R11
|
||||
RESTORE_C_REGS_HELPER 1,0,0,1,1
|
||||
.endm
|
||||
.macro RESTORE_RSI_RDI
|
||||
RESTORE_C_REGS_HELPER 0,0,0,0,0
|
||||
.endm
|
||||
|
@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* C code is not supposed to know about undefined top of stack. Every time
|
||||
* a C function with an pt_regs argument is called from the SYSCALL based
|
||||
* fast path FIXUP_TOP_OF_STACK is needed.
|
||||
* C code is not supposed to know that the iret frame is not populated.
|
||||
* Every time a C function with an pt_regs argument is called from
|
||||
* the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed.
|
||||
* RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
|
||||
* manipulation.
|
||||
*/
|
||||
|
||||
/* %rsp:at FRAMEEND */
|
||||
.macro FIXUP_TOP_OF_STACK tmp offset=0
|
||||
movq PER_CPU_VAR(old_rsp),\tmp
|
||||
movq \tmp,RSP+\offset(%rsp)
|
||||
@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64)
|
||||
movq $__USER_CS,CS+\offset(%rsp)
|
||||
movq RIP+\offset(%rsp),\tmp /* get rip */
|
||||
movq \tmp,RCX+\offset(%rsp) /* copy it to rcx as sysret would do */
|
||||
movq R11+\offset(%rsp),\tmp /* get eflags */
|
||||
movq \tmp,EFLAGS+\offset(%rsp)
|
||||
movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */
|
||||
movq \tmp,R11+\offset(%rsp)
|
||||
.endm
|
||||
|
||||
.macro RESTORE_TOP_OF_STACK tmp offset=0
|
||||
movq RSP+\offset(%rsp),\tmp
|
||||
movq \tmp,PER_CPU_VAR(old_rsp)
|
||||
movq EFLAGS+\offset(%rsp),\tmp
|
||||
movq \tmp,R11+\offset(%rsp)
|
||||
.endm
|
||||
|
||||
/*
|
||||
@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs)
|
||||
*/
|
||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||
ALLOC_PT_GPREGS_ON_STACK 8 /* +8: space for orig_ax */
|
||||
SAVE_C_REGS_EXCEPT_RAX_RCX
|
||||
SAVE_C_REGS_EXCEPT_RAX_RCX_R11
|
||||
movq $-ENOSYS,RAX(%rsp)
|
||||
movq_cfi rax,ORIG_RAX
|
||||
movq %r11,EFLAGS(%rsp)
|
||||
movq %rcx,RIP(%rsp)
|
||||
CFI_REL_OFFSET rip,RIP
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||
@ -277,7 +274,7 @@ system_call_fastpath:
|
||||
movq %rax,RAX(%rsp)
|
||||
/*
|
||||
* Syscall return path ending with SYSRET (fast path)
|
||||
* Has incomplete stack frame and undefined top of stack.
|
||||
* Has incompletely filled pt_regs, iret frame is also incomplete.
|
||||
*/
|
||||
ret_from_sys_call:
|
||||
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
|
||||
@ -291,9 +288,10 @@ ret_from_sys_call:
|
||||
* sysretq will re-enable interrupts:
|
||||
*/
|
||||
TRACE_IRQS_ON
|
||||
RESTORE_C_REGS_EXCEPT_RCX
|
||||
movq RIP(%rsp),%rcx
|
||||
RESTORE_C_REGS_EXCEPT_RCX_R11
|
||||
movq RIP(%rsp),%rcx
|
||||
CFI_REGISTER rip,rcx
|
||||
movq EFLAGS(%rsp),%r11
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
movq PER_CPU_VAR(old_rsp), %rsp
|
||||
/*
|
||||
|
Loading…
Reference in New Issue
Block a user