leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x86: entry_64.S: factor out save_paranoid and paranoid_exit

Browse Source 
Impact: cleanup, shrink kernel image size

Also expand the paranoid_exit0 macro into nmi_exit inside the
nmi stub in the case of enabled irq-tracing.

This gives a few hundred bytes code size reduction.

Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Alexander van Heukelum 2008-11-21 16:43:18 +01:00 committed by Ingo Molnar
parent c002a1e6b6
commit e2f6bc25b9
1 changed files with 102 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
arch/x86/kernel/entry_64.S
View File

@ -353,6 +353,36 @@ ENTRY(save_rest)
CFI_ENDPROC
END(save_rest)
/* save complete stack frame */
ENTRY(save_paranoid)
XCPT_FRAME 1 RDI+8
cld
movq_cfi rdi, RDI+8
movq_cfi rsi, RSI+8
movq_cfi rdx, RDX+8
movq_cfi rcx, RCX+8
movq_cfi rax, RAX+8
movq_cfi r8, R8+8
movq_cfi r9, R9+8
movq_cfi r10, R10+8
movq_cfi r11, R11+8
movq_cfi rbx, RBX+8
movq_cfi rbp, RBP+8
movq_cfi r12, R12+8
movq_cfi r13, R13+8
movq_cfi r14, R14+8
movq_cfi r15, R15+8
movl $1,%ebx
movl $MSR_GS_BASE,%ecx
rdmsr
testl %edx,%edx
js 1f /* negative -> in kernel */
SWAPGS
xorl %ebx,%ebx
1: ret
CFI_ENDPROC
END(save_paranoid)
/*
* A newly forked process directly context switches into this.
*/
@ -1012,24 +1042,15 @@ END(spurious_interrupt)
.endm
/* error code is on the stack already */
/* handle NMI like exceptions that can happen everywhere */
.macro paranoidentry sym, ist=0, irqtrace=1
SAVE_ALL
cld
movl $1,%ebx
movl $MSR_GS_BASE,%ecx
rdmsr
testl %edx,%edx
js 1f
SWAPGS
xorl %ebx,%ebx
1:
.macro paranoidentry sym ist=0
subq $15*8, %rsp
CFI_ADJUST_CFA_OFFSET 15*8
call save_paranoid
DEFAULT_FRAME 0
.if \ist
movq %gs:pda_data_offset, %rbp
.endif
.if \irqtrace
TRACE_IRQS_OFF
.endif
movq %rsp,%rdi
movq ORIG_RAX(%rsp),%rsi
movq $-1,ORIG_RAX(%rsp)
@ -1041,9 +1062,7 @@ END(spurious_interrupt)
addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
.endif
DISABLE_INTERRUPTS(CLBR_NONE)
.if \irqtrace
TRACE_IRQS_OFF
.endif
.endm
/*
@ -1058,57 +1077,48 @@ END(spurious_interrupt)
* is fundamentally NMI-unsafe. (we cannot change the soft and
* hard flags at once, atomically)
*/
.macro paranoidexit trace=1
/* ebx: no swapgs flag */
paranoid_exit\trace:
KPROBE_ENTRY(paranoid_exit)
INTR_FRAME
testl %ebx,%ebx /* swapgs needed? */
jnz paranoid_restore\trace
jnz paranoid_restore
testl $3,CS(%rsp)
jnz paranoid_userspace\trace
paranoid_swapgs\trace:
.if \trace
jnz paranoid_userspace
paranoid_swapgs:
TRACE_IRQS_IRETQ 0
.endif
SWAPGS_UNSAFE_STACK
paranoid_restore\trace:
paranoid_restore:
RESTORE_ALL 8
jmp irq_return
paranoid_userspace\trace:
paranoid_userspace:
GET_THREAD_INFO(%rcx)
movl TI_flags(%rcx),%ebx
andl $_TIF_WORK_MASK,%ebx
jz paranoid_swapgs\trace
jz paranoid_swapgs
movq %rsp,%rdi /* &pt_regs */
call sync_regs
movq %rax,%rsp /* switch stack for scheduling */
testl $_TIF_NEED_RESCHED,%ebx
jnz paranoid_schedule\trace
jnz paranoid_schedule
movl %ebx,%edx /* arg3: thread flags */
.if \trace
TRACE_IRQS_ON
.endif
ENABLE_INTERRUPTS(CLBR_NONE)
xorl %esi,%esi /* arg2: oldset */
movq %rsp,%rdi /* arg1: &pt_regs */
call do_notify_resume
DISABLE_INTERRUPTS(CLBR_NONE)
.if \trace
TRACE_IRQS_OFF
.endif
jmp paranoid_userspace\trace
paranoid_schedule\trace:
.if \trace
jmp paranoid_userspace
paranoid_schedule:
TRACE_IRQS_ON
.endif
ENABLE_INTERRUPTS(CLBR_ANY)
call schedule
DISABLE_INTERRUPTS(CLBR_ANY)
.if \trace
TRACE_IRQS_OFF
.endif
jmp paranoid_userspace\trace
jmp paranoid_userspace
CFI_ENDPROC
.endm
END(paranoid_exit)
/*
* Exception entry point. This expects an error code/orig_rax on the stack.
@ -1326,20 +1336,63 @@ KPROBE_ENTRY(debug)
pushq $0
CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_debug, DEBUG_STACK
paranoidexit
jmp paranoid_exit
CFI_ENDPROC
KPROBE_END(debug)
/* runs on exception stack */
KPROBE_ENTRY(nmi)
INTR_FRAME
PARAVIRT_ADJUST_EXCEPTION_FRAME
pushq $-1
CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_nmi, 0, 0
pushq_cfi $-1
subq $15*8, %rsp
CFI_ADJUST_CFA_OFFSET 15*8
call save_paranoid
DEFAULT_FRAME 0
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
movq %rsp,%rdi
movq ORIG_RAX(%rsp),%rsi
movq $-1,ORIG_RAX(%rsp)
call do_nmi
DISABLE_INTERRUPTS(CLBR_NONE)
#ifdef CONFIG_TRACE_IRQFLAGS
paranoidexit 0
/* paranoidexit; without TRACE_IRQS_OFF */
/* ebx: no swapgs flag */
nmi_exit:
testl %ebx,%ebx /* swapgs needed? */
jnz nmi_restore
testl $3,CS(%rsp)
jnz nmi_userspace
nmi_swapgs:
SWAPGS_UNSAFE_STACK
nmi_restore:
RESTORE_ALL 8
jmp irq_return
nmi_userspace:
GET_THREAD_INFO(%rcx)
movl TI_flags(%rcx),%ebx
andl $_TIF_WORK_MASK,%ebx
jz nmi_swapgs
movq %rsp,%rdi /* &pt_regs */
call sync_regs
movq %rax,%rsp /* switch stack for scheduling */
testl $_TIF_NEED_RESCHED,%ebx
jnz nmi_schedule
movl %ebx,%edx /* arg3: thread flags */
ENABLE_INTERRUPTS(CLBR_NONE)
xorl %esi,%esi /* arg2: oldset */
movq %rsp,%rdi /* arg1: &pt_regs */
call do_notify_resume
DISABLE_INTERRUPTS(CLBR_NONE)
jmp nmi_userspace
nmi_schedule:
ENABLE_INTERRUPTS(CLBR_ANY)
call schedule
DISABLE_INTERRUPTS(CLBR_ANY)
jmp nmi_userspace
CFI_ENDPROC
#else
jmp paranoid_exit1
jmp paranoid_exit
CFI_ENDPROC
#endif
KPROBE_END(nmi)
@ -1350,7 +1403,7 @@ KPROBE_ENTRY(int3)
pushq $0
CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_int3, DEBUG_STACK
jmp paranoid_exit1
jmp paranoid_exit
CFI_ENDPROC
KPROBE_END(int3)
@ -1375,7 +1428,7 @@ ENTRY(double_fault)
XCPT_FRAME
PARAVIRT_ADJUST_EXCEPTION_FRAME
paranoidentry do_double_fault
jmp paranoid_exit1
jmp paranoid_exit
CFI_ENDPROC
END(double_fault)
@ -1392,7 +1445,7 @@ ENTRY(stack_segment)
XCPT_FRAME
PARAVIRT_ADJUST_EXCEPTION_FRAME
paranoidentry do_stack_segment
jmp paranoid_exit1
jmp paranoid_exit
CFI_ENDPROC
END(stack_segment)
@ -1420,7 +1473,7 @@ ENTRY(machine_check)
pushq $0
CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_machine_check
jmp paranoid_exit1
jmp paranoid_exit
CFI_ENDPROC
END(machine_check)
#endif