2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* linux/arch/x86_64/entry.S
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1991, 1992 Linus Torvalds
|
|
|
|
|
* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
|
|
|
|
|
* Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* entry.S contains the system-call and fault low-level handling routines.
|
|
|
|
|
*
|
2011-06-05 17:50:18 +00:00
|
|
|
* Some of this is documented in Documentation/x86/entry_64.txt
|
|
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* NOTE: This code handles signal-recognition, which happens every time
|
|
|
|
|
* after an interrupt and after each system call.
|
2008-11-16 14:29:00 +00:00
|
|
|
*
|
|
|
|
|
* Normal syscalls and interrupts don't save a full stack frame, this is
|
2005-04-16 22:20:36 +00:00
|
|
|
* only done for syscall tracing, signals or fork/exec et.al.
|
2008-11-16 14:29:00 +00:00
|
|
|
*
|
|
|
|
|
* A note on terminology:
|
|
|
|
|
* - top of stack: Architecture defined interrupt frame from SS to RIP
|
|
|
|
|
* at the top of the kernel process stack.
|
2011-03-17 19:24:16 +00:00
|
|
|
* - partial stack frame: partially saved registers up to R11.
|
2008-11-16 14:29:00 +00:00
|
|
|
* - full stack frame: Like partial stack frame, but all register saved.
|
2006-09-26 08:52:29 +00:00
|
|
|
*
|
|
|
|
|
* Some macro usage:
|
|
|
|
|
* - CFI macros are used to generate dwarf2 unwind information for better
|
|
|
|
|
* backtraces. They don't change any code.
|
|
|
|
|
* - SAVE_ALL/RESTORE_ALL - Save/restore all registers
|
|
|
|
|
* - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
|
|
|
|
|
* There are unfortunately lots of special cases where some registers
|
|
|
|
|
* not touched. The macro is a big mess that should be cleaned up.
|
|
|
|
|
* - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
|
|
|
|
|
* Gives a full stack frame.
|
|
|
|
|
* - ENTRY/END Define functions in the symbol table.
|
|
|
|
|
* - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
|
|
|
|
|
* frame that is otherwise undefined after a SYSCALL
|
|
|
|
|
* - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
|
|
|
|
|
* - errorentry/paranoidentry/zeroentry - Define exception entry points.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
|
#include <asm/segment.h>
|
|
|
|
|
#include <asm/cache.h>
|
|
|
|
|
#include <asm/errno.h>
|
|
|
|
|
#include <asm/dwarf2.h>
|
|
|
|
|
#include <asm/calling.h>
|
2005-09-09 19:28:48 +00:00
|
|
|
#include <asm/asm-offsets.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/msr.h>
|
|
|
|
|
#include <asm/unistd.h>
|
|
|
|
|
#include <asm/thread_info.h>
|
|
|
|
|
#include <asm/hw_irq.h>
|
2009-02-13 19:14:01 +00:00
|
|
|
#include <asm/page_types.h>
|
2006-07-03 07:24:45 +00:00
|
|
|
#include <asm/irqflags.h>
|
2008-01-30 12:32:08 +00:00
|
|
|
#include <asm/paravirt.h>
|
2008-06-21 18:17:27 +00:00
|
|
|
#include <asm/ftrace.h>
|
2009-01-13 11:41:35 +00:00
|
|
|
#include <asm/percpu.h>
|
2012-04-20 19:19:50 +00:00
|
|
|
#include <asm/asm.h>
|
2012-07-11 18:26:38 +00:00
|
|
|
#include <asm/rcu.h>
|
2012-09-21 19:43:12 +00:00
|
|
|
#include <asm/smap.h>
|
2012-01-03 19:23:06 +00:00
|
|
|
#include <linux/err.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-06-23 22:37:04 +00:00
|
|
|
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
|
|
|
|
|
#include <linux/elf-em.h>
|
|
|
|
|
#define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
|
|
|
|
|
#define __AUDIT_ARCH_64BIT 0x80000000
|
|
|
|
|
#define __AUDIT_ARCH_LE 0x40000000
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
.code64
|
2011-03-07 18:10:39 +00:00
|
|
|
.section .entry.text, "ax"
|
|
|
|
|
|
2008-10-06 23:06:12 +00:00
|
|
|
#ifdef CONFIG_FUNCTION_TRACER
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
|
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
# define function_hook __fentry__
|
|
|
|
|
#else
|
|
|
|
|
# define function_hook mcount
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-05-12 19:20:43 +00:00
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
|
|
|
|
|
ENTRY(function_hook)
|
2008-05-12 19:20:43 +00:00
|
|
|
retq
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
END(function_hook)
|
2008-05-12 19:20:43 +00:00
|
|
|
|
2012-04-30 20:20:23 +00:00
|
|
|
/* skip is set if stack has been adjusted */
|
|
|
|
|
.macro ftrace_caller_setup skip=0
|
|
|
|
|
MCOUNT_SAVE_FRAME \skip
|
|
|
|
|
|
|
|
|
|
/* Load the ftrace_ops into the 3rd parameter */
|
|
|
|
|
leaq function_trace_op, %rdx
|
|
|
|
|
|
|
|
|
|
/* Load ip into the first parameter */
|
|
|
|
|
movq RIP(%rsp), %rdi
|
|
|
|
|
subq $MCOUNT_INSN_SIZE, %rdi
|
|
|
|
|
/* Load the parent_ip into the second parameter */
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
movq SS+16(%rsp), %rsi
|
|
|
|
|
#else
|
2012-04-30 20:20:23 +00:00
|
|
|
movq 8(%rbp), %rsi
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
#endif
|
2012-04-30 20:20:23 +00:00
|
|
|
.endm
|
2008-05-12 19:20:43 +00:00
|
|
|
|
|
|
|
|
ENTRY(ftrace_caller)
|
2012-04-30 20:20:23 +00:00
|
|
|
/* Check if tracing was disabled (quick check) */
|
2008-11-05 21:05:44 +00:00
|
|
|
cmpl $0, function_trace_stop
|
|
|
|
|
jne ftrace_stub
|
2008-05-12 19:20:43 +00:00
|
|
|
|
2012-04-30 20:20:23 +00:00
|
|
|
ftrace_caller_setup
|
|
|
|
|
/* regs go into 4th parameter (but make it NULL) */
|
|
|
|
|
movq $0, %rcx
|
2008-05-12 19:20:43 +00:00
|
|
|
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(ftrace_call)
|
2008-05-12 19:20:43 +00:00
|
|
|
call ftrace_stub
|
|
|
|
|
|
2008-12-12 21:09:08 +00:00
|
|
|
MCOUNT_RESTORE_FRAME
|
2012-04-30 20:20:23 +00:00
|
|
|
ftrace_return:
|
2008-05-12 19:20:43 +00:00
|
|
|
|
2008-12-01 23:20:39 +00:00
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(ftrace_graph_call)
|
2008-12-01 23:20:39 +00:00
|
|
|
jmp ftrace_stub
|
|
|
|
|
#endif
|
2008-05-12 19:20:43 +00:00
|
|
|
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(ftrace_stub)
|
2008-05-12 19:20:43 +00:00
|
|
|
retq
|
|
|
|
|
END(ftrace_caller)
|
|
|
|
|
|
2012-04-30 20:20:23 +00:00
|
|
|
ENTRY(ftrace_regs_caller)
|
|
|
|
|
/* Save the current flags before compare (in SS location)*/
|
|
|
|
|
pushfq
|
|
|
|
|
|
|
|
|
|
/* Check if tracing was disabled (quick check) */
|
|
|
|
|
cmpl $0, function_trace_stop
|
|
|
|
|
jne ftrace_restore_flags
|
|
|
|
|
|
|
|
|
|
/* skip=8 to skip flags saved in SS */
|
|
|
|
|
ftrace_caller_setup 8
|
|
|
|
|
|
|
|
|
|
/* Save the rest of pt_regs */
|
|
|
|
|
movq %r15, R15(%rsp)
|
|
|
|
|
movq %r14, R14(%rsp)
|
|
|
|
|
movq %r13, R13(%rsp)
|
|
|
|
|
movq %r12, R12(%rsp)
|
|
|
|
|
movq %r11, R11(%rsp)
|
|
|
|
|
movq %r10, R10(%rsp)
|
|
|
|
|
movq %rbp, RBP(%rsp)
|
|
|
|
|
movq %rbx, RBX(%rsp)
|
|
|
|
|
/* Copy saved flags */
|
|
|
|
|
movq SS(%rsp), %rcx
|
|
|
|
|
movq %rcx, EFLAGS(%rsp)
|
|
|
|
|
/* Kernel segments */
|
|
|
|
|
movq $__KERNEL_DS, %rcx
|
|
|
|
|
movq %rcx, SS(%rsp)
|
|
|
|
|
movq $__KERNEL_CS, %rcx
|
|
|
|
|
movq %rcx, CS(%rsp)
|
|
|
|
|
/* Stack - skipping return address */
|
|
|
|
|
leaq SS+16(%rsp), %rcx
|
|
|
|
|
movq %rcx, RSP(%rsp)
|
|
|
|
|
|
|
|
|
|
/* regs go into 4th parameter */
|
|
|
|
|
leaq (%rsp), %rcx
|
|
|
|
|
|
|
|
|
|
GLOBAL(ftrace_regs_call)
|
|
|
|
|
call ftrace_stub
|
|
|
|
|
|
|
|
|
|
/* Copy flags back to SS, to restore them */
|
|
|
|
|
movq EFLAGS(%rsp), %rax
|
|
|
|
|
movq %rax, SS(%rsp)
|
|
|
|
|
|
2012-09-05 14:31:18 +00:00
|
|
|
/* Handlers can change the RIP */
|
|
|
|
|
movq RIP(%rsp), %rax
|
|
|
|
|
movq %rax, SS+8(%rsp)
|
|
|
|
|
|
2012-04-30 20:20:23 +00:00
|
|
|
/* restore the rest of pt_regs */
|
|
|
|
|
movq R15(%rsp), %r15
|
|
|
|
|
movq R14(%rsp), %r14
|
|
|
|
|
movq R13(%rsp), %r13
|
|
|
|
|
movq R12(%rsp), %r12
|
|
|
|
|
movq R10(%rsp), %r10
|
|
|
|
|
movq RBP(%rsp), %rbp
|
|
|
|
|
movq RBX(%rsp), %rbx
|
|
|
|
|
|
|
|
|
|
/* skip=8 to skip flags saved in SS */
|
|
|
|
|
MCOUNT_RESTORE_FRAME 8
|
|
|
|
|
|
|
|
|
|
/* Restore flags */
|
|
|
|
|
popfq
|
|
|
|
|
|
|
|
|
|
jmp ftrace_return
|
|
|
|
|
ftrace_restore_flags:
|
|
|
|
|
popfq
|
|
|
|
|
jmp ftrace_stub
|
|
|
|
|
|
|
|
|
|
END(ftrace_regs_caller)
|
|
|
|
|
|
|
|
|
|
|
2008-05-12 19:20:43 +00:00
|
|
|
#else /* ! CONFIG_DYNAMIC_FTRACE */
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
|
|
|
|
|
ENTRY(function_hook)
|
2008-11-05 21:05:44 +00:00
|
|
|
cmpl $0, function_trace_stop
|
|
|
|
|
jne ftrace_stub
|
|
|
|
|
|
2008-05-12 19:20:42 +00:00
|
|
|
cmpq $ftrace_stub, ftrace_trace_function
|
|
|
|
|
jnz trace
|
2008-12-01 23:20:39 +00:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
|
|
|
cmpq $ftrace_stub, ftrace_graph_return
|
|
|
|
|
jnz ftrace_graph_caller
|
2008-12-03 04:50:05 +00:00
|
|
|
|
|
|
|
|
cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
|
|
|
|
|
jnz ftrace_graph_caller
|
2008-12-01 23:20:39 +00:00
|
|
|
#endif
|
|
|
|
|
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(ftrace_stub)
|
2008-05-12 19:20:42 +00:00
|
|
|
retq
|
|
|
|
|
|
|
|
|
|
trace:
|
2008-12-12 21:09:08 +00:00
|
|
|
MCOUNT_SAVE_FRAME
|
2008-05-12 19:20:42 +00:00
|
|
|
|
2012-04-30 20:20:23 +00:00
|
|
|
movq RIP(%rsp), %rdi
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
movq SS+16(%rsp), %rsi
|
|
|
|
|
#else
|
2008-05-12 19:20:42 +00:00
|
|
|
movq 8(%rbp), %rsi
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
#endif
|
2008-06-21 18:17:27 +00:00
|
|
|
subq $MCOUNT_INSN_SIZE, %rdi
|
2008-05-12 19:20:42 +00:00
|
|
|
|
|
|
|
|
call *ftrace_trace_function
|
|
|
|
|
|
2008-12-12 21:09:08 +00:00
|
|
|
MCOUNT_RESTORE_FRAME
|
2008-05-12 19:20:42 +00:00
|
|
|
|
|
|
|
|
jmp ftrace_stub
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
END(function_hook)
|
2008-05-12 19:20:43 +00:00
|
|
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
2008-10-06 23:06:12 +00:00
|
|
|
#endif /* CONFIG_FUNCTION_TRACER */
|
2008-05-12 19:20:42 +00:00
|
|
|
|
2008-12-01 23:20:39 +00:00
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
|
|
|
ENTRY(ftrace_graph_caller)
|
2008-12-12 21:09:08 +00:00
|
|
|
MCOUNT_SAVE_FRAME
|
2008-12-01 23:20:39 +00:00
|
|
|
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
leaq SS+16(%rsp), %rdi
|
|
|
|
|
movq $0, %rdx /* No framepointers needed */
|
|
|
|
|
#else
|
2008-12-01 23:20:39 +00:00
|
|
|
leaq 8(%rbp), %rdi
|
function-graph: add stack frame test
In case gcc does something funny with the stack frames, or the return
from function code, we would like to detect that.
An arch may implement passing of a variable that is unique to the
function and can be saved on entering a function and can be tested
when exiting the function. Usually the frame pointer can be used for
this purpose.
This patch also implements this for x86. Where it passes in the stack
frame of the parent function, and will test that frame on exit.
There was a case in x86_32 with optimize for size (-Os) where, for a
few functions, gcc would align the stack frame and place a copy of the
return address into it. The function graph tracer modified the copy and
not the actual return address. On return from the funtion, it did not go
to the tracer hook, but returned to the parent. This broke the function
graph tracer, because the return of the parent (where gcc did not do
this funky manipulation) returned to the location that the child function
was suppose to. This caused strange kernel crashes.
This test detected the problem and pointed out where the issue was.
This modifies the parameters of one of the functions that the arch
specific code calls, so it includes changes to arch code to accommodate
the new prototype.
Note, I notice that the parsic arch implements its own push_return_trace.
This is now a generic function and the ftrace_push_return_trace should be
used instead. This patch does not touch that code.
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2009-06-18 16:45:08 +00:00
|
|
|
movq (%rbp), %rdx
|
ftrace/x86: Add support for -mfentry to x86_64
If the kernel is compiled with gcc 4.6.0 which supports -mfentry,
then use that instead of mcount.
With mcount, frame pointers are forced with the -pg option and we
get something like:
<can_vma_merge_before>:
55 push %rbp
48 89 e5 mov %rsp,%rbp
53 push %rbx
41 51 push %r9
e8 fe 6a 39 00 callq ffffffff81483d00 <mcount>
31 c0 xor %eax,%eax
48 89 fb mov %rdi,%rbx
48 89 d7 mov %rdx,%rdi
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
With -mfentry, frame pointers are no longer forced and the call looks
like this:
<can_vma_merge_before>:
e8 33 af 37 00 callq ffffffff81461b40 <__fentry__>
53 push %rbx
48 89 fb mov %rdi,%rbx
31 c0 xor %eax,%eax
48 89 d7 mov %rdx,%rdi
41 51 push %r9
48 33 73 30 xor 0x30(%rbx),%rsi
48 f7 c6 ff ff ff f7 test $0xfffffffff7ffffff,%rsi
This adds the ftrace hook at the beginning of the function before a
frame is set up, and allows the function callbacks to be able to access
parameters. As kprobes now can use function tracing (at least on x86)
this speeds up the kprobe hooks that are at the beginning of the
function.
Link: http://lkml.kernel.org/r/20120807194100.130477900@goodmis.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-02-09 18:32:18 +00:00
|
|
|
#endif
|
|
|
|
|
movq RIP(%rsp), %rsi
|
2008-12-02 20:34:09 +00:00
|
|
|
subq $MCOUNT_INSN_SIZE, %rsi
|
2008-12-01 23:20:39 +00:00
|
|
|
|
|
|
|
|
call prepare_ftrace_return
|
|
|
|
|
|
2008-12-12 21:09:08 +00:00
|
|
|
MCOUNT_RESTORE_FRAME
|
|
|
|
|
|
2008-12-01 23:20:39 +00:00
|
|
|
retq
|
|
|
|
|
END(ftrace_graph_caller)
|
|
|
|
|
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(return_to_handler)
|
2009-07-29 08:58:37 +00:00
|
|
|
subq $24, %rsp
|
2008-12-01 23:20:39 +00:00
|
|
|
|
2009-03-25 18:30:04 +00:00
|
|
|
/* Save the return values */
|
2008-05-12 19:20:42 +00:00
|
|
|
movq %rax, (%rsp)
|
2009-03-25 18:30:04 +00:00
|
|
|
movq %rdx, 8(%rsp)
|
function-graph: add stack frame test
In case gcc does something funny with the stack frames, or the return
from function code, we would like to detect that.
An arch may implement passing of a variable that is unique to the
function and can be saved on entering a function and can be tested
when exiting the function. Usually the frame pointer can be used for
this purpose.
This patch also implements this for x86. Where it passes in the stack
frame of the parent function, and will test that frame on exit.
There was a case in x86_32 with optimize for size (-Os) where, for a
few functions, gcc would align the stack frame and place a copy of the
return address into it. The function graph tracer modified the copy and
not the actual return address. On return from the funtion, it did not go
to the tracer hook, but returned to the parent. This broke the function
graph tracer, because the return of the parent (where gcc did not do
this funky manipulation) returned to the location that the child function
was suppose to. This caused strange kernel crashes.
This test detected the problem and pointed out where the issue was.
This modifies the parameters of one of the functions that the arch
specific code calls, so it includes changes to arch code to accommodate
the new prototype.
Note, I notice that the parsic arch implements its own push_return_trace.
This is now a generic function and the ftrace_push_return_trace should be
used instead. This patch does not touch that code.
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2009-06-18 16:45:08 +00:00
|
|
|
movq %rbp, %rdi
|
2008-05-12 19:20:42 +00:00
|
|
|
|
2008-12-01 23:20:39 +00:00
|
|
|
call ftrace_return_to_handler
|
2008-05-12 19:20:42 +00:00
|
|
|
|
2009-10-13 20:33:50 +00:00
|
|
|
movq %rax, %rdi
|
2009-03-25 18:30:04 +00:00
|
|
|
movq 8(%rsp), %rdx
|
2008-05-12 19:20:42 +00:00
|
|
|
movq (%rsp), %rax
|
2009-10-13 20:33:50 +00:00
|
|
|
addq $24, %rsp
|
|
|
|
|
jmp *%rdi
|
2008-12-01 23:20:39 +00:00
|
|
|
#endif
|
2008-05-12 19:20:42 +00:00
|
|
|
|
|
|
|
|
|
2005-04-16 22:25:05 +00:00
|
|
|
#ifndef CONFIG_PREEMPT
|
2005-04-16 22:20:36 +00:00
|
|
|
#define retint_kernel retint_restore_args
|
2008-11-16 14:29:00 +00:00
|
|
|
#endif
|
2006-07-03 07:24:45 +00:00
|
|
|
|
2008-01-30 12:32:08 +00:00
|
|
|
#ifdef CONFIG_PARAVIRT
|
2008-06-25 04:19:28 +00:00
|
|
|
ENTRY(native_usergs_sysret64)
|
2008-01-30 12:32:08 +00:00
|
|
|
swapgs
|
|
|
|
|
sysretq
|
2009-02-23 19:57:00 +00:00
|
|
|
ENDPROC(native_usergs_sysret64)
|
2008-01-30 12:32:08 +00:00
|
|
|
#endif /* CONFIG_PARAVIRT */
|
|
|
|
|
|
2006-07-03 07:24:45 +00:00
|
|
|
|
|
|
|
|
.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
|
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
|
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
|
|
|
|
|
jnc 1f
|
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
|
1:
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
2012-05-30 15:54:53 +00:00
|
|
|
/*
|
|
|
|
|
* When dynamic function tracer is enabled it will add a breakpoint
|
|
|
|
|
* to all locations that it is about to modify, sync CPUs, update
|
|
|
|
|
* all the code, sync CPUs, then remove the breakpoints. In this time
|
|
|
|
|
* if lockdep is enabled, it might jump back into the debug handler
|
|
|
|
|
* outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
|
|
|
|
|
*
|
|
|
|
|
* We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
|
|
|
|
|
* make sure the stack pointer does not get reset back to the top
|
|
|
|
|
* of the debug stack, and instead just reuses the current stack.
|
|
|
|
|
*/
|
|
|
|
|
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)
|
|
|
|
|
|
|
|
|
|
.macro TRACE_IRQS_OFF_DEBUG
|
|
|
|
|
call debug_stack_set_zero
|
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
|
call debug_stack_reset
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro TRACE_IRQS_ON_DEBUG
|
|
|
|
|
call debug_stack_set_zero
|
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
|
call debug_stack_reset
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro TRACE_IRQS_IRETQ_DEBUG offset=ARGOFFSET
|
|
|
|
|
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
|
|
|
|
|
jnc 1f
|
|
|
|
|
TRACE_IRQS_ON_DEBUG
|
|
|
|
|
1:
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
# define TRACE_IRQS_OFF_DEBUG TRACE_IRQS_OFF
|
|
|
|
|
# define TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON
|
|
|
|
|
# define TRACE_IRQS_IRETQ_DEBUG TRACE_IRQS_IRETQ
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2008-11-16 14:29:00 +00:00
|
|
|
* C code is not supposed to know about undefined top of stack. Every time
|
|
|
|
|
* a C function with an pt_regs argument is called from the SYSCALL based
|
2005-04-16 22:20:36 +00:00
|
|
|
* fast path FIXUP_TOP_OF_STACK is needed.
|
|
|
|
|
* RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
|
|
|
|
|
* manipulation.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/* %rsp:at FRAMEEND */
|
2008-11-21 15:41:55 +00:00
|
|
|
.macro FIXUP_TOP_OF_STACK tmp offset=0
|
2009-01-18 15:38:58 +00:00
|
|
|
movq PER_CPU_VAR(old_rsp),\tmp
|
2008-11-21 15:41:55 +00:00
|
|
|
movq \tmp,RSP+\offset(%rsp)
|
|
|
|
|
movq $__USER_DS,SS+\offset(%rsp)
|
|
|
|
|
movq $__USER_CS,CS+\offset(%rsp)
|
|
|
|
|
movq $-1,RCX+\offset(%rsp)
|
|
|
|
|
movq R11+\offset(%rsp),\tmp /* get eflags */
|
|
|
|
|
movq \tmp,EFLAGS+\offset(%rsp)
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
|
|
|
|
|
2008-11-21 15:41:55 +00:00
|
|
|
.macro RESTORE_TOP_OF_STACK tmp offset=0
|
|
|
|
|
movq RSP+\offset(%rsp),\tmp
|
2009-01-18 15:38:58 +00:00
|
|
|
movq \tmp,PER_CPU_VAR(old_rsp)
|
2008-11-21 15:41:55 +00:00
|
|
|
movq EFLAGS+\offset(%rsp),\tmp
|
|
|
|
|
movq \tmp,R11+\offset(%rsp)
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro FAKE_STACK_FRAME child_rip
|
|
|
|
|
/* push in order ss, rsp, eflags, cs, rip */
|
2005-07-29 04:15:48 +00:00
|
|
|
xorl %eax, %eax
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi $__KERNEL_DS /* ss */
|
2005-09-12 16:49:24 +00:00
|
|
|
/*CFI_REL_OFFSET ss,0*/
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rax /* rsp */
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_REL_OFFSET rsp,0
|
2011-12-06 08:58:14 +00:00
|
|
|
pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_BIT1) /* eflags - interrupts on */
|
2005-09-12 16:49:24 +00:00
|
|
|
/*CFI_REL_OFFSET rflags,0*/
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi $__KERNEL_CS /* cs */
|
2005-09-12 16:49:24 +00:00
|
|
|
/*CFI_REL_OFFSET cs,0*/
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi \child_rip /* rip */
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_REL_OFFSET rip,0
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rax /* orig rax */
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro UNFAKE_STACK_FRAME
|
|
|
|
|
addq $8*6, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET -(6*8)
|
|
|
|
|
.endm
|
|
|
|
|
|
2008-11-20 13:40:11 +00:00
|
|
|
/*
|
|
|
|
|
* initial frame state for interrupts (and exceptions without error code)
|
|
|
|
|
*/
|
|
|
|
|
.macro EMPTY_FRAME start=1 offset=0
|
2005-09-12 16:49:24 +00:00
|
|
|
.if \start
|
2008-11-20 13:40:11 +00:00
|
|
|
CFI_STARTPROC simple
|
2006-09-26 08:52:41 +00:00
|
|
|
CFI_SIGNAL_FRAME
|
2008-11-20 13:40:11 +00:00
|
|
|
CFI_DEF_CFA rsp,8+\offset
|
2005-09-12 16:49:24 +00:00
|
|
|
.else
|
2008-11-20 13:40:11 +00:00
|
|
|
CFI_DEF_CFA_OFFSET 8+\offset
|
2005-09-12 16:49:24 +00:00
|
|
|
.endif
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
|
|
|
|
|
/*
|
2008-11-20 13:40:11 +00:00
|
|
|
* initial frame state for interrupts (and exceptions without error code)
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
*/
|
2008-11-20 13:40:11 +00:00
|
|
|
.macro INTR_FRAME start=1 offset=0
|
2008-11-21 14:11:32 +00:00
|
|
|
EMPTY_FRAME \start, SS+8+\offset-RIP
|
|
|
|
|
/*CFI_REL_OFFSET ss, SS+\offset-RIP*/
|
|
|
|
|
CFI_REL_OFFSET rsp, RSP+\offset-RIP
|
|
|
|
|
/*CFI_REL_OFFSET rflags, EFLAGS+\offset-RIP*/
|
|
|
|
|
/*CFI_REL_OFFSET cs, CS+\offset-RIP*/
|
|
|
|
|
CFI_REL_OFFSET rip, RIP+\offset-RIP
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* initial frame state for exceptions with error code (and interrupts
|
|
|
|
|
* with vector already pushed)
|
|
|
|
|
*/
|
2008-11-20 13:40:11 +00:00
|
|
|
.macro XCPT_FRAME start=1 offset=0
|
2008-11-21 14:11:32 +00:00
|
|
|
INTR_FRAME \start, RIP+\offset-ORIG_RAX
|
2008-11-20 13:40:11 +00:00
|
|
|
/*CFI_REL_OFFSET orig_rax, ORIG_RAX-ORIG_RAX*/
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* frame that enables calling into C.
|
|
|
|
|
*/
|
|
|
|
|
.macro PARTIAL_FRAME start=1 offset=0
|
2008-11-21 14:11:32 +00:00
|
|
|
XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET
|
|
|
|
|
CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET
|
2008-11-20 13:40:11 +00:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* frame that enables passing a complete pt_regs to a C function.
|
|
|
|
|
*/
|
|
|
|
|
.macro DEFAULT_FRAME start=1 offset=0
|
2008-11-21 14:11:32 +00:00
|
|
|
PARTIAL_FRAME \start, R11+\offset-R15
|
2008-11-20 13:40:11 +00:00
|
|
|
CFI_REL_OFFSET rbx, RBX+\offset
|
|
|
|
|
CFI_REL_OFFSET rbp, RBP+\offset
|
|
|
|
|
CFI_REL_OFFSET r12, R12+\offset
|
|
|
|
|
CFI_REL_OFFSET r13, R13+\offset
|
|
|
|
|
CFI_REL_OFFSET r14, R14+\offset
|
|
|
|
|
CFI_REL_OFFSET r15, R15+\offset
|
|
|
|
|
.endm
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
|
|
|
|
|
/* save partial stack frame */
|
2011-06-30 23:51:22 +00:00
|
|
|
.macro SAVE_ARGS_IRQ
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
cld
|
2011-06-30 23:51:22 +00:00
|
|
|
/* start from rbp in pt_regs and jump over */
|
2012-09-26 08:28:22 +00:00
|
|
|
movq_cfi rdi, (RDI-RBP)
|
|
|
|
|
movq_cfi rsi, (RSI-RBP)
|
|
|
|
|
movq_cfi rdx, (RDX-RBP)
|
|
|
|
|
movq_cfi rcx, (RCX-RBP)
|
|
|
|
|
movq_cfi rax, (RAX-RBP)
|
|
|
|
|
movq_cfi r8, (R8-RBP)
|
|
|
|
|
movq_cfi r9, (R9-RBP)
|
|
|
|
|
movq_cfi r10, (R10-RBP)
|
|
|
|
|
movq_cfi r11, (R11-RBP)
|
2011-06-30 23:51:22 +00:00
|
|
|
|
2011-07-02 14:52:45 +00:00
|
|
|
/* Save rbp so that we can unwind from get_irq_regs() */
|
|
|
|
|
movq_cfi rbp, 0
|
|
|
|
|
|
|
|
|
|
/* Save previous stack value */
|
|
|
|
|
movq %rsp, %rsi
|
2011-07-01 00:25:17 +00:00
|
|
|
|
|
|
|
|
leaq -RBP(%rsp),%rdi /* arg1 for handler */
|
2012-02-24 11:55:01 +00:00
|
|
|
testl $3, CS-RBP(%rsi)
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
je 1f
|
|
|
|
|
SWAPGS
|
|
|
|
|
/*
|
2009-01-18 15:38:58 +00:00
|
|
|
* irq_count is used to check if a CPU is already on an interrupt stack
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
* or not. While this is essentially redundant with preempt_count it is
|
|
|
|
|
* a little cheaper to use a separate counter in the PDA (short of
|
|
|
|
|
* moving irq_enter into assembly, which would be too much work)
|
|
|
|
|
*/
|
2009-01-18 15:38:58 +00:00
|
|
|
1: incl PER_CPU_VAR(irq_count)
|
2012-02-24 11:55:01 +00:00
|
|
|
cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
|
2011-09-28 15:57:52 +00:00
|
|
|
CFI_DEF_CFA_REGISTER rsi
|
2011-07-02 14:52:45 +00:00
|
|
|
|
2012-02-24 11:55:01 +00:00
|
|
|
/* Store previous stack value */
|
2011-07-02 14:52:45 +00:00
|
|
|
pushq %rsi
|
2011-09-28 15:57:52 +00:00
|
|
|
CFI_ESCAPE 0x0f /* DW_CFA_def_cfa_expression */, 6, \
|
|
|
|
|
0x77 /* DW_OP_breg7 */, 0, \
|
|
|
|
|
0x06 /* DW_OP_deref */, \
|
|
|
|
|
0x08 /* DW_OP_const1u */, SS+8-RBP, \
|
|
|
|
|
0x22 /* DW_OP_plus */
|
2011-07-02 14:52:45 +00:00
|
|
|
/* We entered an interrupt context - irqs are off: */
|
|
|
|
|
TRACE_IRQS_OFF
|
2011-06-30 23:51:22 +00:00
|
|
|
.endm
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
|
2008-11-21 15:41:55 +00:00
|
|
|
ENTRY(save_rest)
|
2012-09-26 08:28:22 +00:00
|
|
|
PARTIAL_FRAME 1 (REST_SKIP+8)
|
2008-11-21 15:41:55 +00:00
|
|
|
movq 5*8+16(%rsp), %r11 /* save return address */
|
|
|
|
|
movq_cfi rbx, RBX+16
|
|
|
|
|
movq_cfi rbp, RBP+16
|
|
|
|
|
movq_cfi r12, R12+16
|
|
|
|
|
movq_cfi r13, R13+16
|
|
|
|
|
movq_cfi r14, R14+16
|
|
|
|
|
movq_cfi r15, R15+16
|
|
|
|
|
movq %r11, 8(%rsp) /* return address */
|
|
|
|
|
FIXUP_TOP_OF_STACK %r11, 16
|
|
|
|
|
ret
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(save_rest)
|
|
|
|
|
|
2008-11-21 15:43:18 +00:00
|
|
|
/* save complete stack frame */
|
2009-03-12 10:38:55 +00:00
|
|
|
.pushsection .kprobes.text, "ax"
|
2008-11-21 15:43:18 +00:00
|
|
|
ENTRY(save_paranoid)
|
|
|
|
|
XCPT_FRAME 1 RDI+8
|
|
|
|
|
cld
|
|
|
|
|
movq_cfi rdi, RDI+8
|
|
|
|
|
movq_cfi rsi, RSI+8
|
|
|
|
|
movq_cfi rdx, RDX+8
|
|
|
|
|
movq_cfi rcx, RCX+8
|
|
|
|
|
movq_cfi rax, RAX+8
|
|
|
|
|
movq_cfi r8, R8+8
|
|
|
|
|
movq_cfi r9, R9+8
|
|
|
|
|
movq_cfi r10, R10+8
|
|
|
|
|
movq_cfi r11, R11+8
|
|
|
|
|
movq_cfi rbx, RBX+8
|
|
|
|
|
movq_cfi rbp, RBP+8
|
|
|
|
|
movq_cfi r12, R12+8
|
|
|
|
|
movq_cfi r13, R13+8
|
|
|
|
|
movq_cfi r14, R14+8
|
|
|
|
|
movq_cfi r15, R15+8
|
|
|
|
|
movl $1,%ebx
|
|
|
|
|
movl $MSR_GS_BASE,%ecx
|
|
|
|
|
rdmsr
|
|
|
|
|
testl %edx,%edx
|
|
|
|
|
js 1f /* negative -> in kernel */
|
|
|
|
|
SWAPGS
|
|
|
|
|
xorl %ebx,%ebx
|
|
|
|
|
1: ret
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(save_paranoid)
|
2009-03-12 10:38:55 +00:00
|
|
|
.popsection
|
2008-11-21 15:43:18 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2008-11-27 13:41:21 +00:00
|
|
|
* A newly forked process directly context switches into this address.
|
|
|
|
|
*
|
|
|
|
|
* rdi: prev task we switched from
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
ENTRY(ret_from_fork)
|
2008-11-20 13:40:11 +00:00
|
|
|
DEFAULT_FRAME
|
2008-11-27 13:41:21 +00:00
|
|
|
|
2009-01-11 04:00:22 +00:00
|
|
|
LOCK ; btr $TIF_FORK,TI_flags(%r8)
|
|
|
|
|
|
2012-08-24 19:58:47 +00:00
|
|
|
pushq_cfi $0x0002
|
2010-09-02 13:07:16 +00:00
|
|
|
popfq_cfi # reset kernel eflags
|
2008-11-27 13:41:21 +00:00
|
|
|
|
|
|
|
|
call schedule_tail # rdi: 'prev' task parameter
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
GET_THREAD_INFO(%rcx)
|
2008-11-27 13:41:21 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
RESTORE_REST
|
2008-11-27 13:41:21 +00:00
|
|
|
|
|
|
|
|
testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread?
|
2012-09-10 20:44:54 +00:00
|
|
|
jz 1f
|
2008-11-27 13:41:21 +00:00
|
|
|
|
|
|
|
|
testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET
|
2005-04-16 22:20:36 +00:00
|
|
|
jnz int_ret_from_sys_call
|
2008-11-27 13:41:21 +00:00
|
|
|
|
2008-11-21 15:41:55 +00:00
|
|
|
RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET
|
2008-11-27 13:41:21 +00:00
|
|
|
jmp ret_from_sys_call # go to the SYSRET fastpath
|
|
|
|
|
|
2012-09-10 20:44:54 +00:00
|
|
|
1:
|
2012-10-11 01:35:42 +00:00
|
|
|
subq $REST_SKIP, %rsp # leave space for volatiles
|
2012-09-10 20:44:54 +00:00
|
|
|
CFI_ADJUST_CFA_OFFSET REST_SKIP
|
|
|
|
|
movq %rbp, %rdi
|
|
|
|
|
call *%rbx
|
2012-10-11 01:35:42 +00:00
|
|
|
movl $0, RAX(%rsp)
|
|
|
|
|
RESTORE_REST
|
|
|
|
|
jmp int_ret_from_sys_call
|
2005-04-16 22:20:36 +00:00
|
|
|
CFI_ENDPROC
|
2006-06-26 11:56:55 +00:00
|
|
|
END(ret_from_fork)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
2011-03-17 19:24:16 +00:00
|
|
|
* System call entry. Up to 6 arguments in registers are supported.
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
|
* SYSCALL does not save anything on the stack and does not change the
|
2012-09-21 19:43:12 +00:00
|
|
|
* stack pointer. However, it does mask the flags register for us, so
|
|
|
|
|
* CLD and CLAC are not needed.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2008-11-16 14:29:00 +00:00
|
|
|
* Register setup:
|
2005-04-16 22:20:36 +00:00
|
|
|
* rax system call number
|
|
|
|
|
* rdi arg0
|
2008-11-16 14:29:00 +00:00
|
|
|
* rcx return address for syscall/sysret, C arg3
|
2005-04-16 22:20:36 +00:00
|
|
|
* rsi arg1
|
2008-11-16 14:29:00 +00:00
|
|
|
* rdx arg2
|
2005-04-16 22:20:36 +00:00
|
|
|
* r10 arg3 (--> moved to rcx for C)
|
|
|
|
|
* r8 arg4
|
|
|
|
|
* r9 arg5
|
|
|
|
|
* r11 eflags for syscall/sysret, temporary for C
|
2008-11-16 14:29:00 +00:00
|
|
|
* r12-r15,rbp,rbx saved by C code, not touched.
|
|
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Interrupts are off on entry.
|
|
|
|
|
* Only called from user space.
|
|
|
|
|
*
|
|
|
|
|
* XXX if we had a free scratch register we could save the RSP into the stack frame
|
|
|
|
|
* and report it properly in ps. Unfortunately we haven't.
|
2006-04-07 17:50:00 +00:00
|
|
|
*
|
|
|
|
|
* When user can change the frames always force IRET. That is because
|
|
|
|
|
* it deals with uncanonical addresses better. SYSRET has trouble
|
|
|
|
|
* with them due to bugs in both AMD and Intel CPUs.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
ENTRY(system_call)
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_STARTPROC simple
|
2006-09-26 08:52:41 +00:00
|
|
|
CFI_SIGNAL_FRAME
|
2009-01-18 15:38:58 +00:00
|
|
|
CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_REGISTER rip,rcx
|
|
|
|
|
/*CFI_REGISTER rflags,r11*/
|
2008-01-30 12:32:08 +00:00
|
|
|
SWAPGS_UNSAFE_STACK
|
|
|
|
|
/*
|
|
|
|
|
* A hypervisor implementation might want to use a label
|
|
|
|
|
* after the swapgs, so that it can do the swapgs
|
|
|
|
|
* for the guest and jump here on syscall.
|
|
|
|
|
*/
|
2011-11-29 11:24:10 +00:00
|
|
|
GLOBAL(system_call_after_swapgs)
|
2008-01-30 12:32:08 +00:00
|
|
|
|
2009-01-18 15:38:58 +00:00
|
|
|
movq %rsp,PER_CPU_VAR(old_rsp)
|
2009-01-18 15:38:58 +00:00
|
|
|
movq PER_CPU_VAR(kernel_stack),%rsp
|
2006-07-03 07:24:45 +00:00
|
|
|
/*
|
|
|
|
|
* No need to follow this irqs off/on section - it's straight
|
|
|
|
|
* and short:
|
|
|
|
|
*/
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2011-05-31 20:21:52 +00:00
|
|
|
SAVE_ARGS 8,0
|
2008-11-16 14:29:00 +00:00
|
|
|
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
|
2005-09-12 16:49:24 +00:00
|
|
|
movq %rcx,RIP-ARGOFFSET(%rsp)
|
|
|
|
|
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
2011-11-29 11:17:45 +00:00
|
|
|
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
2005-04-16 22:20:36 +00:00
|
|
|
jnz tracesys
|
2008-06-23 22:37:04 +00:00
|
|
|
system_call_fastpath:
|
2012-02-19 15:56:26 +00:00
|
|
|
#if __SYSCALL_MASK == ~0
|
2005-04-16 22:20:36 +00:00
|
|
|
cmpq $__NR_syscall_max,%rax
|
2012-02-19 15:56:26 +00:00
|
|
|
#else
|
|
|
|
|
andl $__SYSCALL_MASK,%eax
|
|
|
|
|
cmpl $__NR_syscall_max,%eax
|
|
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
ja badsys
|
|
|
|
|
movq %r10,%rcx
|
|
|
|
|
call *sys_call_table(,%rax,8) # XXX: rip relative
|
|
|
|
|
movq %rax,RAX-ARGOFFSET(%rsp)
|
|
|
|
|
/*
|
|
|
|
|
* Syscall return path ending with SYSRET (fast path)
|
2008-11-16 14:29:00 +00:00
|
|
|
* Has incomplete stack frame and undefined top of stack.
|
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
ret_from_sys_call:
|
2005-04-16 22:25:02 +00:00
|
|
|
movl $_TIF_ALLWORK_MASK,%edi
|
2005-04-16 22:20:36 +00:00
|
|
|
/* edi: flagmask */
|
2008-11-16 14:29:00 +00:00
|
|
|
sysret_check:
|
2007-10-11 20:11:12 +00:00
|
|
|
LOCKDEP_SYS_EXIT
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2011-11-29 11:17:45 +00:00
|
|
|
movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx
|
2005-04-16 22:20:36 +00:00
|
|
|
andl %edi,%edx
|
2008-11-16 14:29:00 +00:00
|
|
|
jnz sysret_careful
|
2006-12-07 01:14:02 +00:00
|
|
|
CFI_REMEMBER_STATE
|
2006-07-03 07:24:45 +00:00
|
|
|
/*
|
|
|
|
|
* sysretq will re-enable interrupts:
|
|
|
|
|
*/
|
|
|
|
|
TRACE_IRQS_ON
|
2005-04-16 22:20:36 +00:00
|
|
|
movq RIP-ARGOFFSET(%rsp),%rcx
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_REGISTER rip,rcx
|
2011-05-31 20:21:53 +00:00
|
|
|
RESTORE_ARGS 1,-ARG_SKIP,0
|
2005-09-12 16:49:24 +00:00
|
|
|
/*CFI_REGISTER rflags,r11*/
|
2009-01-18 15:38:58 +00:00
|
|
|
movq PER_CPU_VAR(old_rsp), %rsp
|
2008-06-25 04:19:28 +00:00
|
|
|
USERGS_SYSRET64
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-12-07 01:14:02 +00:00
|
|
|
CFI_RESTORE_STATE
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Handle reschedules */
|
2008-11-16 14:29:00 +00:00
|
|
|
/* edx: work, edi: workmask */
|
2005-04-16 22:20:36 +00:00
|
|
|
sysret_careful:
|
|
|
|
|
bt $TIF_NEED_RESCHED,%edx
|
|
|
|
|
jnc sysret_signal
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rdi
|
2012-07-11 18:26:38 +00:00
|
|
|
SCHEDULE_USER
|
2010-09-02 13:07:16 +00:00
|
|
|
popq_cfi %rdi
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp sysret_check
|
|
|
|
|
|
2008-11-16 14:29:00 +00:00
|
|
|
/* Handle a signal */
|
2005-04-16 22:20:36 +00:00
|
|
|
sysret_signal:
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2008-06-23 22:37:04 +00:00
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
|
|
|
|
bt $TIF_SYSCALL_AUDIT,%edx
|
|
|
|
|
jc sysret_audit
|
|
|
|
|
#endif
|
2009-09-22 23:46:34 +00:00
|
|
|
/*
|
|
|
|
|
* We have a signal, or exit tracing or single-step.
|
|
|
|
|
* These all wind up with the iret return path anyway,
|
|
|
|
|
* so just join that path right now.
|
|
|
|
|
*/
|
|
|
|
|
FIXUP_TOP_OF_STACK %r11, -ARGOFFSET
|
|
|
|
|
jmp int_check_syscall_exit_work
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-09-12 16:49:24 +00:00
|
|
|
badsys:
|
|
|
|
|
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
|
|
|
|
|
jmp ret_from_sys_call
|
|
|
|
|
|
2008-06-23 22:37:04 +00:00
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
|
|
|
|
/*
|
|
|
|
|
* Fast path for syscall audit without full syscall trace.
|
2012-01-03 19:23:06 +00:00
|
|
|
* We just call __audit_syscall_entry() directly, and then
|
2008-06-23 22:37:04 +00:00
|
|
|
* jump back to the normal fast path.
|
|
|
|
|
*/
|
|
|
|
|
auditsys:
|
|
|
|
|
movq %r10,%r9 /* 6th arg: 4th syscall arg */
|
|
|
|
|
movq %rdx,%r8 /* 5th arg: 3rd syscall arg */
|
|
|
|
|
movq %rsi,%rcx /* 4th arg: 2nd syscall arg */
|
|
|
|
|
movq %rdi,%rdx /* 3rd arg: 1st syscall arg */
|
|
|
|
|
movq %rax,%rsi /* 2nd arg: syscall number */
|
|
|
|
|
movl $AUDIT_ARCH_X86_64,%edi /* 1st arg: audit arch */
|
2012-01-03 19:23:06 +00:00
|
|
|
call __audit_syscall_entry
|
2008-06-23 22:37:04 +00:00
|
|
|
LOAD_ARGS 0 /* reload call-clobbered registers */
|
|
|
|
|
jmp system_call_fastpath
|
|
|
|
|
|
|
|
|
|
/*
|
2012-01-03 19:23:06 +00:00
|
|
|
* Return fast path for syscall audit. Call __audit_syscall_exit()
|
2008-06-23 22:37:04 +00:00
|
|
|
* directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
|
|
|
|
|
* masked off.
|
|
|
|
|
*/
|
|
|
|
|
sysret_audit:
|
2010-07-22 00:44:12 +00:00
|
|
|
movq RAX-ARGOFFSET(%rsp),%rsi /* second arg, syscall return value */
|
2012-01-03 19:23:06 +00:00
|
|
|
cmpq $-MAX_ERRNO,%rsi /* is it < -MAX_ERRNO? */
|
|
|
|
|
setbe %al /* 1 if so, 0 if not */
|
2008-06-23 22:37:04 +00:00
|
|
|
movzbl %al,%edi /* zero-extend that into %edi */
|
2012-01-03 19:23:06 +00:00
|
|
|
call __audit_syscall_exit
|
2008-06-23 22:37:04 +00:00
|
|
|
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
|
|
|
|
|
jmp sysret_check
|
|
|
|
|
#endif /* CONFIG_AUDITSYSCALL */
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Do syscall tracing */
|
2008-11-16 14:29:00 +00:00
|
|
|
tracesys:
|
2008-06-23 22:37:04 +00:00
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
2011-11-29 11:17:45 +00:00
|
|
|
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
2008-06-23 22:37:04 +00:00
|
|
|
jz auditsys
|
|
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
SAVE_REST
|
2008-03-17 04:59:11 +00:00
|
|
|
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
2005-04-16 22:20:36 +00:00
|
|
|
FIXUP_TOP_OF_STACK %rdi
|
|
|
|
|
movq %rsp,%rdi
|
|
|
|
|
call syscall_trace_enter
|
2008-07-09 09:38:07 +00:00
|
|
|
/*
|
|
|
|
|
* Reload arg registers from stack in case ptrace changed them.
|
|
|
|
|
* We don't reload %rax because syscall_trace_enter() returned
|
|
|
|
|
* the value it wants us to use in the table lookup.
|
|
|
|
|
*/
|
|
|
|
|
LOAD_ARGS ARGOFFSET, 1
|
2005-04-16 22:20:36 +00:00
|
|
|
RESTORE_REST
|
2012-02-19 15:56:26 +00:00
|
|
|
#if __SYSCALL_MASK == ~0
|
2005-04-16 22:20:36 +00:00
|
|
|
cmpq $__NR_syscall_max,%rax
|
2012-02-19 15:56:26 +00:00
|
|
|
#else
|
|
|
|
|
andl $__SYSCALL_MASK,%eax
|
|
|
|
|
cmpl $__NR_syscall_max,%eax
|
|
|
|
|
#endif
|
2008-03-17 04:59:11 +00:00
|
|
|
ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
|
2005-04-16 22:20:36 +00:00
|
|
|
movq %r10,%rcx /* fixup for C */
|
|
|
|
|
call *sys_call_table(,%rax,8)
|
2008-03-17 04:59:11 +00:00
|
|
|
movq %rax,RAX-ARGOFFSET(%rsp)
|
2006-04-07 17:50:00 +00:00
|
|
|
/* Use IRET because user could have changed frame */
|
2008-11-16 14:29:00 +00:00
|
|
|
|
|
|
|
|
/*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Syscall return path ending with IRET.
|
|
|
|
|
* Has correct top of stack, but partial stack frame.
|
2006-12-07 01:14:02 +00:00
|
|
|
*/
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(int_ret_from_sys_call)
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-16 22:20:36 +00:00
|
|
|
movl $_TIF_ALLWORK_MASK,%edi
|
|
|
|
|
/* edi: mask to check */
|
2009-02-23 19:57:01 +00:00
|
|
|
GLOBAL(int_with_check)
|
2007-10-11 20:11:12 +00:00
|
|
|
LOCKDEP_SYS_EXIT_IRQ
|
2005-04-16 22:20:36 +00:00
|
|
|
GET_THREAD_INFO(%rcx)
|
2008-06-24 14:19:35 +00:00
|
|
|
movl TI_flags(%rcx),%edx
|
2005-04-16 22:20:36 +00:00
|
|
|
andl %edi,%edx
|
|
|
|
|
jnz int_careful
|
2008-06-24 14:19:35 +00:00
|
|
|
andl $~TS_COMPAT,TI_status(%rcx)
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp retint_swapgs
|
|
|
|
|
|
|
|
|
|
/* Either reschedule or signal or syscall exit tracking needed. */
|
|
|
|
|
/* First do a reschedule test. */
|
|
|
|
|
/* edx: work, edi: workmask */
|
|
|
|
|
int_careful:
|
|
|
|
|
bt $TIF_NEED_RESCHED,%edx
|
|
|
|
|
jnc int_very_careful
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rdi
|
2012-07-11 18:26:38 +00:00
|
|
|
SCHEDULE_USER
|
2010-09-02 13:07:16 +00:00
|
|
|
popq_cfi %rdi
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp int_with_check
|
|
|
|
|
|
|
|
|
|
/* handle signals and tracing -- both require a full stack frame */
|
|
|
|
|
int_very_careful:
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2009-09-22 23:46:34 +00:00
|
|
|
int_check_syscall_exit_work:
|
2005-04-16 22:20:36 +00:00
|
|
|
SAVE_REST
|
2008-11-16 14:29:00 +00:00
|
|
|
/* Check for syscall exit trace */
|
2008-07-09 09:38:07 +00:00
|
|
|
testl $_TIF_WORK_SYSCALL_EXIT,%edx
|
2005-04-16 22:20:36 +00:00
|
|
|
jz int_signal
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rdi
|
2008-11-16 14:29:00 +00:00
|
|
|
leaq 8(%rsp),%rdi # &ptregs -> arg1
|
2005-04-16 22:20:36 +00:00
|
|
|
call syscall_trace_leave
|
2010-09-02 13:07:16 +00:00
|
|
|
popq_cfi %rdi
|
2008-07-09 09:38:07 +00:00
|
|
|
andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp int_restore_rest
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
int_signal:
|
2008-01-25 20:08:29 +00:00
|
|
|
testl $_TIF_DO_NOTIFY_MASK,%edx
|
2005-04-16 22:20:36 +00:00
|
|
|
jz 1f
|
|
|
|
|
movq %rsp,%rdi # &ptregs -> arg1
|
|
|
|
|
xorl %esi,%esi # oldset -> arg2
|
|
|
|
|
call do_notify_resume
|
x86_64: fix delayed signals
On three of the several paths in entry_64.S that call
do_notify_resume() on the way back to user mode, we fail to properly
check again for newly-arrived work that requires another call to
do_notify_resume() before going to user mode. These paths set the
mask to check only _TIF_NEED_RESCHED, but this is wrong. The other
paths that lead to do_notify_resume() do this correctly already, and
entry_32.S does it correctly in all cases.
All paths back to user mode have to check all the _TIF_WORK_MASK
flags at the last possible stage, with interrupts disabled.
Otherwise, we miss any flags (TIF_SIGPENDING for example) that were
set any time after we entered do_notify_resume(). More work flags
can be set (or left set) synchronously inside do_notify_resume(), as
TIF_SIGPENDING can be, or asynchronously by interrupts or other CPUs
(which then send an asynchronous interrupt).
There are many different scenarios that could hit this bug, most of
them races. The simplest one to demonstrate does not require any
race: when one signal has done handler setup at the check before
returning from a syscall, and there is another signal pending that
should be handled. The second signal's handler should interrupt the
first signal handler before it actually starts (so the interrupted PC
is still at the handler's entry point). Instead, it runs away until
the next kernel entry (next syscall, tick, etc).
This test behaves correctly on 32-bit kernels, and fails on 64-bit
(either 32-bit or 64-bit test binary). With this fix, it works.
#define _GNU_SOURCE
#include <stdio.h>
#include <signal.h>
#include <string.h>
#include <sys/ucontext.h>
#ifndef REG_RIP
#define REG_RIP REG_EIP
#endif
static sig_atomic_t hit1, hit2;
static void
handler (int sig, siginfo_t *info, void *ctx)
{
ucontext_t *uc = ctx;
if ((void *) uc->uc_mcontext.gregs[REG_RIP] == &handler)
{
if (sig == SIGUSR1)
hit1 = 1;
else
hit2 = 1;
}
printf ("%s at %#lx\n", strsignal (sig),
uc->uc_mcontext.gregs[REG_RIP]);
}
int
main (void)
{
struct sigaction sa;
sigset_t set;
sigemptyset (&sa.sa_mask);
sa.sa_flags = SA_SIGINFO;
sa.sa_sigaction = &handler;
if (sigaction (SIGUSR1, &sa, NULL)
|| sigaction (SIGUSR2, &sa, NULL))
return 2;
sigemptyset (&set);
sigaddset (&set, SIGUSR1);
sigaddset (&set, SIGUSR2);
if (sigprocmask (SIG_BLOCK, &set, NULL))
return 3;
printf ("main at %p, handler at %p\n", &main, &handler);
raise (SIGUSR1);
raise (SIGUSR2);
if (sigprocmask (SIG_UNBLOCK, &set, NULL))
return 4;
if (hit1 + hit2 == 1)
{
puts ("PASS");
return 0;
}
puts ("FAIL");
return 1;
}
Signed-off-by: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-10 21:50:39 +00:00
|
|
|
1: movl $_TIF_WORK_MASK,%edi
|
2005-04-16 22:20:36 +00:00
|
|
|
int_restore_rest:
|
|
|
|
|
RESTORE_REST
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp int_with_check
|
|
|
|
|
CFI_ENDPROC
|
2006-12-07 01:14:02 +00:00
|
|
|
END(system_call)
|
2008-11-16 14:29:00 +00:00
|
|
|
|
|
|
|
|
/*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Certain special system calls that need to save a complete full stack frame.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
.macro PTREGSCALL label,func,arg
|
2008-11-21 15:41:55 +00:00
|
|
|
ENTRY(\label)
|
|
|
|
|
PARTIAL_FRAME 1 8 /* offset 8: return address */
|
|
|
|
|
subq $REST_SKIP, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET REST_SKIP
|
|
|
|
|
call save_rest
|
|
|
|
|
DEFAULT_FRAME 0 8 /* offset 8: return address */
|
|
|
|
|
leaq 8(%rsp), \arg /* pt_regs pointer */
|
|
|
|
|
call \func
|
|
|
|
|
jmp ptregscall_common
|
|
|
|
|
CFI_ENDPROC
|
2006-06-26 11:56:55 +00:00
|
|
|
END(\label)
|
2005-04-16 22:20:36 +00:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
PTREGSCALL stub_clone, sys_clone, %r8
|
|
|
|
|
PTREGSCALL stub_fork, sys_fork, %rdi
|
|
|
|
|
PTREGSCALL stub_vfork, sys_vfork, %rdi
|
|
|
|
|
PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
|
|
|
|
|
PTREGSCALL stub_iopl, sys_iopl, %rsi
|
|
|
|
|
|
|
|
|
|
ENTRY(ptregscall_common)
|
2008-11-21 15:41:55 +00:00
|
|
|
DEFAULT_FRAME 1 8 /* offset 8: return address */
|
|
|
|
|
RESTORE_TOP_OF_STACK %r11, 8
|
|
|
|
|
movq_cfi_restore R15+8, r15
|
|
|
|
|
movq_cfi_restore R14+8, r14
|
|
|
|
|
movq_cfi_restore R13+8, r13
|
|
|
|
|
movq_cfi_restore R12+8, r12
|
|
|
|
|
movq_cfi_restore RBP+8, rbp
|
|
|
|
|
movq_cfi_restore RBX+8, rbx
|
|
|
|
|
ret $REST_SKIP /* pop extended registers */
|
2005-04-16 22:20:36 +00:00
|
|
|
CFI_ENDPROC
|
2006-06-26 11:56:55 +00:00
|
|
|
END(ptregscall_common)
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
ENTRY(stub_execve)
|
|
|
|
|
CFI_STARTPROC
|
2010-09-02 12:52:45 +00:00
|
|
|
addq $8, %rsp
|
|
|
|
|
PARTIAL_FRAME 0
|
2005-04-16 22:20:36 +00:00
|
|
|
SAVE_REST
|
|
|
|
|
FIXUP_TOP_OF_STACK %r11
|
|
|
|
|
call sys_execve
|
|
|
|
|
RESTORE_TOP_OF_STACK %r11
|
|
|
|
|
movq %rax,RAX(%rsp)
|
|
|
|
|
RESTORE_REST
|
|
|
|
|
jmp int_ret_from_sys_call
|
|
|
|
|
CFI_ENDPROC
|
2006-06-26 11:56:55 +00:00
|
|
|
END(stub_execve)
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* sigreturn is special because it needs to restore all registers on return.
|
|
|
|
|
* This cannot be done with SYSRET, so use the IRET return path instead.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
ENTRY(stub_rt_sigreturn)
|
|
|
|
|
CFI_STARTPROC
|
2005-09-12 16:49:24 +00:00
|
|
|
addq $8, %rsp
|
2010-09-02 12:52:45 +00:00
|
|
|
PARTIAL_FRAME 0
|
2005-04-16 22:20:36 +00:00
|
|
|
SAVE_REST
|
|
|
|
|
movq %rsp,%rdi
|
|
|
|
|
FIXUP_TOP_OF_STACK %r11
|
|
|
|
|
call sys_rt_sigreturn
|
|
|
|
|
movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
|
|
|
|
|
RESTORE_REST
|
|
|
|
|
jmp int_ret_from_sys_call
|
|
|
|
|
CFI_ENDPROC
|
2006-06-26 11:56:55 +00:00
|
|
|
END(stub_rt_sigreturn)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-02-19 17:41:09 +00:00
|
|
|
#ifdef CONFIG_X86_X32_ABI
|
|
|
|
|
PTREGSCALL stub_x32_sigaltstack, sys32_sigaltstack, %rdx
|
|
|
|
|
|
|
|
|
|
ENTRY(stub_x32_rt_sigreturn)
|
|
|
|
|
CFI_STARTPROC
|
|
|
|
|
addq $8, %rsp
|
|
|
|
|
PARTIAL_FRAME 0
|
|
|
|
|
SAVE_REST
|
|
|
|
|
movq %rsp,%rdi
|
|
|
|
|
FIXUP_TOP_OF_STACK %r11
|
|
|
|
|
call sys32_x32_rt_sigreturn
|
|
|
|
|
movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
|
|
|
|
|
RESTORE_REST
|
|
|
|
|
jmp int_ret_from_sys_call
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(stub_x32_rt_sigreturn)
|
|
|
|
|
|
2012-02-19 18:06:34 +00:00
|
|
|
ENTRY(stub_x32_execve)
|
|
|
|
|
CFI_STARTPROC
|
|
|
|
|
addq $8, %rsp
|
|
|
|
|
PARTIAL_FRAME 0
|
|
|
|
|
SAVE_REST
|
|
|
|
|
FIXUP_TOP_OF_STACK %r11
|
2012-08-02 19:05:11 +00:00
|
|
|
call compat_sys_execve
|
2012-02-19 18:06:34 +00:00
|
|
|
RESTORE_TOP_OF_STACK %r11
|
|
|
|
|
movq %rax,RAX(%rsp)
|
|
|
|
|
RESTORE_REST
|
|
|
|
|
jmp int_ret_from_sys_call
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(stub_x32_execve)
|
|
|
|
|
|
2012-02-19 17:41:09 +00:00
|
|
|
#endif
|
|
|
|
|
|
2008-11-11 21:51:52 +00:00
|
|
|
/*
|
|
|
|
|
* Build the entry stubs and pointer table with some assembler magic.
|
|
|
|
|
* We pack 7 stubs into a single 32-byte chunk, which will fit in a
|
|
|
|
|
* single cache line on all modern x86 implementations.
|
|
|
|
|
*/
|
|
|
|
|
.section .init.rodata,"a"
|
|
|
|
|
ENTRY(interrupt)
|
2011-03-07 18:10:39 +00:00
|
|
|
.section .entry.text
|
2008-11-11 21:51:52 +00:00
|
|
|
.p2align 5
|
|
|
|
|
.p2align CONFIG_X86_L1_CACHE_SHIFT
|
|
|
|
|
ENTRY(irq_entries_start)
|
|
|
|
|
INTR_FRAME
|
|
|
|
|
vector=FIRST_EXTERNAL_VECTOR
|
|
|
|
|
.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
|
|
|
|
|
.balign 32
|
|
|
|
|
.rept 7
|
|
|
|
|
.if vector < NR_VECTORS
|
2008-11-12 18:27:35 +00:00
|
|
|
.if vector <> FIRST_EXTERNAL_VECTOR
|
2008-11-11 21:51:52 +00:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
|
|
|
|
.endif
|
2010-09-02 13:07:16 +00:00
|
|
|
1: pushq_cfi $(~vector+0x80) /* Note: always in signed byte range */
|
2008-11-12 18:27:35 +00:00
|
|
|
.if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
|
2008-11-11 21:51:52 +00:00
|
|
|
jmp 2f
|
|
|
|
|
.endif
|
|
|
|
|
.previous
|
|
|
|
|
.quad 1b
|
2011-03-07 18:10:39 +00:00
|
|
|
.section .entry.text
|
2008-11-11 21:51:52 +00:00
|
|
|
vector=vector+1
|
|
|
|
|
.endif
|
|
|
|
|
.endr
|
|
|
|
|
2: jmp common_interrupt
|
|
|
|
|
.endr
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(irq_entries_start)
|
|
|
|
|
|
|
|
|
|
.previous
|
|
|
|
|
END(interrupt)
|
|
|
|
|
.previous
|
|
|
|
|
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
/*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Interrupt entry/exit.
|
|
|
|
|
*
|
|
|
|
|
* Interrupt entry points save only callee clobbered registers in fast path.
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
*
|
|
|
|
|
* Entry runs with interrupts off.
|
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-11-13 12:50:20 +00:00
|
|
|
/* 0(%rsp): ~(interrupt number) */
|
2005-04-16 22:20:36 +00:00
|
|
|
.macro interrupt func
|
2011-01-06 14:22:47 +00:00
|
|
|
/* reserve pt_regs for scratch regs and rbp */
|
|
|
|
|
subq $ORIG_RAX-RBP, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
|
2011-06-30 23:51:22 +00:00
|
|
|
SAVE_ARGS_IRQ
|
2005-04-16 22:20:36 +00:00
|
|
|
call \func
|
|
|
|
|
.endm
|
|
|
|
|
|
2009-08-27 17:23:25 +00:00
|
|
|
/*
|
|
|
|
|
* Interrupt entry/exit should be protected against kprobes
|
|
|
|
|
*/
|
|
|
|
|
.pushsection .kprobes.text, "ax"
|
2008-11-13 12:50:20 +00:00
|
|
|
/*
|
|
|
|
|
* The interrupt stubs push (~vector+0x80) onto the stack and
|
|
|
|
|
* then jump to common_interrupt.
|
|
|
|
|
*/
|
2008-11-11 21:51:52 +00:00
|
|
|
.p2align CONFIG_X86_L1_CACHE_SHIFT
|
|
|
|
|
common_interrupt:
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2005-09-12 16:49:24 +00:00
|
|
|
XCPT_FRAME
|
2008-11-13 12:50:20 +00:00
|
|
|
addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
|
2005-04-16 22:20:36 +00:00
|
|
|
interrupt do_IRQ
|
2009-01-18 15:38:58 +00:00
|
|
|
/* 0(%rsp): old_rsp-ARGOFFSET */
|
2005-09-12 16:49:24 +00:00
|
|
|
ret_from_intr:
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2009-01-18 15:38:58 +00:00
|
|
|
decl PER_CPU_VAR(irq_count)
|
2011-01-06 14:22:47 +00:00
|
|
|
|
2011-07-02 14:52:45 +00:00
|
|
|
/* Restore saved previous stack */
|
|
|
|
|
popq %rsi
|
2012-02-24 10:32:05 +00:00
|
|
|
CFI_DEF_CFA rsi,SS+8-RBP /* reg/off reset after def_cfa_expr */
|
2011-09-28 15:57:52 +00:00
|
|
|
leaq ARGOFFSET-RBP(%rsi), %rsp
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_DEF_CFA_REGISTER rsp
|
2011-09-28 15:57:52 +00:00
|
|
|
CFI_ADJUST_CFA_OFFSET RBP-ARGOFFSET
|
2011-01-06 14:22:47 +00:00
|
|
|
|
2005-09-12 16:49:24 +00:00
|
|
|
exit_intr:
|
2005-04-16 22:20:36 +00:00
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
|
testl $3,CS-ARGOFFSET(%rsp)
|
|
|
|
|
je retint_kernel
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Interrupt came from user space */
|
|
|
|
|
/*
|
|
|
|
|
* Has a correct top of stack, but a partial stack frame
|
|
|
|
|
* %rcx: thread info. Interrupts off.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
retint_with_reschedule:
|
|
|
|
|
movl $_TIF_WORK_MASK,%edi
|
2005-09-12 16:49:24 +00:00
|
|
|
retint_check:
|
2007-10-11 20:11:12 +00:00
|
|
|
LOCKDEP_SYS_EXIT_IRQ
|
2008-06-24 14:19:35 +00:00
|
|
|
movl TI_flags(%rcx),%edx
|
2005-04-16 22:20:36 +00:00
|
|
|
andl %edi,%edx
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_REMEMBER_STATE
|
2005-04-16 22:20:36 +00:00
|
|
|
jnz retint_careful
|
2007-10-11 20:11:12 +00:00
|
|
|
|
|
|
|
|
retint_swapgs: /* return to user-space */
|
2006-07-03 07:24:45 +00:00
|
|
|
/*
|
|
|
|
|
* The iretq could re-enable interrupts:
|
|
|
|
|
*/
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_IRETQ
|
2008-01-30 12:32:08 +00:00
|
|
|
SWAPGS
|
2006-07-03 07:24:45 +00:00
|
|
|
jmp restore_args
|
|
|
|
|
|
2007-10-11 20:11:12 +00:00
|
|
|
retint_restore_args: /* return to kernel space */
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY)
|
2006-07-03 07:24:45 +00:00
|
|
|
/*
|
|
|
|
|
* The iretq could re-enable interrupts:
|
|
|
|
|
*/
|
|
|
|
|
TRACE_IRQS_IRETQ
|
|
|
|
|
restore_args:
|
2011-05-31 20:21:53 +00:00
|
|
|
RESTORE_ARGS 1,8,1
|
2008-02-09 22:24:08 +00:00
|
|
|
|
2008-02-13 21:29:53 +00:00
|
|
|
irq_return:
|
2008-01-30 12:32:08 +00:00
|
|
|
INTERRUPT_RETURN
|
2012-04-20 19:19:50 +00:00
|
|
|
_ASM_EXTABLE(irq_return, bad_iret)
|
2008-02-09 22:24:08 +00:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_PARAVIRT
|
2008-01-30 12:32:08 +00:00
|
|
|
ENTRY(native_iret)
|
2005-04-16 22:20:36 +00:00
|
|
|
iretq
|
2012-04-20 19:19:50 +00:00
|
|
|
_ASM_EXTABLE(native_iret, bad_iret)
|
2008-02-09 22:24:08 +00:00
|
|
|
#endif
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
.section .fixup,"ax"
|
|
|
|
|
bad_iret:
|
2008-02-06 21:39:43 +00:00
|
|
|
/*
|
|
|
|
|
* The iret traps when the %cs or %ss being restored is bogus.
|
|
|
|
|
* We've lost the original trap vector and error code.
|
|
|
|
|
* #GPF is the most likely one to get for an invalid selector.
|
|
|
|
|
* So pretend we completed the iret and took the #GPF in user mode.
|
|
|
|
|
*
|
|
|
|
|
* We are now running with the kernel GS after exception recovery.
|
|
|
|
|
* But error_entry expects us to have user GS to match the user %cs,
|
|
|
|
|
* so swap back.
|
|
|
|
|
*/
|
|
|
|
|
pushq $0
|
|
|
|
|
|
|
|
|
|
SWAPGS
|
|
|
|
|
jmp general_protection
|
|
|
|
|
|
2008-01-30 12:32:08 +00:00
|
|
|
.previous
|
|
|
|
|
|
2005-09-12 16:49:24 +00:00
|
|
|
/* edi: workmask, edx: work */
|
2005-04-16 22:20:36 +00:00
|
|
|
retint_careful:
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_RESTORE_STATE
|
2005-04-16 22:20:36 +00:00
|
|
|
bt $TIF_NEED_RESCHED,%edx
|
|
|
|
|
jnc retint_signal
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rdi
|
2012-07-11 18:26:38 +00:00
|
|
|
SCHEDULE_USER
|
2010-09-02 13:07:16 +00:00
|
|
|
popq_cfi %rdi
|
2005-04-16 22:20:36 +00:00
|
|
|
GET_THREAD_INFO(%rcx)
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp retint_check
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
retint_signal:
|
2008-01-25 20:08:29 +00:00
|
|
|
testl $_TIF_DO_NOTIFY_MASK,%edx
|
2005-05-17 04:53:19 +00:00
|
|
|
jz retint_swapgs
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 12:32:08 +00:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-16 22:20:36 +00:00
|
|
|
SAVE_REST
|
2008-11-16 14:29:00 +00:00
|
|
|
movq $-1,ORIG_RAX(%rsp)
|
2005-07-29 04:15:48 +00:00
|
|
|
xorl %esi,%esi # oldset
|
2005-04-16 22:20:36 +00:00
|
|
|
movq %rsp,%rdi # &pt_regs
|
|
|
|
|
call do_notify_resume
|
|
|
|
|
RESTORE_REST
|
2008-01-30 12:32:08 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 07:24:45 +00:00
|
|
|
TRACE_IRQS_OFF
|
2005-05-01 15:58:51 +00:00
|
|
|
GET_THREAD_INFO(%rcx)
|
x86_64: fix delayed signals
On three of the several paths in entry_64.S that call
do_notify_resume() on the way back to user mode, we fail to properly
check again for newly-arrived work that requires another call to
do_notify_resume() before going to user mode. These paths set the
mask to check only _TIF_NEED_RESCHED, but this is wrong. The other
paths that lead to do_notify_resume() do this correctly already, and
entry_32.S does it correctly in all cases.
All paths back to user mode have to check all the _TIF_WORK_MASK
flags at the last possible stage, with interrupts disabled.
Otherwise, we miss any flags (TIF_SIGPENDING for example) that were
set any time after we entered do_notify_resume(). More work flags
can be set (or left set) synchronously inside do_notify_resume(), as
TIF_SIGPENDING can be, or asynchronously by interrupts or other CPUs
(which then send an asynchronous interrupt).
There are many different scenarios that could hit this bug, most of
them races. The simplest one to demonstrate does not require any
race: when one signal has done handler setup at the check before
returning from a syscall, and there is another signal pending that
should be handled. The second signal's handler should interrupt the
first signal handler before it actually starts (so the interrupted PC
is still at the handler's entry point). Instead, it runs away until
the next kernel entry (next syscall, tick, etc).
This test behaves correctly on 32-bit kernels, and fails on 64-bit
(either 32-bit or 64-bit test binary). With this fix, it works.
#define _GNU_SOURCE
#include <stdio.h>
#include <signal.h>
#include <string.h>
#include <sys/ucontext.h>
#ifndef REG_RIP
#define REG_RIP REG_EIP
#endif
static sig_atomic_t hit1, hit2;
static void
handler (int sig, siginfo_t *info, void *ctx)
{
ucontext_t *uc = ctx;
if ((void *) uc->uc_mcontext.gregs[REG_RIP] == &handler)
{
if (sig == SIGUSR1)
hit1 = 1;
else
hit2 = 1;
}
printf ("%s at %#lx\n", strsignal (sig),
uc->uc_mcontext.gregs[REG_RIP]);
}
int
main (void)
{
struct sigaction sa;
sigset_t set;
sigemptyset (&sa.sa_mask);
sa.sa_flags = SA_SIGINFO;
sa.sa_sigaction = &handler;
if (sigaction (SIGUSR1, &sa, NULL)
|| sigaction (SIGUSR2, &sa, NULL))
return 2;
sigemptyset (&set);
sigaddset (&set, SIGUSR1);
sigaddset (&set, SIGUSR2);
if (sigprocmask (SIG_BLOCK, &set, NULL))
return 3;
printf ("main at %p, handler at %p\n", &main, &handler);
raise (SIGUSR1);
raise (SIGUSR2);
if (sigprocmask (SIG_UNBLOCK, &set, NULL))
return 4;
if (hit1 + hit2 == 1)
{
puts ("PASS");
return 0;
}
puts ("FAIL");
return 1;
}
Signed-off-by: Roland McGrath <roland@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-10 21:50:39 +00:00
|
|
|
jmp retint_with_reschedule
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
|
/* Returning to kernel space. Check if we need preemption */
|
|
|
|
|
/* rcx: threadinfo. interrupts off. */
|
2006-09-26 08:52:29 +00:00
|
|
|
ENTRY(retint_kernel)
|
2008-06-24 14:19:35 +00:00
|
|
|
cmpl $0,TI_preempt_count(%rcx)
|
2005-04-16 22:20:36 +00:00
|
|
|
jnz retint_restore_args
|
2008-06-24 14:19:35 +00:00
|
|
|
bt $TIF_NEED_RESCHED,TI_flags(%rcx)
|
2005-04-16 22:20:36 +00:00
|
|
|
jnc retint_restore_args
|
|
|
|
|
bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
|
|
|
|
|
jnc retint_restore_args
|
|
|
|
|
call preempt_schedule_irq
|
|
|
|
|
jmp exit_intr
|
2008-11-16 14:29:00 +00:00
|
|
|
#endif
|
2006-06-26 11:56:55 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
CFI_ENDPROC
|
2006-06-26 11:56:55 +00:00
|
|
|
END(common_interrupt)
|
2009-08-27 17:23:25 +00:00
|
|
|
/*
|
|
|
|
|
* End of kprobes section
|
|
|
|
|
*/
|
|
|
|
|
.popsection
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* APIC interrupts.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2008-11-23 09:08:28 +00:00
|
|
|
.macro apicinterrupt num sym do_sym
|
|
|
|
|
ENTRY(\sym)
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2005-09-12 16:49:24 +00:00
|
|
|
INTR_FRAME
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi $~(\num)
|
2011-11-29 11:03:46 +00:00
|
|
|
.Lcommon_\sym:
|
2008-11-23 09:08:28 +00:00
|
|
|
interrupt \do_sym
|
2005-04-16 22:20:36 +00:00
|
|
|
jmp ret_from_intr
|
|
|
|
|
CFI_ENDPROC
|
2008-11-23 09:08:28 +00:00
|
|
|
END(\sym)
|
|
|
|
|
.endm
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-11-23 09:08:28 +00:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \
|
|
|
|
|
irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
|
x86: fix panic with interrupts off (needed for MCE)
For some time each panic() called with interrupts disabled
triggered the !irqs_disabled() WARN_ON in smp_call_function(),
producing ugly backtraces and confusing users.
This is a common situation with machine checks for example which
tend to call panic with interrupts disabled, but will also hit
in other situations e.g. panic during early boot. In fact it
means that panic cannot be called in many circumstances, which
would be bad.
This all started with the new fancy queued smp_call_function,
which is then used by the shutdown path to shut down the other
CPUs.
On closer examination it turned out that the fancy RCU
smp_call_function() does lots of things not suitable in a panic
situation anyways, like allocating memory and relying on complex
system state.
I originally tried to patch this over by checking for panic
there, but it was quite complicated and the original patch
was also not very popular. This also didn't fix some of the
underlying complexity problems.
The new code in post 2.6.29 tries to patch around this by
checking for oops_in_progress, but that is not enough to make
this fully safe and I don't think that's a real solution
because panic has to be reliable.
So instead use an own vector to reboot. This makes the reboot
code extremly straight forward, which is definitely a big plus
in a panic situation where it is important to avoid relying on
too much kernel state. The new simple code is also safe to be
called from interupts off region because it is very very simple.
There can be situations where it is important that panic
is reliable. For example on a fatal machine check the panic
is needed to get the system up again and running as quickly
as possible. So it's important that panic is reliable and
all function it calls simple.
This is why I came up with this simple vector scheme.
It's very hard to beat in simplicity. Vectors are not
particularly precious anymore since all big systems are
using per CPU vectors.
Another possibility would have been to use an NMI similar
to kdump, but there is still the problem that NMIs don't
work reliably on some systems due to BIOS issues. NMIs
would have been able to stop CPUs running with interrupts
off too. In the sake of universal reliability I opted for
using a non NMI vector for now.
I put the reboot vector into the highest priority bucket of
the APIC vectors and moved the 64bit UV_BAU message down
instead into the next lower priority.
[ Impact: bug fix, fixes an old regression ]
Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-05-27 19:56:52 +00:00
|
|
|
apicinterrupt REBOOT_VECTOR \
|
|
|
|
|
reboot_interrupt smp_reboot_interrupt
|
2008-11-23 09:08:28 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-01-20 03:36:04 +00:00
|
|
|
#ifdef CONFIG_X86_UV
|
2008-11-26 21:02:10 +00:00
|
|
|
apicinterrupt UV_BAU_MESSAGE \
|
2008-11-23 09:08:28 +00:00
|
|
|
uv_bau_message_intr1 uv_bau_message_interrupt
|
2009-01-20 03:36:04 +00:00
|
|
|
#endif
|
2008-11-23 09:08:28 +00:00
|
|
|
apicinterrupt LOCAL_TIMER_VECTOR \
|
|
|
|
|
apic_timer_interrupt smp_apic_timer_interrupt
|
2009-10-14 14:22:57 +00:00
|
|
|
apicinterrupt X86_PLATFORM_IPI_VECTOR \
|
|
|
|
|
x86_platform_ipi smp_x86_platform_ipi
|
2005-11-05 16:25:53 +00:00
|
|
|
|
2008-11-23 09:08:28 +00:00
|
|
|
apicinterrupt THRESHOLD_APIC_VECTOR \
|
2009-04-28 21:32:56 +00:00
|
|
|
threshold_interrupt smp_threshold_interrupt
|
2008-11-23 09:08:28 +00:00
|
|
|
apicinterrupt THERMAL_APIC_VECTOR \
|
|
|
|
|
thermal_interrupt smp_thermal_interrupt
|
2008-06-02 13:56:14 +00:00
|
|
|
|
2008-11-23 09:08:28 +00:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \
|
|
|
|
|
call_function_single_interrupt smp_call_function_single_interrupt
|
|
|
|
|
apicinterrupt CALL_FUNCTION_VECTOR \
|
|
|
|
|
call_function_interrupt smp_call_function_interrupt
|
|
|
|
|
apicinterrupt RESCHEDULE_VECTOR \
|
|
|
|
|
reschedule_interrupt smp_reschedule_interrupt
|
|
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-11-23 09:08:28 +00:00
|
|
|
apicinterrupt ERROR_APIC_VECTOR \
|
|
|
|
|
error_interrupt smp_error_interrupt
|
|
|
|
|
apicinterrupt SPURIOUS_APIC_VECTOR \
|
|
|
|
|
spurious_interrupt smp_spurious_interrupt
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2010-10-14 06:01:34 +00:00
|
|
|
#ifdef CONFIG_IRQ_WORK
|
|
|
|
|
apicinterrupt IRQ_WORK_VECTOR \
|
|
|
|
|
irq_work_interrupt smp_irq_work_interrupt
|
2008-12-03 09:39:53 +00:00
|
|
|
#endif
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* Exception entry points.
|
2008-11-16 14:29:00 +00:00
|
|
|
*/
|
2008-11-23 09:08:28 +00:00
|
|
|
.macro zeroentry sym do_sym
|
|
|
|
|
ENTRY(\sym)
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2005-09-12 16:49:24 +00:00
|
|
|
INTR_FRAME
|
2008-06-25 04:19:31 +00:00
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
2008-11-21 14:20:47 +00:00
|
|
|
pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
|
2010-09-02 12:55:11 +00:00
|
|
|
subq $ORIG_RAX-R15, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
call error_entry
|
2008-11-20 13:40:11 +00:00
|
|
|
DEFAULT_FRAME 0
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
|
|
|
xorl %esi,%esi /* no error code */
|
2008-11-23 09:08:28 +00:00
|
|
|
call \do_sym
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
jmp error_exit /* %ebx: no swapgs flag */
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_ENDPROC
|
2008-11-23 09:08:28 +00:00
|
|
|
END(\sym)
|
|
|
|
|
.endm
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-11-23 09:08:28 +00:00
|
|
|
.macro paranoidzeroentry sym do_sym
|
2008-11-24 12:24:28 +00:00
|
|
|
ENTRY(\sym)
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2008-11-21 15:44:28 +00:00
|
|
|
INTR_FRAME
|
|
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
2010-09-02 12:55:11 +00:00
|
|
|
pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
|
|
|
|
|
subq $ORIG_RAX-R15, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
2008-11-21 15:44:28 +00:00
|
|
|
call save_paranoid
|
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
|
|
|
xorl %esi,%esi /* no error code */
|
2008-11-23 09:08:28 +00:00
|
|
|
call \do_sym
|
2008-11-21 15:44:28 +00:00
|
|
|
jmp paranoid_exit /* %ebx: no swapgs flag */
|
|
|
|
|
CFI_ENDPROC
|
2008-11-24 12:24:28 +00:00
|
|
|
END(\sym)
|
2008-11-23 09:08:28 +00:00
|
|
|
.endm
|
2008-11-21 15:44:28 +00:00
|
|
|
|
2010-07-31 16:48:22 +00:00
|
|
|
#define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8)
|
2008-11-23 09:08:28 +00:00
|
|
|
.macro paranoidzeroentry_ist sym do_sym ist
|
2008-11-24 12:24:28 +00:00
|
|
|
ENTRY(\sym)
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2008-11-27 18:10:08 +00:00
|
|
|
INTR_FRAME
|
2008-11-21 15:44:28 +00:00
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
2010-09-02 12:55:11 +00:00
|
|
|
pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
|
|
|
|
|
subq $ORIG_RAX-R15, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
2008-11-21 15:44:28 +00:00
|
|
|
call save_paranoid
|
2012-05-30 15:54:53 +00:00
|
|
|
TRACE_IRQS_OFF_DEBUG
|
2008-11-21 15:44:28 +00:00
|
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
|
|
|
xorl %esi,%esi /* no error code */
|
2010-07-31 16:48:22 +00:00
|
|
|
subq $EXCEPTION_STKSZ, INIT_TSS_IST(\ist)
|
2008-11-23 09:08:28 +00:00
|
|
|
call \do_sym
|
2010-07-31 16:48:22 +00:00
|
|
|
addq $EXCEPTION_STKSZ, INIT_TSS_IST(\ist)
|
2008-11-21 15:44:28 +00:00
|
|
|
jmp paranoid_exit /* %ebx: no swapgs flag */
|
|
|
|
|
CFI_ENDPROC
|
2008-11-24 12:24:28 +00:00
|
|
|
END(\sym)
|
2008-11-23 09:08:28 +00:00
|
|
|
.endm
|
2008-11-21 15:44:28 +00:00
|
|
|
|
2008-11-24 12:24:28 +00:00
|
|
|
.macro errorentry sym do_sym
|
2008-11-23 09:08:28 +00:00
|
|
|
ENTRY(\sym)
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2005-09-12 16:49:24 +00:00
|
|
|
XCPT_FRAME
|
2008-06-25 04:19:31 +00:00
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
2010-09-02 12:55:11 +00:00
|
|
|
subq $ORIG_RAX-R15, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
call error_entry
|
2008-11-20 13:40:11 +00:00
|
|
|
DEFAULT_FRAME 0
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
|
|
|
movq ORIG_RAX(%rsp),%rsi /* get error code */
|
|
|
|
|
movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */
|
2008-11-23 09:08:28 +00:00
|
|
|
call \do_sym
|
x86: move entry_64.S register saving out of the macros
Here is a combined patch that moves "save_args" out-of-line for
the interrupt macro and moves "error_entry" mostly out-of-line
for the zeroentry and errorentry macros.
The save_args function becomes really straightforward and easy
to understand, with the possible exception of the stack switch
code, which now needs to copy the return address of to the
calling function. Normal interrupts arrive with ((~vector)-0x80)
on the stack, which gets adjusted in common_interrupt:
<common_interrupt>:
(5) addq $0xffffffffffffff80,(%rsp) /* -> ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80214290 <do_IRQ>
<ret_from_intr>:
...
An apic interrupt stub now look like this:
<thermal_interrupt>:
(5) pushq $0xffffffffffffff05 /* ~(vector) */
(4) sub $0x50,%rsp /* space for registers */
(5) callq ffffffff80211290 <save_args>
(5) callq ffffffff80212b8f <smp_thermal_interrupt>
(5) jmpq ffffffff80211f93 <ret_from_intr>
Similarly the exception handler register saving function becomes
simpler, without the need of any parameter shuffling. The stub
for an exception without errorcode looks like this:
<overflow>:
(6) callq *0x1cad12(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(2) pushq $0xffffffffffffffff /* no syscall */
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(2) xor %esi,%esi /* no error code */
(5) callq ffffffff80213446 <do_overflow>
(5) jmpq ffffffff8030e460 <error_exit>
And one for an exception with errorcode like this:
<segment_not_present>:
(6) callq *0x1cab92(%rip) # ffffffff803dd448 <pv_irq_ops+0x38>
(4) sub $0x78,%rsp /* space for registers */
(5) callq ffffffff8030e3b0 <error_entry>
(3) mov %rsp,%rdi /* pt_regs pointer */
(5) mov 0x78(%rsp),%rsi /* load error code */
(9) movq $0xffffffffffffffff,0x78(%rsp) /* no syscall */
(5) callq ffffffff80213209 <do_segment_not_present>
(5) jmpq ffffffff8030e460 <error_exit>
Unfortunately, this last type is more than 32 bytes. But the total space
savings due to this patch is about 2500 bytes on an smp-configuration,
and I think the code is clearer than it was before. The tested kernels
were non-paravirt ones (i.e., without the indirect call at the top of
the exception handlers).
Anyhow, I tested this patch on top of a recent -tip. The machine
was an 2x4-core Xeon at 2333MHz. Measured where the delays between
(almost-)adjacent rdtsc instructions. The graphs show how much
time is spent outside of the program as a function of the measured
delay. The area under the graph represents the total time spent
outside the program. Eight instances of the rdtsctest were
started, each pinned to a single cpu. The histogams are added.
For each kernel two measurements were done: one in mostly idle
condition, the other while running "bonnie++ -f", bound to cpu 0.
Each measurement took 40 minutes runtime. See the attached graphs
for the results. The graphs overlap almost everywhere, but there
are small differences.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-19 00:18:11 +00:00
|
|
|
jmp error_exit /* %ebx: no swapgs flag */
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_ENDPROC
|
2008-11-23 09:08:28 +00:00
|
|
|
END(\sym)
|
|
|
|
|
.endm
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* error code is on the stack already */
|
2008-11-24 12:24:28 +00:00
|
|
|
.macro paranoiderrorentry sym do_sym
|
2008-11-23 09:08:28 +00:00
|
|
|
ENTRY(\sym)
|
2012-09-21 19:43:12 +00:00
|
|
|
ASM_CLAC
|
2008-11-21 15:44:28 +00:00
|
|
|
XCPT_FRAME
|
|
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
2010-09-02 12:55:11 +00:00
|
|
|
subq $ORIG_RAX-R15, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
2008-11-21 15:43:18 +00:00
|
|
|
call save_paranoid
|
|
|
|
|
DEFAULT_FRAME 0
|
2008-09-26 12:03:03 +00:00
|
|
|
TRACE_IRQS_OFF
|
2008-11-21 15:44:28 +00:00
|
|
|
movq %rsp,%rdi /* pt_regs pointer */
|
|
|
|
|
movq ORIG_RAX(%rsp),%rsi /* get error code */
|
|
|
|
|
movq $-1,ORIG_RAX(%rsp) /* no syscall to restart */
|
2008-11-23 09:08:28 +00:00
|
|
|
call \do_sym
|
2008-11-21 15:44:28 +00:00
|
|
|
jmp paranoid_exit /* %ebx: no swapgs flag */
|
|
|
|
|
CFI_ENDPROC
|
2008-11-23 09:08:28 +00:00
|
|
|
END(\sym)
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
zeroentry divide_error do_divide_error
|
|
|
|
|
zeroentry overflow do_overflow
|
|
|
|
|
zeroentry bounds do_bounds
|
|
|
|
|
zeroentry invalid_op do_invalid_op
|
|
|
|
|
zeroentry device_not_available do_device_not_available
|
2008-11-24 12:24:28 +00:00
|
|
|
paranoiderrorentry double_fault do_double_fault
|
2008-11-23 09:08:28 +00:00
|
|
|
zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
|
|
|
|
|
errorentry invalid_TSS do_invalid_TSS
|
|
|
|
|
errorentry segment_not_present do_segment_not_present
|
|
|
|
|
zeroentry spurious_interrupt_bug do_spurious_interrupt_bug
|
|
|
|
|
zeroentry coprocessor_error do_coprocessor_error
|
|
|
|
|
errorentry alignment_check do_alignment_check
|
|
|
|
|
zeroentry simd_coprocessor_error do_simd_coprocessor_error
|
2011-06-05 17:50:24 +00:00
|
|
|
|
2006-07-03 07:24:45 +00:00
|
|
|
|
2008-11-27 18:10:08 +00:00
|
|
|
/* Reload gs selector with exception handling */
|
|
|
|
|
/* edi: new selector */
|
2008-06-25 04:19:32 +00:00
|
|
|
ENTRY(native_load_gs_index)
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_STARTPROC
|
2010-09-02 13:07:16 +00:00
|
|
|
pushfq_cfi
|
2009-01-28 22:35:03 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
|
2008-11-27 18:10:08 +00:00
|
|
|
SWAPGS
|
2008-11-16 14:29:00 +00:00
|
|
|
gs_change:
|
2008-11-27 18:10:08 +00:00
|
|
|
movl %edi,%gs
|
2005-04-16 22:20:36 +00:00
|
|
|
2: mfence /* workaround */
|
2008-01-30 12:32:08 +00:00
|
|
|
SWAPGS
|
2010-09-02 13:07:16 +00:00
|
|
|
popfq_cfi
|
2008-11-27 18:10:08 +00:00
|
|
|
ret
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_ENDPROC
|
2008-11-23 09:15:32 +00:00
|
|
|
END(native_load_gs_index)
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2012-04-20 19:19:50 +00:00
|
|
|
_ASM_EXTABLE(gs_change,bad_gs)
|
2008-11-27 18:10:08 +00:00
|
|
|
.section .fixup,"ax"
|
2005-04-16 22:20:36 +00:00
|
|
|
/* running with kernelgs */
|
2008-11-16 14:29:00 +00:00
|
|
|
bad_gs:
|
2008-01-30 12:32:08 +00:00
|
|
|
SWAPGS /* switch back to user gs */
|
2005-04-16 22:20:36 +00:00
|
|
|
xorl %eax,%eax
|
2008-11-27 18:10:08 +00:00
|
|
|
movl %eax,%gs
|
|
|
|
|
jmp 2b
|
|
|
|
|
.previous
|
2008-11-16 14:29:00 +00:00
|
|
|
|
2006-08-02 20:37:28 +00:00
|
|
|
/* Call softirq on interrupt stack. Interrupts are off. */
|
2005-07-29 04:15:49 +00:00
|
|
|
ENTRY(call_softirq)
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_STARTPROC
|
2010-09-02 13:07:16 +00:00
|
|
|
pushq_cfi %rbp
|
2006-08-02 20:37:28 +00:00
|
|
|
CFI_REL_OFFSET rbp,0
|
|
|
|
|
mov %rsp,%rbp
|
|
|
|
|
CFI_DEF_CFA_REGISTER rbp
|
2009-01-18 15:38:58 +00:00
|
|
|
incl PER_CPU_VAR(irq_count)
|
2009-01-18 15:38:58 +00:00
|
|
|
cmove PER_CPU_VAR(irq_stack_ptr),%rsp
|
2006-08-02 20:37:28 +00:00
|
|
|
push %rbp # backlink for old unwinder
|
2005-07-29 04:15:49 +00:00
|
|
|
call __do_softirq
|
2006-08-02 20:37:28 +00:00
|
|
|
leaveq
|
2010-09-02 13:07:16 +00:00
|
|
|
CFI_RESTORE rbp
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_DEF_CFA_REGISTER rsp
|
2006-08-02 20:37:28 +00:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2009-01-18 15:38:58 +00:00
|
|
|
decl PER_CPU_VAR(irq_count)
|
2005-07-29 04:15:49 +00:00
|
|
|
ret
|
2005-09-12 16:49:24 +00:00
|
|
|
CFI_ENDPROC
|
2008-11-23 09:15:32 +00:00
|
|
|
END(call_softirq)
|
2007-06-23 00:29:25 +00:00
|
|
|
|
2008-07-08 22:06:49 +00:00
|
|
|
#ifdef CONFIG_XEN
|
2008-11-23 09:08:28 +00:00
|
|
|
zeroentry xen_hypervisor_callback xen_do_hypervisor_callback
|
2008-07-08 22:06:49 +00:00
|
|
|
|
|
|
|
|
/*
|
2008-11-27 18:10:08 +00:00
|
|
|
* A note on the "critical region" in our callback handler.
|
|
|
|
|
* We want to avoid stacking callback handlers due to events occurring
|
|
|
|
|
* during handling of the last event. To do this, we keep events disabled
|
|
|
|
|
* until we've done all processing. HOWEVER, we must enable events before
|
|
|
|
|
* popping the stack frame (can't be done atomically) and so it would still
|
|
|
|
|
* be possible to get enough handler activations to overflow the stack.
|
|
|
|
|
* Although unlikely, bugs of that kind are hard to track down, so we'd
|
|
|
|
|
* like to avoid the possibility.
|
|
|
|
|
* So, on entry to the handler we detect whether we interrupted an
|
|
|
|
|
* existing activation in its critical region -- if so, we pop the current
|
|
|
|
|
* activation and restart the handler using the previous one.
|
|
|
|
|
*/
|
2008-07-08 22:06:49 +00:00
|
|
|
ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
|
|
|
|
|
CFI_STARTPROC
|
2008-11-27 18:10:08 +00:00
|
|
|
/*
|
|
|
|
|
* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
|
|
|
|
|
* see the correct pointer to the pt_regs
|
|
|
|
|
*/
|
2008-07-08 22:06:49 +00:00
|
|
|
movq %rdi, %rsp # we don't return, adjust the stack frame
|
|
|
|
|
CFI_ENDPROC
|
2008-11-20 13:40:11 +00:00
|
|
|
DEFAULT_FRAME
|
2009-01-18 15:38:58 +00:00
|
|
|
11: incl PER_CPU_VAR(irq_count)
|
2008-07-08 22:06:49 +00:00
|
|
|
movq %rsp,%rbp
|
|
|
|
|
CFI_DEF_CFA_REGISTER rbp
|
2009-01-18 15:38:58 +00:00
|
|
|
cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
|
2008-07-08 22:06:49 +00:00
|
|
|
pushq %rbp # backlink for old unwinder
|
|
|
|
|
call xen_evtchn_do_upcall
|
|
|
|
|
popq %rsp
|
|
|
|
|
CFI_DEF_CFA_REGISTER rsp
|
2009-01-18 15:38:58 +00:00
|
|
|
decl PER_CPU_VAR(irq_count)
|
2008-07-08 22:06:49 +00:00
|
|
|
jmp error_exit
|
|
|
|
|
CFI_ENDPROC
|
x86, binutils, xen: Fix another wrong size directive
The latest binutils (2.21.0.20110302/Ubuntu) breaks the build
yet another time, under CONFIG_XEN=y due to a .size directive that
refers to a slightly differently named (hence, to the now very
strict and unforgiving assembler, non-existent) symbol.
[ mingo:
This unnecessary build breakage caused by new binutils
version 2.21 gets escallated back several kernel releases spanning
several years of Linux history, affecting over 130,000 upstream
kernel commits (!), on CONFIG_XEN=y 64-bit kernels (i.e. essentially
affecting all major Linux distro kernel configs).
Git annotate tells us that this slight debug symbol code mismatch
bug has been introduced in 2008 in commit 3d75e1b8:
3d75e1b8 (Jeremy Fitzhardinge 2008-07-08 15:06:49 -0700 1231) ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
The 'bug' is just a slight assymetry in ENTRY()/END()
debug-symbols sequences, with lots of assembly code between the
ENTRY() and the END():
ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
...
END(do_hypervisor_callback)
Human reviewers almost never catch such small mismatches, and binutils
never even warned about it either.
This new binutils version thus breaks the Xen build on all upstream kernels
since v2.6.27, out of the blue.
This makes a straightforward Git bisection of all 64-bit Xen-enabled kernels
impossible on such binutils, for a bisection window of over hundred
thousand historic commits. (!)
This is a major fail on the side of binutils and binutils needs to turn
this show-stopper build failure into a warning ASAP. ]
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jan Beulich <jbeulich@novell.com>
Cc: H.J. Lu <hjl.tools@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Kees Cook <kees.cook@canonical.com>
LKML-Reference: <1299877178-26063-1-git-send-email-heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2011-03-11 20:59:38 +00:00
|
|
|
END(xen_do_hypervisor_callback)
|
2008-07-08 22:06:49 +00:00
|
|
|
|
|
|
|
|
/*
|
2008-11-27 18:10:08 +00:00
|
|
|
* Hypervisor uses this for application faults while it executes.
|
|
|
|
|
* We get here for two reasons:
|
|
|
|
|
* 1. Fault while reloading DS, ES, FS or GS
|
|
|
|
|
* 2. Fault while executing IRET
|
|
|
|
|
* Category 1 we do not need to fix up as Xen has already reloaded all segment
|
|
|
|
|
* registers that could be reloaded and zeroed the others.
|
|
|
|
|
* Category 2 we fix up by killing the current process. We cannot use the
|
|
|
|
|
* normal Linux return path in this case because if we use the IRET hypercall
|
|
|
|
|
* to pop the stack frame we end up in an infinite loop of failsafe callbacks.
|
|
|
|
|
* We distinguish between categories by comparing each saved segment register
|
|
|
|
|
* with its current contents: any discrepancy means we in category 1.
|
|
|
|
|
*/
|
2008-07-08 22:06:49 +00:00
|
|
|
ENTRY(xen_failsafe_callback)
|
2008-11-20 13:40:11 +00:00
|
|
|
INTR_FRAME 1 (6*8)
|
|
|
|
|
/*CFI_REL_OFFSET gs,GS*/
|
|
|
|
|
/*CFI_REL_OFFSET fs,FS*/
|
|
|
|
|
/*CFI_REL_OFFSET es,ES*/
|
|
|
|
|
/*CFI_REL_OFFSET ds,DS*/
|
|
|
|
|
CFI_REL_OFFSET r11,8
|
|
|
|
|
CFI_REL_OFFSET rcx,0
|
2008-07-08 22:06:49 +00:00
|
|
|
movw %ds,%cx
|
|
|
|
|
cmpw %cx,0x10(%rsp)
|
|
|
|
|
CFI_REMEMBER_STATE
|
|
|
|
|
jne 1f
|
|
|
|
|
movw %es,%cx
|
|
|
|
|
cmpw %cx,0x18(%rsp)
|
|
|
|
|
jne 1f
|
|
|
|
|
movw %fs,%cx
|
|
|
|
|
cmpw %cx,0x20(%rsp)
|
|
|
|
|
jne 1f
|
|
|
|
|
movw %gs,%cx
|
|
|
|
|
cmpw %cx,0x28(%rsp)
|
|
|
|
|
jne 1f
|
|
|
|
|
/* All segments match their saved values => Category 2 (Bad IRET). */
|
|
|
|
|
movq (%rsp),%rcx
|
|
|
|
|
CFI_RESTORE rcx
|
|
|
|
|
movq 8(%rsp),%r11
|
|
|
|
|
CFI_RESTORE r11
|
|
|
|
|
addq $0x30,%rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET -0x30
|
2008-11-21 14:20:47 +00:00
|
|
|
pushq_cfi $0 /* RIP */
|
|
|
|
|
pushq_cfi %r11
|
|
|
|
|
pushq_cfi %rcx
|
2008-07-08 22:07:09 +00:00
|
|
|
jmp general_protection
|
2008-07-08 22:06:49 +00:00
|
|
|
CFI_RESTORE_STATE
|
|
|
|
|
1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
|
|
|
|
|
movq (%rsp),%rcx
|
|
|
|
|
CFI_RESTORE rcx
|
|
|
|
|
movq 8(%rsp),%r11
|
|
|
|
|
CFI_RESTORE r11
|
|
|
|
|
addq $0x30,%rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET -0x30
|
2012-10-19 16:29:07 +00:00
|
|
|
pushq_cfi $-1 /* orig_ax = -1 => not a system call */
|
2008-07-08 22:06:49 +00:00
|
|
|
SAVE_ALL
|
|
|
|
|
jmp error_exit
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(xen_failsafe_callback)
|
|
|
|
|
|
2010-05-14 11:40:51 +00:00
|
|
|
apicinterrupt XEN_HVM_EVTCHN_CALLBACK \
|
|
|
|
|
xen_hvm_callback_vector xen_evtchn_do_upcall
|
|
|
|
|
|
2008-07-08 22:06:49 +00:00
|
|
|
#endif /* CONFIG_XEN */
|
2008-11-24 12:24:28 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Some functions should be protected against kprobes
|
|
|
|
|
*/
|
|
|
|
|
.pushsection .kprobes.text, "ax"
|
|
|
|
|
|
|
|
|
|
paranoidzeroentry_ist debug do_debug DEBUG_STACK
|
|
|
|
|
paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
|
|
|
|
|
paranoiderrorentry stack_segment do_stack_segment
|
2009-03-30 02:56:29 +00:00
|
|
|
#ifdef CONFIG_XEN
|
|
|
|
|
zeroentry xen_debug do_debug
|
|
|
|
|
zeroentry xen_int3 do_int3
|
|
|
|
|
errorentry xen_stack_segment do_stack_segment
|
|
|
|
|
#endif
|
2008-11-24 12:24:28 +00:00
|
|
|
errorentry general_protection do_general_protection
|
|
|
|
|
errorentry page_fault do_page_fault
|
2010-10-14 09:22:52 +00:00
|
|
|
#ifdef CONFIG_KVM_GUEST
|
|
|
|
|
errorentry async_page_fault do_async_page_fault
|
|
|
|
|
#endif
|
2008-11-24 12:24:28 +00:00
|
|
|
#ifdef CONFIG_X86_MCE
|
2009-04-27 17:25:48 +00:00
|
|
|
paranoidzeroentry machine_check *machine_check_vector(%rip)
|
2008-11-24 12:24:28 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
2008-11-27 18:10:08 +00:00
|
|
|
* "Paranoid" exit path from exception stack.
|
|
|
|
|
* Paranoid because this is used by NMIs and cannot take
|
2008-11-24 12:24:28 +00:00
|
|
|
* any kernel state for granted.
|
|
|
|
|
* We don't do kernel preemption checks here, because only
|
|
|
|
|
* NMI should be common and it does not enable IRQs and
|
|
|
|
|
* cannot get reschedule ticks.
|
|
|
|
|
*
|
|
|
|
|
* "trace" is 0 for the NMI handler only, because irq-tracing
|
|
|
|
|
* is fundamentally NMI-unsafe. (we cannot change the soft and
|
|
|
|
|
* hard flags at once, atomically)
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/* ebx: no swapgs flag */
|
|
|
|
|
ENTRY(paranoid_exit)
|
2010-09-02 12:54:32 +00:00
|
|
|
DEFAULT_FRAME
|
2008-11-24 12:24:28 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2012-05-30 15:54:53 +00:00
|
|
|
TRACE_IRQS_OFF_DEBUG
|
2008-11-24 12:24:28 +00:00
|
|
|
testl %ebx,%ebx /* swapgs needed? */
|
|
|
|
|
jnz paranoid_restore
|
|
|
|
|
testl $3,CS(%rsp)
|
|
|
|
|
jnz paranoid_userspace
|
|
|
|
|
paranoid_swapgs:
|
|
|
|
|
TRACE_IRQS_IRETQ 0
|
|
|
|
|
SWAPGS_UNSAFE_STACK
|
2009-04-17 12:33:52 +00:00
|
|
|
RESTORE_ALL 8
|
|
|
|
|
jmp irq_return
|
2008-11-24 12:24:28 +00:00
|
|
|
paranoid_restore:
|
2012-05-30 15:54:53 +00:00
|
|
|
TRACE_IRQS_IRETQ_DEBUG 0
|
2008-11-24 12:24:28 +00:00
|
|
|
RESTORE_ALL 8
|
|
|
|
|
jmp irq_return
|
|
|
|
|
paranoid_userspace:
|
|
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
|
movl TI_flags(%rcx),%ebx
|
|
|
|
|
andl $_TIF_WORK_MASK,%ebx
|
|
|
|
|
jz paranoid_swapgs
|
|
|
|
|
movq %rsp,%rdi /* &pt_regs */
|
|
|
|
|
call sync_regs
|
|
|
|
|
movq %rax,%rsp /* switch stack for scheduling */
|
|
|
|
|
testl $_TIF_NEED_RESCHED,%ebx
|
|
|
|
|
jnz paranoid_schedule
|
|
|
|
|
movl %ebx,%edx /* arg3: thread flags */
|
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
|
|
|
|
xorl %esi,%esi /* arg2: oldset */
|
|
|
|
|
movq %rsp,%rdi /* arg1: &pt_regs */
|
|
|
|
|
call do_notify_resume
|
|
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
|
jmp paranoid_userspace
|
|
|
|
|
paranoid_schedule:
|
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
|
ENABLE_INTERRUPTS(CLBR_ANY)
|
2012-07-11 18:26:38 +00:00
|
|
|
SCHEDULE_USER
|
2008-11-24 12:24:28 +00:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY)
|
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
|
jmp paranoid_userspace
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(paranoid_exit)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Exception entry point. This expects an error code/orig_rax on the stack.
|
|
|
|
|
* returns in "no swapgs flag" in %ebx.
|
|
|
|
|
*/
|
|
|
|
|
ENTRY(error_entry)
|
|
|
|
|
XCPT_FRAME
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET 15*8
|
|
|
|
|
/* oldrax contains error code */
|
|
|
|
|
cld
|
|
|
|
|
movq_cfi rdi, RDI+8
|
|
|
|
|
movq_cfi rsi, RSI+8
|
|
|
|
|
movq_cfi rdx, RDX+8
|
|
|
|
|
movq_cfi rcx, RCX+8
|
|
|
|
|
movq_cfi rax, RAX+8
|
|
|
|
|
movq_cfi r8, R8+8
|
|
|
|
|
movq_cfi r9, R9+8
|
|
|
|
|
movq_cfi r10, R10+8
|
|
|
|
|
movq_cfi r11, R11+8
|
|
|
|
|
movq_cfi rbx, RBX+8
|
|
|
|
|
movq_cfi rbp, RBP+8
|
|
|
|
|
movq_cfi r12, R12+8
|
|
|
|
|
movq_cfi r13, R13+8
|
|
|
|
|
movq_cfi r14, R14+8
|
|
|
|
|
movq_cfi r15, R15+8
|
|
|
|
|
xorl %ebx,%ebx
|
|
|
|
|
testl $3,CS+8(%rsp)
|
|
|
|
|
je error_kernelspace
|
|
|
|
|
error_swapgs:
|
|
|
|
|
SWAPGS
|
|
|
|
|
error_sti:
|
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
|
ret
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* There are two places in the kernel that can potentially fault with
|
|
|
|
|
* usergs. Handle them here. The exception handlers after iret run with
|
|
|
|
|
* kernel gs again, so don't set the user space flag. B stepping K8s
|
|
|
|
|
* sometimes report an truncated RIP for IRET exceptions returning to
|
|
|
|
|
* compat mode. Check for these here too.
|
|
|
|
|
*/
|
|
|
|
|
error_kernelspace:
|
|
|
|
|
incl %ebx
|
|
|
|
|
leaq irq_return(%rip),%rcx
|
|
|
|
|
cmpq %rcx,RIP+8(%rsp)
|
|
|
|
|
je error_swapgs
|
2009-10-12 14:18:23 +00:00
|
|
|
movl %ecx,%eax /* zero extend */
|
|
|
|
|
cmpq %rax,RIP+8(%rsp)
|
|
|
|
|
je bstep_iret
|
2008-11-24 12:24:28 +00:00
|
|
|
cmpq $gs_change,RIP+8(%rsp)
|
2008-11-27 18:10:08 +00:00
|
|
|
je error_swapgs
|
2008-11-24 12:24:28 +00:00
|
|
|
jmp error_sti
|
2009-10-12 14:18:23 +00:00
|
|
|
|
|
|
|
|
bstep_iret:
|
|
|
|
|
/* Fix truncated RIP */
|
|
|
|
|
movq %rcx,RIP+8(%rsp)
|
2009-11-03 19:02:05 +00:00
|
|
|
jmp error_swapgs
|
2010-09-02 12:52:45 +00:00
|
|
|
CFI_ENDPROC
|
2008-11-24 12:24:28 +00:00
|
|
|
END(error_entry)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
|
|
|
|
|
ENTRY(error_exit)
|
|
|
|
|
DEFAULT_FRAME
|
|
|
|
|
movl %ebx,%eax
|
|
|
|
|
RESTORE_REST
|
|
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
|
testl %eax,%eax
|
|
|
|
|
jne retint_kernel
|
|
|
|
|
LOCKDEP_SYS_EXIT_IRQ
|
|
|
|
|
movl TI_flags(%rcx),%edx
|
|
|
|
|
movl $_TIF_WORK_MASK,%edi
|
|
|
|
|
andl %edi,%edx
|
|
|
|
|
jnz retint_careful
|
|
|
|
|
jmp retint_swapgs
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(error_exit)
|
|
|
|
|
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
/*
|
|
|
|
|
* Test if a given stack is an NMI stack or not.
|
|
|
|
|
*/
|
|
|
|
|
.macro test_in_nmi reg stack nmi_ret normal_ret
|
|
|
|
|
cmpq %\reg, \stack
|
|
|
|
|
ja \normal_ret
|
|
|
|
|
subq $EXCEPTION_STKSZ, %\reg
|
|
|
|
|
cmpq %\reg, \stack
|
|
|
|
|
jb \normal_ret
|
|
|
|
|
jmp \nmi_ret
|
|
|
|
|
.endm
|
2008-11-24 12:24:28 +00:00
|
|
|
|
|
|
|
|
/* runs on exception stack */
|
|
|
|
|
ENTRY(nmi)
|
|
|
|
|
INTR_FRAME
|
|
|
|
|
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
/*
|
|
|
|
|
* We allow breakpoints in NMIs. If a breakpoint occurs, then
|
|
|
|
|
* the iretq it performs will take us out of NMI context.
|
|
|
|
|
* This means that we can have nested NMIs where the next
|
|
|
|
|
* NMI is using the top of the stack of the previous NMI. We
|
|
|
|
|
* can't let it execute because the nested NMI will corrupt the
|
|
|
|
|
* stack of the previous NMI. NMI handlers are not re-entrant
|
|
|
|
|
* anyway.
|
|
|
|
|
*
|
|
|
|
|
* To handle this case we do the following:
|
|
|
|
|
* Check the a special location on the stack that contains
|
|
|
|
|
* a variable that is set when NMIs are executing.
|
|
|
|
|
* The interrupted task's stack is also checked to see if it
|
|
|
|
|
* is an NMI stack.
|
|
|
|
|
* If the variable is not set and the stack is not the NMI
|
|
|
|
|
* stack then:
|
|
|
|
|
* o Set the special variable on the stack
|
|
|
|
|
* o Copy the interrupt frame into a "saved" location on the stack
|
|
|
|
|
* o Copy the interrupt frame into a "copy" location on the stack
|
|
|
|
|
* o Continue processing the NMI
|
|
|
|
|
* If the variable is set or the previous stack is the NMI stack:
|
|
|
|
|
* o Modify the "copy" location to jump to the repeate_nmi
|
|
|
|
|
* o return back to the first NMI
|
|
|
|
|
*
|
|
|
|
|
* Now on exit of the first NMI, we first clear the stack variable
|
|
|
|
|
* The NMI stack will tell any nested NMIs at that point that it is
|
|
|
|
|
* nested. Then we pop the stack normally with iret, and if there was
|
|
|
|
|
* a nested NMI that updated the copy interrupt stack frame, a
|
|
|
|
|
* jump will be made to the repeat_nmi code that will handle the second
|
|
|
|
|
* NMI.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/* Use %rdx as out temp variable throughout */
|
|
|
|
|
pushq_cfi %rdx
|
2012-02-24 14:54:37 +00:00
|
|
|
CFI_REL_OFFSET rdx, 0
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
|
2012-02-19 21:43:37 +00:00
|
|
|
/*
|
|
|
|
|
* If %cs was not the kernel segment, then the NMI triggered in user
|
|
|
|
|
* space, which means it is definitely not nested.
|
|
|
|
|
*/
|
2012-02-20 20:29:34 +00:00
|
|
|
cmpl $__KERNEL_CS, 16(%rsp)
|
2012-02-19 21:43:37 +00:00
|
|
|
jne first_nmi
|
|
|
|
|
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
/*
|
|
|
|
|
* Check the special variable on the stack to see if NMIs are
|
|
|
|
|
* executing.
|
|
|
|
|
*/
|
2012-02-20 20:29:34 +00:00
|
|
|
cmpl $1, -8(%rsp)
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
je nested_nmi
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Now test if the previous stack was an NMI stack.
|
|
|
|
|
* We need the double check. We check the NMI stack to satisfy the
|
|
|
|
|
* race when the first NMI clears the variable before returning.
|
|
|
|
|
* We check the variable because the first NMI could be in a
|
|
|
|
|
* breakpoint routine using a breakpoint stack.
|
|
|
|
|
*/
|
|
|
|
|
lea 6*8(%rsp), %rdx
|
|
|
|
|
test_in_nmi rdx, 4*8(%rsp), nested_nmi, first_nmi
|
2012-02-24 14:54:37 +00:00
|
|
|
CFI_REMEMBER_STATE
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
|
|
|
|
|
nested_nmi:
|
|
|
|
|
/*
|
|
|
|
|
* Do nothing if we interrupted the fixup in repeat_nmi.
|
|
|
|
|
* It's about to repeat the NMI handler, so we are fine
|
|
|
|
|
* with ignoring this one.
|
|
|
|
|
*/
|
|
|
|
|
movq $repeat_nmi, %rdx
|
|
|
|
|
cmpq 8(%rsp), %rdx
|
|
|
|
|
ja 1f
|
|
|
|
|
movq $end_repeat_nmi, %rdx
|
|
|
|
|
cmpq 8(%rsp), %rdx
|
|
|
|
|
ja nested_nmi_out
|
|
|
|
|
|
|
|
|
|
1:
|
|
|
|
|
/* Set up the interrupted NMIs stack to jump to repeat_nmi */
|
|
|
|
|
leaq -6*8(%rsp), %rdx
|
|
|
|
|
movq %rdx, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET 6*8
|
|
|
|
|
pushq_cfi $__KERNEL_DS
|
|
|
|
|
pushq_cfi %rdx
|
|
|
|
|
pushfq_cfi
|
|
|
|
|
pushq_cfi $__KERNEL_CS
|
|
|
|
|
pushq_cfi $repeat_nmi
|
|
|
|
|
|
|
|
|
|
/* Put stack back */
|
|
|
|
|
addq $(11*8), %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET -11*8
|
|
|
|
|
|
|
|
|
|
nested_nmi_out:
|
|
|
|
|
popq_cfi %rdx
|
2012-02-24 14:54:37 +00:00
|
|
|
CFI_RESTORE rdx
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
|
|
|
|
|
/* No need to check faults here */
|
|
|
|
|
INTERRUPT_RETURN
|
|
|
|
|
|
2012-02-24 14:54:37 +00:00
|
|
|
CFI_RESTORE_STATE
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
first_nmi:
|
|
|
|
|
/*
|
|
|
|
|
* Because nested NMIs will use the pushed location that we
|
|
|
|
|
* stored in rdx, we must keep that space available.
|
|
|
|
|
* Here's what our stack frame will look like:
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
* | original SS |
|
|
|
|
|
* | original Return RSP |
|
|
|
|
|
* | original RFLAGS |
|
|
|
|
|
* | original CS |
|
|
|
|
|
* | original RIP |
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
* | temp storage for rdx |
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
* | NMI executing variable |
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
* | Saved SS |
|
|
|
|
|
* | Saved Return RSP |
|
|
|
|
|
* | Saved RFLAGS |
|
|
|
|
|
* | Saved CS |
|
|
|
|
|
* | Saved RIP |
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
* | copied SS |
|
|
|
|
|
* | copied Return RSP |
|
|
|
|
|
* | copied RFLAGS |
|
|
|
|
|
* | copied CS |
|
|
|
|
|
* | copied RIP |
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
* | pt_regs |
|
|
|
|
|
* +-------------------------+
|
|
|
|
|
*
|
2012-02-24 20:55:13 +00:00
|
|
|
* The saved stack frame is used to fix up the copied stack frame
|
|
|
|
|
* that a nested NMI may change to make the interrupted NMI iret jump
|
|
|
|
|
* to the repeat_nmi. The original stack frame and the temp storage
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
* is also used by nested NMIs and can not be trusted on exit.
|
|
|
|
|
*/
|
2012-02-24 20:55:13 +00:00
|
|
|
/* Do not pop rdx, nested NMIs will corrupt that part of the stack */
|
2012-02-24 14:54:37 +00:00
|
|
|
movq (%rsp), %rdx
|
|
|
|
|
CFI_RESTORE rdx
|
|
|
|
|
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
/* Set the NMI executing variable on the stack. */
|
|
|
|
|
pushq_cfi $1
|
|
|
|
|
|
|
|
|
|
/* Copy the stack frame to the Saved frame */
|
|
|
|
|
.rept 5
|
|
|
|
|
pushq_cfi 6*8(%rsp)
|
|
|
|
|
.endr
|
2012-02-24 14:54:37 +00:00
|
|
|
CFI_DEF_CFA_OFFSET SS+8-RIP
|
|
|
|
|
|
2012-02-24 20:55:13 +00:00
|
|
|
/* Everything up to here is safe from nested NMIs */
|
|
|
|
|
|
2012-02-24 14:54:37 +00:00
|
|
|
/*
|
|
|
|
|
* If there was a nested NMI, the first NMI's iret will return
|
|
|
|
|
* here. But NMIs are still enabled and we can take another
|
|
|
|
|
* nested NMI. The nested NMI checks the interrupted RIP to see
|
|
|
|
|
* if it is between repeat_nmi and end_repeat_nmi, and if so
|
|
|
|
|
* it will just return, as we are about to repeat an NMI anyway.
|
|
|
|
|
* This makes it safe to copy to the stack frame that a nested
|
|
|
|
|
* NMI will update.
|
|
|
|
|
*/
|
|
|
|
|
repeat_nmi:
|
|
|
|
|
/*
|
|
|
|
|
* Update the stack variable to say we are still in NMI (the update
|
|
|
|
|
* is benign for the non-repeat case, where 1 was pushed just above
|
|
|
|
|
* to this very stack slot).
|
|
|
|
|
*/
|
|
|
|
|
movq $1, 5*8(%rsp)
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
|
|
|
|
|
/* Make another copy, this one may be modified by nested NMIs */
|
|
|
|
|
.rept 5
|
|
|
|
|
pushq_cfi 4*8(%rsp)
|
|
|
|
|
.endr
|
2012-02-24 14:54:37 +00:00
|
|
|
CFI_DEF_CFA_OFFSET SS+8-RIP
|
|
|
|
|
end_repeat_nmi:
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Everything below this point can be preempted by a nested
|
2012-02-24 20:55:13 +00:00
|
|
|
* NMI if the first NMI took an exception and reset our iret stack
|
|
|
|
|
* so that we repeat another NMI.
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
*/
|
2011-12-08 17:32:27 +00:00
|
|
|
pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
|
2010-09-02 12:55:11 +00:00
|
|
|
subq $ORIG_RAX-R15, %rsp
|
|
|
|
|
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
|
2011-12-08 17:32:27 +00:00
|
|
|
/*
|
|
|
|
|
* Use save_paranoid to handle SWAPGS, but no need to use paranoid_exit
|
|
|
|
|
* as we should not be calling schedule in NMI context.
|
|
|
|
|
* Even with normal interrupts enabled. An NMI should not be
|
|
|
|
|
* setting NEED_RESCHED or anything that normal interrupts and
|
|
|
|
|
* exceptions might do.
|
|
|
|
|
*/
|
2008-11-24 12:24:28 +00:00
|
|
|
call save_paranoid
|
|
|
|
|
DEFAULT_FRAME 0
|
2012-06-07 14:21:21 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Save off the CR2 register. If we take a page fault in the NMI then
|
|
|
|
|
* it could corrupt the CR2 value. If the NMI preempts a page fault
|
|
|
|
|
* handler before it was able to read the CR2 register, and then the
|
|
|
|
|
* NMI itself takes a page fault, the page fault that was preempted
|
|
|
|
|
* will read the information from the NMI page fault and not the
|
|
|
|
|
* origin fault. Save it off and restore it if it changes.
|
|
|
|
|
* Use the r12 callee-saved register.
|
|
|
|
|
*/
|
|
|
|
|
movq %cr2, %r12
|
|
|
|
|
|
2008-11-24 12:24:28 +00:00
|
|
|
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
|
|
|
|
|
movq %rsp,%rdi
|
|
|
|
|
movq $-1,%rsi
|
|
|
|
|
call do_nmi
|
2012-06-07 14:21:21 +00:00
|
|
|
|
|
|
|
|
/* Did the NMI take a page fault? Restore cr2 if it did */
|
|
|
|
|
movq %cr2, %rcx
|
|
|
|
|
cmpq %rcx, %r12
|
|
|
|
|
je 1f
|
|
|
|
|
movq %r12, %cr2
|
|
|
|
|
1:
|
|
|
|
|
|
2008-11-24 12:24:28 +00:00
|
|
|
testl %ebx,%ebx /* swapgs needed? */
|
|
|
|
|
jnz nmi_restore
|
|
|
|
|
nmi_swapgs:
|
|
|
|
|
SWAPGS_UNSAFE_STACK
|
|
|
|
|
nmi_restore:
|
|
|
|
|
RESTORE_ALL 8
|
x86: Add workaround to NMI iret woes
In x86, when an NMI goes off, the CPU goes into an NMI context that
prevents other NMIs to trigger on that CPU. If an NMI is suppose to
trigger, it has to wait till the previous NMI leaves NMI context.
At that time, the next NMI can trigger (note, only one more NMI will
trigger, as only one can be latched at a time).
The way x86 gets out of NMI context is by calling iret. The problem
with this is that this causes problems if the NMI handle either
triggers an exception, or a breakpoint. Both the exception and the
breakpoint handlers will finish with an iret. If this happens while
in NMI context, the CPU will leave NMI context and a new NMI may come
in. As NMI handlers are not made to be re-entrant, this can cause
havoc with the system, not to mention, the nested NMI will write
all over the previous NMI's stack.
Linus Torvalds proposed the following workaround to this problem:
https://lkml.org/lkml/2010/7/14/264
"In fact, I wonder if we couldn't just do a software NMI disable
instead? Hav ea per-cpu variable (in the _core_ percpu areas that get
allocated statically) that points to the NMI stack frame, and just
make the NMI code itself do something like
NMI entry:
- load percpu NMI stack frame pointer
- if non-zero we know we're nested, and should ignore this NMI:
- we're returning to kernel mode, so return immediately by using
"popf/ret", which also keeps NMI's disabled in the hardware until the
"real" NMI iret happens.
- before the popf/iret, use the NMI stack pointer to make the NMI
return stack be invalid and cause a fault
- set the NMI stack pointer to the current stack pointer
NMI exit (not the above "immediate exit because we nested"):
clear the percpu NMI stack pointer
Just do the iret.
Now, the thing is, now the "iret" is atomic. If we had a nested NMI,
we'll take a fault, and that re-does our "delayed" NMI - and NMI's
will stay masked.
And if we didn't have a nested NMI, that iret will now unmask NMI's,
and everything is happy."
I first tried to follow this advice but as I started implementing this
code, a few gotchas showed up.
One, is accessing per-cpu variables in the NMI handler.
The problem is that per-cpu variables use the %gs register to get the
variable for the given CPU. But as the NMI may happen in userspace,
we must first perform a SWAPGS to get to it. The NMI handler already
does this later in the code, but its too late as we have saved off
all the registers and we don't want to do that for a disabled NMI.
Peter Zijlstra suggested to keep all variables on the stack. This
simplifies things greatly and it has the added benefit of cache locality.
Two, faulting on the iret.
I really wanted to make this work, but it was becoming very hacky, and
I never got it to be stable. The iret already had a fault handler for
userspace faulting with bad segment registers, and getting NMI to trigger
a fault and detect it was very tricky. But for strange reasons, the system
would usually take a double fault and crash. I never figured out why
and decided to go with a simple "jmp" approach. The new approach I took
also simplified things.
Finally, the last problem with Linus's approach was to have the nested
NMI handler do a ret instead of an iret to give the first NMI NMI-context
again.
The problem is that ret is much more limited than an iret. I couldn't figure
out how to get the stack back where it belonged. I could have copied the
current stack, pushed the return onto it, but my fear here is that there
may be some place that writes data below the stack pointer. I know that
is not something code should depend on, but I don't want to chance it.
I may add this feature later, but for now, an NMI handler that loses NMI
context will not get it back.
Here's what is done:
When an NMI comes in, the HW pushes the interrupt stack frame onto the
per cpu NMI stack that is selected by the IST.
A special location on the NMI stack holds a variable that is set when
the first NMI handler runs. If this variable is set then we know that
this is a nested NMI and we process the nested NMI code.
There is still a race when this variable is cleared and an NMI comes
in just before the first NMI does the return. For this case, if the
variable is cleared, we also check if the interrupted stack is the
NMI stack. If it is, then we process the nested NMI code.
Why the two tests and not just test the interrupted stack?
If the first NMI hits a breakpoint and loses NMI context, and then it
hits another breakpoint and while processing that breakpoint we get a
nested NMI. When processing a breakpoint, the stack changes to the
breakpoint stack. If another NMI comes in here we can't rely on the
interrupted stack to be the NMI stack.
If the variable is not set and the interrupted task's stack is not the
NMI stack, then we know this is the first NMI and we can process things
normally. But in order to do so, we need to do a few things first.
1) Set the stack variable that tells us that we are in an NMI handler
2) Make two copies of the interrupt stack frame.
One copy is used to return on iret
The other is used to restore the first one if we have a nested NMI.
This is what the stack will look like:
+-------------------------+
| original SS |
| original Return RSP |
| original RFLAGS |
| original CS |
| original RIP |
+-------------------------+
| temp storage for rdx |
+-------------------------+
| NMI executing variable |
+-------------------------+
| Saved SS |
| Saved Return RSP |
| Saved RFLAGS |
| Saved CS |
| Saved RIP |
+-------------------------+
| copied SS |
| copied Return RSP |
| copied RFLAGS |
| copied CS |
| copied RIP |
+-------------------------+
| pt_regs |
+-------------------------+
The original stack frame contains what the HW put in when we entered
the NMI.
We store %rdx as a temp variable to use. Both the original HW stack
frame and this %rdx storage will be clobbered by nested NMIs so we
can not rely on them later in the first NMI handler.
The next item is the special stack variable that is set when we execute
the rest of the NMI handler.
Then we have two copies of the interrupt stack. The second copy is
modified by any nested NMIs to let the first NMI know that we triggered
a second NMI (latched) and that we should repeat the NMI handler.
If the first NMI hits an exception or breakpoint that takes it out of
NMI context, if a second NMI comes in before the first one finishes,
it will update the copied interrupt stack to point to a fix up location
to trigger another NMI.
When the first NMI calls iret, it will instead jump to the fix up
location. This fix up location will copy the saved interrupt stack back
to the copy and execute the nmi handler again.
Note, the nested NMI knows enough to check if it preempted a previous
NMI handler while it is in the fixup location. If it has, it will not
modify the copied interrupt stack and will just leave as if nothing
happened. As the NMI handle is about to execute again, there's no reason
to latch now.
To test all this, I forced the NMI handler to call iret and take itself
out of NMI context. I also added assemble code to write to the serial to
make sure that it hits the nested path as well as the fix up path.
Everything seems to be working fine.
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: H. Peter Anvin <hpa@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2011-12-08 17:36:23 +00:00
|
|
|
/* Clear the NMI executing stack variable */
|
|
|
|
|
movq $0, 10*8(%rsp)
|
2008-11-24 12:24:28 +00:00
|
|
|
jmp irq_return
|
2008-11-27 18:10:08 +00:00
|
|
|
CFI_ENDPROC
|
2008-11-24 12:24:28 +00:00
|
|
|
END(nmi)
|
|
|
|
|
|
|
|
|
|
ENTRY(ignore_sysret)
|
|
|
|
|
CFI_STARTPROC
|
|
|
|
|
mov $-ENOSYS,%eax
|
|
|
|
|
sysret
|
|
|
|
|
CFI_ENDPROC
|
|
|
|
|
END(ignore_sysret)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* End of kprobes section
|
|
|
|
|
*/
|
|
|
|
|
.popsection
|
