x86,objtool: Split UNWIND_HINT_EMPTY in two

Mark reported that the ORC unwinder incorrectly marks an unwind as
reliable when the unwind terminates prematurely in the dark corners of
return_to_handler() due to lack of information about the next frame.

The problem is UNWIND_HINT_EMPTY is used in two different situations:

  1) The end of the kernel stack unwind before hitting user entry, boot
     code, or fork entry

  2) A blind spot in ORC coverage where the unwinder has to bail due to
     lack of information about the next frame

The ORC unwinder has no way to tell the difference between the two.
When it encounters an undefined stack state with 'end=1', it blindly
marks the stack reliable, which can break the livepatch consistency
model.

Fix it by splitting UNWIND_HINT_EMPTY into UNWIND_HINT_UNDEFINED and
UNWIND_HINT_END_OF_STACK.

Reported-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/fd6212c8b450d3564b855e1cb48404d6277b4d9f.1677683419.git.jpoimboe@kernel.org
This commit is contained in:
Josh Poimboeuf 2023-03-01 07:13:12 -08:00 committed by Peter Zijlstra
parent 4708ea14be
commit fb799447ae
20 changed files with 116 additions and 95 deletions

View File

@ -183,7 +183,7 @@ trampoline or return trampoline. For example, considering the x86_64
.. code-block:: none .. code-block:: none
SYM_CODE_START(return_to_handler) SYM_CODE_START(return_to_handler)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
subq $24, %rsp subq $24, %rsp
/* Save the return values */ /* Save the return values */

View File

@ -205,7 +205,7 @@ syscall_return_via_sysret:
*/ */
movq %rsp, %rdi movq %rsp, %rdi
movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
pushq RSP-RDI(%rdi) /* RSP */ pushq RSP-RDI(%rdi) /* RSP */
pushq (%rdi) /* RDI */ pushq (%rdi) /* RDI */
@ -286,7 +286,7 @@ SYM_FUNC_END(__switch_to_asm)
.pushsection .text, "ax" .pushsection .text, "ax"
__FUNC_ALIGN __FUNC_ALIGN
SYM_CODE_START_NOALIGN(ret_from_fork) SYM_CODE_START_NOALIGN(ret_from_fork)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR // copy_thread ANNOTATE_NOENDBR // copy_thread
CALL_DEPTH_ACCOUNT CALL_DEPTH_ACCOUNT
movq %rax, %rdi movq %rax, %rdi
@ -303,7 +303,7 @@ SYM_CODE_START_NOALIGN(ret_from_fork)
1: 1:
/* kernel thread */ /* kernel thread */
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
movq %r12, %rdi movq %r12, %rdi
CALL_NOSPEC rbx CALL_NOSPEC rbx
/* /*
@ -643,7 +643,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
*/ */
movq %rsp, %rdi movq %rsp, %rdi
movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
/* Copy the IRET frame to the trampoline stack. */ /* Copy the IRET frame to the trampoline stack. */
pushq 6*8(%rdi) /* SS */ pushq 6*8(%rdi) /* SS */
@ -869,7 +869,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback)
*/ */
__FUNC_ALIGN __FUNC_ALIGN
SYM_CODE_START_NOALIGN(xen_failsafe_callback) SYM_CODE_START_NOALIGN(xen_failsafe_callback)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ENDBR ENDBR
movl %ds, %ecx movl %ds, %ecx
cmpw %cx, 0x10(%rsp) cmpw %cx, 0x10(%rsp)
@ -1520,7 +1520,7 @@ SYM_CODE_END(asm_exc_nmi)
* MSRs to fully disable 32-bit SYSCALL. * MSRs to fully disable 32-bit SYSCALL.
*/ */
SYM_CODE_START(ignore_sysret) SYM_CODE_START(ignore_sysret)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ENDBR ENDBR
mov $-ENOSYS, %eax mov $-ENOSYS, %eax
sysretl sysretl

View File

@ -39,9 +39,11 @@
#define ORC_REG_SP_INDIRECT 9 #define ORC_REG_SP_INDIRECT 9
#define ORC_REG_MAX 15 #define ORC_REG_MAX 15
#define ORC_TYPE_CALL 0 #define ORC_TYPE_UNDEFINED 0
#define ORC_TYPE_REGS 1 #define ORC_TYPE_END_OF_STACK 1
#define ORC_TYPE_REGS_PARTIAL 2 #define ORC_TYPE_CALL 2
#define ORC_TYPE_REGS 3
#define ORC_TYPE_REGS_PARTIAL 4
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <asm/byteorder.h> #include <asm/byteorder.h>
@ -60,16 +62,14 @@ struct orc_entry {
#if defined(__LITTLE_ENDIAN_BITFIELD) #if defined(__LITTLE_ENDIAN_BITFIELD)
unsigned sp_reg:4; unsigned sp_reg:4;
unsigned bp_reg:4; unsigned bp_reg:4;
unsigned type:2; unsigned type:3;
unsigned signal:1; unsigned signal:1;
unsigned end:1;
#elif defined(__BIG_ENDIAN_BITFIELD) #elif defined(__BIG_ENDIAN_BITFIELD)
unsigned bp_reg:4; unsigned bp_reg:4;
unsigned sp_reg:4; unsigned sp_reg:4;
unsigned unused:4; unsigned unused:4;
unsigned end:1;
unsigned signal:1; unsigned signal:1;
unsigned type:2; unsigned type:3;
#endif #endif
} __packed; } __packed;

View File

@ -7,13 +7,17 @@
#ifdef __ASSEMBLY__ #ifdef __ASSEMBLY__
.macro UNWIND_HINT_EMPTY .macro UNWIND_HINT_END_OF_STACK
UNWIND_HINT type=UNWIND_HINT_TYPE_CALL end=1 UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK
.endm
.macro UNWIND_HINT_UNDEFINED
UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED
.endm .endm
.macro UNWIND_HINT_ENTRY .macro UNWIND_HINT_ENTRY
VALIDATE_UNRET_BEGIN VALIDATE_UNRET_BEGIN
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
.endm .endm
.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1 .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1
@ -73,7 +77,7 @@
#else #else
#define UNWIND_HINT_FUNC \ #define UNWIND_HINT_FUNC \
UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0, 0) UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0)
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */

View File

@ -340,7 +340,7 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_CODE_START(return_to_handler) SYM_CODE_START(return_to_handler)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
subq $16, %rsp subq $16, %rsp

View File

@ -42,7 +42,7 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map)
__HEAD __HEAD
.code64 .code64
SYM_CODE_START_NOALIGN(startup_64) SYM_CODE_START_NOALIGN(startup_64)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
/* /*
* At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
* and someone has loaded an identity mapped page table * and someone has loaded an identity mapped page table
@ -105,7 +105,7 @@ SYM_CODE_START_NOALIGN(startup_64)
lretq lretq
.Lon_kernel_cs: .Lon_kernel_cs:
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
/* Sanitize CPU configuration */ /* Sanitize CPU configuration */
call verify_cpu call verify_cpu
@ -127,7 +127,7 @@ SYM_CODE_START_NOALIGN(startup_64)
SYM_CODE_END(startup_64) SYM_CODE_END(startup_64)
SYM_CODE_START(secondary_startup_64) SYM_CODE_START(secondary_startup_64)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
/* /*
* At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
@ -156,7 +156,7 @@ SYM_CODE_START(secondary_startup_64)
* verify_cpu() above to make sure NX is enabled. * verify_cpu() above to make sure NX is enabled.
*/ */
SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
/* /*
@ -238,7 +238,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
ANNOTATE_RETPOLINE_SAFE ANNOTATE_RETPOLINE_SAFE
jmp *%rax jmp *%rax
1: 1:
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR // above ANNOTATE_NOENDBR // above
/* /*
@ -371,7 +371,7 @@ SYM_CODE_END(secondary_startup_64)
*/ */
SYM_CODE_START(start_cpu0) SYM_CODE_START(start_cpu0)
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
movq initial_stack(%rip), %rsp movq initial_stack(%rip), %rsp
jmp .Ljump_to_C_code jmp .Ljump_to_C_code
SYM_CODE_END(start_cpu0) SYM_CODE_END(start_cpu0)

View File

@ -43,7 +43,7 @@
.code64 .code64
SYM_CODE_START_NOALIGN(relocate_range) SYM_CODE_START_NOALIGN(relocate_range)
SYM_CODE_START_NOALIGN(relocate_kernel) SYM_CODE_START_NOALIGN(relocate_kernel)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
/* /*
* %rdi indirection_page * %rdi indirection_page
@ -113,7 +113,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
SYM_CODE_END(relocate_kernel) SYM_CODE_END(relocate_kernel)
SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
/* set return address to 0 if not preserving context */ /* set return address to 0 if not preserving context */
pushq $0 pushq $0
/* store the start address on the stack */ /* store the start address on the stack */
@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
SYM_CODE_END(identity_mapped) SYM_CODE_END(identity_mapped)
SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR // RET target, above ANNOTATE_NOENDBR // RET target, above
movq RSP(%r8), %rsp movq RSP(%r8), %rsp
movq CR4(%r8), %rax movq CR4(%r8), %rax
@ -256,8 +256,8 @@ SYM_CODE_END(virtual_mapped)
/* Do the copies */ /* Do the copies */
SYM_CODE_START_LOCAL_NOALIGN(swap_pages) SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
movq %rdi, %rcx /* Put the page_list in %rcx */ movq %rdi, %rcx /* Put the page_list in %rcx */
xorl %edi, %edi xorl %edi, %edi
xorl %esi, %esi xorl %esi, %esi
jmp 1f jmp 1f

View File

@ -158,7 +158,6 @@ static struct orc_entry orc_fp_entry = {
.sp_offset = 16, .sp_offset = 16,
.bp_reg = ORC_REG_PREV_SP, .bp_reg = ORC_REG_PREV_SP,
.bp_offset = -16, .bp_offset = -16,
.end = 0,
}; };
static struct orc_entry *orc_find(unsigned long ip) static struct orc_entry *orc_find(unsigned long ip)
@ -250,13 +249,13 @@ static int orc_sort_cmp(const void *_a, const void *_b)
return -1; return -1;
/* /*
* The "weak" section terminator entries need to always be on the left * The "weak" section terminator entries need to always be first
* to ensure the lookup code skips them in favor of real entries. * to ensure the lookup code skips them in favor of real entries.
* These terminator entries exist to handle any gaps created by * These terminator entries exist to handle any gaps created by
* whitelisted .o files which didn't get objtool generation. * whitelisted .o files which didn't get objtool generation.
*/ */
orc_a = cur_orc_table + (a - cur_orc_ip_table); orc_a = cur_orc_table + (a - cur_orc_ip_table);
return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
} }
void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
@ -474,14 +473,12 @@ bool unwind_next_frame(struct unwind_state *state)
*/ */
orc = &orc_fp_entry; orc = &orc_fp_entry;
state->error = true; state->error = true;
} } else {
if (orc->type == ORC_TYPE_UNDEFINED)
/* End-of-stack check for kernel threads: */
if (orc->sp_reg == ORC_REG_UNDEFINED) {
if (!orc->end)
goto err; goto err;
goto the_end; if (orc->type == ORC_TYPE_END_OF_STACK)
goto the_end;
} }
state->signal = orc->signal; state->signal = orc->signal;

View File

@ -33,7 +33,7 @@
.align RETPOLINE_THUNK_SIZE .align RETPOLINE_THUNK_SIZE
SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ ALTERNATIVE_2 __stringify(RETPOLINE \reg), \
@ -75,7 +75,7 @@ SYM_CODE_END(__x86_indirect_thunk_array)
.align RETPOLINE_THUNK_SIZE .align RETPOLINE_THUNK_SIZE
SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL) SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
CALL_DEPTH_ACCOUNT CALL_DEPTH_ACCOUNT
@ -103,7 +103,7 @@ SYM_CODE_END(__x86_indirect_call_thunk_array)
.align RETPOLINE_THUNK_SIZE .align RETPOLINE_THUNK_SIZE
SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL) SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
POLINE \reg POLINE \reg
ANNOTATE_UNRET_SAFE ANNOTATE_UNRET_SAFE

View File

@ -50,7 +50,7 @@
#define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8) #define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8)
SYM_CODE_START_LOCAL(pvh_start_xen) SYM_CODE_START_LOCAL(pvh_start_xen)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
cld cld
lgdt (_pa(gdt)) lgdt (_pa(gdt))

View File

@ -165,7 +165,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback
SYM_CODE_START(xen_early_idt_handler_array) SYM_CODE_START(xen_early_idt_handler_array)
i = 0 i = 0
.rept NUM_EXCEPTION_VECTORS .rept NUM_EXCEPTION_VECTORS
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ENDBR ENDBR
pop %rcx pop %rcx
pop %r11 pop %r11
@ -193,7 +193,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
* rsp->rax } * rsp->rax }
*/ */
SYM_CODE_START(xen_iret) SYM_CODE_START(xen_iret)
UNWIND_HINT_EMPTY UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
pushq $0 pushq $0
jmp hypercall_iret jmp hypercall_iret

View File

@ -45,7 +45,7 @@ SYM_CODE_END(hypercall_page)
#ifdef CONFIG_XEN_PV #ifdef CONFIG_XEN_PV
__INIT __INIT
SYM_CODE_START(startup_xen) SYM_CODE_START(startup_xen)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR ANNOTATE_NOENDBR
cld cld
@ -71,7 +71,7 @@ SYM_CODE_END(startup_xen)
#ifdef CONFIG_XEN_PV_SMP #ifdef CONFIG_XEN_PV_SMP
.pushsection .text .pushsection .text
SYM_CODE_START(asm_cpu_bringup_and_idle) SYM_CODE_START(asm_cpu_bringup_and_idle)
UNWIND_HINT_EMPTY UNWIND_HINT_END_OF_STACK
ENDBR ENDBR
call cpu_bringup_and_idle call cpu_bringup_and_idle

View File

@ -10,7 +10,7 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) \ #define UNWIND_HINT(type, sp_reg, sp_offset, signal) \
"987: \n\t" \ "987: \n\t" \
".pushsection .discard.unwind_hints\n\t" \ ".pushsection .discard.unwind_hints\n\t" \
/* struct unwind_hint */ \ /* struct unwind_hint */ \
@ -19,7 +19,6 @@
".byte " __stringify(sp_reg) "\n\t" \ ".byte " __stringify(sp_reg) "\n\t" \
".byte " __stringify(type) "\n\t" \ ".byte " __stringify(type) "\n\t" \
".byte " __stringify(signal) "\n\t" \ ".byte " __stringify(signal) "\n\t" \
".byte " __stringify(end) "\n\t" \
".balign 4 \n\t" \ ".balign 4 \n\t" \
".popsection\n\t" ".popsection\n\t"
@ -91,7 +90,7 @@
* the debuginfo as necessary. It will also warn if it sees any * the debuginfo as necessary. It will also warn if it sees any
* inconsistencies. * inconsistencies.
*/ */
.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
.Lhere_\@: .Lhere_\@:
.pushsection .discard.unwind_hints .pushsection .discard.unwind_hints
/* struct unwind_hint */ /* struct unwind_hint */
@ -100,7 +99,6 @@
.byte \sp_reg .byte \sp_reg
.byte \type .byte \type
.byte \signal .byte \signal
.byte \end
.balign 4 .balign 4
.popsection .popsection
.endm .endm
@ -153,14 +151,14 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) "\n\t" #define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t"
#define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD(func)
#define STACK_FRAME_NON_STANDARD_FP(func) #define STACK_FRAME_NON_STANDARD_FP(func)
#define ANNOTATE_NOENDBR #define ANNOTATE_NOENDBR
#define ASM_REACHABLE #define ASM_REACHABLE
#else #else
#define ANNOTATE_INTRA_FUNCTION_CALL #define ANNOTATE_INTRA_FUNCTION_CALL
.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
.endm .endm
.macro STACK_FRAME_NON_STANDARD func:req .macro STACK_FRAME_NON_STANDARD func:req
.endm .endm

View File

@ -16,12 +16,18 @@ struct unwind_hint {
u8 sp_reg; u8 sp_reg;
u8 type; u8 type;
u8 signal; u8 signal;
u8 end;
}; };
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
/* /*
* UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in
* a truncated and unreliable stack unwind.
*
* UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before
* hitting user entry, boot code, or fork entry (when there are no pt_regs
* available).
*
* UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP
* (the caller's SP right before it made the call). Used for all callable * (the caller's SP right before it made the call). Used for all callable
* functions, i.e. all C code and all callable asm functions. * functions, i.e. all C code and all callable asm functions.
@ -32,17 +38,20 @@ struct unwind_hint {
* UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that
* sp_reg+sp_offset points to the iret return frame. * sp_reg+sp_offset points to the iret return frame.
* *
* UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function.
* Useful for code which doesn't have an ELF function annotation. * Useful for code which doesn't have an ELF function annotation.
* *
* UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain
* location so that it can be restored later.
*/ */
#define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_UNDEFINED 0
#define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_END_OF_STACK 1
#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 #define UNWIND_HINT_TYPE_CALL 2
#define UNWIND_HINT_TYPE_REGS 3
#define UNWIND_HINT_TYPE_REGS_PARTIAL 4
/* The below hint types don't have corresponding ORC types */ /* The below hint types don't have corresponding ORC types */
#define UNWIND_HINT_TYPE_FUNC 3 #define UNWIND_HINT_TYPE_FUNC 5
#define UNWIND_HINT_TYPE_SAVE 4 #define UNWIND_HINT_TYPE_SAVE 6
#define UNWIND_HINT_TYPE_RESTORE 5 #define UNWIND_HINT_TYPE_RESTORE 7
#endif /* _LINUX_OBJTOOL_TYPES_H */ #endif /* _LINUX_OBJTOOL_TYPES_H */

View File

@ -128,7 +128,7 @@ static int orc_sort_cmp(const void *_a, const void *_b)
* whitelisted .o files which didn't get objtool generation. * whitelisted .o files which didn't get objtool generation.
*/ */
orc_a = g_orc_table + (a - g_orc_ip_table); orc_a = g_orc_table + (a - g_orc_ip_table);
return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
} }
static void *sort_orctable(void *arg) static void *sort_orctable(void *arg)

View File

@ -39,9 +39,11 @@
#define ORC_REG_SP_INDIRECT 9 #define ORC_REG_SP_INDIRECT 9
#define ORC_REG_MAX 15 #define ORC_REG_MAX 15
#define ORC_TYPE_CALL 0 #define ORC_TYPE_UNDEFINED 0
#define ORC_TYPE_REGS 1 #define ORC_TYPE_END_OF_STACK 1
#define ORC_TYPE_REGS_PARTIAL 2 #define ORC_TYPE_CALL 2
#define ORC_TYPE_REGS 3
#define ORC_TYPE_REGS_PARTIAL 4
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <asm/byteorder.h> #include <asm/byteorder.h>
@ -60,16 +62,14 @@ struct orc_entry {
#if defined(__LITTLE_ENDIAN_BITFIELD) #if defined(__LITTLE_ENDIAN_BITFIELD)
unsigned sp_reg:4; unsigned sp_reg:4;
unsigned bp_reg:4; unsigned bp_reg:4;
unsigned type:2; unsigned type:3;
unsigned signal:1; unsigned signal:1;
unsigned end:1;
#elif defined(__BIG_ENDIAN_BITFIELD) #elif defined(__BIG_ENDIAN_BITFIELD)
unsigned bp_reg:4; unsigned bp_reg:4;
unsigned sp_reg:4; unsigned sp_reg:4;
unsigned unused:4; unsigned unused:4;
unsigned end:1;
unsigned signal:1; unsigned signal:1;
unsigned type:2; unsigned type:3;
#endif #endif
} __packed; } __packed;

View File

@ -16,12 +16,18 @@ struct unwind_hint {
u8 sp_reg; u8 sp_reg;
u8 type; u8 type;
u8 signal; u8 signal;
u8 end;
}; };
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
/* /*
* UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in
* a truncated and unreliable stack unwind.
*
* UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before
* hitting user entry, boot code, or fork entry (when there are no pt_regs
* available).
*
* UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP
* (the caller's SP right before it made the call). Used for all callable * (the caller's SP right before it made the call). Used for all callable
* functions, i.e. all C code and all callable asm functions. * functions, i.e. all C code and all callable asm functions.
@ -32,17 +38,20 @@ struct unwind_hint {
* UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that
* sp_reg+sp_offset points to the iret return frame. * sp_reg+sp_offset points to the iret return frame.
* *
* UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function.
* Useful for code which doesn't have an ELF function annotation. * Useful for code which doesn't have an ELF function annotation.
* *
* UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain
* location so that it can be restored later.
*/ */
#define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_UNDEFINED 0
#define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_END_OF_STACK 1
#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 #define UNWIND_HINT_TYPE_CALL 2
#define UNWIND_HINT_TYPE_REGS 3
#define UNWIND_HINT_TYPE_REGS_PARTIAL 4
/* The below hint types don't have corresponding ORC types */ /* The below hint types don't have corresponding ORC types */
#define UNWIND_HINT_TYPE_FUNC 3 #define UNWIND_HINT_TYPE_FUNC 5
#define UNWIND_HINT_TYPE_SAVE 4 #define UNWIND_HINT_TYPE_SAVE 6
#define UNWIND_HINT_TYPE_RESTORE 5 #define UNWIND_HINT_TYPE_RESTORE 7
#endif /* _LINUX_OBJTOOL_TYPES_H */ #endif /* _LINUX_OBJTOOL_TYPES_H */

View File

@ -2243,6 +2243,7 @@ static void set_func_state(struct cfi_state *state)
memcpy(&state->regs, &initial_func_cfi.regs, memcpy(&state->regs, &initial_func_cfi.regs,
CFI_NUM_REGS * sizeof(struct cfi_reg)); CFI_NUM_REGS * sizeof(struct cfi_reg));
state->stack_size = initial_func_cfi.cfa.offset; state->stack_size = initial_func_cfi.cfa.offset;
state->type = UNWIND_HINT_TYPE_CALL;
} }
static int read_unwind_hints(struct objtool_file *file) static int read_unwind_hints(struct objtool_file *file)
@ -2327,7 +2328,6 @@ static int read_unwind_hints(struct objtool_file *file)
cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
cfi.type = hint->type; cfi.type = hint->type;
cfi.signal = hint->signal; cfi.signal = hint->signal;
cfi.end = hint->end;
insn->cfi = cfi_hash_find_or_add(&cfi); insn->cfi = cfi_hash_find_or_add(&cfi);
} }

View File

@ -38,6 +38,10 @@ static const char *reg_name(unsigned int reg)
static const char *orc_type_name(unsigned int type) static const char *orc_type_name(unsigned int type)
{ {
switch (type) { switch (type) {
case ORC_TYPE_UNDEFINED:
return "(und)";
case ORC_TYPE_END_OF_STACK:
return "end";
case ORC_TYPE_CALL: case ORC_TYPE_CALL:
return "call"; return "call";
case ORC_TYPE_REGS: case ORC_TYPE_REGS:
@ -201,6 +205,7 @@ int orc_dump(const char *_objname)
printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i])); printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i]));
} }
printf("type:%s", orc_type_name(orc[i].type));
printf(" sp:"); printf(" sp:");
@ -210,8 +215,7 @@ int orc_dump(const char *_objname)
print_reg(orc[i].bp_reg, bswap_if_needed(&dummy_elf, orc[i].bp_offset)); print_reg(orc[i].bp_reg, bswap_if_needed(&dummy_elf, orc[i].bp_offset));
printf(" type:%s signal:%d end:%d\n", printf(" signal:%d\n", orc[i].signal);
orc_type_name(orc[i].type), orc[i].signal, orc[i].end);
} }
elf_end(elf); elf_end(elf);

View File

@ -21,12 +21,22 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi,
memset(orc, 0, sizeof(*orc)); memset(orc, 0, sizeof(*orc));
if (!cfi) { if (!cfi) {
orc->end = 0; /*
orc->sp_reg = ORC_REG_UNDEFINED; * This is usually either unreachable nops/traps (which don't
* trigger unreachable instruction warnings), or
* STACK_FRAME_NON_STANDARD functions.
*/
orc->type = ORC_TYPE_UNDEFINED;
return 0; return 0;
} }
switch (cfi->type) { switch (cfi->type) {
case UNWIND_HINT_TYPE_UNDEFINED:
orc->type = ORC_TYPE_UNDEFINED;
return 0;
case UNWIND_HINT_TYPE_END_OF_STACK:
orc->type = ORC_TYPE_END_OF_STACK;
return 0;
case UNWIND_HINT_TYPE_CALL: case UNWIND_HINT_TYPE_CALL:
orc->type = ORC_TYPE_CALL; orc->type = ORC_TYPE_CALL;
break; break;
@ -42,14 +52,8 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi,
return -1; return -1;
} }
orc->end = cfi->end;
orc->signal = cfi->signal; orc->signal = cfi->signal;
if (cfi->cfa.base == CFI_UNDEFINED) {
orc->sp_reg = ORC_REG_UNDEFINED;
return 0;
}
switch (cfi->cfa.base) { switch (cfi->cfa.base) {
case CFI_SP: case CFI_SP:
orc->sp_reg = ORC_REG_SP; orc->sp_reg = ORC_REG_SP;
@ -163,11 +167,7 @@ int orc_create(struct objtool_file *file)
struct orc_list_entry *entry; struct orc_list_entry *entry;
struct list_head orc_list; struct list_head orc_list;
struct orc_entry null = { struct orc_entry null = { .type = ORC_TYPE_UNDEFINED };
.sp_reg = ORC_REG_UNDEFINED,
.bp_reg = ORC_REG_UNDEFINED,
.type = ORC_TYPE_CALL,
};
/* Build a deduplicated list of ORC entries: */ /* Build a deduplicated list of ORC entries: */
INIT_LIST_HEAD(&orc_list); INIT_LIST_HEAD(&orc_list);