mirror of
https://github.com/torvalds/linux.git
synced 2024-11-29 15:41:36 +00:00
2aa6ac0351
This builds up on the CALL_OPS work which extends the ftrace patchsite
on arm64 with an ops pointer usable by the ftrace trampoline.
This ops pointer is valid at all time. Indeed, it is either pointing to
ftrace_list_ops or to the single ops which should be called from that
patchsite.
There are a few cases to distinguish:
- If a direct call ops is the only one tracing a function:
- If the direct called trampoline is within the reach of a BL
instruction
-> the ftrace patchsite jumps to the trampoline
- Else
-> the ftrace patchsite jumps to the ftrace_caller trampoline which
reads the ops pointer in the patchsite and jumps to the direct
call address stored in the ops
- Else
-> the ftrace patchsite jumps to the ftrace_caller trampoline and its
ops literal points to ftrace_list_ops so it iterates over all
registered ftrace ops, including the direct call ops and calls its
call_direct_funcs handler which stores the direct called
trampoline's address in the ftrace_regs and the ftrace_caller
trampoline will return to that address instead of returning to the
traced function
Signed-off-by: Florent Revest <revest@chromium.org>
Co-developed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20230405180250.2046566-2-revest@chromium.org
Signed-off-by: Will Deacon <will@kernel.org>
353 lines
9.7 KiB
ArmAsm
353 lines
9.7 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* arch/arm64/kernel/entry-ftrace.S
|
|
*
|
|
* Copyright (C) 2013 Linaro Limited
|
|
* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <linux/cfi_types.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/assembler.h>
|
|
#include <asm/ftrace.h>
|
|
#include <asm/insn.h>
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
|
|
/*
|
|
* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
|
|
* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
|
|
* ftrace_make_call() have patched those NOPs to:
|
|
*
|
|
* MOV X9, LR
|
|
* BL ftrace_caller
|
|
*
|
|
* Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
|
|
* live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
|
|
* clobber.
|
|
*
|
|
* We save the callsite's context into a struct ftrace_regs before invoking any
|
|
* ftrace callbacks. So that we can get a sensible backtrace, we create frame
|
|
* records for the callsite and the ftrace entry assembly. This is not
|
|
* sufficient for reliable stacktrace: until we create the callsite stack
|
|
* record, its caller is missing from the LR and existing chain of frame
|
|
* records.
|
|
*/
|
|
SYM_CODE_START(ftrace_caller)
|
|
bti c
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
|
|
/*
|
|
* The literal pointer to the ops is at an 8-byte aligned boundary
|
|
* which is either 12 or 16 bytes before the BL instruction in the call
|
|
* site. See ftrace_call_adjust() for details.
|
|
*
|
|
* Therefore here the LR points at `literal + 16` or `literal + 20`,
|
|
* and we can find the address of the literal in either case by
|
|
* aligning to an 8-byte boundary and subtracting 16. We do the
|
|
* alignment first as this allows us to fold the subtraction into the
|
|
* LDR.
|
|
*/
|
|
bic x11, x30, 0x7
|
|
ldr x11, [x11, #-(4 * AARCH64_INSN_SIZE)] // op
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
|
/*
|
|
* If the op has a direct call, handle it immediately without
|
|
* saving/restoring registers.
|
|
*/
|
|
ldr x17, [x11, #FTRACE_OPS_DIRECT_CALL] // op->direct_call
|
|
cbnz x17, ftrace_caller_direct
|
|
#endif
|
|
#endif
|
|
|
|
/* Save original SP */
|
|
mov x10, sp
|
|
|
|
/* Make room for ftrace regs, plus two frame records */
|
|
sub sp, sp, #(FREGS_SIZE + 32)
|
|
|
|
/* Save function arguments */
|
|
stp x0, x1, [sp, #FREGS_X0]
|
|
stp x2, x3, [sp, #FREGS_X2]
|
|
stp x4, x5, [sp, #FREGS_X4]
|
|
stp x6, x7, [sp, #FREGS_X6]
|
|
str x8, [sp, #FREGS_X8]
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
|
str xzr, [sp, #FREGS_DIRECT_TRAMP]
|
|
#endif
|
|
|
|
/* Save the callsite's FP, LR, SP */
|
|
str x29, [sp, #FREGS_FP]
|
|
str x9, [sp, #FREGS_LR]
|
|
str x10, [sp, #FREGS_SP]
|
|
|
|
/* Save the PC after the ftrace callsite */
|
|
str x30, [sp, #FREGS_PC]
|
|
|
|
/* Create a frame record for the callsite above the ftrace regs */
|
|
stp x29, x9, [sp, #FREGS_SIZE + 16]
|
|
add x29, sp, #FREGS_SIZE + 16
|
|
|
|
/* Create our frame record above the ftrace regs */
|
|
stp x29, x30, [sp, #FREGS_SIZE]
|
|
add x29, sp, #FREGS_SIZE
|
|
|
|
/* Prepare arguments for the the tracer func */
|
|
sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
|
|
mov x1, x9 // parent_ip (callsite's LR)
|
|
mov x3, sp // regs
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
|
|
mov x2, x11 // op
|
|
ldr x4, [x2, #FTRACE_OPS_FUNC] // op->func
|
|
blr x4 // op->func(ip, parent_ip, op, regs)
|
|
|
|
#else
|
|
ldr_l x2, function_trace_op // op
|
|
|
|
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
|
|
bl ftrace_stub // func(ip, parent_ip, op, regs)
|
|
#endif
|
|
|
|
/*
|
|
* At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
|
|
* x19-x29 per the AAPCS, and we created frame records upon entry, so we need
|
|
* to restore x0-x8, x29, and x30.
|
|
*/
|
|
/* Restore function arguments */
|
|
ldp x0, x1, [sp, #FREGS_X0]
|
|
ldp x2, x3, [sp, #FREGS_X2]
|
|
ldp x4, x5, [sp, #FREGS_X4]
|
|
ldp x6, x7, [sp, #FREGS_X6]
|
|
ldr x8, [sp, #FREGS_X8]
|
|
|
|
/* Restore the callsite's FP */
|
|
ldr x29, [sp, #FREGS_FP]
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
|
ldr x17, [sp, #FREGS_DIRECT_TRAMP]
|
|
cbnz x17, ftrace_caller_direct_late
|
|
#endif
|
|
|
|
/* Restore the callsite's LR and PC */
|
|
ldr x30, [sp, #FREGS_LR]
|
|
ldr x9, [sp, #FREGS_PC]
|
|
|
|
/* Restore the callsite's SP */
|
|
add sp, sp, #FREGS_SIZE + 32
|
|
|
|
ret x9
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
|
SYM_INNER_LABEL(ftrace_caller_direct_late, SYM_L_LOCAL)
|
|
/*
|
|
* Head to a direct trampoline in x17 after having run other tracers.
|
|
* The ftrace_regs are live, and x0-x8 and FP have been restored. The
|
|
* LR, PC, and SP have not been restored.
|
|
*/
|
|
|
|
/*
|
|
* Restore the callsite's LR and PC matching the trampoline calling
|
|
* convention.
|
|
*/
|
|
ldr x9, [sp, #FREGS_LR]
|
|
ldr x30, [sp, #FREGS_PC]
|
|
|
|
/* Restore the callsite's SP */
|
|
add sp, sp, #FREGS_SIZE + 32
|
|
|
|
SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL)
|
|
/*
|
|
* Head to a direct trampoline in x17.
|
|
*
|
|
* We use `BR X17` as this can safely land on a `BTI C` or `PACIASP` in
|
|
* the trampoline, and will not unbalance any return stack.
|
|
*/
|
|
br x17
|
|
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
|
SYM_CODE_END(ftrace_caller)
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
|
SYM_CODE_START(ftrace_stub_direct_tramp)
|
|
bti c
|
|
mov x10, x30
|
|
mov x30, x9
|
|
ret x10
|
|
SYM_CODE_END(ftrace_stub_direct_tramp)
|
|
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
|
|
|
#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
|
|
|
|
/*
|
|
* Gcc with -pg will put the following code in the beginning of each function:
|
|
* mov x0, x30
|
|
* bl _mcount
|
|
* [function's body ...]
|
|
* "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
|
|
* ftrace is enabled.
|
|
*
|
|
* Please note that x0 as an argument will not be used here because we can
|
|
* get lr(x30) of instrumented function at any time by winding up call stack
|
|
* as long as the kernel is compiled without -fomit-frame-pointer.
|
|
* (or CONFIG_FRAME_POINTER, this is forced on arm64)
|
|
*
|
|
* stack layout after mcount_enter in _mcount():
|
|
*
|
|
* current sp/fp => 0:+-----+
|
|
* in _mcount() | x29 | -> instrumented function's fp
|
|
* +-----+
|
|
* | x30 | -> _mcount()'s lr (= instrumented function's pc)
|
|
* old sp => +16:+-----+
|
|
* when instrumented | |
|
|
* function calls | ... |
|
|
* _mcount() | |
|
|
* | |
|
|
* instrumented => +xx:+-----+
|
|
* function's fp | x29 | -> parent's fp
|
|
* +-----+
|
|
* | x30 | -> instrumented function's lr (= parent's pc)
|
|
* +-----+
|
|
* | ... |
|
|
*/
|
|
|
|
.macro mcount_enter
|
|
stp x29, x30, [sp, #-16]!
|
|
mov x29, sp
|
|
.endm
|
|
|
|
.macro mcount_exit
|
|
ldp x29, x30, [sp], #16
|
|
ret
|
|
.endm
|
|
|
|
.macro mcount_adjust_addr rd, rn
|
|
sub \rd, \rn, #AARCH64_INSN_SIZE
|
|
.endm
|
|
|
|
/* for instrumented function's parent */
|
|
.macro mcount_get_parent_fp reg
|
|
ldr \reg, [x29]
|
|
ldr \reg, [\reg]
|
|
.endm
|
|
|
|
/* for instrumented function */
|
|
.macro mcount_get_pc0 reg
|
|
mcount_adjust_addr \reg, x30
|
|
.endm
|
|
|
|
.macro mcount_get_pc reg
|
|
ldr \reg, [x29, #8]
|
|
mcount_adjust_addr \reg, \reg
|
|
.endm
|
|
|
|
.macro mcount_get_lr reg
|
|
ldr \reg, [x29]
|
|
ldr \reg, [\reg, #8]
|
|
.endm
|
|
|
|
.macro mcount_get_lr_addr reg
|
|
ldr \reg, [x29]
|
|
add \reg, \reg, #8
|
|
.endm
|
|
|
|
/*
|
|
* _mcount() is used to build the kernel with -pg option, but all the branch
|
|
* instructions to _mcount() are replaced to NOP initially at kernel start up,
|
|
* and later on, NOP to branch to ftrace_caller() when enabled or branch to
|
|
* NOP when disabled per-function base.
|
|
*/
|
|
SYM_FUNC_START(_mcount)
|
|
ret
|
|
SYM_FUNC_END(_mcount)
|
|
EXPORT_SYMBOL(_mcount)
|
|
NOKPROBE(_mcount)
|
|
|
|
/*
|
|
* void ftrace_caller(unsigned long return_address)
|
|
* @return_address: return address to instrumented function
|
|
*
|
|
* This function is a counterpart of _mcount() in 'static' ftrace, and
|
|
* makes calls to:
|
|
* - tracer function to probe instrumented function's entry,
|
|
* - ftrace_graph_caller to set up an exit hook
|
|
*/
|
|
SYM_FUNC_START(ftrace_caller)
|
|
mcount_enter
|
|
|
|
mcount_get_pc0 x0 // function's pc
|
|
mcount_get_lr x1 // function's lr
|
|
|
|
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr);
|
|
nop // This will be replaced with "bl xxx"
|
|
// where xxx can be any kind of tracer.
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
|
|
nop // If enabled, this will be replaced
|
|
// "b ftrace_graph_caller"
|
|
#endif
|
|
|
|
mcount_exit
|
|
SYM_FUNC_END(ftrace_caller)
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
/*
|
|
* void ftrace_graph_caller(void)
|
|
*
|
|
* Called from _mcount() or ftrace_caller() when function_graph tracer is
|
|
* selected.
|
|
* This function w/ prepare_ftrace_return() fakes link register's value on
|
|
* the call stack in order to intercept instrumented function's return path
|
|
* and run return_to_handler() later on its exit.
|
|
*/
|
|
SYM_FUNC_START(ftrace_graph_caller)
|
|
mcount_get_pc x0 // function's pc
|
|
mcount_get_lr_addr x1 // pointer to function's saved lr
|
|
mcount_get_parent_fp x2 // parent's fp
|
|
bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp)
|
|
|
|
mcount_exit
|
|
SYM_FUNC_END(ftrace_graph_caller)
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|
|
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
|
|
|
|
SYM_TYPED_FUNC_START(ftrace_stub)
|
|
ret
|
|
SYM_FUNC_END(ftrace_stub)
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
SYM_TYPED_FUNC_START(ftrace_stub_graph)
|
|
ret
|
|
SYM_FUNC_END(ftrace_stub_graph)
|
|
|
|
/*
|
|
* void return_to_handler(void)
|
|
*
|
|
* Run ftrace_return_to_handler() before going back to parent.
|
|
* @fp is checked against the value passed by ftrace_graph_caller().
|
|
*/
|
|
SYM_CODE_START(return_to_handler)
|
|
/* save return value regs */
|
|
sub sp, sp, #64
|
|
stp x0, x1, [sp]
|
|
stp x2, x3, [sp, #16]
|
|
stp x4, x5, [sp, #32]
|
|
stp x6, x7, [sp, #48]
|
|
|
|
mov x0, x29 // parent's fp
|
|
bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
|
|
mov x30, x0 // restore the original return address
|
|
|
|
/* restore return value regs */
|
|
ldp x0, x1, [sp]
|
|
ldp x2, x3, [sp, #16]
|
|
ldp x4, x5, [sp, #32]
|
|
ldp x6, x7, [sp, #48]
|
|
add sp, sp, #64
|
|
|
|
ret
|
|
SYM_CODE_END(return_to_handler)
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|