Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-10-23 13:40:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

function_graph: Use static_call and branch to optimize return function

Browse Source 
In most cases function graph is used by a single user. Instead of calling
a loop to call function graph callbacks in this case, call the function
return callback directly.

Use the static_key that is set when the function graph tracer has less
than 2 callbacks registered. It will do the direct call in that case, and
will do the loop over all callers when there are 2 or more callbacks
registered.

Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.921460797@goodmis.org

Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Florent Revest <revest@chromium.org>
Cc: Martin KaFai Lau <martin.lau@linux.dev>
Cc: bpf <bpf@vger.kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Alan Maguire <alan.maguire@oracle.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Guo Ren <guoren@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
This commit is contained in:
Steven Rostedt (Google) 2024-06-03 15:07:29 -04:00
parent cc60ee813b
commit fe835e3ca4
1 changed files with 18 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
kernel/trace/fgraph.c
View File

@ -514,6 +514,7 @@ static struct fgraph_ops fgraph_stub = {
static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub;
DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub);
DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub);
DEFINE_STATIC_KEY_TRUE(fgraph_do_direct);
/**
@ -808,13 +809,21 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs
bitmap = get_bitmap_bits(current, offset);
for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) {
struct fgraph_ops *gops = fgraph_array[i];
#ifdef CONFIG_HAVE_STATIC_CALL
if (static_branch_likely(&fgraph_do_direct)) {
if (test_bit(fgraph_direct_gops->idx, &bitmap))
static_call(fgraph_retfunc)(&trace, fgraph_direct_gops);
} else
#endif
{
for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) {
struct fgraph_ops *gops = fgraph_array[i];
if (gops == &fgraph_stub)
continue;
if (gops == &fgraph_stub)
continue;
gops->retfunc(&trace, gops);
gops->retfunc(&trace, gops);
}
}
/*
@ -1232,17 +1241,20 @@ static void init_task_vars(int idx)
static void ftrace_graph_enable_direct(bool enable_branch)
{
trace_func_graph_ent_t func = NULL;
trace_func_graph_ret_t retfunc = NULL;
int i;
for_each_set_bit(i, &fgraph_array_bitmask,
sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) {
func = fgraph_array[i]->entryfunc;
retfunc = fgraph_array[i]->retfunc;
fgraph_direct_gops = fgraph_array[i];
}
if (WARN_ON_ONCE(!func))
return;
static_call_update(fgraph_func, func);
static_call_update(fgraph_retfunc, retfunc);
if (enable_branch)
static_branch_disable(&fgraph_do_direct);
}
@ -1252,6 +1264,7 @@ static void ftrace_graph_disable_direct(bool disable_branch)
if (disable_branch)
static_branch_disable(&fgraph_do_direct);
static_call_update(fgraph_func, ftrace_graph_entry_stub);
static_call_update(fgraph_retfunc, ftrace_graph_ret_stub);
fgraph_direct_gops = &fgraph_stub;
}