perf/core improvements and fixes:

User visible fixes:
 
 - Fallback to kallsyms when using the minimal 'ELF' loader (Arnaldo Carvalho de Melo)
 
 - Fix annotation with kcore (Adrian Hunter)
 
 - Fix up srcline histogram key formatting (Arnaldo Carvalho de Melo)
 
 - Add missing handler for PERF_RECORD_MMAP2 events in 'perf diff' (Kan Liang)
 
 User visible changes/new features:
 
 - Only print base source file for srcline histogram sort key (Andi Kleen)
 
 - Support source line numbers in annotate using a hotkey (Andi Kleen)
 
 Infrastructure:
 
 - Do not poll events that use the system_wide flag (Adrian Hunter)
 
 - Add perf-read-vdso32 and perf-read-vdsox32 to .gitignore (Adrian Hunter)
 
   perf tools: Only override the default :tid comm entry (Adrian Hunter)
 
 - Factor out adding new call chain entries (Andi Kleen)
 
 - Use al.addr to set up call chain (Andi Kleen)
 
 - Use a common function to resolve symbol or name (Andi Kleen)
 
 - Fix ftrace:function event recording (Jiri Olsa)
 
 - Move disable_buildid_cache() to util/build-id.c (Namhyung Kim)
 
 - Clean up libelf feature support code (Namhyung Kim)
 
 - fix typo in python 'perf test' (WANG Chao)
 
 Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
 iQEcBAABAgAGBQJUbL59AAoJEBpxZoYYoA71W84H+QHPIOA4dVb+HkP248yibgul
 OrkFmT7QCVfnKFTW1eOdVoXGDquzuC7svxT7ePAAjlIb/0JeKXCopQOPPs0o2PTb
 0Gv53hAD7Y3dxCq/PS0Nn6pBw/4ZKk3e/IZGIHoVUxiZjoAd1tFPjiljJgkrsj9F
 jR4cYmNshOkPV6ltGRBXGvJe9X0slGqxrX0z3gOxf4+NiZNclV6LhU5GR+Bx/Q0H
 k7VR0zUihBkG0W2mbdoJsFipUcQM+wJKAUo5iu1a1o64e8yhwiGwuHzBKe4xzHxy
 6Xpq26TeP949/r+AIPNVjIH3a4Jn53lu0LrPGF/LCE4EcdQfNi8IX+6plXENr/A=
 =sNl0
 -----END PGP SIGNATURE-----

Merge tag 'perf-core-for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux into perf/core

Pull perf/core improvements and fixes from Arnaldo Carvalho de Melo:

User visible fixes:

 - Fallback to kallsyms when using the minimal 'ELF' loader (Arnaldo Carvalho de Melo)

 - Fix annotation with kcore (Adrian Hunter)

 - Fix up srcline histogram key formatting (Arnaldo Carvalho de Melo)

 - Add missing handler for PERF_RECORD_MMAP2 events in 'perf diff' (Kan Liang)

User visible changes/new features:

 - Only print base source file for srcline histogram sort key (Andi Kleen)

 - Support source line numbers in annotate using a hotkey (Andi Kleen)

Infrastructure changes and fixes:

 - Do not poll events that use the system_wide flag (Adrian Hunter)

 - Add perf-read-vdso32 and perf-read-vdsox32 to .gitignore (Adrian Hunter)

 - Only override the default :tid comm entry (Adrian Hunter)

 - Factor out adding new call chain entries (Andi Kleen)

 - Use al.addr to set up call chain (Andi Kleen)

 - Use a common function to resolve symbol or name (Andi Kleen)

 - Fix ftrace:function event recording (Jiri Olsa)

 - Move disable_buildid_cache() to util/build-id.c (Namhyung Kim)

 - Clean up libelf feature support code (Namhyung Kim)

 - Fix typo in python 'perf test' (WANG Chao)

Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Ingo Molnar 2014-11-20 08:32:01 +01:00
commit 4e6e311e59
24 changed files with 145 additions and 88 deletions

View File

@ -2,6 +2,8 @@ PERF-CFLAGS
PERF-GUI-VARS
PERF-VERSION-FILE
perf
perf-read-vdso32
perf-read-vdsox32
perf-help
perf-record
perf-report

View File

@ -497,8 +497,6 @@ ifneq ($(OUTPUT),)
endif
ifdef NO_LIBELF
EXTLIBS := $(filter-out -lelf,$(EXTLIBS))
# Remove ELF/DWARF dependent codes
LIB_OBJS := $(filter-out $(OUTPUT)util/symbol-elf.o,$(LIB_OBJS))
LIB_OBJS := $(filter-out $(OUTPUT)util/dwarf-aux.o,$(LIB_OBJS))

View File

@ -357,6 +357,7 @@ static int diff__process_sample_event(struct perf_tool *tool __maybe_unused,
static struct perf_tool tool = {
.sample = diff__process_sample_event,
.mmap = perf_event__process_mmap,
.mmap2 = perf_event__process_mmap2,
.comm = perf_event__process_comm,
.exit = perf_event__process_exit,
.fork = perf_event__process_fork,

View File

@ -150,7 +150,7 @@ CFLAGS += -std=gnu99
# adding assembler files missing the .GNU-stack linker note.
LDFLAGS += -Wl,-z,noexecstack
EXTLIBS = -lelf -lpthread -lrt -lm -ldl
EXTLIBS = -lpthread -lrt -lm -ldl
ifneq ($(OUTPUT),)
OUTPUT_FEATURES = $(OUTPUT)config/feature-checks/
@ -354,6 +354,7 @@ endif # NO_LIBELF
ifndef NO_LIBELF
CFLAGS += -DHAVE_LIBELF_SUPPORT
EXTLIBS += -lelf
ifeq ($(feature-libelf-mmap), 1)
CFLAGS += -DHAVE_LIBELF_MMAP_SUPPORT
@ -373,7 +374,7 @@ ifndef NO_LIBELF
else
CFLAGS += -DHAVE_DWARF_SUPPORT $(LIBDW_CFLAGS)
LDFLAGS += $(LIBDW_LDFLAGS)
EXTLIBS += -lelf -ldw
EXTLIBS += -ldw
endif # PERF_HAVE_DWARF_REGS
endif # NO_DWARF
endif # NO_LIBELF

View File

@ -85,7 +85,7 @@ static struct test {
.func = test__hists_link,
},
{
.desc = "Try 'use perf' in python, checking link problems",
.desc = "Try 'import perf' in python, checking link problems",
.func = test__python_use,
},
{

View File

@ -27,6 +27,7 @@ static struct annotate_browser_opt {
bool hide_src_code,
use_offset,
jump_arrows,
show_linenr,
show_nr_jumps;
} annotate_browser__opts = {
.use_offset = true,
@ -128,7 +129,11 @@ static void annotate_browser__write(struct ui_browser *browser, void *entry, int
if (!*dl->line)
slsmg_write_nstring(" ", width - pcnt_width);
else if (dl->offset == -1) {
printed = scnprintf(bf, sizeof(bf), "%*s ",
if (dl->line_nr && annotate_browser__opts.show_linenr)
printed = scnprintf(bf, sizeof(bf), "%-*d ",
ab->addr_width + 1, dl->line_nr);
else
printed = scnprintf(bf, sizeof(bf), "%*s ",
ab->addr_width, " ");
slsmg_write_nstring(bf, printed);
slsmg_write_nstring(dl->line, width - printed - pcnt_width + 1);
@ -733,6 +738,7 @@ static int annotate_browser__run(struct annotate_browser *browser,
"o Toggle disassembler output/simplified view\n"
"s Toggle source code view\n"
"/ Search string\n"
"k Toggle line numbers\n"
"r Run available scripts\n"
"? Search string backwards\n");
continue;
@ -741,6 +747,10 @@ static int annotate_browser__run(struct annotate_browser *browser,
script_browse(NULL);
continue;
}
case 'k':
annotate_browser__opts.show_linenr =
!annotate_browser__opts.show_linenr;
break;
case 'H':
nd = browser->curr_hot;
break;
@ -984,6 +994,7 @@ static struct annotate_config {
} annotate__configs[] = {
ANNOTATE_CFG(hide_src_code),
ANNOTATE_CFG(jump_arrows),
ANNOTATE_CFG(show_linenr),
ANNOTATE_CFG(show_nr_jumps),
ANNOTATE_CFG(use_offset),
};

View File

@ -463,23 +463,6 @@ out:
return key;
}
static char *callchain_list__sym_name(struct callchain_list *cl,
char *bf, size_t bfsize, bool show_dso)
{
int printed;
if (cl->ms.sym)
printed = scnprintf(bf, bfsize, "%s", cl->ms.sym->name);
else
printed = scnprintf(bf, bfsize, "%#" PRIx64, cl->ip);
if (show_dso)
scnprintf(bf + printed, bfsize - printed, " %s",
cl->ms.map ? cl->ms.map->dso->short_name : "unknown");
return bf;
}
struct callchain_print_arg {
/* for hists browser */
off_t row_offset;

View File

@ -89,15 +89,6 @@ void perf_gtk__init_hpp(void)
perf_gtk__hpp_color_overhead_acc;
}
static void callchain_list__sym_name(struct callchain_list *cl,
char *bf, size_t bfsize)
{
if (cl->ms.sym)
scnprintf(bf, bfsize, "%s", cl->ms.sym->name);
else
scnprintf(bf, bfsize, "%#" PRIx64, cl->ip);
}
static void perf_gtk__add_callchain(struct rb_root *root, GtkTreeStore *store,
GtkTreeIter *parent, int col, u64 total)
{
@ -128,7 +119,7 @@ static void perf_gtk__add_callchain(struct rb_root *root, GtkTreeStore *store,
scnprintf(buf, sizeof(buf), "%5.2f%%", percent);
gtk_tree_store_set(store, &iter, 0, buf, -1);
callchain_list__sym_name(chain, buf, sizeof(buf));
callchain_list__sym_name(chain, buf, sizeof(buf), false);
gtk_tree_store_set(store, &iter, col, buf, -1);
if (need_new_parent) {

View File

@ -41,6 +41,7 @@ static size_t ipchain__fprintf_graph(FILE *fp, struct callchain_list *chain,
{
int i;
size_t ret = 0;
char bf[1024];
ret += callchain__fprintf_left_margin(fp, left_margin);
for (i = 0; i < depth; i++) {
@ -56,11 +57,8 @@ static size_t ipchain__fprintf_graph(FILE *fp, struct callchain_list *chain,
} else
ret += fprintf(fp, "%s", " ");
}
if (chain->ms.sym)
ret += fprintf(fp, "%s\n", chain->ms.sym->name);
else
ret += fprintf(fp, "0x%0" PRIx64 "\n", chain->ip);
fputs(callchain_list__sym_name(chain, bf, sizeof(bf), false), fp);
fputc('\n', fp);
return ret;
}
@ -168,6 +166,7 @@ static size_t callchain__fprintf_graph(FILE *fp, struct rb_root *root,
struct rb_node *node;
int i = 0;
int ret = 0;
char bf[1024];
/*
* If have one single callchain root, don't bother printing
@ -196,10 +195,8 @@ static size_t callchain__fprintf_graph(FILE *fp, struct rb_root *root,
} else
ret += callchain__fprintf_left_margin(fp, left_margin);
if (chain->ms.sym)
ret += fprintf(fp, " %s\n", chain->ms.sym->name);
else
ret += fprintf(fp, " %p\n", (void *)(long)chain->ip);
ret += fprintf(fp, "%s\n", callchain_list__sym_name(chain, bf, sizeof(bf),
false));
if (++entries_printed == callchain_param.print_limit)
break;
@ -219,6 +216,7 @@ static size_t __callchain__fprintf_flat(FILE *fp, struct callchain_node *node,
{
struct callchain_list *chain;
size_t ret = 0;
char bf[1024];
if (!node)
return 0;
@ -229,11 +227,8 @@ static size_t __callchain__fprintf_flat(FILE *fp, struct callchain_node *node,
list_for_each_entry(chain, &node->val, list) {
if (chain->ip >= PERF_CONTEXT_MAX)
continue;
if (chain->ms.sym)
ret += fprintf(fp, " %s\n", chain->ms.sym->name);
else
ret += fprintf(fp, " %p\n",
(void *)(long)chain->ip);
ret += fprintf(fp, " %s\n", callchain_list__sym_name(chain,
bf, sizeof(bf), false));
}
return ret;

View File

@ -17,11 +17,13 @@
#include "debug.h"
#include "annotate.h"
#include "evsel.h"
#include <regex.h>
#include <pthread.h>
#include <linux/bitops.h>
const char *disassembler_style;
const char *objdump_path;
static regex_t file_lineno;
static struct ins *ins__find(const char *name);
static int disasm_line__parse(char *line, char **namep, char **rawp);
@ -570,13 +572,15 @@ out_free_name:
return -1;
}
static struct disasm_line *disasm_line__new(s64 offset, char *line, size_t privsize)
static struct disasm_line *disasm_line__new(s64 offset, char *line,
size_t privsize, int line_nr)
{
struct disasm_line *dl = zalloc(sizeof(*dl) + privsize);
if (dl != NULL) {
dl->offset = offset;
dl->line = strdup(line);
dl->line_nr = line_nr;
if (dl->line == NULL)
goto out_delete;
@ -788,13 +792,15 @@ static int disasm_line__print(struct disasm_line *dl, struct symbol *sym, u64 st
* The ops.raw part will be parsed further according to type of the instruction.
*/
static int symbol__parse_objdump_line(struct symbol *sym, struct map *map,
FILE *file, size_t privsize)
FILE *file, size_t privsize,
int *line_nr)
{
struct annotation *notes = symbol__annotation(sym);
struct disasm_line *dl;
char *line = NULL, *parsed_line, *tmp, *tmp2, *c;
size_t line_len;
s64 line_ip, offset = -1;
regmatch_t match[2];
if (getline(&line, &line_len, file) < 0)
return -1;
@ -812,6 +818,12 @@ static int symbol__parse_objdump_line(struct symbol *sym, struct map *map,
line_ip = -1;
parsed_line = line;
/* /filename:linenr ? Save line number and ignore. */
if (regexec(&file_lineno, line, 2, match, 0) == 0) {
*line_nr = atoi(line + match[1].rm_so);
return 0;
}
/*
* Strip leading spaces:
*/
@ -842,8 +854,9 @@ static int symbol__parse_objdump_line(struct symbol *sym, struct map *map,
parsed_line = tmp2 + 1;
}
dl = disasm_line__new(offset, parsed_line, privsize);
dl = disasm_line__new(offset, parsed_line, privsize, *line_nr);
free(line);
(*line_nr)++;
if (dl == NULL)
return -1;
@ -869,6 +882,11 @@ static int symbol__parse_objdump_line(struct symbol *sym, struct map *map,
return 0;
}
static __attribute__((constructor)) void symbol__init_regexpr(void)
{
regcomp(&file_lineno, "^/[^:]+:([0-9]+)", REG_EXTENDED);
}
static void delete_last_nop(struct symbol *sym)
{
struct annotation *notes = symbol__annotation(sym);
@ -904,6 +922,7 @@ int symbol__annotate(struct symbol *sym, struct map *map, size_t privsize)
char symfs_filename[PATH_MAX];
struct kcore_extract kce;
bool delete_extract = false;
int lineno = 0;
if (filename)
symbol__join_symfs(symfs_filename, filename);
@ -915,6 +934,8 @@ int symbol__annotate(struct symbol *sym, struct map *map, size_t privsize)
return -ENOMEM;
}
goto fallback;
} else if (dso__is_kcore(dso)) {
goto fallback;
} else if (readlink(symfs_filename, command, sizeof(command)) < 0 ||
strstr(command, "[kernel.kallsyms]") ||
access(symfs_filename, R_OK)) {
@ -982,7 +1003,7 @@ fallback:
snprintf(command, sizeof(command),
"%s %s%s --start-address=0x%016" PRIx64
" --stop-address=0x%016" PRIx64
" -d %s %s -C %s 2>/dev/null|grep -v %s|expand",
" -l -d %s %s -C %s 2>/dev/null|grep -v %s|expand",
objdump_path ? objdump_path : "objdump",
disassembler_style ? "-M " : "",
disassembler_style ? disassembler_style : "",
@ -999,7 +1020,8 @@ fallback:
goto out_free_filename;
while (!feof(file))
if (symbol__parse_objdump_line(sym, map, file, privsize) < 0)
if (symbol__parse_objdump_line(sym, map, file, privsize,
&lineno) < 0)
break;
/*

View File

@ -58,6 +58,7 @@ struct disasm_line {
char *line;
char *name;
struct ins *ins;
int line_nr;
struct ins_operands ops;
};

View File

@ -18,6 +18,9 @@
#include "header.h"
#include "vdso.h"
static bool no_buildid_cache;
int build_id__mark_dso_hit(struct perf_tool *tool __maybe_unused,
union perf_event *event,
struct perf_sample *sample,
@ -251,6 +254,11 @@ int dsos__hit_all(struct perf_session *session)
return 0;
}
void disable_buildid_cache(void)
{
no_buildid_cache = true;
}
int build_id_cache__add_s(const char *sbuild_id, const char *debugdir,
const char *name, bool is_kallsyms, bool is_vdso)
{
@ -404,6 +412,9 @@ int perf_session__cache_build_ids(struct perf_session *session)
int ret;
char debugdir[PATH_MAX];
if (no_buildid_cache)
return 0;
snprintf(debugdir, sizeof(debugdir), "%s", buildid_dir);
if (mkdir(debugdir, 0755) != 0 && errno != EEXIST)

View File

@ -25,5 +25,6 @@ int perf_session__cache_build_ids(struct perf_session *session);
int build_id_cache__add_s(const char *sbuild_id, const char *debugdir,
const char *name, bool is_kallsyms, bool is_vdso);
int build_id_cache__remove_s(const char *sbuild_id, const char *debugdir);
void disable_buildid_cache(void);
#endif

View File

@ -808,3 +808,22 @@ int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *
out:
return 1;
}
char *callchain_list__sym_name(struct callchain_list *cl,
char *bf, size_t bfsize, bool show_dso)
{
int printed;
if (cl->ms.sym) {
printed = scnprintf(bf, bfsize, "%s", cl->ms.sym->name);
} else
printed = scnprintf(bf, bfsize, "%#" PRIx64, cl->ip);
if (show_dso)
scnprintf(bf + printed, bfsize - printed, " %s",
cl->ms.map ?
cl->ms.map->dso->short_name :
"unknown");
return bf;
}

View File

@ -193,4 +193,7 @@ static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused,
}
#endif
char *callchain_list__sym_name(struct callchain_list *cl,
char *bf, size_t bfsize, bool show_dso);
#endif /* __PERF_CALLCHAIN_H */

View File

@ -816,7 +816,15 @@ static int perf_evlist__mmap_per_evsel(struct perf_evlist *evlist, int idx,
perf_evlist__mmap_get(evlist, idx);
}
if (__perf_evlist__add_pollfd(evlist, fd, idx) < 0) {
/*
* The system_wide flag causes a selected event to be opened
* always without a pid. Consequently it will never get a
* POLLHUP, but it is used for tracking in combination with
* other events, so it should not need to be polled anyway.
* Therefore don't add it for polling.
*/
if (!evsel->system_wide &&
__perf_evlist__add_pollfd(evlist, fd, idx) < 0) {
perf_evlist__mmap_put(evlist, idx);
return -1;
}

View File

@ -658,6 +658,14 @@ void perf_evsel__config(struct perf_evsel *evsel, struct record_opts *opts)
attr->mmap_data = track;
}
/*
* We don't allow user space callchains for function trace
* event, due to issues with page faults while tracing page
* fault handler and its overall trickiness nature.
*/
if (perf_evsel__is_function_event(evsel))
evsel->attr.exclude_callchain_user = 1;
if (callchain_param.enabled && !evsel->no_aux_samples)
perf_evsel__config_callgraph(evsel);

View File

@ -24,8 +24,6 @@
#include "build-id.h"
#include "data.h"
static bool no_buildid_cache = false;
static u32 header_argc;
static const char **header_argv;
@ -191,8 +189,7 @@ static int write_build_id(int fd, struct perf_header *h,
pr_debug("failed to write buildid table\n");
return err;
}
if (!no_buildid_cache)
perf_session__cache_build_ids(session);
perf_session__cache_build_ids(session);
return 0;
}
@ -2791,8 +2788,3 @@ int perf_event__process_build_id(struct perf_tool *tool __maybe_unused,
session);
return 0;
}
void disable_buildid_cache(void)
{
no_buildid_cache = true;
}

View File

@ -1381,6 +1381,34 @@ struct mem_info *sample__resolve_mem(struct perf_sample *sample,
return mi;
}
static int add_callchain_ip(struct thread *thread,
struct symbol **parent,
struct addr_location *root_al,
int cpumode,
u64 ip)
{
struct addr_location al;
al.filtered = 0;
al.sym = NULL;
thread__find_addr_location(thread, cpumode, MAP__FUNCTION,
ip, &al);
if (al.sym != NULL) {
if (sort__has_parent && !*parent &&
symbol__match_regex(al.sym, &parent_regex))
*parent = al.sym;
else if (have_ignore_callees && root_al &&
symbol__match_regex(al.sym, &ignore_callees_regex)) {
/* Treat this symbol as the root,
forgetting its callees. */
*root_al = al;
callchain_cursor_reset(&callchain_cursor);
}
}
return callchain_cursor_append(&callchain_cursor, al.addr, al.map, al.sym);
}
struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
struct addr_location *al)
{
@ -1427,7 +1455,6 @@ static int thread__resolve_callchain_sample(struct thread *thread,
for (i = 0; i < chain_nr; i++) {
u64 ip;
struct addr_location al;
if (callchain_param.order == ORDER_CALLEE)
j = i;
@ -1464,24 +1491,10 @@ static int thread__resolve_callchain_sample(struct thread *thread,
continue;
}
al.filtered = 0;
thread__find_addr_location(thread, cpumode,
MAP__FUNCTION, ip, &al);
if (al.sym != NULL) {
if (sort__has_parent && !*parent &&
symbol__match_regex(al.sym, &parent_regex))
*parent = al.sym;
else if (have_ignore_callees && root_al &&
symbol__match_regex(al.sym, &ignore_callees_regex)) {
/* Treat this symbol as the root,
forgetting its callees. */
*root_al = al;
callchain_cursor_reset(&callchain_cursor);
}
}
err = callchain_cursor_append(&callchain_cursor,
ip, al.map, al.sym);
err = add_callchain_ip(thread, parent, root_al,
cpumode, ip);
if (err == -EINVAL)
break;
if (err)
return err;
}

View File

@ -309,7 +309,7 @@ sort__srcline_cmp(struct hist_entry *left, struct hist_entry *right)
static int hist_entry__srcline_snprintf(struct hist_entry *he, char *bf,
size_t size, unsigned int width)
{
return repsep_snprintf(bf, size, "%*.*-s", width, width, he->srcline);
return repsep_snprintf(bf, size, "%-*.*s", width, width, he->srcline);
}
struct sort_entry sort_srcline = {

View File

@ -274,7 +274,7 @@ char *get_srcline(struct dso *dso, unsigned long addr)
if (!addr2line(dso_name, addr, &file, &line, dso))
goto out;
if (asprintf(&srcline, "%s:%u", file, line) < 0) {
if (asprintf(&srcline, "%s:%u", basename(file), line) < 0) {
free(file);
goto out;
}

View File

@ -341,7 +341,6 @@ int dso__load_sym(struct dso *dso, struct map *map __maybe_unused,
if (filename__read_build_id(ss->name, build_id, BUILD_ID_SIZE) > 0) {
dso__set_build_id(dso, build_id);
return 1;
}
return 0;
}

View File

@ -103,15 +103,14 @@ struct comm *thread__exec_comm(const struct thread *thread)
return last;
}
/* CHECKME: time should always be 0 if event aren't ordered */
int __thread__set_comm(struct thread *thread, const char *str, u64 timestamp,
bool exec)
{
struct comm *new, *curr = thread__comm(thread);
int err;
/* Override latest entry if it had no specific time coverage */
if (!curr->start && !curr->exec) {
/* Override the default :tid entry */
if (!thread->comm_set) {
err = comm__override(curr, str, timestamp, exec);
if (err)
return err;

View File

@ -154,7 +154,6 @@ extern void set_die_routine(void (*routine)(const char *err, va_list params) NOR
extern int prefixcmp(const char *str, const char *prefix);
extern void set_buildid_dir(void);
extern void disable_buildid_cache(void);
static inline const char *skip_prefix(const char *str, const char *prefix)
{