2009-12-14 22:09:31 +00:00
|
|
|
/*
|
|
|
|
* builtin-diff.c
|
|
|
|
*
|
|
|
|
* Builtin diff command: Analyze two perf.data input files, look up and read
|
|
|
|
* DSOs and symbol information, sort them and produce a diff.
|
|
|
|
*/
|
|
|
|
#include "builtin.h"
|
|
|
|
|
|
|
|
#include "util/debug.h"
|
|
|
|
#include "util/event.h"
|
|
|
|
#include "util/hist.h"
|
2011-11-28 09:56:39 +00:00
|
|
|
#include "util/evsel.h"
|
2012-09-06 15:46:55 +00:00
|
|
|
#include "util/evlist.h"
|
2009-12-14 22:09:31 +00:00
|
|
|
#include "util/session.h"
|
2011-11-28 10:30:20 +00:00
|
|
|
#include "util/tool.h"
|
2009-12-14 22:09:31 +00:00
|
|
|
#include "util/sort.h"
|
|
|
|
#include "util/symbol.h"
|
|
|
|
#include "util/util.h"
|
|
|
|
|
|
|
|
#include <stdlib.h>
|
|
|
|
|
perf diff: Use perf_session__fprintf_hists just like 'perf record'
That means that almost everything you can do with 'perf report'
can be done with 'perf diff', for instance:
$ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2699
samples) ] $ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2687
samples) ] perf diff | head -8
9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal
2.91% -1.00% find [kernel] [k] __kmalloc
2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent
1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock
2.44% find [kernel] [k] half_md4_transform
$
So if you want to zoom into libc:
$ perf diff --dsos libc-2.10.1.so | head -8
37.34% find [.] _IO_vfprintf_internal
10.34% find [.] __GI_memmove
8.25% +2.00% find [.] _int_malloc
5.07% -1.00% find [.] __GI_mempcpy
7.62% +2.00% find [.] _int_free
$
And if there were multiple commands using libc, it is also
possible to aggregate them all by using --sort symbol:
$ perf diff --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% [.] __GI_mempcpy
7.62% +2.00% [.] _int_free
$
The displacement column now is off by default, to use it:
perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% +2 [.] __GI_mempcpy
7.62% +2.00% -1 [.] _int_free
$
Using -t/--field-separator can be used for scripting:
$ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34, , ,[.] _IO_vfprintf_internal
10.34, , ,[.] __GI_memmove
8.25,+2.00%, ,[.] _int_malloc
5.07,-1.00%, +2,[.] __GI_mempcpy
7.62,+2.00%, -1,[.] _int_free
6.99,+1.00%, -1,[.] _IO_new_file_xsputn
1.89,-2.00%, +4,[.] __readdir64
$
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 15:49:27 +00:00
|
|
|
static char const *input_old = "perf.data.old",
|
|
|
|
*input_new = "perf.data";
|
2009-12-16 16:09:53 +00:00
|
|
|
static char diff__default_sort_order[] = "dso,symbol";
|
2010-04-13 08:37:33 +00:00
|
|
|
static bool force;
|
perf diff: Use perf_session__fprintf_hists just like 'perf record'
That means that almost everything you can do with 'perf report'
can be done with 'perf diff', for instance:
$ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2699
samples) ] $ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2687
samples) ] perf diff | head -8
9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal
2.91% -1.00% find [kernel] [k] __kmalloc
2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent
1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock
2.44% find [kernel] [k] half_md4_transform
$
So if you want to zoom into libc:
$ perf diff --dsos libc-2.10.1.so | head -8
37.34% find [.] _IO_vfprintf_internal
10.34% find [.] __GI_memmove
8.25% +2.00% find [.] _int_malloc
5.07% -1.00% find [.] __GI_mempcpy
7.62% +2.00% find [.] _int_free
$
And if there were multiple commands using libc, it is also
possible to aggregate them all by using --sort symbol:
$ perf diff --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% [.] __GI_mempcpy
7.62% +2.00% [.] _int_free
$
The displacement column now is off by default, to use it:
perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% +2 [.] __GI_mempcpy
7.62% +2.00% -1 [.] _int_free
$
Using -t/--field-separator can be used for scripting:
$ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34, , ,[.] _IO_vfprintf_internal
10.34, , ,[.] __GI_memmove
8.25,+2.00%, ,[.] _int_malloc
5.07,-1.00%, +2,[.] __GI_mempcpy
7.62,+2.00%, -1,[.] _int_free
6.99,+1.00%, -1,[.] _IO_new_file_xsputn
1.89,-2.00%, +4,[.] __readdir64
$
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 15:49:27 +00:00
|
|
|
static bool show_displacement;
|
2009-12-14 22:09:31 +00:00
|
|
|
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
static int hists__add_entry(struct hists *self,
|
2010-05-14 17:19:35 +00:00
|
|
|
struct addr_location *al, u64 period)
|
2009-12-14 22:09:31 +00:00
|
|
|
{
|
2010-05-14 17:19:35 +00:00
|
|
|
if (__hists__add_entry(self, al, NULL, period) != NULL)
|
2010-05-09 16:02:23 +00:00
|
|
|
return 0;
|
|
|
|
return -ENOMEM;
|
2009-12-14 22:09:31 +00:00
|
|
|
}
|
|
|
|
|
2012-09-10 22:15:03 +00:00
|
|
|
static int diff__process_sample_event(struct perf_tool *tool __maybe_unused,
|
2011-11-25 10:19:45 +00:00
|
|
|
union perf_event *event,
|
2011-01-29 15:02:00 +00:00
|
|
|
struct perf_sample *sample,
|
2012-09-06 15:46:55 +00:00
|
|
|
struct perf_evsel *evsel,
|
2011-11-28 09:56:39 +00:00
|
|
|
struct machine *machine)
|
2009-12-14 22:09:31 +00:00
|
|
|
{
|
|
|
|
struct addr_location al;
|
|
|
|
|
2011-11-28 09:56:39 +00:00
|
|
|
if (perf_event__preprocess_sample(event, machine, &al, sample, NULL) < 0) {
|
2009-12-14 22:09:31 +00:00
|
|
|
pr_warning("problem processing %d event, skipping it.\n",
|
|
|
|
event->header.type);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2009-12-29 00:48:35 +00:00
|
|
|
if (al.filtered || al.sym == NULL)
|
2009-12-15 22:04:41 +00:00
|
|
|
return 0;
|
|
|
|
|
2012-09-06 15:46:55 +00:00
|
|
|
if (hists__add_entry(&evsel->hists, &al, sample->period)) {
|
2010-05-14 17:19:35 +00:00
|
|
|
pr_warning("problem incrementing symbol period, skipping event\n");
|
2009-12-14 22:09:31 +00:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2012-09-06 15:46:55 +00:00
|
|
|
evsel->hists.stats.total_period += sample->period;
|
2009-12-14 22:09:31 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-09-06 15:46:55 +00:00
|
|
|
static struct perf_tool tool = {
|
|
|
|
.sample = diff__process_sample_event,
|
|
|
|
.mmap = perf_event__process_mmap,
|
|
|
|
.comm = perf_event__process_comm,
|
|
|
|
.exit = perf_event__process_task,
|
|
|
|
.fork = perf_event__process_task,
|
|
|
|
.lost = perf_event__process_lost,
|
|
|
|
.ordered_samples = true,
|
|
|
|
.ordering_requires_timestamps = true,
|
2009-12-14 22:09:31 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static void perf_session__insert_hist_entry_by_name(struct rb_root *root,
|
|
|
|
struct hist_entry *he)
|
|
|
|
{
|
|
|
|
struct rb_node **p = &root->rb_node;
|
|
|
|
struct rb_node *parent = NULL;
|
|
|
|
struct hist_entry *iter;
|
|
|
|
|
|
|
|
while (*p != NULL) {
|
|
|
|
parent = *p;
|
|
|
|
iter = rb_entry(parent, struct hist_entry, rb_node);
|
2009-12-29 00:48:36 +00:00
|
|
|
if (hist_entry__cmp(he, iter) < 0)
|
2009-12-14 22:09:31 +00:00
|
|
|
p = &(*p)->rb_left;
|
2009-12-29 00:48:36 +00:00
|
|
|
else
|
2009-12-14 22:09:31 +00:00
|
|
|
p = &(*p)->rb_right;
|
|
|
|
}
|
|
|
|
|
|
|
|
rb_link_node(&he->rb_node, parent, p);
|
|
|
|
rb_insert_color(&he->rb_node, root);
|
|
|
|
}
|
|
|
|
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
static void hists__resort_entries(struct hists *self)
|
2009-12-14 22:09:31 +00:00
|
|
|
{
|
|
|
|
unsigned long position = 1;
|
|
|
|
struct rb_root tmp = RB_ROOT;
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
struct rb_node *next = rb_first(&self->entries);
|
2009-12-14 22:09:31 +00:00
|
|
|
|
|
|
|
while (next != NULL) {
|
|
|
|
struct hist_entry *n = rb_entry(next, struct hist_entry, rb_node);
|
|
|
|
|
|
|
|
next = rb_next(&n->rb_node);
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
rb_erase(&n->rb_node, &self->entries);
|
2009-12-14 22:09:31 +00:00
|
|
|
n->position = position++;
|
|
|
|
perf_session__insert_hist_entry_by_name(&tmp, n);
|
|
|
|
}
|
|
|
|
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
self->entries = tmp;
|
2009-12-14 22:09:31 +00:00
|
|
|
}
|
|
|
|
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
static struct hist_entry *hists__find_entry(struct hists *self,
|
|
|
|
struct hist_entry *he)
|
2009-12-14 22:09:31 +00:00
|
|
|
{
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
struct rb_node *n = self->entries.rb_node;
|
2009-12-14 22:09:31 +00:00
|
|
|
|
|
|
|
while (n) {
|
|
|
|
struct hist_entry *iter = rb_entry(n, struct hist_entry, rb_node);
|
2009-12-29 00:48:36 +00:00
|
|
|
int64_t cmp = hist_entry__cmp(he, iter);
|
2009-12-14 22:09:31 +00:00
|
|
|
|
2009-12-29 00:48:36 +00:00
|
|
|
if (cmp < 0)
|
2009-12-14 22:09:31 +00:00
|
|
|
n = n->rb_left;
|
2009-12-29 00:48:36 +00:00
|
|
|
else if (cmp > 0)
|
2009-12-14 22:09:31 +00:00
|
|
|
n = n->rb_right;
|
2009-12-29 00:48:36 +00:00
|
|
|
else
|
|
|
|
return iter;
|
2009-12-14 22:09:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
static void hists__match(struct hists *older, struct hists *newer)
|
2009-12-14 22:09:31 +00:00
|
|
|
{
|
|
|
|
struct rb_node *nd;
|
|
|
|
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
for (nd = rb_first(&newer->entries); nd; nd = rb_next(nd)) {
|
2009-12-14 22:09:31 +00:00
|
|
|
struct hist_entry *pos = rb_entry(nd, struct hist_entry, rb_node);
|
perf hist: Introduce hists class and move lots of methods to it
In cbbc79a we introduced support for multiple events by introducing a
new "event_stat_id" struct and then made several perf_session methods
receive a point to it instead of a pointer to perf_session, and kept the
event_stats and hists rb_tree in perf_session.
While working on the new newt based browser, I realised that it would be
better to introduce a new class, "hists" (short for "histograms"),
renaming the "event_stat_id" struct and the perf_session methods that
were really "hists" methods, as they manipulate only struct hists
members, not touching anything in the other perf_session members.
Other optimizations, such as calculating the maximum lenght of a symbol
name present in an hists instance will be possible as we add them,
avoiding a re-traversal just for finding that information.
The rationale for the name "hists" to replace "event_stat_id" is that we
may have multiple sets of hists for the same event_stat id, as, for
instance, the 'perf diff' tool has, so event stat id is not what
characterizes what this struct and the functions that manipulate it do.
Cc: Eric B Munson <ebmunson@us.ibm.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Tom Zanussi <tzanussi@gmail.com>
LKML-Reference: <new-submission>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2010-05-10 16:04:11 +00:00
|
|
|
pos->pair = hists__find_entry(older, pos);
|
2009-12-14 22:09:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-09-06 15:46:55 +00:00
|
|
|
static struct perf_evsel *evsel_match(struct perf_evsel *evsel,
|
|
|
|
struct perf_evlist *evlist)
|
|
|
|
{
|
|
|
|
struct perf_evsel *e;
|
|
|
|
|
|
|
|
list_for_each_entry(e, &evlist->entries, node)
|
|
|
|
if (perf_evsel__match2(evsel, e))
|
|
|
|
return e;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2009-12-14 22:09:31 +00:00
|
|
|
static int __cmd_diff(void)
|
|
|
|
{
|
|
|
|
int ret, i;
|
2012-03-22 13:37:26 +00:00
|
|
|
#define older (session[0])
|
|
|
|
#define newer (session[1])
|
2009-12-14 22:09:31 +00:00
|
|
|
struct perf_session *session[2];
|
2012-09-06 15:46:55 +00:00
|
|
|
struct perf_evlist *evlist_new, *evlist_old;
|
|
|
|
struct perf_evsel *evsel;
|
|
|
|
bool first = true;
|
2009-12-14 22:09:31 +00:00
|
|
|
|
2012-03-22 13:37:26 +00:00
|
|
|
older = perf_session__new(input_old, O_RDONLY, force, false,
|
2012-09-06 15:46:55 +00:00
|
|
|
&tool);
|
2012-03-22 13:37:26 +00:00
|
|
|
newer = perf_session__new(input_new, O_RDONLY, force, false,
|
2012-09-06 15:46:55 +00:00
|
|
|
&tool);
|
2009-12-14 22:09:31 +00:00
|
|
|
if (session[0] == NULL || session[1] == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for (i = 0; i < 2; ++i) {
|
2012-09-06 15:46:55 +00:00
|
|
|
ret = perf_session__process_events(session[i], &tool);
|
2009-12-14 22:09:31 +00:00
|
|
|
if (ret)
|
|
|
|
goto out_delete;
|
|
|
|
}
|
|
|
|
|
2012-09-06 15:46:55 +00:00
|
|
|
evlist_old = older->evlist;
|
|
|
|
evlist_new = newer->evlist;
|
|
|
|
|
|
|
|
list_for_each_entry(evsel, &evlist_new->entries, node)
|
|
|
|
hists__output_resort(&evsel->hists);
|
|
|
|
|
|
|
|
list_for_each_entry(evsel, &evlist_old->entries, node) {
|
|
|
|
hists__output_resort(&evsel->hists);
|
|
|
|
|
|
|
|
if (show_displacement)
|
|
|
|
hists__resort_entries(&evsel->hists);
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(evsel, &evlist_new->entries, node) {
|
|
|
|
struct perf_evsel *evsel_old;
|
|
|
|
|
|
|
|
evsel_old = evsel_match(evsel, evlist_old);
|
|
|
|
if (!evsel_old)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
fprintf(stdout, "%s# Event '%s'\n#\n", first ? "" : "\n",
|
|
|
|
perf_evsel__name(evsel));
|
|
|
|
|
|
|
|
first = false;
|
|
|
|
|
|
|
|
hists__match(&evsel_old->hists, &evsel->hists);
|
|
|
|
hists__fprintf(&evsel->hists, &evsel_old->hists,
|
|
|
|
show_displacement, true, 0, 0, stdout);
|
|
|
|
}
|
2009-12-29 00:48:36 +00:00
|
|
|
|
2009-12-14 22:09:31 +00:00
|
|
|
out_delete:
|
|
|
|
for (i = 0; i < 2; ++i)
|
|
|
|
perf_session__delete(session[i]);
|
|
|
|
return ret;
|
2012-03-22 13:37:26 +00:00
|
|
|
#undef older
|
|
|
|
#undef newer
|
2009-12-14 22:09:31 +00:00
|
|
|
}
|
|
|
|
|
2009-12-18 18:35:58 +00:00
|
|
|
static const char * const diff_usage[] = {
|
2009-12-14 22:09:31 +00:00
|
|
|
"perf diff [<options>] [old_file] [new_file]",
|
2009-12-18 18:35:58 +00:00
|
|
|
NULL,
|
2009-12-14 22:09:31 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static const struct option options[] = {
|
2010-04-13 08:37:33 +00:00
|
|
|
OPT_INCR('v', "verbose", &verbose,
|
2009-12-14 22:09:31 +00:00
|
|
|
"be more verbose (show symbol address, etc)"),
|
2010-12-01 01:57:11 +00:00
|
|
|
OPT_BOOLEAN('M', "displacement", &show_displacement,
|
perf diff: Use perf_session__fprintf_hists just like 'perf record'
That means that almost everything you can do with 'perf report'
can be done with 'perf diff', for instance:
$ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2699
samples) ] $ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2687
samples) ] perf diff | head -8
9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal
2.91% -1.00% find [kernel] [k] __kmalloc
2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent
1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock
2.44% find [kernel] [k] half_md4_transform
$
So if you want to zoom into libc:
$ perf diff --dsos libc-2.10.1.so | head -8
37.34% find [.] _IO_vfprintf_internal
10.34% find [.] __GI_memmove
8.25% +2.00% find [.] _int_malloc
5.07% -1.00% find [.] __GI_mempcpy
7.62% +2.00% find [.] _int_free
$
And if there were multiple commands using libc, it is also
possible to aggregate them all by using --sort symbol:
$ perf diff --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% [.] __GI_mempcpy
7.62% +2.00% [.] _int_free
$
The displacement column now is off by default, to use it:
perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% +2 [.] __GI_mempcpy
7.62% +2.00% -1 [.] _int_free
$
Using -t/--field-separator can be used for scripting:
$ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34, , ,[.] _IO_vfprintf_internal
10.34, , ,[.] __GI_memmove
8.25,+2.00%, ,[.] _int_malloc
5.07,-1.00%, +2,[.] __GI_mempcpy
7.62,+2.00%, -1,[.] _int_free
6.99,+1.00%, -1,[.] _IO_new_file_xsputn
1.89,-2.00%, +4,[.] __readdir64
$
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 15:49:27 +00:00
|
|
|
"Show position displacement relative to baseline"),
|
2009-12-14 22:09:31 +00:00
|
|
|
OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
|
|
|
|
"dump raw trace in ASCII"),
|
|
|
|
OPT_BOOLEAN('f', "force", &force, "don't complain, do it"),
|
|
|
|
OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules,
|
|
|
|
"load module symbols - WARNING: use only with -k and LIVE kernel"),
|
2009-12-15 22:04:41 +00:00
|
|
|
OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
|
|
|
|
"only consider symbols in these dsos"),
|
|
|
|
OPT_STRING('C', "comms", &symbol_conf.comm_list_str, "comm[,comm...]",
|
|
|
|
"only consider symbols in these comms"),
|
|
|
|
OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]",
|
|
|
|
"only consider these symbols"),
|
perf diff: Use perf_session__fprintf_hists just like 'perf record'
That means that almost everything you can do with 'perf report'
can be done with 'perf diff', for instance:
$ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2699
samples) ] $ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2687
samples) ] perf diff | head -8
9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal
2.91% -1.00% find [kernel] [k] __kmalloc
2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent
1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock
2.44% find [kernel] [k] half_md4_transform
$
So if you want to zoom into libc:
$ perf diff --dsos libc-2.10.1.so | head -8
37.34% find [.] _IO_vfprintf_internal
10.34% find [.] __GI_memmove
8.25% +2.00% find [.] _int_malloc
5.07% -1.00% find [.] __GI_mempcpy
7.62% +2.00% find [.] _int_free
$
And if there were multiple commands using libc, it is also
possible to aggregate them all by using --sort symbol:
$ perf diff --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% [.] __GI_mempcpy
7.62% +2.00% [.] _int_free
$
The displacement column now is off by default, to use it:
perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% +2 [.] __GI_mempcpy
7.62% +2.00% -1 [.] _int_free
$
Using -t/--field-separator can be used for scripting:
$ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34, , ,[.] _IO_vfprintf_internal
10.34, , ,[.] __GI_memmove
8.25,+2.00%, ,[.] _int_malloc
5.07,-1.00%, +2,[.] __GI_mempcpy
7.62,+2.00%, -1,[.] _int_free
6.99,+1.00%, -1,[.] _IO_new_file_xsputn
1.89,-2.00%, +4,[.] __readdir64
$
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 15:49:27 +00:00
|
|
|
OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
|
|
|
|
"sort by key(s): pid, comm, dso, symbol, parent"),
|
|
|
|
OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator",
|
|
|
|
"separator for columns, no spaces will be added between "
|
|
|
|
"columns '.' is reserved."),
|
2010-12-09 20:27:07 +00:00
|
|
|
OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
|
|
|
|
"Look for files with symbols relative to this directory"),
|
2009-12-14 22:09:31 +00:00
|
|
|
OPT_END()
|
|
|
|
};
|
|
|
|
|
2012-09-10 22:15:03 +00:00
|
|
|
int cmd_diff(int argc, const char **argv, const char *prefix __maybe_unused)
|
2009-12-14 22:09:31 +00:00
|
|
|
{
|
2009-12-16 16:09:53 +00:00
|
|
|
sort_order = diff__default_sort_order;
|
2009-12-14 22:09:31 +00:00
|
|
|
argc = parse_options(argc, argv, options, diff_usage, 0);
|
|
|
|
if (argc) {
|
|
|
|
if (argc > 2)
|
|
|
|
usage_with_options(diff_usage, options);
|
|
|
|
if (argc == 2) {
|
|
|
|
input_old = argv[0];
|
|
|
|
input_new = argv[1];
|
|
|
|
} else
|
|
|
|
input_new = argv[0];
|
2010-04-19 05:32:50 +00:00
|
|
|
} else if (symbol_conf.default_guest_vmlinux_name ||
|
|
|
|
symbol_conf.default_guest_kallsyms) {
|
|
|
|
input_old = "perf.data.host";
|
|
|
|
input_new = "perf.data.guest";
|
2009-12-14 22:09:31 +00:00
|
|
|
}
|
|
|
|
|
perf diff: Use perf_session__fprintf_hists just like 'perf record'
That means that almost everything you can do with 'perf report'
can be done with 'perf diff', for instance:
$ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2699
samples) ] $ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2687
samples) ] perf diff | head -8
9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal
2.91% -1.00% find [kernel] [k] __kmalloc
2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent
1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock
2.44% find [kernel] [k] half_md4_transform
$
So if you want to zoom into libc:
$ perf diff --dsos libc-2.10.1.so | head -8
37.34% find [.] _IO_vfprintf_internal
10.34% find [.] __GI_memmove
8.25% +2.00% find [.] _int_malloc
5.07% -1.00% find [.] __GI_mempcpy
7.62% +2.00% find [.] _int_free
$
And if there were multiple commands using libc, it is also
possible to aggregate them all by using --sort symbol:
$ perf diff --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% [.] __GI_mempcpy
7.62% +2.00% [.] _int_free
$
The displacement column now is off by default, to use it:
perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% +2 [.] __GI_mempcpy
7.62% +2.00% -1 [.] _int_free
$
Using -t/--field-separator can be used for scripting:
$ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34, , ,[.] _IO_vfprintf_internal
10.34, , ,[.] __GI_memmove
8.25,+2.00%, ,[.] _int_malloc
5.07,-1.00%, +2,[.] __GI_mempcpy
7.62,+2.00%, -1,[.] _int_free
6.99,+1.00%, -1,[.] _IO_new_file_xsputn
1.89,-2.00%, +4,[.] __readdir64
$
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 15:49:27 +00:00
|
|
|
symbol_conf.exclude_other = false;
|
2009-12-15 22:04:40 +00:00
|
|
|
if (symbol__init() < 0)
|
|
|
|
return -1;
|
|
|
|
|
2012-09-03 02:53:06 +00:00
|
|
|
perf_hpp__init(true, show_displacement);
|
2009-12-15 22:04:40 +00:00
|
|
|
setup_sorting(diff_usage, options);
|
2009-12-14 22:09:31 +00:00
|
|
|
setup_pager();
|
perf diff: Use perf_session__fprintf_hists just like 'perf record'
That means that almost everything you can do with 'perf report'
can be done with 'perf diff', for instance:
$ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2699
samples) ] $ perf record -f find / > /dev/null
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.062 MB perf.data (~2687
samples) ] perf diff | head -8
9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal
2.91% -1.00% find [kernel] [k] __kmalloc
2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent
1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock
2.44% find [kernel] [k] half_md4_transform
$
So if you want to zoom into libc:
$ perf diff --dsos libc-2.10.1.so | head -8
37.34% find [.] _IO_vfprintf_internal
10.34% find [.] __GI_memmove
8.25% +2.00% find [.] _int_malloc
5.07% -1.00% find [.] __GI_mempcpy
7.62% +2.00% find [.] _int_free
$
And if there were multiple commands using libc, it is also
possible to aggregate them all by using --sort symbol:
$ perf diff --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% [.] __GI_mempcpy
7.62% +2.00% [.] _int_free
$
The displacement column now is off by default, to use it:
perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34% [.] _IO_vfprintf_internal
10.34% [.] __GI_memmove
8.25% +2.00% [.] _int_malloc
5.07% -1.00% +2 [.] __GI_mempcpy
7.62% +2.00% -1 [.] _int_free
$
Using -t/--field-separator can be used for scripting:
$ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8
37.34, , ,[.] _IO_vfprintf_internal
10.34, , ,[.] __GI_memmove
8.25,+2.00%, ,[.] _int_malloc
5.07,-1.00%, +2,[.] __GI_mempcpy
7.62,+2.00%, -1,[.] _int_free
6.99,+1.00%, -1,[.] _IO_new_file_xsputn
1.89,-2.00%, +4,[.] __readdir64
$
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frédéric Weisbecker <fweisbec@gmail.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 15:49:27 +00:00
|
|
|
|
|
|
|
sort_entry__setup_elide(&sort_dso, symbol_conf.dso_list, "dso", NULL);
|
|
|
|
sort_entry__setup_elide(&sort_comm, symbol_conf.comm_list, "comm", NULL);
|
|
|
|
sort_entry__setup_elide(&sort_sym, symbol_conf.sym_list, "symbol", NULL);
|
|
|
|
|
2009-12-14 22:09:31 +00:00
|
|
|
return __cmd_diff();
|
|
|
|
}
|