2008-12-03 09:39:53 +00:00
|
|
|
/*
|
|
|
|
* Performance counter x86 architecture code
|
|
|
|
*
|
|
|
|
* Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
|
|
|
|
* Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
|
2009-02-27 12:39:09 +00:00
|
|
|
* Copyright(C) 2009 Jaswinder Singh Rajput
|
2009-04-29 10:47:05 +00:00
|
|
|
* Copyright(C) 2009 Advanced Micro Devices, Inc., Robert Richter
|
2008-12-03 09:39:53 +00:00
|
|
|
*
|
|
|
|
* For licencing details see kernel-base/COPYING
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/perf_counter.h>
|
|
|
|
#include <linux/capability.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/kprobes.h>
|
2008-12-09 20:43:39 +00:00
|
|
|
#include <linux/module.h>
|
2008-12-03 09:39:53 +00:00
|
|
|
#include <linux/kdebug.h>
|
|
|
|
#include <linux/sched.h>
|
2009-03-30 17:07:15 +00:00
|
|
|
#include <linux/uaccess.h>
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
#include <asm/apic.h>
|
2009-03-30 17:07:15 +00:00
|
|
|
#include <asm/stacktrace.h>
|
2009-03-30 17:07:16 +00:00
|
|
|
#include <asm/nmi.h>
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
static bool perf_counters_initialized __read_mostly;
|
2008-12-17 12:09:20 +00:00
|
|
|
static u64 perf_counter_mask __read_mostly;
|
2008-12-17 09:51:15 +00:00
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
struct cpu_hw_counters {
|
2008-12-17 12:09:20 +00:00
|
|
|
struct perf_counter *counters[X86_PMC_IDX_MAX];
|
|
|
|
unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
|
2009-04-29 10:47:15 +00:00
|
|
|
unsigned long active[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
|
2009-01-23 13:36:16 +00:00
|
|
|
unsigned long interrupts;
|
2009-03-05 17:08:27 +00:00
|
|
|
u64 throttle_ctrl;
|
|
|
|
int enabled;
|
2008-12-03 09:39:53 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
2009-04-29 10:47:04 +00:00
|
|
|
* struct x86_pmu - generic x86 pmu
|
2008-12-03 09:39:53 +00:00
|
|
|
*/
|
2009-04-29 10:47:04 +00:00
|
|
|
struct x86_pmu {
|
2009-04-29 10:47:13 +00:00
|
|
|
const char *name;
|
|
|
|
int version;
|
2009-04-29 10:47:05 +00:00
|
|
|
int (*handle_irq)(struct pt_regs *, int);
|
2009-02-28 13:07:49 +00:00
|
|
|
u64 (*save_disable_all)(void);
|
2009-03-05 17:08:27 +00:00
|
|
|
void (*restore_all)(u64);
|
|
|
|
void (*enable)(int, u64);
|
|
|
|
void (*disable)(int, u64);
|
2009-02-28 13:07:49 +00:00
|
|
|
unsigned eventsel;
|
|
|
|
unsigned perfctr;
|
2009-03-05 17:08:27 +00:00
|
|
|
u64 (*event_map)(int);
|
|
|
|
u64 (*raw_event)(u64);
|
2009-02-28 13:07:49 +00:00
|
|
|
int max_events;
|
2009-04-29 10:47:12 +00:00
|
|
|
int num_counters;
|
|
|
|
int num_counters_fixed;
|
|
|
|
int counter_bits;
|
|
|
|
u64 counter_mask;
|
2009-02-27 12:39:09 +00:00
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
static struct x86_pmu x86_pmu __read_mostly;
|
2009-02-27 12:39:09 +00:00
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
|
|
|
|
.enabled = 1,
|
|
|
|
};
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-02-27 12:39:09 +00:00
|
|
|
/*
|
|
|
|
* Intel PerfMon v3. Used on Core2 and later.
|
|
|
|
*/
|
2009-03-05 17:08:27 +00:00
|
|
|
static const u64 intel_perfmon_event_map[] =
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
2008-12-23 11:17:29 +00:00
|
|
|
[PERF_COUNT_CPU_CYCLES] = 0x003c,
|
2008-12-03 09:39:53 +00:00
|
|
|
[PERF_COUNT_INSTRUCTIONS] = 0x00c0,
|
|
|
|
[PERF_COUNT_CACHE_REFERENCES] = 0x4f2e,
|
|
|
|
[PERF_COUNT_CACHE_MISSES] = 0x412e,
|
|
|
|
[PERF_COUNT_BRANCH_INSTRUCTIONS] = 0x00c4,
|
|
|
|
[PERF_COUNT_BRANCH_MISSES] = 0x00c5,
|
2008-12-23 11:17:29 +00:00
|
|
|
[PERF_COUNT_BUS_CYCLES] = 0x013c,
|
2008-12-03 09:39:53 +00:00
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static u64 intel_pmu_event_map(int event)
|
2009-02-27 12:39:09 +00:00
|
|
|
{
|
|
|
|
return intel_perfmon_event_map[event];
|
|
|
|
}
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static u64 intel_pmu_raw_event(u64 event)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
2009-03-13 11:21:31 +00:00
|
|
|
#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
|
|
|
|
#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
|
|
|
|
#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL
|
2009-03-05 17:08:27 +00:00
|
|
|
|
|
|
|
#define CORE_EVNTSEL_MASK \
|
|
|
|
(CORE_EVNTSEL_EVENT_MASK | \
|
|
|
|
CORE_EVNTSEL_UNIT_MASK | \
|
|
|
|
CORE_EVNTSEL_COUNTER_MASK)
|
|
|
|
|
|
|
|
return event & CORE_EVNTSEL_MASK;
|
|
|
|
}
|
|
|
|
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
/*
|
|
|
|
* AMD Performance Monitor K7 and later.
|
|
|
|
*/
|
2009-03-05 17:08:27 +00:00
|
|
|
static const u64 amd_perfmon_event_map[] =
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
{
|
|
|
|
[PERF_COUNT_CPU_CYCLES] = 0x0076,
|
|
|
|
[PERF_COUNT_INSTRUCTIONS] = 0x00c0,
|
|
|
|
[PERF_COUNT_CACHE_REFERENCES] = 0x0080,
|
|
|
|
[PERF_COUNT_CACHE_MISSES] = 0x0081,
|
|
|
|
[PERF_COUNT_BRANCH_INSTRUCTIONS] = 0x00c4,
|
|
|
|
[PERF_COUNT_BRANCH_MISSES] = 0x00c5,
|
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static u64 amd_pmu_event_map(int event)
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
{
|
|
|
|
return amd_perfmon_event_map[event];
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static u64 amd_pmu_raw_event(u64 event)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
2009-03-13 11:21:31 +00:00
|
|
|
#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
|
|
|
|
#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
|
|
|
|
#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
|
2009-03-05 17:08:27 +00:00
|
|
|
|
|
|
|
#define K7_EVNTSEL_MASK \
|
|
|
|
(K7_EVNTSEL_EVENT_MASK | \
|
|
|
|
K7_EVNTSEL_UNIT_MASK | \
|
|
|
|
K7_EVNTSEL_COUNTER_MASK)
|
|
|
|
|
|
|
|
return event & K7_EVNTSEL_MASK;
|
|
|
|
}
|
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
/*
|
|
|
|
* Propagate counter elapsed time into the generic counter.
|
|
|
|
* Can only be executed on the CPU where the counter is active.
|
|
|
|
* Returns the delta events processed.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
x86_perf_counter_update(struct perf_counter *counter,
|
|
|
|
struct hw_perf_counter *hwc, int idx)
|
|
|
|
{
|
|
|
|
u64 prev_raw_count, new_raw_count, delta;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Careful: an NMI might modify the previous counter value.
|
|
|
|
*
|
|
|
|
* Our tactic to handle this is to first atomically read and
|
|
|
|
* exchange a new raw count - then add that new-prev delta
|
|
|
|
* count to the generic counter atomically:
|
|
|
|
*/
|
|
|
|
again:
|
|
|
|
prev_raw_count = atomic64_read(&hwc->prev_count);
|
|
|
|
rdmsrl(hwc->counter_base + idx, new_raw_count);
|
|
|
|
|
|
|
|
if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
|
|
|
|
new_raw_count) != prev_raw_count)
|
|
|
|
goto again;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now we have the new raw value and have updated the prev
|
|
|
|
* timestamp already. We can now calculate the elapsed delta
|
|
|
|
* (counter-)time and add that to the generic counter.
|
|
|
|
*
|
|
|
|
* Careful, not all hw sign-extends above the physical width
|
|
|
|
* of the count, so we do that by clipping the delta to 32 bits:
|
|
|
|
*/
|
|
|
|
delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
|
|
|
|
|
|
|
|
atomic64_add(delta, &counter->count);
|
|
|
|
atomic64_sub(delta, &hwc->period_left);
|
|
|
|
}
|
|
|
|
|
2009-03-30 17:07:16 +00:00
|
|
|
static atomic_t num_counters;
|
|
|
|
static DEFINE_MUTEX(pmc_reserve_mutex);
|
|
|
|
|
|
|
|
static bool reserve_pmc_hardware(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (nmi_watchdog == NMI_LOCAL_APIC)
|
|
|
|
disable_lapic_nmi_watchdog();
|
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
for (i = 0; i < x86_pmu.num_counters; i++) {
|
2009-04-29 10:47:11 +00:00
|
|
|
if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
|
2009-03-30 17:07:16 +00:00
|
|
|
goto perfctr_fail;
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
for (i = 0; i < x86_pmu.num_counters; i++) {
|
2009-04-29 10:47:11 +00:00
|
|
|
if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
|
2009-03-30 17:07:16 +00:00
|
|
|
goto eventsel_fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
eventsel_fail:
|
|
|
|
for (i--; i >= 0; i--)
|
2009-04-29 10:47:11 +00:00
|
|
|
release_evntsel_nmi(x86_pmu.eventsel + i);
|
2009-03-30 17:07:16 +00:00
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
i = x86_pmu.num_counters;
|
2009-03-30 17:07:16 +00:00
|
|
|
|
|
|
|
perfctr_fail:
|
|
|
|
for (i--; i >= 0; i--)
|
2009-04-29 10:47:11 +00:00
|
|
|
release_perfctr_nmi(x86_pmu.perfctr + i);
|
2009-03-30 17:07:16 +00:00
|
|
|
|
|
|
|
if (nmi_watchdog == NMI_LOCAL_APIC)
|
|
|
|
enable_lapic_nmi_watchdog();
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void release_pmc_hardware(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
for (i = 0; i < x86_pmu.num_counters; i++) {
|
2009-04-29 10:47:11 +00:00
|
|
|
release_perfctr_nmi(x86_pmu.perfctr + i);
|
|
|
|
release_evntsel_nmi(x86_pmu.eventsel + i);
|
2009-03-30 17:07:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (nmi_watchdog == NMI_LOCAL_APIC)
|
|
|
|
enable_lapic_nmi_watchdog();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hw_perf_counter_destroy(struct perf_counter *counter)
|
|
|
|
{
|
|
|
|
if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
|
|
|
|
release_pmc_hardware();
|
|
|
|
mutex_unlock(&pmc_reserve_mutex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
/*
|
|
|
|
* Setup the hardware configuration for a given hw_event_type
|
|
|
|
*/
|
2008-12-11 11:46:46 +00:00
|
|
|
static int __hw_perf_counter_init(struct perf_counter *counter)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
2008-12-10 11:33:23 +00:00
|
|
|
struct perf_counter_hw_event *hw_event = &counter->hw_event;
|
2008-12-03 09:39:53 +00:00
|
|
|
struct hw_perf_counter *hwc = &counter->hw;
|
2009-03-30 17:07:16 +00:00
|
|
|
int err;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:05 +00:00
|
|
|
/* disable temporarily */
|
|
|
|
if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
|
|
|
|
return -ENOSYS;
|
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2009-03-30 17:07:16 +00:00
|
|
|
err = 0;
|
|
|
|
if (atomic_inc_not_zero(&num_counters)) {
|
|
|
|
mutex_lock(&pmc_reserve_mutex);
|
|
|
|
if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
|
|
|
|
err = -EBUSY;
|
|
|
|
else
|
|
|
|
atomic_inc(&num_counters);
|
|
|
|
mutex_unlock(&pmc_reserve_mutex);
|
|
|
|
}
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
/*
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
* Generate PMC IRQs:
|
2008-12-03 09:39:53 +00:00
|
|
|
* (keep 'enabled' bit clear for now)
|
|
|
|
*/
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
hwc->config = ARCH_PERFMON_EVENTSEL_INT;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
/*
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
* Count user and OS events unless requested not to.
|
2008-12-03 09:39:53 +00:00
|
|
|
*/
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
if (!hw_event->exclude_user)
|
|
|
|
hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
|
|
|
|
if (!hw_event->exclude_kernel)
|
2008-12-03 09:39:53 +00:00
|
|
|
hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If privileged enough, allow NMI events:
|
|
|
|
*/
|
|
|
|
hwc->nmi = 0;
|
|
|
|
if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
|
|
|
|
hwc->nmi = 1;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-10 11:33:23 +00:00
|
|
|
hwc->irq_period = hw_event->irq_period;
|
2008-12-03 09:39:53 +00:00
|
|
|
/*
|
|
|
|
* Intel PMCs cannot be accessed sanely above 32 bit width,
|
|
|
|
* so we install an artificial 1<<31 period regardless of
|
|
|
|
* the generic counter period:
|
|
|
|
*/
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
|
|
|
|
if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
|
|
|
|
hwc->irq_period = 0x7FFFFFFF;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
atomic64_set(&hwc->period_left, hwc->irq_period);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
/*
|
2008-12-08 18:35:37 +00:00
|
|
|
* Raw event type provide the config in the event structure
|
2008-12-03 09:39:53 +00:00
|
|
|
*/
|
2009-03-23 17:22:06 +00:00
|
|
|
if (perf_event_raw(hw_event)) {
|
2009-04-29 10:47:11 +00:00
|
|
|
hwc->config |= x86_pmu.raw_event(perf_event_config(hw_event));
|
2008-12-03 09:39:53 +00:00
|
|
|
} else {
|
2009-04-29 10:47:11 +00:00
|
|
|
if (perf_event_id(hw_event) >= x86_pmu.max_events)
|
2008-12-03 09:39:53 +00:00
|
|
|
return -EINVAL;
|
|
|
|
/*
|
|
|
|
* The generic map:
|
|
|
|
*/
|
2009-04-29 10:47:11 +00:00
|
|
|
hwc->config |= x86_pmu.event_map(perf_event_id(hw_event));
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
2009-03-30 17:07:16 +00:00
|
|
|
counter->destroy = hw_perf_counter_destroy;
|
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static u64 intel_pmu_save_disable_all(void)
|
2008-12-09 20:43:39 +00:00
|
|
|
{
|
|
|
|
u64 ctrl;
|
|
|
|
|
|
|
|
rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
|
2008-12-17 12:09:20 +00:00
|
|
|
wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
|
2008-12-14 17:36:30 +00:00
|
|
|
|
2008-12-09 20:43:39 +00:00
|
|
|
return ctrl;
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
2009-02-27 12:39:09 +00:00
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static u64 amd_pmu_save_disable_all(void)
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
{
|
2009-03-05 17:08:27 +00:00
|
|
|
struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
|
|
|
|
int enabled, idx;
|
|
|
|
|
|
|
|
enabled = cpuc->enabled;
|
|
|
|
cpuc->enabled = 0;
|
2009-03-13 11:21:30 +00:00
|
|
|
/*
|
|
|
|
* ensure we write the disable before we start disabling the
|
2009-04-29 10:47:04 +00:00
|
|
|
* counters proper, so that amd_pmu_enable_counter() does the
|
|
|
|
* right thing.
|
2009-03-13 11:21:30 +00:00
|
|
|
*/
|
2009-03-05 17:08:27 +00:00
|
|
|
barrier();
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
for (idx = 0; idx < x86_pmu.num_counters; idx++) {
|
2009-03-05 17:08:27 +00:00
|
|
|
u64 val;
|
|
|
|
|
2009-04-29 10:47:15 +00:00
|
|
|
if (!test_bit(idx, cpuc->active))
|
2009-04-29 10:47:01 +00:00
|
|
|
continue;
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
|
2009-04-29 10:47:01 +00:00
|
|
|
if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
|
|
|
|
continue;
|
|
|
|
val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
|
|
|
|
wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
}
|
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
return enabled;
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
}
|
|
|
|
|
2009-02-27 12:39:09 +00:00
|
|
|
u64 hw_perf_save_disable(void)
|
|
|
|
{
|
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return 0;
|
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
return x86_pmu.save_disable_all();
|
2009-02-27 12:39:09 +00:00
|
|
|
}
|
2009-03-05 17:08:27 +00:00
|
|
|
/*
|
|
|
|
* Exported because of ACPI idle
|
|
|
|
*/
|
2008-12-11 12:45:51 +00:00
|
|
|
EXPORT_SYMBOL_GPL(hw_perf_save_disable);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static void intel_pmu_restore_all(u64 ctrl)
|
2009-02-27 12:39:09 +00:00
|
|
|
{
|
|
|
|
wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static void amd_pmu_restore_all(u64 ctrl)
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
{
|
2009-03-05 17:08:27 +00:00
|
|
|
struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
int idx;
|
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
cpuc->enabled = ctrl;
|
|
|
|
barrier();
|
|
|
|
if (!ctrl)
|
|
|
|
return;
|
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
for (idx = 0; idx < x86_pmu.num_counters; idx++) {
|
2009-04-29 10:47:01 +00:00
|
|
|
u64 val;
|
2009-03-05 17:08:27 +00:00
|
|
|
|
2009-04-29 10:47:15 +00:00
|
|
|
if (!test_bit(idx, cpuc->active))
|
2009-04-29 10:47:01 +00:00
|
|
|
continue;
|
|
|
|
rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
|
|
|
|
if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
|
|
|
|
continue;
|
|
|
|
val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
|
|
|
|
wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
void hw_perf_restore(u64 ctrl)
|
|
|
|
{
|
2008-12-14 17:36:30 +00:00
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return;
|
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
x86_pmu.restore_all(ctrl);
|
2008-12-13 08:00:03 +00:00
|
|
|
}
|
2009-03-05 17:08:27 +00:00
|
|
|
/*
|
|
|
|
* Exported because of ACPI idle
|
|
|
|
*/
|
2008-12-13 08:00:03 +00:00
|
|
|
EXPORT_SYMBOL_GPL(hw_perf_restore);
|
|
|
|
|
2009-04-29 10:47:06 +00:00
|
|
|
static inline u64 intel_pmu_get_status(u64 mask)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
|
|
|
u64 status;
|
|
|
|
|
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return 0;
|
2009-04-29 10:47:06 +00:00
|
|
|
rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
|
2009-03-05 17:08:27 +00:00
|
|
|
|
2009-04-29 10:47:06 +00:00
|
|
|
return status;
|
2009-03-05 17:08:27 +00:00
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:07 +00:00
|
|
|
static inline void intel_pmu_ack_status(u64 ack)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
|
|
|
wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static void intel_pmu_enable_counter(int idx, u64 config)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
|
|
|
wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
|
|
|
|
config | ARCH_PERFMON_EVENTSEL0_ENABLE);
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static void amd_pmu_enable_counter(int idx, u64 config)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
|
|
|
struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
|
|
|
|
|
|
|
|
if (cpuc->enabled)
|
|
|
|
config |= ARCH_PERFMON_EVENTSEL0_ENABLE;
|
|
|
|
|
|
|
|
wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hw_perf_enable(int idx, u64 config)
|
|
|
|
{
|
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return;
|
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
x86_pmu.enable(idx, config);
|
2009-03-05 17:08:27 +00:00
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static void intel_pmu_disable_counter(int idx, u64 config)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
|
|
|
wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static void amd_pmu_disable_counter(int idx, u64 config)
|
2009-03-05 17:08:27 +00:00
|
|
|
{
|
|
|
|
wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hw_perf_disable(int idx, u64 config)
|
|
|
|
{
|
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return;
|
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
x86_pmu.disable(idx, config);
|
2009-03-05 17:08:27 +00:00
|
|
|
}
|
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
static inline void
|
|
|
|
__pmc_fixed_disable(struct perf_counter *counter,
|
2009-04-29 10:47:17 +00:00
|
|
|
struct hw_perf_counter *hwc, int __idx)
|
2008-12-22 10:10:42 +00:00
|
|
|
{
|
|
|
|
int idx = __idx - X86_PMC_IDX_FIXED;
|
|
|
|
u64 ctrl_val, mask;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
mask = 0xfULL << (idx * 4);
|
|
|
|
|
|
|
|
rdmsrl(hwc->config_base, ctrl_val);
|
|
|
|
ctrl_val &= ~mask;
|
|
|
|
err = checking_wrmsrl(hwc->config_base, ctrl_val);
|
|
|
|
}
|
|
|
|
|
2008-12-09 10:40:46 +00:00
|
|
|
static inline void
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_disable(struct perf_counter *counter,
|
2009-04-29 10:47:17 +00:00
|
|
|
struct hw_perf_counter *hwc, int idx)
|
2008-12-09 10:40:46 +00:00
|
|
|
{
|
2008-12-22 10:10:42 +00:00
|
|
|
if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
|
2008-12-27 13:45:43 +00:00
|
|
|
__pmc_fixed_disable(counter, hwc, idx);
|
|
|
|
else
|
2009-03-05 17:08:27 +00:00
|
|
|
hw_perf_disable(idx, hwc->config);
|
2008-12-09 10:40:46 +00:00
|
|
|
}
|
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
/*
|
|
|
|
* Set the next IRQ period, based on the hwc->period_left value.
|
|
|
|
* To be called with the counter disabled in hw:
|
|
|
|
*/
|
|
|
|
static void
|
2009-04-29 10:47:08 +00:00
|
|
|
x86_perf_counter_set_period(struct perf_counter *counter,
|
2008-12-13 08:00:03 +00:00
|
|
|
struct hw_perf_counter *hwc, int idx)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
2008-12-22 10:10:42 +00:00
|
|
|
s64 left = atomic64_read(&hwc->period_left);
|
2009-03-13 11:21:28 +00:00
|
|
|
s64 period = hwc->irq_period;
|
2008-12-22 10:10:42 +00:00
|
|
|
int err;
|
2008-12-13 08:00:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we are way outside a reasoable range then just skip forward:
|
|
|
|
*/
|
|
|
|
if (unlikely(left <= -period)) {
|
|
|
|
left = period;
|
|
|
|
atomic64_set(&hwc->period_left, left);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlikely(left <= 0)) {
|
|
|
|
left += period;
|
|
|
|
atomic64_set(&hwc->period_left, left);
|
|
|
|
}
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
per_cpu(prev_left[idx], smp_processor_id()) = left;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The hw counter starts counting from this counter offset,
|
|
|
|
* mark it to be able to extra future deltas:
|
|
|
|
*/
|
2008-12-22 10:10:42 +00:00
|
|
|
atomic64_set(&hwc->prev_count, (u64)-left);
|
2008-12-13 08:00:03 +00:00
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
err = checking_wrmsrl(hwc->counter_base + idx,
|
2009-04-29 10:47:12 +00:00
|
|
|
(u64)(-left) & x86_pmu.counter_mask);
|
2008-12-22 10:10:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
__pmc_fixed_enable(struct perf_counter *counter,
|
2009-04-29 10:47:17 +00:00
|
|
|
struct hw_perf_counter *hwc, int __idx)
|
2008-12-22 10:10:42 +00:00
|
|
|
{
|
|
|
|
int idx = __idx - X86_PMC_IDX_FIXED;
|
|
|
|
u64 ctrl_val, bits, mask;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
/*
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
* Enable IRQ generation (0x8),
|
|
|
|
* and enable ring-3 counting (0x2) and ring-0 counting (0x1)
|
|
|
|
* if requested:
|
2008-12-22 10:10:42 +00:00
|
|
|
*/
|
perf_counters: allow users to count user, kernel and/or hypervisor events
Impact: new perf_counter feature
This extends the perf_counter_hw_event struct with bits that specify
that events in user, kernel and/or hypervisor mode should not be
counted (i.e. should be excluded), and adds code to program the PMU
mode selection bits accordingly on x86 and powerpc.
For software counters, we don't currently have the infrastructure to
distinguish which mode an event occurs in, so we currently fail the
counter initialization if the setting of the hw_event.exclude_* bits
would require us to distinguish. Context switches and CPU migrations
are currently considered to occur in kernel mode.
On x86, this changes the previous policy that only root can count
kernel events. Now non-root users can count kernel events or exclude
them. Non-root users still can't use NMI events, though. On x86 we
don't appear to have any way to control whether hypervisor events are
counted or not, so hw_event.exclude_hv is ignored.
On powerpc, the selection of whether to count events in user, kernel
and/or hypervisor mode is PMU-wide, not per-counter, so this adds a
check that the hw_event.exclude_* settings are the same as other events
on the PMU. Counters being added to a group have to have the same
settings as the other hardware counters in the group. Counters and
groups can only be enabled in hw_perf_group_sched_in or power_perf_enable
if they have the same settings as any other counters already on the
PMU. If we are not running on a hypervisor, the exclude_hv setting
is ignored (by forcing it to 0) since we can't ever get any
hypervisor events.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-02-11 03:35:35 +00:00
|
|
|
bits = 0x8ULL;
|
|
|
|
if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
|
|
|
|
bits |= 0x2;
|
2008-12-22 10:10:42 +00:00
|
|
|
if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
|
|
|
|
bits |= 0x1;
|
|
|
|
bits <<= (idx * 4);
|
|
|
|
mask = 0xfULL << (idx * 4);
|
|
|
|
|
|
|
|
rdmsrl(hwc->config_base, ctrl_val);
|
|
|
|
ctrl_val &= ~mask;
|
|
|
|
ctrl_val |= bits;
|
|
|
|
err = checking_wrmsrl(hwc->config_base, ctrl_val);
|
2008-12-09 10:40:46 +00:00
|
|
|
}
|
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
static void
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_enable(struct perf_counter *counter,
|
|
|
|
struct hw_perf_counter *hwc, int idx)
|
2008-12-09 10:40:46 +00:00
|
|
|
{
|
2008-12-22 10:10:42 +00:00
|
|
|
if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
|
2008-12-27 13:45:43 +00:00
|
|
|
__pmc_fixed_enable(counter, hwc, idx);
|
|
|
|
else
|
2009-03-05 17:08:27 +00:00
|
|
|
hw_perf_enable(idx, hwc->config);
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
static int
|
|
|
|
fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
|
2008-12-17 12:09:20 +00:00
|
|
|
{
|
2008-12-22 10:10:42 +00:00
|
|
|
unsigned int event;
|
|
|
|
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
|
|
|
|
return -1;
|
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
if (unlikely(hwc->nmi))
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
event = hwc->config & ARCH_PERFMON_EVENT_MASK;
|
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
|
2008-12-22 10:10:42 +00:00
|
|
|
return X86_PMC_IDX_FIXED_INSTRUCTIONS;
|
2009-04-29 10:47:11 +00:00
|
|
|
if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
|
2008-12-22 10:10:42 +00:00
|
|
|
return X86_PMC_IDX_FIXED_CPU_CYCLES;
|
2009-04-29 10:47:11 +00:00
|
|
|
if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
|
2008-12-22 10:10:42 +00:00
|
|
|
return X86_PMC_IDX_FIXED_BUS_CYCLES;
|
|
|
|
|
2008-12-17 12:09:20 +00:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
/*
|
|
|
|
* Find a PMC slot for the freshly enabled / scheduled in counter:
|
|
|
|
*/
|
2009-04-29 10:47:03 +00:00
|
|
|
static int x86_pmu_enable(struct perf_counter *counter)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
|
|
|
struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
|
|
|
|
struct hw_perf_counter *hwc = &counter->hw;
|
2008-12-22 10:10:42 +00:00
|
|
|
int idx;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
idx = fixed_mode_idx(counter, hwc);
|
|
|
|
if (idx >= 0) {
|
|
|
|
/*
|
|
|
|
* Try to get the fixed counter, if that is already taken
|
|
|
|
* then try to get a generic counter:
|
|
|
|
*/
|
|
|
|
if (test_and_set_bit(idx, cpuc->used))
|
|
|
|
goto try_generic;
|
2008-12-23 11:28:12 +00:00
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
|
|
|
|
/*
|
|
|
|
* We set it so that counter_base + idx in wrmsr/rdmsr maps to
|
|
|
|
* MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
|
|
|
|
*/
|
|
|
|
hwc->counter_base =
|
|
|
|
MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
|
2008-12-03 09:39:53 +00:00
|
|
|
hwc->idx = idx;
|
2008-12-22 10:10:42 +00:00
|
|
|
} else {
|
|
|
|
idx = hwc->idx;
|
|
|
|
/* Try to get the previous generic counter again */
|
|
|
|
if (test_and_set_bit(idx, cpuc->used)) {
|
|
|
|
try_generic:
|
2009-04-29 10:47:12 +00:00
|
|
|
idx = find_first_zero_bit(cpuc->used,
|
|
|
|
x86_pmu.num_counters);
|
|
|
|
if (idx == x86_pmu.num_counters)
|
2008-12-22 10:10:42 +00:00
|
|
|
return -EAGAIN;
|
|
|
|
|
|
|
|
set_bit(idx, cpuc->used);
|
|
|
|
hwc->idx = idx;
|
|
|
|
}
|
2009-04-29 10:47:11 +00:00
|
|
|
hwc->config_base = x86_pmu.eventsel;
|
|
|
|
hwc->counter_base = x86_pmu.perfctr;
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
perf_counters_lapic_init(hwc->nmi);
|
|
|
|
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_disable(counter, hwc, idx);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-17 12:09:20 +00:00
|
|
|
cpuc->counters[idx] = counter;
|
2009-04-29 10:47:16 +00:00
|
|
|
set_bit(idx, cpuc->active);
|
2008-12-09 10:40:46 +00:00
|
|
|
|
2009-04-29 10:47:08 +00:00
|
|
|
x86_perf_counter_set_period(counter, hwc, idx);
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_enable(counter, hwc, idx);
|
2008-12-21 12:50:42 +00:00
|
|
|
|
|
|
|
return 0;
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void perf_counter_print_debug(void)
|
|
|
|
{
|
2008-12-22 10:10:42 +00:00
|
|
|
u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
|
2008-12-23 11:28:12 +00:00
|
|
|
struct cpu_hw_counters *cpuc;
|
2008-12-09 11:18:18 +00:00
|
|
|
int cpu, idx;
|
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
if (!x86_pmu.num_counters)
|
2008-12-09 11:18:18 +00:00
|
|
|
return;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
|
|
|
|
cpu = smp_processor_id();
|
2008-12-23 11:28:12 +00:00
|
|
|
cpuc = &per_cpu(cpu_hw_counters, cpu);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:13 +00:00
|
|
|
if (x86_pmu.version >= 2) {
|
2009-02-28 13:15:39 +00:00
|
|
|
rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
|
|
|
|
rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
|
|
|
|
rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
|
|
|
|
rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
|
|
|
|
|
|
|
|
pr_info("\n");
|
|
|
|
pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
|
|
|
|
pr_info("CPU#%d: status: %016llx\n", cpu, status);
|
|
|
|
pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
|
|
|
|
pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
}
|
2009-02-28 13:15:39 +00:00
|
|
|
pr_info("CPU#%d: used: %016llx\n", cpu, *(u64 *)cpuc->used);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
for (idx = 0; idx < x86_pmu.num_counters; idx++) {
|
2009-04-29 10:47:11 +00:00
|
|
|
rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
|
|
|
|
rdmsrl(x86_pmu.perfctr + idx, pmc_count);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
prev_left = per_cpu(prev_left[idx], cpu);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-02-28 13:15:39 +00:00
|
|
|
pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
|
2008-12-03 09:39:53 +00:00
|
|
|
cpu, idx, pmc_ctrl);
|
2009-02-28 13:15:39 +00:00
|
|
|
pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
|
2008-12-03 09:39:53 +00:00
|
|
|
cpu, idx, pmc_count);
|
2009-02-28 13:15:39 +00:00
|
|
|
pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
|
2008-12-13 08:00:03 +00:00
|
|
|
cpu, idx, prev_left);
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
2009-04-29 10:47:12 +00:00
|
|
|
for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
|
2008-12-22 10:10:42 +00:00
|
|
|
rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
|
|
|
|
|
2009-02-28 13:15:39 +00:00
|
|
|
pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
|
2008-12-22 10:10:42 +00:00
|
|
|
cpu, idx, pmc_count);
|
|
|
|
}
|
2008-12-03 09:39:53 +00:00
|
|
|
local_irq_enable();
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:03 +00:00
|
|
|
static void x86_pmu_disable(struct perf_counter *counter)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
|
|
|
struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
|
|
|
|
struct hw_perf_counter *hwc = &counter->hw;
|
2009-04-29 10:47:17 +00:00
|
|
|
int idx = hwc->idx;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:16 +00:00
|
|
|
/*
|
|
|
|
* Must be done before we disable, otherwise the nmi handler
|
|
|
|
* could reenable again:
|
|
|
|
*/
|
|
|
|
clear_bit(idx, cpuc->active);
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_disable(counter, hwc, idx);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
/*
|
|
|
|
* Make sure the cleared pointer becomes visible before we
|
|
|
|
* (potentially) free the counter:
|
|
|
|
*/
|
2009-04-29 10:47:02 +00:00
|
|
|
barrier();
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
/*
|
|
|
|
* Drain the remaining delta count out of a counter
|
|
|
|
* that we are disabling:
|
|
|
|
*/
|
|
|
|
x86_perf_counter_update(counter, hwc, idx);
|
2009-04-29 10:47:16 +00:00
|
|
|
cpuc->counters[idx] = NULL;
|
|
|
|
clear_bit(idx, cpuc->used);
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
2008-12-09 10:40:46 +00:00
|
|
|
/*
|
2008-12-13 08:00:03 +00:00
|
|
|
* Save and restart an expired counter. Called by NMI contexts,
|
|
|
|
* so it has to be careful about preempting normal counter ops:
|
2008-12-09 10:40:46 +00:00
|
|
|
*/
|
2009-04-29 10:47:09 +00:00
|
|
|
static void intel_pmu_save_and_restart(struct perf_counter *counter)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
|
|
|
struct hw_perf_counter *hwc = &counter->hw;
|
|
|
|
int idx = hwc->idx;
|
|
|
|
|
2008-12-13 08:00:03 +00:00
|
|
|
x86_perf_counter_update(counter, hwc, idx);
|
2009-04-29 10:47:08 +00:00
|
|
|
x86_perf_counter_set_period(counter, hwc, idx);
|
2008-12-09 10:40:46 +00:00
|
|
|
|
2008-12-22 10:10:42 +00:00
|
|
|
if (counter->state == PERF_COUNTER_STATE_ACTIVE)
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_enable(counter, hwc, idx);
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
2009-01-23 13:36:16 +00:00
|
|
|
/*
|
|
|
|
* Maximum interrupt frequency of 100KHz per CPU
|
|
|
|
*/
|
2009-02-28 13:07:49 +00:00
|
|
|
#define PERFMON_MAX_INTERRUPTS (100000/HZ)
|
2009-01-23 13:36:16 +00:00
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
/*
|
|
|
|
* This handler is triggered by the local APIC, so the APIC IRQ handling
|
|
|
|
* rules apply:
|
|
|
|
*/
|
2009-04-29 10:47:05 +00:00
|
|
|
static int intel_pmu_handle_irq(struct pt_regs *regs, int nmi)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
|
|
|
int bit, cpu = smp_processor_id();
|
2009-01-23 13:36:16 +00:00
|
|
|
u64 ack, status;
|
2009-01-23 09:13:01 +00:00
|
|
|
struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
|
2009-03-05 17:08:27 +00:00
|
|
|
int ret = 0;
|
2008-12-09 11:23:59 +00:00
|
|
|
|
2009-04-29 10:47:09 +00:00
|
|
|
cpuc->throttle_ctrl = intel_pmu_save_disable_all();
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:06 +00:00
|
|
|
status = intel_pmu_get_status(cpuc->throttle_ctrl);
|
2008-12-08 13:20:16 +00:00
|
|
|
if (!status)
|
|
|
|
goto out;
|
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
ret = 1;
|
2008-12-03 09:39:53 +00:00
|
|
|
again:
|
2009-02-09 06:38:50 +00:00
|
|
|
inc_irq_stat(apic_perf_irqs);
|
2008-12-03 09:39:53 +00:00
|
|
|
ack = status;
|
2008-12-22 10:10:42 +00:00
|
|
|
for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
|
2008-12-17 12:09:20 +00:00
|
|
|
struct perf_counter *counter = cpuc->counters[bit];
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
clear_bit(bit, (unsigned long *) &status);
|
2009-04-29 10:47:16 +00:00
|
|
|
if (!test_bit(bit, cpuc->active))
|
2008-12-03 09:39:53 +00:00
|
|
|
continue;
|
|
|
|
|
2009-04-29 10:47:09 +00:00
|
|
|
intel_pmu_save_and_restart(counter);
|
2009-04-08 13:01:33 +00:00
|
|
|
if (perf_counter_overflow(counter, nmi, regs, 0))
|
2009-04-29 10:47:03 +00:00
|
|
|
__x86_pmu_disable(counter, &counter->hw, bit);
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:07 +00:00
|
|
|
intel_pmu_ack_status(ack);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Repeat if there is more work to be done:
|
|
|
|
*/
|
2009-04-29 10:47:06 +00:00
|
|
|
status = intel_pmu_get_status(cpuc->throttle_ctrl);
|
2008-12-03 09:39:53 +00:00
|
|
|
if (status)
|
|
|
|
goto again;
|
2008-12-08 13:20:16 +00:00
|
|
|
out:
|
2008-12-03 09:39:53 +00:00
|
|
|
/*
|
2009-01-23 09:13:01 +00:00
|
|
|
* Restore - do not reenable when global enable is off or throttled:
|
2008-12-03 09:39:53 +00:00
|
|
|
*/
|
2009-01-23 13:36:16 +00:00
|
|
|
if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
|
2009-04-29 10:47:09 +00:00
|
|
|
intel_pmu_restore_all(cpuc->throttle_ctrl);
|
2009-03-05 17:08:27 +00:00
|
|
|
|
|
|
|
return ret;
|
2009-01-23 09:13:01 +00:00
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:05 +00:00
|
|
|
static int amd_pmu_handle_irq(struct pt_regs *regs, int nmi) { return 0; }
|
|
|
|
|
2009-01-23 09:13:01 +00:00
|
|
|
void perf_counter_unthrottle(void)
|
|
|
|
{
|
|
|
|
struct cpu_hw_counters *cpuc;
|
|
|
|
|
|
|
|
if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (unlikely(!perf_counters_initialized))
|
|
|
|
return;
|
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
cpuc = &__get_cpu_var(cpu_hw_counters);
|
2009-01-23 13:36:16 +00:00
|
|
|
if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
|
2009-01-23 09:13:01 +00:00
|
|
|
if (printk_ratelimit())
|
2009-01-23 13:36:16 +00:00
|
|
|
printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
|
2009-03-05 17:08:27 +00:00
|
|
|
hw_perf_restore(cpuc->throttle_ctrl);
|
2009-01-23 09:13:01 +00:00
|
|
|
}
|
2009-01-23 13:36:16 +00:00
|
|
|
cpuc->interrupts = 0;
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void smp_perf_counter_interrupt(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
irq_enter();
|
|
|
|
apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
|
2009-03-05 17:08:27 +00:00
|
|
|
ack_APIC_irq();
|
2009-04-29 10:47:11 +00:00
|
|
|
x86_pmu.handle_irq(regs, 0);
|
2008-12-03 09:39:53 +00:00
|
|
|
irq_exit();
|
|
|
|
}
|
|
|
|
|
2009-04-06 09:45:03 +00:00
|
|
|
void smp_perf_pending_interrupt(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
irq_enter();
|
|
|
|
ack_APIC_irq();
|
|
|
|
inc_irq_stat(apic_pending_irqs);
|
|
|
|
perf_counter_do_pending();
|
|
|
|
irq_exit();
|
|
|
|
}
|
|
|
|
|
|
|
|
void set_perf_counter_pending(void)
|
|
|
|
{
|
|
|
|
apic->send_IPI_self(LOCAL_PENDING_VECTOR);
|
|
|
|
}
|
|
|
|
|
2009-01-23 13:16:53 +00:00
|
|
|
void perf_counters_lapic_init(int nmi)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
|
|
|
u32 apic_val;
|
|
|
|
|
|
|
|
if (!perf_counters_initialized)
|
|
|
|
return;
|
|
|
|
/*
|
|
|
|
* Enable the performance counter vector in the APIC LVT:
|
|
|
|
*/
|
|
|
|
apic_val = apic_read(APIC_LVTERR);
|
|
|
|
|
|
|
|
apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
|
|
|
|
if (nmi)
|
|
|
|
apic_write(APIC_LVTPC, APIC_DM_NMI);
|
|
|
|
else
|
|
|
|
apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
|
|
|
|
apic_write(APIC_LVTERR, apic_val);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __kprobes
|
|
|
|
perf_counter_nmi_handler(struct notifier_block *self,
|
|
|
|
unsigned long cmd, void *__args)
|
|
|
|
{
|
|
|
|
struct die_args *args = __args;
|
|
|
|
struct pt_regs *regs;
|
2009-03-05 17:08:27 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
switch (cmd) {
|
|
|
|
case DIE_NMI:
|
|
|
|
case DIE_NMI_IPI:
|
|
|
|
break;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
default:
|
2008-12-03 09:39:53 +00:00
|
|
|
return NOTIFY_DONE;
|
2009-03-05 17:08:27 +00:00
|
|
|
}
|
2008-12-03 09:39:53 +00:00
|
|
|
|
|
|
|
regs = args->regs;
|
|
|
|
|
|
|
|
apic_write(APIC_LVTPC, APIC_DM_NMI);
|
2009-04-29 10:47:11 +00:00
|
|
|
ret = x86_pmu.handle_irq(regs, 1);
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-03-05 17:08:27 +00:00
|
|
|
return ret ? NOTIFY_STOP : NOTIFY_OK;
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
|
2009-02-04 16:11:34 +00:00
|
|
|
.notifier_call = perf_counter_nmi_handler,
|
|
|
|
.next = NULL,
|
|
|
|
.priority = 1
|
2008-12-03 09:39:53 +00:00
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static struct x86_pmu intel_pmu = {
|
2009-04-29 10:47:13 +00:00
|
|
|
.name = "Intel",
|
2009-04-29 10:47:05 +00:00
|
|
|
.handle_irq = intel_pmu_handle_irq,
|
2009-04-29 10:47:04 +00:00
|
|
|
.save_disable_all = intel_pmu_save_disable_all,
|
|
|
|
.restore_all = intel_pmu_restore_all,
|
|
|
|
.enable = intel_pmu_enable_counter,
|
|
|
|
.disable = intel_pmu_disable_counter,
|
2009-02-27 12:39:09 +00:00
|
|
|
.eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
|
|
|
|
.perfctr = MSR_ARCH_PERFMON_PERFCTR0,
|
2009-04-29 10:47:04 +00:00
|
|
|
.event_map = intel_pmu_event_map,
|
|
|
|
.raw_event = intel_pmu_raw_event,
|
2009-02-27 12:39:09 +00:00
|
|
|
.max_events = ARRAY_SIZE(intel_perfmon_event_map),
|
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:04 +00:00
|
|
|
static struct x86_pmu amd_pmu = {
|
2009-04-29 10:47:13 +00:00
|
|
|
.name = "AMD",
|
2009-04-29 10:47:05 +00:00
|
|
|
.handle_irq = amd_pmu_handle_irq,
|
2009-04-29 10:47:04 +00:00
|
|
|
.save_disable_all = amd_pmu_save_disable_all,
|
|
|
|
.restore_all = amd_pmu_restore_all,
|
|
|
|
.enable = amd_pmu_enable_counter,
|
|
|
|
.disable = amd_pmu_disable_counter,
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
.eventsel = MSR_K7_EVNTSEL0,
|
|
|
|
.perfctr = MSR_K7_PERFCTR0,
|
2009-04-29 10:47:04 +00:00
|
|
|
.event_map = amd_pmu_event_map,
|
|
|
|
.raw_event = amd_pmu_raw_event,
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
.max_events = ARRAY_SIZE(amd_perfmon_event_map),
|
2009-04-29 10:47:12 +00:00
|
|
|
.num_counters = 4,
|
|
|
|
.counter_bits = 48,
|
|
|
|
.counter_mask = (1ULL << 48) - 1,
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:10 +00:00
|
|
|
static int intel_pmu_init(void)
|
2008-12-03 09:39:53 +00:00
|
|
|
{
|
2009-03-18 07:59:21 +00:00
|
|
|
union cpuid10_edx edx;
|
2008-12-03 09:39:53 +00:00
|
|
|
union cpuid10_eax eax;
|
2008-12-17 09:51:15 +00:00
|
|
|
unsigned int unused;
|
2009-03-18 07:59:21 +00:00
|
|
|
unsigned int ebx;
|
2009-04-29 10:47:13 +00:00
|
|
|
int version;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:46:58 +00:00
|
|
|
if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
|
2009-04-29 10:47:10 +00:00
|
|
|
return -ENODEV;
|
2009-04-29 10:46:58 +00:00
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
/*
|
|
|
|
* Check whether the Architectural PerfMon supports
|
|
|
|
* Branch Misses Retired Event or not.
|
|
|
|
*/
|
2008-12-17 09:51:15 +00:00
|
|
|
cpuid(10, &eax.full, &ebx, &unused, &edx.full);
|
2008-12-03 09:39:53 +00:00
|
|
|
if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
|
2009-04-29 10:47:10 +00:00
|
|
|
return -ENODEV;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:13 +00:00
|
|
|
version = eax.split.version_id;
|
|
|
|
if (version < 2)
|
2009-04-29 10:47:10 +00:00
|
|
|
return -ENODEV;
|
2009-03-18 07:59:21 +00:00
|
|
|
|
2009-04-29 10:47:11 +00:00
|
|
|
x86_pmu = intel_pmu;
|
2009-04-29 10:47:13 +00:00
|
|
|
x86_pmu.version = version;
|
2009-04-29 10:47:12 +00:00
|
|
|
x86_pmu.num_counters = eax.split.num_counters;
|
|
|
|
x86_pmu.num_counters_fixed = edx.split.num_counters_fixed;
|
|
|
|
x86_pmu.counter_bits = eax.split.bit_width;
|
|
|
|
x86_pmu.counter_mask = (1ULL << eax.split.bit_width) - 1;
|
2009-02-27 12:39:09 +00:00
|
|
|
|
2009-04-29 10:47:10 +00:00
|
|
|
return 0;
|
2009-02-27 12:39:09 +00:00
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:10 +00:00
|
|
|
static int amd_pmu_init(void)
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
{
|
2009-04-29 10:47:11 +00:00
|
|
|
x86_pmu = amd_pmu;
|
2009-04-29 10:47:10 +00:00
|
|
|
return 0;
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
}
|
|
|
|
|
2009-02-27 12:39:09 +00:00
|
|
|
void __init init_hw_perf_counters(void)
|
|
|
|
{
|
2009-04-29 10:47:10 +00:00
|
|
|
int err;
|
|
|
|
|
2009-02-27 12:39:09 +00:00
|
|
|
switch (boot_cpu_data.x86_vendor) {
|
|
|
|
case X86_VENDOR_INTEL:
|
2009-04-29 10:47:10 +00:00
|
|
|
err = intel_pmu_init();
|
2009-02-27 12:39:09 +00:00
|
|
|
break;
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
case X86_VENDOR_AMD:
|
2009-04-29 10:47:10 +00:00
|
|
|
err = amd_pmu_init();
|
x86: AMD Support for perf_counter
Supported basic performance counter for AMD K7 and later:
$ perfstat -e 0,1,2,3,4,5,-1,-2,-3,-4,-5 ls > /dev/null
Performance counter stats for 'ls':
12.298610 task clock ticks (msecs)
3298477 CPU cycles (events)
1406354 instructions (events)
749035 cache references (events)
16939 cache misses (events)
100589 branches (events)
11159 branch misses (events)
7.627540 cpu clock ticks (msecs)
12.298610 task clock ticks (msecs)
500 pagefaults (events)
6 context switches (events)
3 CPU migrations (events)
Wall-clock time elapsed: 8.672290 msecs
Signed-off-by: Jaswinder Singh Rajput <jaswinderrajput@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-27 14:45:14 +00:00
|
|
|
break;
|
2009-04-29 10:47:00 +00:00
|
|
|
default:
|
|
|
|
return;
|
2009-02-27 12:39:09 +00:00
|
|
|
}
|
2009-04-29 10:47:10 +00:00
|
|
|
if (err != 0)
|
2009-02-27 12:39:09 +00:00
|
|
|
return;
|
|
|
|
|
2009-04-29 10:47:13 +00:00
|
|
|
pr_info("%s Performance Monitoring support detected.\n", x86_pmu.name);
|
|
|
|
pr_info("... version: %d\n", x86_pmu.version);
|
|
|
|
pr_info("... bit width: %d\n", x86_pmu.counter_bits);
|
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
pr_info("... num counters: %d\n", x86_pmu.num_counters);
|
|
|
|
if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
|
|
|
|
x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
|
2008-12-03 09:39:53 +00:00
|
|
|
WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
|
2009-04-29 10:47:12 +00:00
|
|
|
x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
|
2008-12-03 09:39:53 +00:00
|
|
|
}
|
2009-04-29 10:47:12 +00:00
|
|
|
perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
|
|
|
|
perf_max_counters = x86_pmu.num_counters;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
pr_info("... value mask: %016Lx\n", x86_pmu.counter_mask);
|
2008-12-22 10:10:42 +00:00
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
|
|
|
|
x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
|
2008-12-17 09:51:15 +00:00
|
|
|
WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
|
2009-04-29 10:47:12 +00:00
|
|
|
x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
|
2008-12-17 09:51:15 +00:00
|
|
|
}
|
2009-04-29 10:47:12 +00:00
|
|
|
pr_info("... fixed counters: %d\n", x86_pmu.num_counters_fixed);
|
2008-12-17 12:09:20 +00:00
|
|
|
|
2009-04-29 10:47:12 +00:00
|
|
|
perf_counter_mask |=
|
|
|
|
((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
|
2008-12-03 09:39:53 +00:00
|
|
|
|
2009-02-28 13:15:39 +00:00
|
|
|
pr_info("... counter mask: %016Lx\n", perf_counter_mask);
|
2008-12-14 20:58:46 +00:00
|
|
|
perf_counters_initialized = true;
|
|
|
|
|
2008-12-03 09:39:53 +00:00
|
|
|
perf_counters_lapic_init(0);
|
|
|
|
register_die_notifier(&perf_counter_nmi_notifier);
|
|
|
|
}
|
2008-12-11 11:46:46 +00:00
|
|
|
|
2009-04-29 10:47:14 +00:00
|
|
|
static inline void x86_pmu_read(struct perf_counter *counter)
|
2008-12-13 08:00:03 +00:00
|
|
|
{
|
|
|
|
x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
|
|
|
|
}
|
|
|
|
|
2009-04-29 10:47:03 +00:00
|
|
|
static const struct pmu pmu = {
|
|
|
|
.enable = x86_pmu_enable,
|
|
|
|
.disable = x86_pmu_disable,
|
|
|
|
.read = x86_pmu_read,
|
2008-12-11 11:46:46 +00:00
|
|
|
};
|
|
|
|
|
2009-04-29 10:47:03 +00:00
|
|
|
const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
|
2008-12-11 11:46:46 +00:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = __hw_perf_counter_init(counter);
|
|
|
|
if (err)
|
2009-03-30 17:07:09 +00:00
|
|
|
return ERR_PTR(err);
|
2008-12-11 11:46:46 +00:00
|
|
|
|
2009-04-29 10:47:03 +00:00
|
|
|
return &pmu;
|
2008-12-11 11:46:46 +00:00
|
|
|
}
|
2009-03-30 17:07:15 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* callchain support
|
|
|
|
*/
|
|
|
|
|
|
|
|
static inline
|
|
|
|
void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
|
|
|
|
{
|
|
|
|
if (entry->nr < MAX_STACK_DEPTH)
|
|
|
|
entry->ip[entry->nr++] = ip;
|
|
|
|
}
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
|
|
|
|
static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
|
|
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
|
|
|
|
{
|
|
|
|
/* Ignore warnings */
|
|
|
|
}
|
|
|
|
|
|
|
|
static void backtrace_warning(void *data, char *msg)
|
|
|
|
{
|
|
|
|
/* Ignore warnings */
|
|
|
|
}
|
|
|
|
|
|
|
|
static int backtrace_stack(void *data, char *name)
|
|
|
|
{
|
|
|
|
/* Don't bother with IRQ stacks for now */
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void backtrace_address(void *data, unsigned long addr, int reliable)
|
|
|
|
{
|
|
|
|
struct perf_callchain_entry *entry = data;
|
|
|
|
|
|
|
|
if (reliable)
|
|
|
|
callchain_store(entry, addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct stacktrace_ops backtrace_ops = {
|
|
|
|
.warning = backtrace_warning,
|
|
|
|
.warning_symbol = backtrace_warning_symbol,
|
|
|
|
.stack = backtrace_stack,
|
|
|
|
.address = backtrace_address,
|
|
|
|
};
|
|
|
|
|
|
|
|
static void
|
|
|
|
perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
|
|
|
|
{
|
|
|
|
unsigned long bp;
|
|
|
|
char *stack;
|
2009-04-02 09:12:03 +00:00
|
|
|
int nr = entry->nr;
|
2009-03-30 17:07:15 +00:00
|
|
|
|
|
|
|
callchain_store(entry, instruction_pointer(regs));
|
|
|
|
|
|
|
|
stack = ((char *)regs + sizeof(struct pt_regs));
|
|
|
|
#ifdef CONFIG_FRAME_POINTER
|
|
|
|
bp = frame_pointer(regs);
|
|
|
|
#else
|
|
|
|
bp = 0;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
|
2009-04-02 09:12:03 +00:00
|
|
|
|
|
|
|
entry->kernel = entry->nr - nr;
|
2009-03-30 17:07:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
struct stack_frame {
|
|
|
|
const void __user *next_fp;
|
|
|
|
unsigned long return_address;
|
|
|
|
};
|
|
|
|
|
|
|
|
static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ret = 1;
|
|
|
|
pagefault_disable();
|
|
|
|
if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
|
|
|
|
ret = 0;
|
|
|
|
pagefault_enable();
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
|
|
|
|
{
|
|
|
|
struct stack_frame frame;
|
|
|
|
const void __user *fp;
|
2009-04-02 09:12:03 +00:00
|
|
|
int nr = entry->nr;
|
2009-03-30 17:07:15 +00:00
|
|
|
|
|
|
|
regs = (struct pt_regs *)current->thread.sp0 - 1;
|
|
|
|
fp = (void __user *)regs->bp;
|
|
|
|
|
|
|
|
callchain_store(entry, regs->ip);
|
|
|
|
|
|
|
|
while (entry->nr < MAX_STACK_DEPTH) {
|
|
|
|
frame.next_fp = NULL;
|
|
|
|
frame.return_address = 0;
|
|
|
|
|
|
|
|
if (!copy_stack_frame(fp, &frame))
|
|
|
|
break;
|
|
|
|
|
|
|
|
if ((unsigned long)fp < user_stack_pointer(regs))
|
|
|
|
break;
|
|
|
|
|
|
|
|
callchain_store(entry, frame.return_address);
|
|
|
|
fp = frame.next_fp;
|
|
|
|
}
|
2009-04-02 09:12:03 +00:00
|
|
|
|
|
|
|
entry->user = entry->nr - nr;
|
2009-03-30 17:07:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
|
|
|
|
{
|
|
|
|
int is_user;
|
|
|
|
|
|
|
|
if (!regs)
|
|
|
|
return;
|
|
|
|
|
|
|
|
is_user = user_mode(regs);
|
|
|
|
|
|
|
|
if (!current || current->pid == 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (is_user && current->state != TASK_RUNNING)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!is_user)
|
|
|
|
perf_callchain_kernel(regs, entry);
|
|
|
|
|
|
|
|
if (current->mm)
|
|
|
|
perf_callchain_user(regs, entry);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct perf_callchain_entry *entry;
|
|
|
|
|
|
|
|
if (in_nmi())
|
|
|
|
entry = &__get_cpu_var(nmi_entry);
|
|
|
|
else
|
|
|
|
entry = &__get_cpu_var(irq_entry);
|
|
|
|
|
|
|
|
entry->nr = 0;
|
2009-04-02 09:12:03 +00:00
|
|
|
entry->hv = 0;
|
|
|
|
entry->kernel = 0;
|
|
|
|
entry->user = 0;
|
2009-03-30 17:07:15 +00:00
|
|
|
|
|
|
|
perf_do_callchain(regs, entry);
|
|
|
|
|
|
|
|
return entry;
|
|
|
|
}
|