Incorporates the new performance counter reservation system in oprofile.
Also cleans up a lot of the initialization code. The code original zero'd
out every register associated with performance counters regardless if those
registers were used or not. This causes issues with the nmi watchdog.
Now oprofile tries to reserve registers and gives up if it can't get them.
Cc: levon@movementarian.org
Cc: oprofile-list@lists.sf.net
Signed-off-by: Don Zickus <dzickus@redhat.com>
Signed-off-by: Andi Kleen <ak@suse.de>
unsigned int i;
for (i = 0; i < nr_ctrs; ++i) {
- rdmsr(counters[i].addr,
- counters[i].saved.low,
- counters[i].saved.high);
+ if (counters[i].addr){
+ rdmsr(counters[i].addr,
+ counters[i].saved.low,
+ counters[i].saved.high);
+ }
}
for (i = 0; i < nr_ctrls; ++i) {
- rdmsr(controls[i].addr,
- controls[i].saved.low,
- controls[i].saved.high);
+ if (controls[i].addr){
+ rdmsr(controls[i].addr,
+ controls[i].saved.low,
+ controls[i].saved.high);
+ }
}
}
unsigned int i;
for (i = 0; i < nr_ctrls; ++i) {
- wrmsr(controls[i].addr,
- controls[i].saved.low,
- controls[i].saved.high);
+ if (controls[i].addr){
+ wrmsr(controls[i].addr,
+ controls[i].saved.low,
+ controls[i].saved.high);
+ }
}
for (i = 0; i < nr_ctrs; ++i) {
- wrmsr(counters[i].addr,
- counters[i].saved.low,
- counters[i].saved.high);
+ if (counters[i].addr){
+ wrmsr(counters[i].addr,
+ counters[i].saved.low,
+ counters[i].saved.high);
+ }
}
}
apic_write(APIC_LVTPC, saved_lvtpc[cpu]);
apic_write(APIC_LVTERR, v);
nmi_restore_registers(msrs);
+ model->shutdown(msrs);
}
struct dentry * dir;
char buf[4];
+ /* quick little hack to _not_ expose a counter if it is not
+ * available for use. This should protect userspace app.
+ * NOTE: assumes 1:1 mapping here (that counters are organized
+ * sequentially in their struct assignment).
+ */
+ if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
+ continue;
+
snprintf(buf, sizeof(buf), "%d", i);
dir = oprofilefs_mkdir(sb, root, buf);
oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled);
#define NUM_COUNTERS 4
#define NUM_CONTROLS 4
+#define CTR_IS_RESERVED(msrs,c) (msrs->counters[(c)].addr ? 1 : 0)
#define CTR_READ(l,h,msrs,c) do {rdmsr(msrs->counters[(c)].addr, (l), (h));} while (0)
#define CTR_WRITE(l,msrs,c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1);} while (0)
#define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
+#define CTRL_IS_RESERVED(msrs,c) (msrs->controls[(c)].addr ? 1 : 0)
#define CTRL_READ(l,h,msrs,c) do {rdmsr(msrs->controls[(c)].addr, (l), (h));} while (0)
#define CTRL_WRITE(l,h,msrs,c) do {wrmsr(msrs->controls[(c)].addr, (l), (h));} while (0)
#define CTRL_SET_ACTIVE(n) (n |= (1<<22))
static void athlon_fill_in_addresses(struct op_msrs * const msrs)
{
- msrs->counters[0].addr = MSR_K7_PERFCTR0;
- msrs->counters[1].addr = MSR_K7_PERFCTR1;
- msrs->counters[2].addr = MSR_K7_PERFCTR2;
- msrs->counters[3].addr = MSR_K7_PERFCTR3;
-
- msrs->controls[0].addr = MSR_K7_EVNTSEL0;
- msrs->controls[1].addr = MSR_K7_EVNTSEL1;
- msrs->controls[2].addr = MSR_K7_EVNTSEL2;
- msrs->controls[3].addr = MSR_K7_EVNTSEL3;
+ int i;
+
+ for (i=0; i < NUM_COUNTERS; i++) {
+ if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
+ msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
+ else
+ msrs->counters[i].addr = 0;
+ }
+
+ for (i=0; i < NUM_CONTROLS; i++) {
+ if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
+ msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
+ else
+ msrs->controls[i].addr = 0;
+ }
}
/* clear all counters */
for (i = 0 ; i < NUM_CONTROLS; ++i) {
+ if (unlikely(!CTRL_IS_RESERVED(msrs,i)))
+ continue;
CTRL_READ(low, high, msrs, i);
CTRL_CLEAR(low);
CTRL_WRITE(low, high, msrs, i);
}
-
+
/* avoid a false detection of ctr overflows in NMI handler */
for (i = 0; i < NUM_COUNTERS; ++i) {
+ if (unlikely(!CTR_IS_RESERVED(msrs,i)))
+ continue;
CTR_WRITE(1, msrs, i);
}
/* enable active counters */
for (i = 0; i < NUM_COUNTERS; ++i) {
- if (counter_config[i].enabled) {
+ if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs,i))) {
reset_value[i] = counter_config[i].count;
CTR_WRITE(counter_config[i].count, msrs, i);
int i;
for (i = 0 ; i < NUM_COUNTERS; ++i) {
+ if (!reset_value[i])
+ continue;
CTR_READ(low, high, msrs, i);
if (CTR_OVERFLOWED(low)) {
oprofile_add_sample(regs, i);
/* Subtle: stop on all counters to avoid race with
* setting our pm callback */
for (i = 0 ; i < NUM_COUNTERS ; ++i) {
+ if (!reset_value[i])
+ continue;
CTRL_READ(low, high, msrs, i);
CTRL_SET_INACTIVE(low);
CTRL_WRITE(low, high, msrs, i);
}
}
+static void athlon_shutdown(struct op_msrs const * const msrs)
+{
+ int i;
+
+ for (i = 0 ; i < NUM_COUNTERS ; ++i) {
+ if (CTR_IS_RESERVED(msrs,i))
+ release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
+ }
+ for (i = 0 ; i < NUM_CONTROLS ; ++i) {
+ if (CTRL_IS_RESERVED(msrs,i))
+ release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
+ }
+}
struct op_x86_model_spec const op_athlon_spec = {
.num_counters = NUM_COUNTERS,
.setup_ctrs = &athlon_setup_ctrs,
.check_ctrs = &athlon_check_ctrs,
.start = &athlon_start,
- .stop = &athlon_stop
+ .stop = &athlon_stop,
+ .shutdown = &athlon_shutdown
};
#define NUM_CONTROLS_HT2 (NUM_ESCRS_HT2 + NUM_CCCRS_HT2)
static unsigned int num_counters = NUM_COUNTERS_NON_HT;
-
+static unsigned int num_controls = NUM_CONTROLS_NON_HT;
/* this has to be checked dynamically since the
hyper-threadedness of a chip is discovered at
static inline void setup_num_counters(void)
{
#ifdef CONFIG_SMP
- if (smp_num_siblings == 2)
+ if (smp_num_siblings == 2){
num_counters = NUM_COUNTERS_HT2;
+ num_controls = NUM_CONTROLS_HT2;
+ }
#endif
}
#define NUM_UNUSED_CCCRS NUM_CCCRS_NON_HT - NUM_COUNTERS_NON_HT
-/* All cccr we don't use. */
-static int p4_unused_cccr[NUM_UNUSED_CCCRS] = {
- MSR_P4_BPU_CCCR1, MSR_P4_BPU_CCCR3,
- MSR_P4_MS_CCCR1, MSR_P4_MS_CCCR3,
- MSR_P4_FLAME_CCCR1, MSR_P4_FLAME_CCCR3,
- MSR_P4_IQ_CCCR0, MSR_P4_IQ_CCCR1,
- MSR_P4_IQ_CCCR2, MSR_P4_IQ_CCCR3
-};
-
/* p4 event codes in libop/op_event.h are indices into this table. */
static struct p4_event_binding p4_events[NUM_EVENTS] = {
#define CCCR_OVF_P(cccr) ((cccr) & (1U<<31))
#define CCCR_CLEAR_OVF(cccr) ((cccr) &= (~(1U<<31)))
+#define CTRL_IS_RESERVED(msrs,c) (msrs->controls[(c)].addr ? 1 : 0)
+#define CTR_IS_RESERVED(msrs,c) (msrs->counters[(c)].addr ? 1 : 0)
#define CTR_READ(l,h,i) do {rdmsr(p4_counters[(i)].counter_address, (l), (h));} while (0)
#define CTR_WRITE(l,i) do {wrmsr(p4_counters[(i)].counter_address, -(u32)(l), -1);} while (0)
#define CTR_OVERFLOW_P(ctr) (!((ctr) & 0x80000000))
static void p4_fill_in_addresses(struct op_msrs * const msrs)
{
unsigned int i;
- unsigned int addr, stag;
+ unsigned int addr, cccraddr, stag;
setup_num_counters();
stag = get_stagger();
- /* the counter registers we pay attention to */
+ /* initialize some registers */
for (i = 0; i < num_counters; ++i) {
- msrs->counters[i].addr =
- p4_counters[VIRT_CTR(stag, i)].counter_address;
+ msrs->counters[i].addr = 0;
}
-
- /* FIXME: bad feeling, we don't save the 10 counters we don't use. */
-
- /* 18 CCCR registers */
- for (i = 0, addr = MSR_P4_BPU_CCCR0 + stag;
- addr <= MSR_P4_IQ_CCCR5; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ for (i = 0; i < num_controls; ++i) {
+ msrs->controls[i].addr = 0;
}
+ /* the counter & cccr registers we pay attention to */
+ for (i = 0; i < num_counters; ++i) {
+ addr = p4_counters[VIRT_CTR(stag, i)].counter_address;
+ cccraddr = p4_counters[VIRT_CTR(stag, i)].cccr_address;
+ if (reserve_perfctr_nmi(addr)){
+ msrs->counters[i].addr = addr;
+ msrs->controls[i].addr = cccraddr;
+ }
+ }
+
/* 43 ESCR registers in three or four discontiguous group */
for (addr = MSR_P4_BSU_ESCR0 + stag;
addr < MSR_P4_IQ_ESCR0; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ if (reserve_evntsel_nmi(addr))
+ msrs->controls[i].addr = addr;
}
/* no IQ_ESCR0/1 on some models, we save a seconde time BSU_ESCR0/1
if (boot_cpu_data.x86_model >= 0x3) {
for (addr = MSR_P4_BSU_ESCR0 + stag;
addr <= MSR_P4_BSU_ESCR1; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ if (reserve_evntsel_nmi(addr))
+ msrs->controls[i].addr = addr;
}
} else {
for (addr = MSR_P4_IQ_ESCR0 + stag;
addr <= MSR_P4_IQ_ESCR1; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ if (reserve_evntsel_nmi(addr))
+ msrs->controls[i].addr = addr;
}
}
for (addr = MSR_P4_RAT_ESCR0 + stag;
addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ if (reserve_evntsel_nmi(addr))
+ msrs->controls[i].addr = addr;
}
for (addr = MSR_P4_MS_ESCR0 + stag;
addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ if (reserve_evntsel_nmi(addr))
+ msrs->controls[i].addr = addr;
}
for (addr = MSR_P4_IX_ESCR0 + stag;
addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) {
- msrs->controls[i].addr = addr;
+ if (reserve_evntsel_nmi(addr))
+ msrs->controls[i].addr = addr;
}
/* there are 2 remaining non-contiguously located ESCRs */
if (num_counters == NUM_COUNTERS_NON_HT) {
/* standard non-HT CPUs handle both remaining ESCRs*/
- msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
- msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;
+ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR5))
+ msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
+ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR4))
+ msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;
} else if (stag == 0) {
/* HT CPUs give the first remainder to the even thread, as
the 32nd control register */
- msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;
+ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR4))
+ msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;
} else {
/* and two copies of the second to the odd thread,
for the 22st and 23nd control registers */
- msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
- msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
+ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR5)) {
+ msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
+ msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
+ }
}
}
{
unsigned int i;
unsigned int low, high;
- unsigned int addr;
unsigned int stag;
stag = get_stagger();
/* clear the cccrs we will use */
for (i = 0 ; i < num_counters ; i++) {
+ if (unlikely(!CTRL_IS_RESERVED(msrs,i)))
+ continue;
rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
CCCR_CLEAR(low);
CCCR_SET_REQUIRED_BITS(low);
wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
}
- /* clear cccrs outside our concern */
- for (i = stag ; i < NUM_UNUSED_CCCRS ; i += addr_increment()) {
- rdmsr(p4_unused_cccr[i], low, high);
- CCCR_CLEAR(low);
- CCCR_SET_REQUIRED_BITS(low);
- wrmsr(p4_unused_cccr[i], low, high);
- }
-
/* clear all escrs (including those outside our concern) */
- for (addr = MSR_P4_BSU_ESCR0 + stag;
- addr < MSR_P4_IQ_ESCR0; addr += addr_increment()) {
- wrmsr(addr, 0, 0);
- }
-
- /* On older models clear also MSR_P4_IQ_ESCR0/1 */
- if (boot_cpu_data.x86_model < 0x3) {
- wrmsr(MSR_P4_IQ_ESCR0, 0, 0);
- wrmsr(MSR_P4_IQ_ESCR1, 0, 0);
- }
-
- for (addr = MSR_P4_RAT_ESCR0 + stag;
- addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) {
- wrmsr(addr, 0, 0);
- }
-
- for (addr = MSR_P4_MS_ESCR0 + stag;
- addr <= MSR_P4_TC_ESCR1; addr += addr_increment()){
- wrmsr(addr, 0, 0);
- }
-
- for (addr = MSR_P4_IX_ESCR0 + stag;
- addr <= MSR_P4_CRU_ESCR3; addr += addr_increment()){
- wrmsr(addr, 0, 0);
+ for (i = num_counters; i < num_controls; i++) {
+ if (unlikely(!CTRL_IS_RESERVED(msrs,i)))
+ continue;
+ wrmsr(msrs->controls[i].addr, 0, 0);
}
- if (num_counters == NUM_COUNTERS_NON_HT) {
- wrmsr(MSR_P4_CRU_ESCR4, 0, 0);
- wrmsr(MSR_P4_CRU_ESCR5, 0, 0);
- } else if (stag == 0) {
- wrmsr(MSR_P4_CRU_ESCR4, 0, 0);
- } else {
- wrmsr(MSR_P4_CRU_ESCR5, 0, 0);
- }
-
/* setup all counters */
for (i = 0 ; i < num_counters ; ++i) {
- if (counter_config[i].enabled) {
+ if ((counter_config[i].enabled) && (CTRL_IS_RESERVED(msrs,i))) {
reset_value[i] = counter_config[i].count;
pmc_setup_one_p4_counter(i);
CTR_WRITE(counter_config[i].count, VIRT_CTR(stag, i));
stag = get_stagger();
for (i = 0; i < num_counters; ++i) {
+ if (!reset_value[i])
+ continue;
CCCR_READ(low, high, VIRT_CTR(stag, i));
CCCR_SET_DISABLE(low);
CCCR_WRITE(low, high, VIRT_CTR(stag, i));
}
}
+static void p4_shutdown(struct op_msrs const * const msrs)
+{
+ int i;
+
+ for (i = 0 ; i < num_counters ; ++i) {
+ if (CTR_IS_RESERVED(msrs,i))
+ release_perfctr_nmi(msrs->counters[i].addr);
+ }
+ /* some of the control registers are specially reserved in
+ * conjunction with the counter registers (hence the starting offset).
+ * This saves a few bits.
+ */
+ for (i = num_counters ; i < num_controls ; ++i) {
+ if (CTRL_IS_RESERVED(msrs,i))
+ release_evntsel_nmi(msrs->controls[i].addr);
+ }
+}
+
#ifdef CONFIG_SMP
struct op_x86_model_spec const op_p4_ht2_spec = {
.setup_ctrs = &p4_setup_ctrs,
.check_ctrs = &p4_check_ctrs,
.start = &p4_start,
- .stop = &p4_stop
+ .stop = &p4_stop,
+ .shutdown = &p4_shutdown
};
#endif
.setup_ctrs = &p4_setup_ctrs,
.check_ctrs = &p4_check_ctrs,
.start = &p4_start,
- .stop = &p4_stop
+ .stop = &p4_stop,
+ .shutdown = &p4_shutdown
};
#define NUM_COUNTERS 2
#define NUM_CONTROLS 2
+#define CTR_IS_RESERVED(msrs,c) (msrs->counters[(c)].addr ? 1 : 0)
#define CTR_READ(l,h,msrs,c) do {rdmsr(msrs->counters[(c)].addr, (l), (h));} while (0)
#define CTR_WRITE(l,msrs,c) do {wrmsr(msrs->counters[(c)].addr, -(u32)(l), -1);} while (0)
#define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
+#define CTRL_IS_RESERVED(msrs,c) (msrs->controls[(c)].addr ? 1 : 0)
#define CTRL_READ(l,h,msrs,c) do {rdmsr((msrs->controls[(c)].addr), (l), (h));} while (0)
#define CTRL_WRITE(l,h,msrs,c) do {wrmsr((msrs->controls[(c)].addr), (l), (h));} while (0)
#define CTRL_SET_ACTIVE(n) (n |= (1<<22))
static void ppro_fill_in_addresses(struct op_msrs * const msrs)
{
- msrs->counters[0].addr = MSR_P6_PERFCTR0;
- msrs->counters[1].addr = MSR_P6_PERFCTR1;
+ int i;
+
+ for (i=0; i < NUM_COUNTERS; i++) {
+ if (reserve_perfctr_nmi(MSR_P6_PERFCTR0 + i))
+ msrs->counters[i].addr = MSR_P6_PERFCTR0 + i;
+ else
+ msrs->counters[i].addr = 0;
+ }
- msrs->controls[0].addr = MSR_P6_EVNTSEL0;
- msrs->controls[1].addr = MSR_P6_EVNTSEL1;
+ for (i=0; i < NUM_CONTROLS; i++) {
+ if (reserve_evntsel_nmi(MSR_P6_EVNTSEL0 + i))
+ msrs->controls[i].addr = MSR_P6_EVNTSEL0 + i;
+ else
+ msrs->controls[i].addr = 0;
+ }
}
/* clear all counters */
for (i = 0 ; i < NUM_CONTROLS; ++i) {
+ if (unlikely(!CTRL_IS_RESERVED(msrs,i)))
+ continue;
CTRL_READ(low, high, msrs, i);
CTRL_CLEAR(low);
CTRL_WRITE(low, high, msrs, i);
/* avoid a false detection of ctr overflows in NMI handler */
for (i = 0; i < NUM_COUNTERS; ++i) {
+ if (unlikely(!CTR_IS_RESERVED(msrs,i)))
+ continue;
CTR_WRITE(1, msrs, i);
}
/* enable active counters */
for (i = 0; i < NUM_COUNTERS; ++i) {
- if (counter_config[i].enabled) {
+ if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs,i))) {
reset_value[i] = counter_config[i].count;
CTR_WRITE(counter_config[i].count, msrs, i);
CTRL_SET_UM(low, counter_config[i].unit_mask);
CTRL_SET_EVENT(low, counter_config[i].event);
CTRL_WRITE(low, high, msrs, i);
+ } else {
+ reset_value[i] = 0;
}
}
}
int i;
for (i = 0 ; i < NUM_COUNTERS; ++i) {
+ if (!reset_value[i])
+ continue;
CTR_READ(low, high, msrs, i);
if (CTR_OVERFLOWED(low)) {
oprofile_add_sample(regs, i);
static void ppro_start(struct op_msrs const * const msrs)
{
unsigned int low,high;
- CTRL_READ(low, high, msrs, 0);
- CTRL_SET_ACTIVE(low);
- CTRL_WRITE(low, high, msrs, 0);
+
+ if (reset_value[0]) {
+ CTRL_READ(low, high, msrs, 0);
+ CTRL_SET_ACTIVE(low);
+ CTRL_WRITE(low, high, msrs, 0);
+ }
}
static void ppro_stop(struct op_msrs const * const msrs)
{
unsigned int low,high;
- CTRL_READ(low, high, msrs, 0);
- CTRL_SET_INACTIVE(low);
- CTRL_WRITE(low, high, msrs, 0);
+
+ if (reset_value[0]) {
+ CTRL_READ(low, high, msrs, 0);
+ CTRL_SET_INACTIVE(low);
+ CTRL_WRITE(low, high, msrs, 0);
+ }
+}
+
+static void ppro_shutdown(struct op_msrs const * const msrs)
+{
+ int i;
+
+ for (i = 0 ; i < NUM_COUNTERS ; ++i) {
+ if (CTR_IS_RESERVED(msrs,i))
+ release_perfctr_nmi(MSR_P6_PERFCTR0 + i);
+ }
+ for (i = 0 ; i < NUM_CONTROLS ; ++i) {
+ if (CTRL_IS_RESERVED(msrs,i))
+ release_evntsel_nmi(MSR_P6_EVNTSEL0 + i);
+ }
}
.setup_ctrs = &ppro_setup_ctrs,
.check_ctrs = &ppro_check_ctrs,
.start = &ppro_start,
- .stop = &ppro_stop
+ .stop = &ppro_stop,
+ .shutdown = &ppro_shutdown
};
struct op_msrs const * const msrs);
void (*start)(struct op_msrs const * const msrs);
void (*stop)(struct op_msrs const * const msrs);
+ void (*shutdown)(struct op_msrs const * const msrs);
};
extern struct op_x86_model_spec const op_ppro_spec;
projects / linux-2.6 / commitdiff