1b716916679e72054d436afadce2f94dcad71cfadRobert Richter/* 2b716916679e72054d436afadce2f94dcad71cfadRobert Richter * Performance events - AMD IBS 3b716916679e72054d436afadce2f94dcad71cfadRobert Richter * 4b716916679e72054d436afadce2f94dcad71cfadRobert Richter * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter 5b716916679e72054d436afadce2f94dcad71cfadRobert Richter * 6b716916679e72054d436afadce2f94dcad71cfadRobert Richter * For licencing details see kernel-base/COPYING 7b716916679e72054d436afadce2f94dcad71cfadRobert Richter */ 8b716916679e72054d436afadce2f94dcad71cfadRobert Richter 9b716916679e72054d436afadce2f94dcad71cfadRobert Richter#include <linux/perf_event.h> 10b716916679e72054d436afadce2f94dcad71cfadRobert Richter#include <linux/module.h> 11b716916679e72054d436afadce2f94dcad71cfadRobert Richter#include <linux/pci.h> 12d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter#include <linux/ptrace.h> 13bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter#include <linux/syscore_ops.h> 14b716916679e72054d436afadce2f94dcad71cfadRobert Richter 15b716916679e72054d436afadce2f94dcad71cfadRobert Richter#include <asm/apic.h> 16b716916679e72054d436afadce2f94dcad71cfadRobert Richter 17d07bdfd322d307789f15b427dbcc39257665356fPeter Zijlstra#include "perf_event.h" 18d07bdfd322d307789f15b427dbcc39257665356fPeter Zijlstra 19b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic u32 ibs_caps; 20b716916679e72054d436afadce2f94dcad71cfadRobert Richter 21b716916679e72054d436afadce2f94dcad71cfadRobert Richter#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) 22b716916679e72054d436afadce2f94dcad71cfadRobert Richter 23b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter#include <linux/kprobes.h> 24b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter#include <linux/hardirq.h> 25b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 26b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter#include <asm/nmi.h> 27b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 28510419435c6948fb32959d691bf84eaba41ca474Robert Richter#define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) 29510419435c6948fb32959d691bf84eaba41ca474Robert Richter#define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT 30510419435c6948fb32959d691bf84eaba41ca474Robert Richter 314db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richterenum ibs_states { 324db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter IBS_ENABLED = 0, 334db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter IBS_STARTED = 1, 344db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter IBS_STOPPING = 2, 354db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 364db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter IBS_MAX_STATES, 374db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter}; 384db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 394db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richterstruct cpu_perf_ibs { 404db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct perf_event *event; 414db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)]; 424db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter}; 434db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 44510419435c6948fb32959d691bf84eaba41ca474Robert Richterstruct perf_ibs { 452e132b12f78d88672711ae1d87624951de1089caRobert Richter struct pmu pmu; 462e132b12f78d88672711ae1d87624951de1089caRobert Richter unsigned int msr; 472e132b12f78d88672711ae1d87624951de1089caRobert Richter u64 config_mask; 482e132b12f78d88672711ae1d87624951de1089caRobert Richter u64 cnt_mask; 492e132b12f78d88672711ae1d87624951de1089caRobert Richter u64 enable_mask; 502e132b12f78d88672711ae1d87624951de1089caRobert Richter u64 valid_mask; 512e132b12f78d88672711ae1d87624951de1089caRobert Richter u64 max_period; 522e132b12f78d88672711ae1d87624951de1089caRobert Richter unsigned long offset_mask[1]; 532e132b12f78d88672711ae1d87624951de1089caRobert Richter int offset_max; 542e132b12f78d88672711ae1d87624951de1089caRobert Richter struct cpu_perf_ibs __percpu *pcpu; 552e132b12f78d88672711ae1d87624951de1089caRobert Richter 562e132b12f78d88672711ae1d87624951de1089caRobert Richter struct attribute **format_attrs; 572e132b12f78d88672711ae1d87624951de1089caRobert Richter struct attribute_group format_group; 582e132b12f78d88672711ae1d87624951de1089caRobert Richter const struct attribute_group *attr_groups[2]; 592e132b12f78d88672711ae1d87624951de1089caRobert Richter 602e132b12f78d88672711ae1d87624951de1089caRobert Richter u64 (*get_count)(u64 config); 61b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter}; 62b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 63b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richterstruct perf_ibs_data { 64b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter u32 size; 65b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter union { 66b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter u32 data[0]; /* data buffer starts here */ 67b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter u32 caps; 68b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter }; 69b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; 70510419435c6948fb32959d691bf84eaba41ca474Robert Richter}; 71510419435c6948fb32959d691bf84eaba41ca474Robert Richter 72db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterstatic int 7398112d2e957e0d348f06d8a40f2f720204a70b55Robert Richterperf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) 74db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 75db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter s64 left = local64_read(&hwc->period_left); 76db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter s64 period = hwc->sample_period; 77db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter int overflow = 0; 78db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 79db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter /* 80db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * If we are way outside a reasonable range then just skip forward: 81db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter */ 82db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (unlikely(left <= -period)) { 83db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter left = period; 84db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter local64_set(&hwc->period_left, left); 85db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter hwc->last_period = period; 86db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter overflow = 1; 87db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter } 88db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 89fc006cf7cc7471e1bdf34e40111971e03622af6cRobert Richter if (unlikely(left < (s64)min)) { 90db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter left += period; 91db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter local64_set(&hwc->period_left, left); 92db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter hwc->last_period = period; 93db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter overflow = 1; 94db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter } 95db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 967caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter /* 977caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter * If the hw period that triggers the sw overflow is too short 987caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter * we might hit the irq handler. This biases the results. 997caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter * Thus we shorten the next-to-last period and set the last 1007caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter * period to the max period. 1017caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter */ 1027caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter if (left > max) { 1037caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter left -= max; 1047caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter if (left > max) 1057caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter left = max; 1067caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter else if (left < min) 1077caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter left = min; 1087caaf4d8241feecafb87919402b0a6dbb1b71d9eRobert Richter } 109db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 11098112d2e957e0d348f06d8a40f2f720204a70b55Robert Richter *hw_period = (u64)left; 111db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 112db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter return overflow; 113db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 114db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 115db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterstatic int 116db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterperf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) 117db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 118db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter struct hw_perf_event *hwc = &event->hw; 119db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter int shift = 64 - width; 120db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter u64 prev_raw_count; 121db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter u64 delta; 122db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 123db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter /* 124db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * Careful: an NMI might modify the previous event value. 125db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * 126db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * Our tactic to handle this is to first atomically read and 127db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * exchange a new raw count - then add that new-prev delta 128db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * count to the generic event atomically: 129db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter */ 130db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter prev_raw_count = local64_read(&hwc->prev_count); 131db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 132db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter new_raw_count) != prev_raw_count) 133db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter return 0; 134db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 135db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter /* 136db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * Now we have the new raw value and have updated the prev 137db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * timestamp already. We can now calculate the elapsed delta 138db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * (event-)time and add that to the generic event. 139db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * 140db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * Careful, not all hw sign-extends above the physical width 141db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * of the count. 142db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter */ 143db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter delta = (new_raw_count << shift) - (prev_raw_count << shift); 144db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter delta >>= shift; 145db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 146db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter local64_add(delta, &event->count); 147db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter local64_sub(delta, &hwc->period_left); 148db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 149db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter return 1; 150db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 151db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 152510419435c6948fb32959d691bf84eaba41ca474Robert Richterstatic struct perf_ibs perf_ibs_fetch; 153510419435c6948fb32959d691bf84eaba41ca474Robert Richterstatic struct perf_ibs perf_ibs_op; 154510419435c6948fb32959d691bf84eaba41ca474Robert Richter 155510419435c6948fb32959d691bf84eaba41ca474Robert Richterstatic struct perf_ibs *get_ibs_pmu(int type) 156510419435c6948fb32959d691bf84eaba41ca474Robert Richter{ 157510419435c6948fb32959d691bf84eaba41ca474Robert Richter if (perf_ibs_fetch.pmu.type == type) 158510419435c6948fb32959d691bf84eaba41ca474Robert Richter return &perf_ibs_fetch; 159510419435c6948fb32959d691bf84eaba41ca474Robert Richter if (perf_ibs_op.pmu.type == type) 160510419435c6948fb32959d691bf84eaba41ca474Robert Richter return &perf_ibs_op; 161510419435c6948fb32959d691bf84eaba41ca474Robert Richter return NULL; 162510419435c6948fb32959d691bf84eaba41ca474Robert Richter} 163b716916679e72054d436afadce2f94dcad71cfadRobert Richter 164450bbd493d436f9eadd1b7828158f37559f26674Robert Richter/* 165450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * Use IBS for precise event sampling: 166450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * 167450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count 168450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * perf record -a -e r076:p ... # same as -e cpu-cycles:p 169450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * perf record -a -e r0C1:p ... # use ibs op counting micro-ops 170450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * 171450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, 172450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * MSRC001_1033) is used to select either cycle or micro-ops counting 173450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * mode. 174450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * 175450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * The rip of IBS samples has skid 0. Thus, IBS supports precise 176450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the 177450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * rip is invalid when IBS was not able to record the rip correctly. 178450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. 179450bbd493d436f9eadd1b7828158f37559f26674Robert Richter * 180450bbd493d436f9eadd1b7828158f37559f26674Robert Richter */ 181450bbd493d436f9eadd1b7828158f37559f26674Robert Richterstatic int perf_ibs_precise_event(struct perf_event *event, u64 *config) 182450bbd493d436f9eadd1b7828158f37559f26674Robert Richter{ 183450bbd493d436f9eadd1b7828158f37559f26674Robert Richter switch (event->attr.precise_ip) { 184450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case 0: 185450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return -ENOENT; 186450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case 1: 187450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case 2: 188450bbd493d436f9eadd1b7828158f37559f26674Robert Richter break; 189450bbd493d436f9eadd1b7828158f37559f26674Robert Richter default: 190450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return -EOPNOTSUPP; 191450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } 192450bbd493d436f9eadd1b7828158f37559f26674Robert Richter 193450bbd493d436f9eadd1b7828158f37559f26674Robert Richter switch (event->attr.type) { 194450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case PERF_TYPE_HARDWARE: 195450bbd493d436f9eadd1b7828158f37559f26674Robert Richter switch (event->attr.config) { 196450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case PERF_COUNT_HW_CPU_CYCLES: 197450bbd493d436f9eadd1b7828158f37559f26674Robert Richter *config = 0; 198450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return 0; 199450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } 200450bbd493d436f9eadd1b7828158f37559f26674Robert Richter break; 201450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case PERF_TYPE_RAW: 202450bbd493d436f9eadd1b7828158f37559f26674Robert Richter switch (event->attr.config) { 203450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case 0x0076: 204450bbd493d436f9eadd1b7828158f37559f26674Robert Richter *config = 0; 205450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return 0; 206450bbd493d436f9eadd1b7828158f37559f26674Robert Richter case 0x00C1: 207450bbd493d436f9eadd1b7828158f37559f26674Robert Richter *config = IBS_OP_CNT_CTL; 208450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return 0; 209450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } 210450bbd493d436f9eadd1b7828158f37559f26674Robert Richter break; 211450bbd493d436f9eadd1b7828158f37559f26674Robert Richter default: 212450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return -ENOENT; 213450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } 214450bbd493d436f9eadd1b7828158f37559f26674Robert Richter 215450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return -EOPNOTSUPP; 216450bbd493d436f9eadd1b7828158f37559f26674Robert Richter} 217450bbd493d436f9eadd1b7828158f37559f26674Robert Richter 218bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richterstatic const struct perf_event_attr ibs_notsupp = { 219bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter .exclude_user = 1, 220bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter .exclude_kernel = 1, 221bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter .exclude_hv = 1, 222bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter .exclude_idle = 1, 223bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter .exclude_host = 1, 224bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter .exclude_guest = 1, 225bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter}; 226bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter 227b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic int perf_ibs_init(struct perf_event *event) 228b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 229510419435c6948fb32959d691bf84eaba41ca474Robert Richter struct hw_perf_event *hwc = &event->hw; 230510419435c6948fb32959d691bf84eaba41ca474Robert Richter struct perf_ibs *perf_ibs; 231510419435c6948fb32959d691bf84eaba41ca474Robert Richter u64 max_cnt, config; 232450bbd493d436f9eadd1b7828158f37559f26674Robert Richter int ret; 233510419435c6948fb32959d691bf84eaba41ca474Robert Richter 234510419435c6948fb32959d691bf84eaba41ca474Robert Richter perf_ibs = get_ibs_pmu(event->attr.type); 235450bbd493d436f9eadd1b7828158f37559f26674Robert Richter if (perf_ibs) { 236450bbd493d436f9eadd1b7828158f37559f26674Robert Richter config = event->attr.config; 237450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } else { 238450bbd493d436f9eadd1b7828158f37559f26674Robert Richter perf_ibs = &perf_ibs_op; 239450bbd493d436f9eadd1b7828158f37559f26674Robert Richter ret = perf_ibs_precise_event(event, &config); 240450bbd493d436f9eadd1b7828158f37559f26674Robert Richter if (ret) 241450bbd493d436f9eadd1b7828158f37559f26674Robert Richter return ret; 242450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } 243450bbd493d436f9eadd1b7828158f37559f26674Robert Richter 244450bbd493d436f9eadd1b7828158f37559f26674Robert Richter if (event->pmu != &perf_ibs->pmu) 245b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -ENOENT; 246510419435c6948fb32959d691bf84eaba41ca474Robert Richter 247bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter if (perf_flags(&event->attr) & perf_flags(&ibs_notsupp)) 248bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter return -EINVAL; 249bad9ac2d7f878a31cf1ae8c1ee3768077d222bcbRobert Richter 250510419435c6948fb32959d691bf84eaba41ca474Robert Richter if (config & ~perf_ibs->config_mask) 251510419435c6948fb32959d691bf84eaba41ca474Robert Richter return -EINVAL; 252510419435c6948fb32959d691bf84eaba41ca474Robert Richter 253510419435c6948fb32959d691bf84eaba41ca474Robert Richter if (hwc->sample_period) { 254510419435c6948fb32959d691bf84eaba41ca474Robert Richter if (config & perf_ibs->cnt_mask) 255510419435c6948fb32959d691bf84eaba41ca474Robert Richter /* raw max_cnt may not be set */ 256510419435c6948fb32959d691bf84eaba41ca474Robert Richter return -EINVAL; 2576accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter if (!event->attr.sample_freq && hwc->sample_period & 0x0f) 2586accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter /* 2596accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter * lower 4 bits can not be set in ibs max cnt, 2606accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter * but allowing it in case we adjust the 2616accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter * sample period to set a frequency. 2626accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter */ 263510419435c6948fb32959d691bf84eaba41ca474Robert Richter return -EINVAL; 2646accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter hwc->sample_period &= ~0x0FULL; 2656accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter if (!hwc->sample_period) 2666accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter hwc->sample_period = 0x10; 267510419435c6948fb32959d691bf84eaba41ca474Robert Richter } else { 268510419435c6948fb32959d691bf84eaba41ca474Robert Richter max_cnt = config & perf_ibs->cnt_mask; 269db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter config &= ~perf_ibs->cnt_mask; 270510419435c6948fb32959d691bf84eaba41ca474Robert Richter event->attr.sample_period = max_cnt << 4; 271510419435c6948fb32959d691bf84eaba41ca474Robert Richter hwc->sample_period = event->attr.sample_period; 272510419435c6948fb32959d691bf84eaba41ca474Robert Richter } 273510419435c6948fb32959d691bf84eaba41ca474Robert Richter 274db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (!hwc->sample_period) 275510419435c6948fb32959d691bf84eaba41ca474Robert Richter return -EINVAL; 276510419435c6948fb32959d691bf84eaba41ca474Robert Richter 2776accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter /* 2786accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter * If we modify hwc->sample_period, we also need to update 2796accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter * hwc->last_period and hwc->period_left. 2806accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter */ 2816accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter hwc->last_period = hwc->sample_period; 2826accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter local64_set(&hwc->period_left, hwc->sample_period); 2836accb9cf76080422d400a641d9068b6b2a2c216fRobert Richter 284510419435c6948fb32959d691bf84eaba41ca474Robert Richter hwc->config_base = perf_ibs->msr; 285510419435c6948fb32959d691bf84eaba41ca474Robert Richter hwc->config = config; 286510419435c6948fb32959d691bf84eaba41ca474Robert Richter 287b716916679e72054d436afadce2f94dcad71cfadRobert Richter return 0; 288b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 289b716916679e72054d436afadce2f94dcad71cfadRobert Richter 290db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterstatic int perf_ibs_set_period(struct perf_ibs *perf_ibs, 291db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter struct hw_perf_event *hwc, u64 *period) 292db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 29398112d2e957e0d348f06d8a40f2f720204a70b55Robert Richter int overflow; 294db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 295db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter /* ignore lower 4 bits in min count: */ 29698112d2e957e0d348f06d8a40f2f720204a70b55Robert Richter overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period); 297db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter local64_set(&hwc->prev_count, 0); 298db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 29998112d2e957e0d348f06d8a40f2f720204a70b55Robert Richter return overflow; 300db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 301db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 302db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterstatic u64 get_ibs_fetch_count(u64 config) 303db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 304db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter return (config & IBS_FETCH_CNT) >> 12; 305db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 306db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 307db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterstatic u64 get_ibs_op_count(u64 config) 308db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 3098b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter u64 count = 0; 3108b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter 3118b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter if (config & IBS_OP_VAL) 3128b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter count += (config & IBS_OP_MAX_CNT) << 4; /* cnt rolled over */ 3138b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter 3148b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter if (ibs_caps & IBS_CAPS_RDWROPCNT) 3158b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter count += (config & IBS_OP_CUR_CNT) >> 32; 3168b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter 3178b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter return count; 318db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 319db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 320db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterstatic void 321db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richterperf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, 322c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter u64 *config) 323db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 324c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter u64 count = perf_ibs->get_count(*config); 325db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 3268b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter /* 3278b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter * Set width to 64 since we do not overflow on max width but 3288b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter * instead on max count. In perf_ibs_set_period() we clear 3298b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter * prev count manually on overflow. 3308b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter */ 3318b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter while (!perf_event_try_update(event, count, 64)) { 332c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter rdmsrl(event->hw.config_base, *config); 333c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter count = perf_ibs->get_count(*config); 334db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter } 335db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 336db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 337c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richterstatic inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, 338c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter struct hw_perf_event *hwc, u64 config) 339db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter{ 340c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter wrmsrl(hwc->config_base, hwc->config | config | perf_ibs->enable_mask); 341c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter} 342c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter 343c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter/* 344c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter * Erratum #420 Instruction-Based Sampling Engine May Generate 345c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter * Interrupt that Cannot Be Cleared: 346c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter * 347c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter * Must clear counter mask first, then clear the enable bit. See 348c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter * Revision Guide for AMD Family 10h Processors, Publication #41322. 349c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter */ 350c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richterstatic inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, 351c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter struct hw_perf_event *hwc, u64 config) 352c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter{ 353c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter config &= ~perf_ibs->cnt_mask; 354c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter wrmsrl(hwc->config_base, config); 355c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter config &= ~perf_ibs->enable_mask; 356c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter wrmsrl(hwc->config_base, config); 357db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter} 358db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 359db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter/* 360db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * We cannot restore the ibs pmu state, so we always needs to update 361db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * the event while stopping it and then reset the state when starting 362db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in 363db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter * perf_ibs_start()/perf_ibs_stop() and instead always do it. 364db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter */ 3654db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richterstatic void perf_ibs_start(struct perf_event *event, int flags) 3664db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter{ 3674db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct hw_perf_event *hwc = &event->hw; 3684db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 3694db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 370c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter u64 period; 3714db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 372db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 3734db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter return; 3744db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 375db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 376db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter hwc->state = 0; 377db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 378c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter perf_ibs_set_period(perf_ibs, hwc, &period); 379db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter set_bit(IBS_STARTED, pcpu->state); 380c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter perf_ibs_enable_event(perf_ibs, hwc, period >> 4); 381db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 382db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter perf_event_update_userpage(event); 3834db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter} 3844db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 3854db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richterstatic void perf_ibs_stop(struct perf_event *event, int flags) 3864db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter{ 3874db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct hw_perf_event *hwc = &event->hw; 3884db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 3894db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 390c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter u64 config; 391db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter int stopping; 3924db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 393db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter stopping = test_and_clear_bit(IBS_STARTED, pcpu->state); 3944db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 395db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (!stopping && (hwc->state & PERF_HES_UPTODATE)) 396db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter return; 3974db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 398c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter rdmsrl(hwc->config_base, config); 399db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 400db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (stopping) { 401db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter set_bit(IBS_STOPPING, pcpu->state); 402c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter perf_ibs_disable_event(perf_ibs, hwc, config); 403db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 404db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter hwc->state |= PERF_HES_STOPPED; 405db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter } 406db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 407db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter if (hwc->state & PERF_HES_UPTODATE) 408db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter return; 409db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 4108b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter /* 4118b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter * Clear valid bit to not count rollovers on update, rollovers 4128b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter * are only updated in the irq handler. 4138b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter */ 4148b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter config &= ~perf_ibs->valid_mask; 4158b1e13638d465863572c8207a5cfceeef0cf0441Robert Richter 416c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter perf_ibs_event_update(perf_ibs, event, &config); 417db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter hwc->state |= PERF_HES_UPTODATE; 4184db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter} 4194db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 420b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic int perf_ibs_add(struct perf_event *event, int flags) 421b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 4224db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 4234db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 4244db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 4254db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter if (test_and_set_bit(IBS_ENABLED, pcpu->state)) 4264db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter return -ENOSPC; 4274db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 428db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 429db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 4304db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter pcpu->event = event; 4314db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 4324db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter if (flags & PERF_EF_START) 4334db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter perf_ibs_start(event, PERF_EF_RELOAD); 4344db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 435b716916679e72054d436afadce2f94dcad71cfadRobert Richter return 0; 436b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 437b716916679e72054d436afadce2f94dcad71cfadRobert Richter 438b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic void perf_ibs_del(struct perf_event *event, int flags) 439b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 4404db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 4414db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 4424db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 4434db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) 4444db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter return; 4454db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 446db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter perf_ibs_stop(event, PERF_EF_UPDATE); 4474db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 4484db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter pcpu->event = NULL; 449db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 450db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter perf_event_update_userpage(event); 451b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 452b716916679e72054d436afadce2f94dcad71cfadRobert Richter 4534db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richterstatic void perf_ibs_read(struct perf_event *event) { } 4544db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 4552e132b12f78d88672711ae1d87624951de1089caRobert RichterPMU_FORMAT_ATTR(rand_en, "config:57"); 4562e132b12f78d88672711ae1d87624951de1089caRobert RichterPMU_FORMAT_ATTR(cnt_ctl, "config:19"); 4572e132b12f78d88672711ae1d87624951de1089caRobert Richter 4582e132b12f78d88672711ae1d87624951de1089caRobert Richterstatic struct attribute *ibs_fetch_format_attrs[] = { 4592e132b12f78d88672711ae1d87624951de1089caRobert Richter &format_attr_rand_en.attr, 4602e132b12f78d88672711ae1d87624951de1089caRobert Richter NULL, 4612e132b12f78d88672711ae1d87624951de1089caRobert Richter}; 4622e132b12f78d88672711ae1d87624951de1089caRobert Richter 4632e132b12f78d88672711ae1d87624951de1089caRobert Richterstatic struct attribute *ibs_op_format_attrs[] = { 4642e132b12f78d88672711ae1d87624951de1089caRobert Richter NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */ 4652e132b12f78d88672711ae1d87624951de1089caRobert Richter NULL, 4662e132b12f78d88672711ae1d87624951de1089caRobert Richter}; 4672e132b12f78d88672711ae1d87624951de1089caRobert Richter 468510419435c6948fb32959d691bf84eaba41ca474Robert Richterstatic struct perf_ibs perf_ibs_fetch = { 469510419435c6948fb32959d691bf84eaba41ca474Robert Richter .pmu = { 470510419435c6948fb32959d691bf84eaba41ca474Robert Richter .task_ctx_nr = perf_invalid_context, 471510419435c6948fb32959d691bf84eaba41ca474Robert Richter 472510419435c6948fb32959d691bf84eaba41ca474Robert Richter .event_init = perf_ibs_init, 473510419435c6948fb32959d691bf84eaba41ca474Robert Richter .add = perf_ibs_add, 474510419435c6948fb32959d691bf84eaba41ca474Robert Richter .del = perf_ibs_del, 4754db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter .start = perf_ibs_start, 4764db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter .stop = perf_ibs_stop, 4774db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter .read = perf_ibs_read, 478510419435c6948fb32959d691bf84eaba41ca474Robert Richter }, 479510419435c6948fb32959d691bf84eaba41ca474Robert Richter .msr = MSR_AMD64_IBSFETCHCTL, 480510419435c6948fb32959d691bf84eaba41ca474Robert Richter .config_mask = IBS_FETCH_CONFIG_MASK, 481510419435c6948fb32959d691bf84eaba41ca474Robert Richter .cnt_mask = IBS_FETCH_MAX_CNT, 482510419435c6948fb32959d691bf84eaba41ca474Robert Richter .enable_mask = IBS_FETCH_ENABLE, 483b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter .valid_mask = IBS_FETCH_VAL, 484db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter .max_period = IBS_FETCH_MAX_CNT << 4, 485b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, 486b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, 4872e132b12f78d88672711ae1d87624951de1089caRobert Richter .format_attrs = ibs_fetch_format_attrs, 488db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 489db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter .get_count = get_ibs_fetch_count, 490510419435c6948fb32959d691bf84eaba41ca474Robert Richter}; 491510419435c6948fb32959d691bf84eaba41ca474Robert Richter 492510419435c6948fb32959d691bf84eaba41ca474Robert Richterstatic struct perf_ibs perf_ibs_op = { 493510419435c6948fb32959d691bf84eaba41ca474Robert Richter .pmu = { 494510419435c6948fb32959d691bf84eaba41ca474Robert Richter .task_ctx_nr = perf_invalid_context, 495510419435c6948fb32959d691bf84eaba41ca474Robert Richter 496510419435c6948fb32959d691bf84eaba41ca474Robert Richter .event_init = perf_ibs_init, 497510419435c6948fb32959d691bf84eaba41ca474Robert Richter .add = perf_ibs_add, 498510419435c6948fb32959d691bf84eaba41ca474Robert Richter .del = perf_ibs_del, 4994db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter .start = perf_ibs_start, 5004db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter .stop = perf_ibs_stop, 5014db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter .read = perf_ibs_read, 502510419435c6948fb32959d691bf84eaba41ca474Robert Richter }, 503510419435c6948fb32959d691bf84eaba41ca474Robert Richter .msr = MSR_AMD64_IBSOPCTL, 504510419435c6948fb32959d691bf84eaba41ca474Robert Richter .config_mask = IBS_OP_CONFIG_MASK, 505510419435c6948fb32959d691bf84eaba41ca474Robert Richter .cnt_mask = IBS_OP_MAX_CNT, 506510419435c6948fb32959d691bf84eaba41ca474Robert Richter .enable_mask = IBS_OP_ENABLE, 507b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter .valid_mask = IBS_OP_VAL, 508db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter .max_period = IBS_OP_MAX_CNT << 4, 509b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, 510b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter .offset_max = MSR_AMD64_IBSOP_REG_COUNT, 5112e132b12f78d88672711ae1d87624951de1089caRobert Richter .format_attrs = ibs_op_format_attrs, 512db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 513db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter .get_count = get_ibs_op_count, 514b716916679e72054d436afadce2f94dcad71cfadRobert Richter}; 515b716916679e72054d436afadce2f94dcad71cfadRobert Richter 516b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richterstatic int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) 517b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter{ 5184db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 5194db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct perf_event *event = pcpu->event; 520b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter struct hw_perf_event *hwc = &event->hw; 521b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter struct perf_sample_data data; 522b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter struct perf_raw_record raw; 523b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter struct pt_regs regs; 524b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter struct perf_ibs_data ibs_data; 525d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter int offset, size, check_rip, offset_max, throttle = 0; 526b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter unsigned int msr; 527c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter u64 *buf, *config, period; 528b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 5294db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter if (!test_bit(IBS_STARTED, pcpu->state)) { 530fc5fb2b5e1874e5894e2ac503bfb744220db89a1Robert Richter /* 531fc5fb2b5e1874e5894e2ac503bfb744220db89a1Robert Richter * Catch spurious interrupts after stopping IBS: After 532d82603c6da7579c50ebe3fe7da6e3e267d9f6427Jorrit Schippers * disabling IBS there could be still incoming NMIs 533fc5fb2b5e1874e5894e2ac503bfb744220db89a1Robert Richter * with samples that even have the valid bit cleared. 534fc5fb2b5e1874e5894e2ac503bfb744220db89a1Robert Richter * Mark all this NMIs as handled. 535fc5fb2b5e1874e5894e2ac503bfb744220db89a1Robert Richter */ 536fc5fb2b5e1874e5894e2ac503bfb744220db89a1Robert Richter return test_and_clear_bit(IBS_STOPPING, pcpu->state) ? 1 : 0; 5374db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter } 5384db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 539b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter msr = hwc->config_base; 540b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter buf = ibs_data.regs; 541b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter rdmsrl(msr, *buf); 542b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter if (!(*buf++ & perf_ibs->valid_mask)) 543b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter return 0; 544b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 545c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter config = &ibs_data.regs[0]; 546c75841a398d667d9968245b9519d93cedbfb4780Robert Richter perf_ibs_event_update(perf_ibs, event, config); 547fd0d000b2c34aa43d4e92dcf0dfaeda7e123008aRobert Richter perf_sample_data_init(&data, 0, hwc->last_period); 548c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter if (!perf_ibs_set_period(perf_ibs, hwc, &period)) 549d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter goto out; /* no sw counter overflow */ 550d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter 551d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter ibs_data.caps = ibs_caps; 552d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter size = 1; 553d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter offset = 1; 554d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); 555d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter if (event->attr.sample_type & PERF_SAMPLE_RAW) 556d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter offset_max = perf_ibs->offset_max; 557d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter else if (check_rip) 558d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter offset_max = 2; 559d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter else 560d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter offset_max = 1; 561d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter do { 562d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter rdmsrl(msr + offset, *buf++); 563d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter size++; 564d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter offset = find_next_bit(perf_ibs->offset_mask, 565d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter perf_ibs->offset_max, 566d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter offset + 1); 567d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter } while (offset < offset_max); 568d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter ibs_data.size = sizeof(u64) * size; 569d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter 570d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter regs = *iregs; 571450bbd493d436f9eadd1b7828158f37559f26674Robert Richter if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { 572450bbd493d436f9eadd1b7828158f37559f26674Robert Richter regs.flags &= ~PERF_EFLAGS_EXACT; 573450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } else { 574d07bdfd322d307789f15b427dbcc39257665356fPeter Zijlstra set_linear_ip(®s, ibs_data.regs[1]); 575450bbd493d436f9eadd1b7828158f37559f26674Robert Richter regs.flags |= PERF_EFLAGS_EXACT; 576450bbd493d436f9eadd1b7828158f37559f26674Robert Richter } 577c75841a398d667d9968245b9519d93cedbfb4780Robert Richter 578b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter if (event->attr.sample_type & PERF_SAMPLE_RAW) { 579d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter raw.size = sizeof(u32) + ibs_data.size; 580b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter raw.data = ibs_data.data; 581b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter data.raw = &raw; 582b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter } 583b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 584d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richter throttle = perf_event_overflow(event, &data, ®s); 585d47e8238cd76f1ffa7c8cd30e08b8e9074fd597eRobert Richterout: 586c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter if (throttle) 587c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter perf_ibs_disable_event(perf_ibs, hwc, *config); 588c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter else 589c9574fe0bdb9ac9a2698e02a712088ce8431e9f8Robert Richter perf_ibs_enable_event(perf_ibs, hwc, period >> 4); 590db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter 591db98c5faf8cb350212ea3af786cb3ba0d4e7a01eRobert Richter perf_event_update_userpage(event); 592b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 593b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter return 1; 594b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter} 595b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 5969326638cbee2d36b051ed2a69f3e4e107e5f86bdMasami Hiramatsustatic int 597b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richterperf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) 598b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter{ 599b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter int handled = 0; 600b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 601b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs); 602b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter handled += perf_ibs_handle_irq(&perf_ibs_op, regs); 603b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 604b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter if (handled) 605b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter inc_irq_stat(apic_perf_irqs); 606b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 607b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter return handled; 608b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter} 6099326638cbee2d36b051ed2a69f3e4e107e5f86bdMasami HiramatsuNOKPROBE_SYMBOL(perf_ibs_nmi_handler); 610b7074f1fbd6149eac1ec25063e4a364c39a85473Robert Richter 6114db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richterstatic __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) 6124db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter{ 6134db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter struct cpu_perf_ibs __percpu *pcpu; 6144db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter int ret; 6154db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 6164db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter pcpu = alloc_percpu(struct cpu_perf_ibs); 6174db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter if (!pcpu) 6184db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter return -ENOMEM; 6194db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 6204db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter perf_ibs->pcpu = pcpu; 6214db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 6222e132b12f78d88672711ae1d87624951de1089caRobert Richter /* register attributes */ 6232e132b12f78d88672711ae1d87624951de1089caRobert Richter if (perf_ibs->format_attrs[0]) { 6242e132b12f78d88672711ae1d87624951de1089caRobert Richter memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group)); 6252e132b12f78d88672711ae1d87624951de1089caRobert Richter perf_ibs->format_group.name = "format"; 6262e132b12f78d88672711ae1d87624951de1089caRobert Richter perf_ibs->format_group.attrs = perf_ibs->format_attrs; 6272e132b12f78d88672711ae1d87624951de1089caRobert Richter 6282e132b12f78d88672711ae1d87624951de1089caRobert Richter memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups)); 6292e132b12f78d88672711ae1d87624951de1089caRobert Richter perf_ibs->attr_groups[0] = &perf_ibs->format_group; 6302e132b12f78d88672711ae1d87624951de1089caRobert Richter perf_ibs->pmu.attr_groups = perf_ibs->attr_groups; 6312e132b12f78d88672711ae1d87624951de1089caRobert Richter } 6322e132b12f78d88672711ae1d87624951de1089caRobert Richter 6334db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter ret = perf_pmu_register(&perf_ibs->pmu, name, -1); 6344db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter if (ret) { 6354db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter perf_ibs->pcpu = NULL; 6364db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter free_percpu(pcpu); 6374db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter } 6384db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 6394db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter return ret; 6404db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter} 6414db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter 642b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic __init int perf_event_ibs_init(void) 643b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 6442e132b12f78d88672711ae1d87624951de1089caRobert Richter struct attribute **attr = ibs_op_format_attrs; 6452e132b12f78d88672711ae1d87624951de1089caRobert Richter 646b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!ibs_caps) 647b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -ENODEV; /* ibs not supported by the cpu */ 648b716916679e72054d436afadce2f94dcad71cfadRobert Richter 6494db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); 6502e132b12f78d88672711ae1d87624951de1089caRobert Richter 6512e132b12f78d88672711ae1d87624951de1089caRobert Richter if (ibs_caps & IBS_CAPS_OPCNT) { 6527bf352384fda3f678a283928c6c5b2cd9da877e4Robert Richter perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; 6532e132b12f78d88672711ae1d87624951de1089caRobert Richter *attr++ = &format_attr_cnt_ctl.attr; 6542e132b12f78d88672711ae1d87624951de1089caRobert Richter } 6554db2e8e6500d9ba6406f2714fa3968b39a325274Robert Richter perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); 6562e132b12f78d88672711ae1d87624951de1089caRobert Richter 657fab06992de6433af097c4a1d2d1b119812753ca7Ingo Molnar register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); 658b716916679e72054d436afadce2f94dcad71cfadRobert Richter printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps); 659b716916679e72054d436afadce2f94dcad71cfadRobert Richter 660b716916679e72054d436afadce2f94dcad71cfadRobert Richter return 0; 661b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 662b716916679e72054d436afadce2f94dcad71cfadRobert Richter 663b716916679e72054d436afadce2f94dcad71cfadRobert Richter#else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ 664b716916679e72054d436afadce2f94dcad71cfadRobert Richter 665b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic __init int perf_event_ibs_init(void) { return 0; } 666b716916679e72054d436afadce2f94dcad71cfadRobert Richter 667b716916679e72054d436afadce2f94dcad71cfadRobert Richter#endif 668b716916679e72054d436afadce2f94dcad71cfadRobert Richter 669b716916679e72054d436afadce2f94dcad71cfadRobert Richter/* IBS - apic initialization, for perf and oprofile */ 670b716916679e72054d436afadce2f94dcad71cfadRobert Richter 671b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic __init u32 __get_ibs_caps(void) 672b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 673b716916679e72054d436afadce2f94dcad71cfadRobert Richter u32 caps; 674b716916679e72054d436afadce2f94dcad71cfadRobert Richter unsigned int max_level; 675b716916679e72054d436afadce2f94dcad71cfadRobert Richter 676b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!boot_cpu_has(X86_FEATURE_IBS)) 677b716916679e72054d436afadce2f94dcad71cfadRobert Richter return 0; 678b716916679e72054d436afadce2f94dcad71cfadRobert Richter 679b716916679e72054d436afadce2f94dcad71cfadRobert Richter /* check IBS cpuid feature flags */ 680b716916679e72054d436afadce2f94dcad71cfadRobert Richter max_level = cpuid_eax(0x80000000); 681b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (max_level < IBS_CPUID_FEATURES) 682b716916679e72054d436afadce2f94dcad71cfadRobert Richter return IBS_CAPS_DEFAULT; 683b716916679e72054d436afadce2f94dcad71cfadRobert Richter 684b716916679e72054d436afadce2f94dcad71cfadRobert Richter caps = cpuid_eax(IBS_CPUID_FEATURES); 685b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!(caps & IBS_CAPS_AVAIL)) 686b716916679e72054d436afadce2f94dcad71cfadRobert Richter /* cpuid flags not valid */ 687b716916679e72054d436afadce2f94dcad71cfadRobert Richter return IBS_CAPS_DEFAULT; 688b716916679e72054d436afadce2f94dcad71cfadRobert Richter 689b716916679e72054d436afadce2f94dcad71cfadRobert Richter return caps; 690b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 691b716916679e72054d436afadce2f94dcad71cfadRobert Richter 692b716916679e72054d436afadce2f94dcad71cfadRobert Richteru32 get_ibs_caps(void) 693b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 694b716916679e72054d436afadce2f94dcad71cfadRobert Richter return ibs_caps; 695b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 696b716916679e72054d436afadce2f94dcad71cfadRobert Richter 697b716916679e72054d436afadce2f94dcad71cfadRobert RichterEXPORT_SYMBOL(get_ibs_caps); 698b716916679e72054d436afadce2f94dcad71cfadRobert Richter 699b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic inline int get_eilvt(int offset) 700b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 701b716916679e72054d436afadce2f94dcad71cfadRobert Richter return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); 702b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 703b716916679e72054d436afadce2f94dcad71cfadRobert Richter 704b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic inline int put_eilvt(int offset) 705b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 706b716916679e72054d436afadce2f94dcad71cfadRobert Richter return !setup_APIC_eilvt(offset, 0, 0, 1); 707b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 708b716916679e72054d436afadce2f94dcad71cfadRobert Richter 709b716916679e72054d436afadce2f94dcad71cfadRobert Richter/* 710b716916679e72054d436afadce2f94dcad71cfadRobert Richter * Check and reserve APIC extended interrupt LVT offset for IBS if available. 711b716916679e72054d436afadce2f94dcad71cfadRobert Richter */ 712b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic inline int ibs_eilvt_valid(void) 713b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 714b716916679e72054d436afadce2f94dcad71cfadRobert Richter int offset; 715b716916679e72054d436afadce2f94dcad71cfadRobert Richter u64 val; 716b716916679e72054d436afadce2f94dcad71cfadRobert Richter int valid = 0; 717b716916679e72054d436afadce2f94dcad71cfadRobert Richter 718b716916679e72054d436afadce2f94dcad71cfadRobert Richter preempt_disable(); 719b716916679e72054d436afadce2f94dcad71cfadRobert Richter 720b716916679e72054d436afadce2f94dcad71cfadRobert Richter rdmsrl(MSR_AMD64_IBSCTL, val); 721b716916679e72054d436afadce2f94dcad71cfadRobert Richter offset = val & IBSCTL_LVT_OFFSET_MASK; 722b716916679e72054d436afadce2f94dcad71cfadRobert Richter 723b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!(val & IBSCTL_LVT_OFFSET_VALID)) { 724b716916679e72054d436afadce2f94dcad71cfadRobert Richter pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", 725b716916679e72054d436afadce2f94dcad71cfadRobert Richter smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 726b716916679e72054d436afadce2f94dcad71cfadRobert Richter goto out; 727b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 728b716916679e72054d436afadce2f94dcad71cfadRobert Richter 729b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!get_eilvt(offset)) { 730b716916679e72054d436afadce2f94dcad71cfadRobert Richter pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", 731b716916679e72054d436afadce2f94dcad71cfadRobert Richter smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 732b716916679e72054d436afadce2f94dcad71cfadRobert Richter goto out; 733b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 734b716916679e72054d436afadce2f94dcad71cfadRobert Richter 735b716916679e72054d436afadce2f94dcad71cfadRobert Richter valid = 1; 736b716916679e72054d436afadce2f94dcad71cfadRobert Richterout: 737b716916679e72054d436afadce2f94dcad71cfadRobert Richter preempt_enable(); 738b716916679e72054d436afadce2f94dcad71cfadRobert Richter 739b716916679e72054d436afadce2f94dcad71cfadRobert Richter return valid; 740b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 741b716916679e72054d436afadce2f94dcad71cfadRobert Richter 742b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic int setup_ibs_ctl(int ibs_eilvt_off) 743b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 744b716916679e72054d436afadce2f94dcad71cfadRobert Richter struct pci_dev *cpu_cfg; 745b716916679e72054d436afadce2f94dcad71cfadRobert Richter int nodes; 746b716916679e72054d436afadce2f94dcad71cfadRobert Richter u32 value = 0; 747b716916679e72054d436afadce2f94dcad71cfadRobert Richter 748b716916679e72054d436afadce2f94dcad71cfadRobert Richter nodes = 0; 749b716916679e72054d436afadce2f94dcad71cfadRobert Richter cpu_cfg = NULL; 750b716916679e72054d436afadce2f94dcad71cfadRobert Richter do { 751b716916679e72054d436afadce2f94dcad71cfadRobert Richter cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, 752b716916679e72054d436afadce2f94dcad71cfadRobert Richter PCI_DEVICE_ID_AMD_10H_NB_MISC, 753b716916679e72054d436afadce2f94dcad71cfadRobert Richter cpu_cfg); 754b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!cpu_cfg) 755b716916679e72054d436afadce2f94dcad71cfadRobert Richter break; 756b716916679e72054d436afadce2f94dcad71cfadRobert Richter ++nodes; 757b716916679e72054d436afadce2f94dcad71cfadRobert Richter pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off 758b716916679e72054d436afadce2f94dcad71cfadRobert Richter | IBSCTL_LVT_OFFSET_VALID); 759b716916679e72054d436afadce2f94dcad71cfadRobert Richter pci_read_config_dword(cpu_cfg, IBSCTL, &value); 760b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { 761b716916679e72054d436afadce2f94dcad71cfadRobert Richter pci_dev_put(cpu_cfg); 762b716916679e72054d436afadce2f94dcad71cfadRobert Richter printk(KERN_DEBUG "Failed to setup IBS LVT offset, " 763b716916679e72054d436afadce2f94dcad71cfadRobert Richter "IBSCTL = 0x%08x\n", value); 764b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -EINVAL; 765b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 766b716916679e72054d436afadce2f94dcad71cfadRobert Richter } while (1); 767b716916679e72054d436afadce2f94dcad71cfadRobert Richter 768b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!nodes) { 769b716916679e72054d436afadce2f94dcad71cfadRobert Richter printk(KERN_DEBUG "No CPU node configured for IBS\n"); 770b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -ENODEV; 771b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 772b716916679e72054d436afadce2f94dcad71cfadRobert Richter 773b716916679e72054d436afadce2f94dcad71cfadRobert Richter return 0; 774b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 775b716916679e72054d436afadce2f94dcad71cfadRobert Richter 776b716916679e72054d436afadce2f94dcad71cfadRobert Richter/* 777b716916679e72054d436afadce2f94dcad71cfadRobert Richter * This runs only on the current cpu. We try to find an LVT offset and 778b716916679e72054d436afadce2f94dcad71cfadRobert Richter * setup the local APIC. For this we must disable preemption. On 779b716916679e72054d436afadce2f94dcad71cfadRobert Richter * success we initialize all nodes with this offset. This updates then 780b716916679e72054d436afadce2f94dcad71cfadRobert Richter * the offset in the IBS_CTL per-node msr. The per-core APIC setup of 781b716916679e72054d436afadce2f94dcad71cfadRobert Richter * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that 782b716916679e72054d436afadce2f94dcad71cfadRobert Richter * is using the new offset. 783b716916679e72054d436afadce2f94dcad71cfadRobert Richter */ 784b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic int force_ibs_eilvt_setup(void) 785b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 786b716916679e72054d436afadce2f94dcad71cfadRobert Richter int offset; 787b716916679e72054d436afadce2f94dcad71cfadRobert Richter int ret; 788b716916679e72054d436afadce2f94dcad71cfadRobert Richter 789b716916679e72054d436afadce2f94dcad71cfadRobert Richter preempt_disable(); 790b716916679e72054d436afadce2f94dcad71cfadRobert Richter /* find the next free available EILVT entry, skip offset 0 */ 791b716916679e72054d436afadce2f94dcad71cfadRobert Richter for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { 792b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (get_eilvt(offset)) 793b716916679e72054d436afadce2f94dcad71cfadRobert Richter break; 794b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 795b716916679e72054d436afadce2f94dcad71cfadRobert Richter preempt_enable(); 796b716916679e72054d436afadce2f94dcad71cfadRobert Richter 797b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (offset == APIC_EILVT_NR_MAX) { 798b716916679e72054d436afadce2f94dcad71cfadRobert Richter printk(KERN_DEBUG "No EILVT entry available\n"); 799b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -EBUSY; 800b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 801b716916679e72054d436afadce2f94dcad71cfadRobert Richter 802b716916679e72054d436afadce2f94dcad71cfadRobert Richter ret = setup_ibs_ctl(offset); 803b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (ret) 804b716916679e72054d436afadce2f94dcad71cfadRobert Richter goto out; 805b716916679e72054d436afadce2f94dcad71cfadRobert Richter 806b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!ibs_eilvt_valid()) { 807b716916679e72054d436afadce2f94dcad71cfadRobert Richter ret = -EFAULT; 808b716916679e72054d436afadce2f94dcad71cfadRobert Richter goto out; 809b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 810b716916679e72054d436afadce2f94dcad71cfadRobert Richter 81116e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter pr_info("IBS: LVT offset %d assigned\n", offset); 812b716916679e72054d436afadce2f94dcad71cfadRobert Richter 813b716916679e72054d436afadce2f94dcad71cfadRobert Richter return 0; 814b716916679e72054d436afadce2f94dcad71cfadRobert Richterout: 815b716916679e72054d436afadce2f94dcad71cfadRobert Richter preempt_disable(); 816b716916679e72054d436afadce2f94dcad71cfadRobert Richter put_eilvt(offset); 817b716916679e72054d436afadce2f94dcad71cfadRobert Richter preempt_enable(); 818b716916679e72054d436afadce2f94dcad71cfadRobert Richter return ret; 819b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 820b716916679e72054d436afadce2f94dcad71cfadRobert Richter 821bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richterstatic void ibs_eilvt_setup(void) 822bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter{ 823bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter /* 824bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter * Force LVT offset assignment for family 10h: The offsets are 825bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter * not assigned by the BIOS for this family, so the OS is 826bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter * responsible for doing it. If the OS assignment fails, fall 827bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter * back to BIOS settings and try to setup this. 828bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter */ 829bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter if (boot_cpu_data.x86 == 0x10) 830bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter force_ibs_eilvt_setup(); 831bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter} 832bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 833b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic inline int get_ibs_lvt_offset(void) 834b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 835b716916679e72054d436afadce2f94dcad71cfadRobert Richter u64 val; 836b716916679e72054d436afadce2f94dcad71cfadRobert Richter 837b716916679e72054d436afadce2f94dcad71cfadRobert Richter rdmsrl(MSR_AMD64_IBSCTL, val); 838b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!(val & IBSCTL_LVT_OFFSET_VALID)) 839b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -EINVAL; 840b716916679e72054d436afadce2f94dcad71cfadRobert Richter 841b716916679e72054d436afadce2f94dcad71cfadRobert Richter return val & IBSCTL_LVT_OFFSET_MASK; 842b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 843b716916679e72054d436afadce2f94dcad71cfadRobert Richter 844b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic void setup_APIC_ibs(void *dummy) 845b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 846b716916679e72054d436afadce2f94dcad71cfadRobert Richter int offset; 847b716916679e72054d436afadce2f94dcad71cfadRobert Richter 848b716916679e72054d436afadce2f94dcad71cfadRobert Richter offset = get_ibs_lvt_offset(); 849b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (offset < 0) 850b716916679e72054d436afadce2f94dcad71cfadRobert Richter goto failed; 851b716916679e72054d436afadce2f94dcad71cfadRobert Richter 852b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) 853b716916679e72054d436afadce2f94dcad71cfadRobert Richter return; 854b716916679e72054d436afadce2f94dcad71cfadRobert Richterfailed: 855b716916679e72054d436afadce2f94dcad71cfadRobert Richter pr_warn("perf: IBS APIC setup failed on cpu #%d\n", 856b716916679e72054d436afadce2f94dcad71cfadRobert Richter smp_processor_id()); 857b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 858b716916679e72054d436afadce2f94dcad71cfadRobert Richter 859b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic void clear_APIC_ibs(void *dummy) 860b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 861b716916679e72054d436afadce2f94dcad71cfadRobert Richter int offset; 862b716916679e72054d436afadce2f94dcad71cfadRobert Richter 863b716916679e72054d436afadce2f94dcad71cfadRobert Richter offset = get_ibs_lvt_offset(); 864b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (offset >= 0) 865b716916679e72054d436afadce2f94dcad71cfadRobert Richter setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); 866b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 867b716916679e72054d436afadce2f94dcad71cfadRobert Richter 868bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter#ifdef CONFIG_PM 869bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 870bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richterstatic int perf_ibs_suspend(void) 871bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter{ 872bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter clear_APIC_ibs(NULL); 873bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter return 0; 874bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter} 875bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 876bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richterstatic void perf_ibs_resume(void) 877bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter{ 878bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter ibs_eilvt_setup(); 879bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter setup_APIC_ibs(NULL); 880bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter} 881bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 882bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richterstatic struct syscore_ops perf_ibs_syscore_ops = { 883bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter .resume = perf_ibs_resume, 884bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter .suspend = perf_ibs_suspend, 885bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter}; 886bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 887bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richterstatic void perf_ibs_pm_init(void) 888bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter{ 889bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter register_syscore_ops(&perf_ibs_syscore_ops); 890bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter} 891bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 892bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter#else 893bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 894bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richterstatic inline void perf_ibs_pm_init(void) { } 895bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 896bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter#endif 897bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter 898148f9bb87745ed45f7a11b2cbd3bc0f017d5d257Paul Gortmakerstatic int 899b716916679e72054d436afadce2f94dcad71cfadRobert Richterperf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) 900b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 901b716916679e72054d436afadce2f94dcad71cfadRobert Richter switch (action & ~CPU_TASKS_FROZEN) { 902b716916679e72054d436afadce2f94dcad71cfadRobert Richter case CPU_STARTING: 903b716916679e72054d436afadce2f94dcad71cfadRobert Richter setup_APIC_ibs(NULL); 904b716916679e72054d436afadce2f94dcad71cfadRobert Richter break; 905b716916679e72054d436afadce2f94dcad71cfadRobert Richter case CPU_DYING: 906b716916679e72054d436afadce2f94dcad71cfadRobert Richter clear_APIC_ibs(NULL); 907b716916679e72054d436afadce2f94dcad71cfadRobert Richter break; 908b716916679e72054d436afadce2f94dcad71cfadRobert Richter default: 909b716916679e72054d436afadce2f94dcad71cfadRobert Richter break; 910b716916679e72054d436afadce2f94dcad71cfadRobert Richter } 911b716916679e72054d436afadce2f94dcad71cfadRobert Richter 912b716916679e72054d436afadce2f94dcad71cfadRobert Richter return NOTIFY_OK; 913b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 914b716916679e72054d436afadce2f94dcad71cfadRobert Richter 915b716916679e72054d436afadce2f94dcad71cfadRobert Richterstatic __init int amd_ibs_init(void) 916b716916679e72054d436afadce2f94dcad71cfadRobert Richter{ 917b716916679e72054d436afadce2f94dcad71cfadRobert Richter u32 caps; 91816e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter int ret = -EINVAL; 919b716916679e72054d436afadce2f94dcad71cfadRobert Richter 920b716916679e72054d436afadce2f94dcad71cfadRobert Richter caps = __get_ibs_caps(); 921b716916679e72054d436afadce2f94dcad71cfadRobert Richter if (!caps) 922b716916679e72054d436afadce2f94dcad71cfadRobert Richter return -ENODEV; /* ibs not supported by the cpu */ 923b716916679e72054d436afadce2f94dcad71cfadRobert Richter 924bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter ibs_eilvt_setup(); 92516e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter 92616e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter if (!ibs_eilvt_valid()) 92716e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter goto out; 928b716916679e72054d436afadce2f94dcad71cfadRobert Richter 929bee09ed91cacdbffdbcd3b05de8409c77ec9fcd6Robert Richter perf_ibs_pm_init(); 930047868ce2952bb6a86ccea3ecc6fd59faa9062a7Srivatsa S. Bhat cpu_notifier_register_begin(); 931b716916679e72054d436afadce2f94dcad71cfadRobert Richter ibs_caps = caps; 932b716916679e72054d436afadce2f94dcad71cfadRobert Richter /* make ibs_caps visible to other cpus: */ 933b716916679e72054d436afadce2f94dcad71cfadRobert Richter smp_mb(); 934b716916679e72054d436afadce2f94dcad71cfadRobert Richter smp_call_function(setup_APIC_ibs, NULL, 1); 935047868ce2952bb6a86ccea3ecc6fd59faa9062a7Srivatsa S. Bhat __perf_cpu_notifier(perf_ibs_cpu_notifier); 936047868ce2952bb6a86ccea3ecc6fd59faa9062a7Srivatsa S. Bhat cpu_notifier_register_done(); 937b716916679e72054d436afadce2f94dcad71cfadRobert Richter 93816e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter ret = perf_event_ibs_init(); 93916e5294e5f8303756a179cf218e37dfb9ed34417Robert Richterout: 94016e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter if (ret) 94116e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter pr_err("Failed to setup IBS, %d\n", ret); 94216e5294e5f8303756a179cf218e37dfb9ed34417Robert Richter return ret; 943b716916679e72054d436afadce2f94dcad71cfadRobert Richter} 944b716916679e72054d436afadce2f94dcad71cfadRobert Richter 945b716916679e72054d436afadce2f94dcad71cfadRobert Richter/* Since we need the pci subsystem to init ibs we can't do this earlier: */ 946b716916679e72054d436afadce2f94dcad71cfadRobert Richterdevice_initcall(amd_ibs_init); 947