11b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles#undef DEBUG 21b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 31b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles/* 41b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * ARM performance counter support. 51b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * 61b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles 743eab87828fee65f89f4088736b2b7a187390a2fWill Deacon * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com> 8796d12959ad374cae8eb77faaf4243455a305433Jean PIHET * 91b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * This code is based on the sparc64 perf event code, which is in turn based 101b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * on the x86 code. Callchain code is based on the ARM OProfile backtrace 111b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * code. 121b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 131b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles#define pr_fmt(fmt) "hw perfevents: " fmt 141b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 151b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles#include <linux/kernel.h> 1649c006b93769a86bec2b32b9234abf016ac0d50eWill Deacon#include <linux/platform_device.h> 177be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter#include <linux/pm_runtime.h> 185505b206ca006d0506d1d3b3c494aa86234f66e2Will Deacon#include <linux/uaccess.h> 19bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd#include <linux/irq.h> 20bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd#include <linux/irqdesc.h> 211b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 221b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles#include <asm/irq_regs.h> 231b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles#include <asm/pmu.h> 241b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles#include <asm/stacktrace.h> 251b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 261b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 27e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutlandarmpmu_map_cache_event(const unsigned (*cache_map) 28e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland [PERF_COUNT_HW_CACHE_MAX] 29e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland [PERF_COUNT_HW_CACHE_OP_MAX] 30e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland [PERF_COUNT_HW_CACHE_RESULT_MAX], 31e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland u64 config) 321b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 331b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles unsigned int cache_type, cache_op, cache_result, ret; 341b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 351b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles cache_type = (config >> 0) & 0xff; 361b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 371b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return -EINVAL; 381b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 391b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles cache_op = (config >> 8) & 0xff; 401b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 411b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return -EINVAL; 421b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 431b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles cache_result = (config >> 16) & 0xff; 441b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 451b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return -EINVAL; 461b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 47e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; 481b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 491b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (ret == CACHE_OP_UNSUPPORTED) 501b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return -ENOENT; 511b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 521b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return ret; 531b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 541b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 551b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 566dbc00297095122ea89e016ce6affad0b7c0ddacWill Deaconarmpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) 5784fee97a026ca085f08381054513f9e24689a303Will Deacon{ 58d9f966357b14e356dbd83b8f4a197a287ab4ff83Stephen Boyd int mapping; 59d9f966357b14e356dbd83b8f4a197a287ab4ff83Stephen Boyd 60d9f966357b14e356dbd83b8f4a197a287ab4ff83Stephen Boyd if (config >= PERF_COUNT_HW_MAX) 61d9f966357b14e356dbd83b8f4a197a287ab4ff83Stephen Boyd return -EINVAL; 62d9f966357b14e356dbd83b8f4a197a287ab4ff83Stephen Boyd 63d9f966357b14e356dbd83b8f4a197a287ab4ff83Stephen Boyd mapping = (*event_map)[config]; 64e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; 6584fee97a026ca085f08381054513f9e24689a303Will Deacon} 6684fee97a026ca085f08381054513f9e24689a303Will Deacon 6784fee97a026ca085f08381054513f9e24689a303Will Deaconstatic int 68e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutlandarmpmu_map_raw_event(u32 raw_event_mask, u64 config) 6984fee97a026ca085f08381054513f9e24689a303Will Deacon{ 70e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland return (int)(config & raw_event_mask); 71e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland} 72e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland 736dbc00297095122ea89e016ce6affad0b7c0ddacWill Deaconint 746dbc00297095122ea89e016ce6affad0b7c0ddacWill Deaconarmpmu_map_event(struct perf_event *event, 756dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon const unsigned (*event_map)[PERF_COUNT_HW_MAX], 766dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon const unsigned (*cache_map) 776dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon [PERF_COUNT_HW_CACHE_MAX] 786dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon [PERF_COUNT_HW_CACHE_OP_MAX] 796dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon [PERF_COUNT_HW_CACHE_RESULT_MAX], 806dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon u32 raw_event_mask) 81e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland{ 82e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland u64 config = event->attr.config; 83e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland 84e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland switch (event->attr.type) { 85e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland case PERF_TYPE_HARDWARE: 866dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon return armpmu_map_hw_event(event_map, config); 87e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland case PERF_TYPE_HW_CACHE: 88e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland return armpmu_map_cache_event(cache_map, config); 89e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland case PERF_TYPE_RAW: 90e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland return armpmu_map_raw_event(raw_event_mask, config); 91e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland } 92e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland 93e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland return -ENOENT; 9484fee97a026ca085f08381054513f9e24689a303Will Deacon} 9584fee97a026ca085f08381054513f9e24689a303Will Deacon 96ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNageshaint armpmu_event_set_period(struct perf_event *event) 971b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 988a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 99ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha struct hw_perf_event *hwc = &event->hw; 100e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra s64 left = local64_read(&hwc->period_left); 1011b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles s64 period = hwc->sample_period; 1021b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles int ret = 0; 1031b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1041b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (unlikely(left <= -period)) { 1051b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles left = period; 106e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra local64_set(&hwc->period_left, left); 1071b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles hwc->last_period = period; 1081b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles ret = 1; 1091b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 1101b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1111b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (unlikely(left <= 0)) { 1121b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles left += period; 113e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra local64_set(&hwc->period_left, left); 1141b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles hwc->last_period = period; 1151b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles ret = 1; 1161b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 1171b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1181b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (left > (s64)armpmu->max_period) 1191b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles left = armpmu->max_period; 1201b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 121e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra local64_set(&hwc->prev_count, (u64)-left); 1221b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 123ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->write_counter(event, (u64)(-left) & 0xffffffff); 1241b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1251b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles perf_event_update_userpage(event); 1261b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1271b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return ret; 1281b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 1291b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 130ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNageshau64 armpmu_event_update(struct perf_event *event) 1311b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 1328a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 133ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha struct hw_perf_event *hwc = &event->hw; 134a737823d37666255e3e74ce84bc9611a038e0888Will Deacon u64 delta, prev_raw_count, new_raw_count; 1351b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1361b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesagain: 137e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra prev_raw_count = local64_read(&hwc->prev_count); 138ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha new_raw_count = armpmu->read_counter(event); 1391b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 140e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 1411b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles new_raw_count) != prev_raw_count) 1421b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles goto again; 1431b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1445727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon delta = (new_raw_count - prev_raw_count) & armpmu->max_period; 1451b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 146e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra local64_add(delta, &event->count); 147e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra local64_sub(delta, &hwc->period_left); 1481b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1491b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return new_raw_count; 1501b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 1511b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1521b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic void 153a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstraarmpmu_read(struct perf_event *event) 1541b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 155ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu_event_update(event); 1561b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 1571b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 1581b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic void 159a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstraarmpmu_stop(struct perf_event *event, int flags) 1601b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 1618a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 1621b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct hw_perf_event *hwc = &event->hw; 1631b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 164a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra /* 165a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra * ARM pmu always has to update the counter, so ignore 166a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra * PERF_EF_UPDATE, see comments in armpmu_start(). 167a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra */ 168a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra if (!(hwc->state & PERF_HES_STOPPED)) { 169ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->disable(event); 170ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu_event_update(event); 171a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 172a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra } 1731b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 1741b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 175ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNageshastatic void armpmu_start(struct perf_event *event, int flags) 1761b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 1778a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 1781b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct hw_perf_event *hwc = &event->hw; 1791b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 180a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra /* 181a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra * ARM pmu always has to reprogram the period, so ignore 182a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra * PERF_EF_RELOAD, see the comment below. 183a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra */ 184a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra if (flags & PERF_EF_RELOAD) 185a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 186a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra 187a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra hwc->state = 0; 1881b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* 1891b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Set the period again. Some counters can't be stopped, so when we 190a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra * were stopped we simply disabled the IRQ source and the counter 1911b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * may have been left counting. If we don't do this step then we may 1921b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * get an interrupt too soon or *way* too late if the overflow has 1931b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * happened since disabling. 1941b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 195ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu_event_set_period(event); 196ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->enable(event); 1971b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 1981b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 199a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstrastatic void 200a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstraarmpmu_del(struct perf_event *event, int flags) 201a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra{ 2028a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 2038be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland struct pmu_hw_events *hw_events = armpmu->get_hw_events(); 204a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra struct hw_perf_event *hwc = &event->hw; 205a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra int idx = hwc->idx; 206a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra 207a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra armpmu_stop(event, PERF_EF_UPDATE); 2088be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland hw_events->events[idx] = NULL; 2098be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland clear_bit(idx, hw_events->used_mask); 210eab443ef391d18772710dc2c156f7ee05e51f754Stephen Boyd if (armpmu->clear_event_idx) 211eab443ef391d18772710dc2c156f7ee05e51f754Stephen Boyd armpmu->clear_event_idx(hw_events, event); 212a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra 213a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra perf_event_update_userpage(event); 214a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra} 215a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra 2161b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 217a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstraarmpmu_add(struct perf_event *event, int flags) 2181b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 2198a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 2208be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland struct pmu_hw_events *hw_events = armpmu->get_hw_events(); 2211b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct hw_perf_event *hwc = &event->hw; 2221b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles int idx; 2231b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles int err = 0; 2241b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 22533696fc0d141bbbcb12f75b69608ea83282e3117Peter Zijlstra perf_pmu_disable(event->pmu); 22624cd7f54a0d47e1d5b3de29e2456bfbd2d8447b7Peter Zijlstra 2271b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* If we don't have a space for the counter then finish early. */ 228ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha idx = armpmu->get_event_idx(hw_events, event); 2291b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (idx < 0) { 2301b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles err = idx; 2311b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles goto out; 2321b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 2331b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2341b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* 2351b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * If there is an event in the counter we are going to use then make 2361b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * sure it is disabled. 2371b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 2381b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles event->hw.idx = idx; 239ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->disable(event); 2408be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland hw_events->events[idx] = event; 2411b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 242a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 243a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra if (flags & PERF_EF_START) 244a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstra armpmu_start(event, PERF_EF_RELOAD); 2451b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2461b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* Propagate our changes to the userspace mapping. */ 2471b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles perf_event_update_userpage(event); 2481b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2491b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesout: 25033696fc0d141bbbcb12f75b69608ea83282e3117Peter Zijlstra perf_pmu_enable(event->pmu); 2511b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return err; 2521b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 2531b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2541b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 2558be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutlandvalidate_event(struct pmu_hw_events *hw_events, 2561b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct perf_event *event) 2571b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 2588a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 2591b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 260c95eb3184ea1a3a2551df57190c81da695e2144bWill Deacon if (is_software_event(event)) 261c95eb3184ea1a3a2551df57190c81da695e2144bWill Deacon return 1; 262c95eb3184ea1a3a2551df57190c81da695e2144bWill Deacon 2632dfcb802d6bd54a2353678c6434846d94b058f2cWill Deacon if (event->state < PERF_EVENT_STATE_OFF) 264cb2d8b342aa084d1f3ac29966245dec9163677fbWill Deacon return 1; 265cb2d8b342aa084d1f3ac29966245dec9163677fbWill Deacon 266cb2d8b342aa084d1f3ac29966245dec9163677fbWill Deacon if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 26765b4711ff513767341aa1915c822de6ec0de65cbWill Deacon return 1; 2681b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 269ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha return armpmu->get_event_idx(hw_events, event) >= 0; 2701b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 2711b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2721b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 2731b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesvalidate_group(struct perf_event *event) 2741b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 2751b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct perf_event *sibling, *leader = event->group_leader; 2768be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland struct pmu_hw_events fake_pmu; 277bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS); 2781b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 279bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon /* 280bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon * Initialise the fake PMU. We only need to populate the 281bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon * used_mask for the purposes of validation. 282bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon */ 283bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon memset(fake_used_mask, 0, sizeof(fake_used_mask)); 284bce34d14428d35d9a06ddc10cd46ecef311764c9Will Deacon fake_pmu.used_mask = fake_used_mask; 2851b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2861b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (!validate_event(&fake_pmu, leader)) 287aa2bc1ade59003a379ffc485d6da2d92ea3370a6Peter Zijlstra return -EINVAL; 2881b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2891b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 2901b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (!validate_event(&fake_pmu, sibling)) 291aa2bc1ade59003a379ffc485d6da2d92ea3370a6Peter Zijlstra return -EINVAL; 2921b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 2931b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2941b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (!validate_event(&fake_pmu, event)) 295aa2bc1ade59003a379ffc485d6da2d92ea3370a6Peter Zijlstra return -EINVAL; 2961b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 2971b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return 0; 2981b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 2991b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 300051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNageshastatic irqreturn_t armpmu_dispatch_irq(int irq, void *dev) 3010e25a5c98067286fc727cf142fc0dadf95790921Rabin Vincent{ 302bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd struct arm_pmu *armpmu; 303bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd struct platform_device *plat_device; 304bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd struct arm_pmu_platdata *plat; 3055f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon int ret; 3065f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon u64 start_clock, finish_clock; 307bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd 308bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd if (irq_is_percpu(irq)) 309bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd dev = *(void **)dev; 310bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd armpmu = dev; 311bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd plat_device = armpmu->plat_device; 312bbd64559376fa25732994c4181c8ec493fa57871Stephen Boyd plat = dev_get_platdata(&plat_device->dev); 3130e25a5c98067286fc727cf142fc0dadf95790921Rabin Vincent 3145f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon start_clock = sched_clock(); 315051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha if (plat && plat->handle_irq) 3165f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon ret = plat->handle_irq(irq, dev, armpmu->handle_irq); 317051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha else 3185f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon ret = armpmu->handle_irq(irq, dev); 3195f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon finish_clock = sched_clock(); 3205f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon 3215f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon perf_sample_event_took(finish_clock - start_clock); 3225f5092e72cc25a6a5785308270e0085b2b2772ccWill Deacon return ret; 3230e25a5c98067286fc727cf142fc0dadf95790921Rabin Vincent} 3240e25a5c98067286fc727cf142fc0dadf95790921Rabin Vincent 3250b390e2126e03b6ec41f96fb0550b1526d00e203Will Deaconstatic void 3268a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutlandarmpmu_release_hardware(struct arm_pmu *armpmu) 3270b390e2126e03b6ec41f96fb0550b1526d00e203Will Deacon{ 328ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->free_irq(armpmu); 329051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha pm_runtime_put_sync(&armpmu->plat_device->dev); 3300b390e2126e03b6ec41f96fb0550b1526d00e203Will Deacon} 3310b390e2126e03b6ec41f96fb0550b1526d00e203Will Deacon 3321b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 3338a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutlandarmpmu_reserve_hardware(struct arm_pmu *armpmu) 3341b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 335051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha int err; 336a9356a04fab912289b886824cb4b1d461987a910Mark Rutland struct platform_device *pmu_device = armpmu->plat_device; 3371b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 338e5a21327644adba32816f74a415114d11c57f2e9Will Deacon if (!pmu_device) 339e5a21327644adba32816f74a415114d11c57f2e9Will Deacon return -ENODEV; 340e5a21327644adba32816f74a415114d11c57f2e9Will Deacon 3417be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter pm_runtime_get_sync(&pmu_device->dev); 342ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha err = armpmu->request_irq(armpmu, armpmu_dispatch_irq); 343051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha if (err) { 344051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha armpmu_release_hardware(armpmu); 345051f1b13144dd8553d5a5104dde94c7263ae3ba7Sudeep KarkadaNagesha return err; 34649c006b93769a86bec2b32b9234abf016ac0d50eWill Deacon } 3471b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 3480b390e2126e03b6ec41f96fb0550b1526d00e203Will Deacon return 0; 3491b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 3501b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 3511b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic void 3521b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ileshw_perf_event_destroy(struct perf_event *event) 3531b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 3548a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 35503b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland atomic_t *active_events = &armpmu->active_events; 35603b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex; 35703b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland 35803b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) { 3598a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland armpmu_release_hardware(armpmu); 36003b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland mutex_unlock(pmu_reserve_mutex); 3611b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 3621b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 3631b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 3641b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 36505d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deaconevent_requires_mode_exclusion(struct perf_event_attr *attr) 36605d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon{ 36705d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon return attr->exclude_idle || attr->exclude_user || 36805d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon attr->exclude_kernel || attr->exclude_hv; 36905d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon} 37005d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon 37105d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deaconstatic int 3721b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles__hw_perf_event_init(struct perf_event *event) 3731b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 3748a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 3751b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct hw_perf_event *hwc = &event->hw; 3769dcbf466559f6f2f55d60eb5a1bbebc8e694b52aMark Rutland int mapping; 3771b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 378e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland mapping = armpmu->map_event(event); 3791b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 3801b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (mapping < 0) { 3811b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles pr_debug("event %x:%llx not supported\n", event->attr.type, 3821b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles event->attr.config); 3831b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return mapping; 3841b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 3851b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 3861b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* 38705d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon * We don't assign an index until we actually place the event onto 38805d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon * hardware. Use -1 to signify that we haven't decided where to put it 38905d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon * yet. For SMP systems, each core has it's own PMU so we can't do any 39005d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon * clever allocation or constraints checking at this point. 39105d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon */ 39205d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon hwc->idx = -1; 39305d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon hwc->config_base = 0; 39405d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon hwc->config = 0; 39505d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon hwc->event_base = 0; 39605d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon 39705d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon /* 3981b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Check whether we need to exclude the counter from certain modes. 3991b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 40005d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon if ((!armpmu->set_event_filter || 40105d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon armpmu->set_event_filter(hwc, &event->attr)) && 40205d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon event_requires_mode_exclusion(&event->attr)) { 4031b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles pr_debug("ARM performance counters do not support " 4041b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles "mode exclusion\n"); 405fdeb8e35fd59e79dec385f98eb4b6d2e3398264bWill Deacon return -EOPNOTSUPP; 4061b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 4071b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4081b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* 40905d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon * Store the event encoding into the config_base field. 4101b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 41105d22fde3c0b86c8395d8f12ac01fbbc524d73caWill Deacon hwc->config_base |= (unsigned long)mapping; 4121b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 413edcb4d3c36a6429caa03ddfeab4cbb153c7002b2Vince Weaver if (!is_sampling_event(event)) { 4145727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon /* 4155727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon * For non-sampling runs, limit the sample_period to half 4165727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon * of the counter width. That way, the new counter value 4175727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon * is far less likely to overtake the previous one unless 4185727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon * you have some serious IRQ latency issues. 4195727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon */ 4205727347180ebc6b4a866fcbe00dcb39cc03acb37Will Deacon hwc->sample_period = armpmu->max_period >> 1; 4211b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles hwc->last_period = hwc->sample_period; 422e78505958cf123048fb48cb56b79cebb8edd15fbPeter Zijlstra local64_set(&hwc->period_left, hwc->sample_period); 4231b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 4241b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4251b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (event->group_leader != event) { 426e595ede6050b1ce982d74f7084f93715bcc32359Chen Gang if (validate_group(event) != 0) 4271b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return -EINVAL; 4281b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 4291b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4309dcbf466559f6f2f55d60eb5a1bbebc8e694b52aMark Rutland return 0; 4311b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 4321b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 433b0a873ebbf87bf38bf70b5e39a7cadc96099fa13Peter Zijlstrastatic int armpmu_event_init(struct perf_event *event) 4341b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 4358a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(event->pmu); 4361b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles int err = 0; 43703b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland atomic_t *active_events = &armpmu->active_events; 4381b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4392481c5fa6db0237e4f0168f88913178b2b495b7cStephane Eranian /* does not support taken branch sampling */ 4402481c5fa6db0237e4f0168f88913178b2b495b7cStephane Eranian if (has_branch_stack(event)) 4412481c5fa6db0237e4f0168f88913178b2b495b7cStephane Eranian return -EOPNOTSUPP; 4422481c5fa6db0237e4f0168f88913178b2b495b7cStephane Eranian 443e1f431b57ef9e4a68281540933fa74865cbb7a74Mark Rutland if (armpmu->map_event(event) == -ENOENT) 444b0a873ebbf87bf38bf70b5e39a7cadc96099fa13Peter Zijlstra return -ENOENT; 445b0a873ebbf87bf38bf70b5e39a7cadc96099fa13Peter Zijlstra 4461b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles event->destroy = hw_perf_event_destroy; 4471b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 44803b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland if (!atomic_inc_not_zero(active_events)) { 44903b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland mutex_lock(&armpmu->reserve_mutex); 45003b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland if (atomic_read(active_events) == 0) 4518a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland err = armpmu_reserve_hardware(armpmu); 4521b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4531b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (!err) 45403b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland atomic_inc(active_events); 45503b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland mutex_unlock(&armpmu->reserve_mutex); 4561b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles } 4571b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4581b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (err) 459b0a873ebbf87bf38bf70b5e39a7cadc96099fa13Peter Zijlstra return err; 4601b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4611b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles err = __hw_perf_event_init(event); 4621b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (err) 4631b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles hw_perf_event_destroy(event); 4641b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 465b0a873ebbf87bf38bf70b5e39a7cadc96099fa13Peter Zijlstra return err; 4661b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 4671b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 468a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstrastatic void armpmu_enable(struct pmu *pmu) 4691b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 4708be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland struct arm_pmu *armpmu = to_arm_pmu(pmu); 4718be3f9a2385f91f7bf5c58f351e24b9247898e8fMark Rutland struct pmu_hw_events *hw_events = armpmu->get_hw_events(); 4727325eaec439cd0cc8c9b61b59d41d99abace1b23Mark Rutland int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events); 4731b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 474f4f38430c94c38187db73a2cf3892cc8b12a2713Will Deacon if (enabled) 475ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->start(armpmu); 4761b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 4771b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 478a4eaf7f14675cb512d69f0c928055e73d0c6d252Peter Zijlstrastatic void armpmu_disable(struct pmu *pmu) 4791b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 4808a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland struct arm_pmu *armpmu = to_arm_pmu(pmu); 481ed6f2a522398c26559f4da23a80aa6195e6284c7Sudeep KarkadaNagesha armpmu->stop(armpmu); 4821b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 4831b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 4847be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter#ifdef CONFIG_PM_RUNTIME 4857be2958e97b37256b8016db39ac6cf51f711e390Jon Hunterstatic int armpmu_runtime_resume(struct device *dev) 4867be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter{ 4877be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter struct arm_pmu_platdata *plat = dev_get_platdata(dev); 4887be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter 4897be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter if (plat && plat->runtime_resume) 4907be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter return plat->runtime_resume(dev); 4917be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter 4927be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter return 0; 4937be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter} 4947be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter 4957be2958e97b37256b8016db39ac6cf51f711e390Jon Hunterstatic int armpmu_runtime_suspend(struct device *dev) 4967be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter{ 4977be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter struct arm_pmu_platdata *plat = dev_get_platdata(dev); 4987be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter 4997be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter if (plat && plat->runtime_suspend) 5007be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter return plat->runtime_suspend(dev); 5017be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter 5027be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter return 0; 5037be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter} 5047be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter#endif 5057be2958e97b37256b8016db39ac6cf51f711e390Jon Hunter 5066dbc00297095122ea89e016ce6affad0b7c0ddacWill Deaconconst struct dev_pm_ops armpmu_dev_pm_ops = { 5076dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon SET_RUNTIME_PM_OPS(armpmu_runtime_suspend, armpmu_runtime_resume, NULL) 5086dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon}; 5096dbc00297095122ea89e016ce6affad0b7c0ddacWill Deacon 51044d6b1fc3e3c6a3af8e599b724972e881c81e1c9Stephen Boydstatic void armpmu_init(struct arm_pmu *armpmu) 51103b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland{ 51203b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland atomic_set(&armpmu->active_events, 0); 51303b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland mutex_init(&armpmu->reserve_mutex); 5148a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland 5158a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland armpmu->pmu = (struct pmu) { 5168a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .pmu_enable = armpmu_enable, 5178a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .pmu_disable = armpmu_disable, 5188a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .event_init = armpmu_event_init, 5198a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .add = armpmu_add, 5208a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .del = armpmu_del, 5218a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .start = armpmu_start, 5228a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .stop = armpmu_stop, 5238a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland .read = armpmu_read, 5248a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland }; 5258a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland} 5268a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland 5270305230a3d92d6829db89c9e0c096d4d8733f317Will Deaconint armpmu_register(struct arm_pmu *armpmu, int type) 5288a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland{ 5298a16b34e21199eb5fcf2c5050d3bc414fc5d6563Mark Rutland armpmu_init(armpmu); 5302ac29a14a8b6b4a37c09c50db88dc893e6e7fc75Jon Hunter pm_runtime_enable(&armpmu->plat_device->dev); 53104236f9fe07462849215c67cae6147661368bfadWill Deacon pr_info("enabled with %s PMU driver, %d counters available\n", 53204236f9fe07462849215c67cae6147661368bfadWill Deacon armpmu->name, armpmu->num_events); 5330305230a3d92d6829db89c9e0c096d4d8733f317Will Deacon return perf_pmu_register(&armpmu->pmu, armpmu->name, type); 53403b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland} 53503b7898d300de62078cc130fbc83b84b1d1e0f8dMark Rutland 5361b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles/* 5371b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Callchain handling code. 5381b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 5391b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 5401b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles/* 5411b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * The registers we're interested in are at the end of the variable 5421b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * length saved register structure. The fp points at the end of this 5431b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * structure so the address of this struct is: 5441b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * (struct frame_tail *)(xxx->fp)-1 5451b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * 5461b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * This code has been adapted from the ARM OProfile support. 5471b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 5481b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstruct frame_tail { 5494d6b7a779be34e1df296abc1dc555134a8cf34afWill Deacon struct frame_tail __user *fp; 5504d6b7a779be34e1df296abc1dc555134a8cf34afWill Deacon unsigned long sp; 5514d6b7a779be34e1df296abc1dc555134a8cf34afWill Deacon unsigned long lr; 5521b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} __attribute__((packed)); 5531b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 5541b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles/* 5551b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Get the return address for a single stackframe and return a pointer to the 5561b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * next frame tail. 5571b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 5584d6b7a779be34e1df296abc1dc555134a8cf34afWill Deaconstatic struct frame_tail __user * 5594d6b7a779be34e1df296abc1dc555134a8cf34afWill Deaconuser_backtrace(struct frame_tail __user *tail, 5601b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct perf_callchain_entry *entry) 5611b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 5621b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct frame_tail buftail; 5634b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet unsigned long err; 5641b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 5651b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) 5661b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return NULL; 5674b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet 5684b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet pagefault_disable(); 5694b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail)); 5704b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet pagefault_enable(); 5714b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet 5724b2974fa6a4a32d390a50e23381a2270a2e0d444Jean Pihet if (err) 5731b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return NULL; 5741b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 57570791ce9ba68a5921c9905ef05d23f62a90bc10cFrederic Weisbecker perf_callchain_store(entry, buftail.lr); 5761b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 5771b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles /* 5781b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Frame pointers should strictly progress back up the stack 5791b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * (towards higher addresses). 5801b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 581cb06199b1df492fcfbaedd2256b5054f944b664fRabin Vincent if (tail + 1 >= buftail.fp) 5821b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return NULL; 5831b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 5841b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return buftail.fp - 1; 5851b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 5861b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 58756962b4449af34070bb1994621ef4f0265eed4d8Frederic Weisbeckervoid 58856962b4449af34070bb1994621ef4f0265eed4d8Frederic Weisbeckerperf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) 5891b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 5904d6b7a779be34e1df296abc1dc555134a8cf34afWill Deacon struct frame_tail __user *tail; 5911b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 592e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 593e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier /* We don't support guest os callchain now */ 594e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier return; 595e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier } 5961b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 597c5f927a6f62196226915f12194c9d0df4e2210d7Jed Davis perf_callchain_store(entry, regs->ARM_pc); 598a7cc91001e36a4a4152c3ada6c8fe38adc5badbcJean Pihet 599a7cc91001e36a4a4152c3ada6c8fe38adc5badbcJean Pihet if (!current->mm) 600a7cc91001e36a4a4152c3ada6c8fe38adc5badbcJean Pihet return; 601a7cc91001e36a4a4152c3ada6c8fe38adc5badbcJean Pihet 6024d6b7a779be34e1df296abc1dc555134a8cf34afWill Deacon tail = (struct frame_tail __user *)regs->ARM_fp - 1; 6031b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 604860ad7823fdc00cd61dc70e7f35e07fb327cc9a4Sonny Rao while ((entry->nr < PERF_MAX_STACK_DEPTH) && 605860ad7823fdc00cd61dc70e7f35e07fb327cc9a4Sonny Rao tail && !((unsigned long)tail & 0x3)) 6061b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles tail = user_backtrace(tail, entry); 6071b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 6081b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 6091b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles/* 6101b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * Gets called by walk_stackframe() for every stackframe. This will be called 6111b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * whist unwinding the stackframe and is like a subroutine return so we use 6121b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles * the PC. 6131b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles */ 6141b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilesstatic int 6151b8873a0c6ec511870c106c80b94658f857c47f2Jamie Ilescallchain_trace(struct stackframe *fr, 6161b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles void *data) 6171b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 6181b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct perf_callchain_entry *entry = data; 61970791ce9ba68a5921c9905ef05d23f62a90bc10cFrederic Weisbecker perf_callchain_store(entry, fr->pc); 6201b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles return 0; 6211b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 6221b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 62356962b4449af34070bb1994621ef4f0265eed4d8Frederic Weisbeckervoid 62456962b4449af34070bb1994621ef4f0265eed4d8Frederic Weisbeckerperf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs) 6251b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles{ 6261b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles struct stackframe fr; 6271b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles 628e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 629e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier /* We don't support guest os callchain now */ 630e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier return; 631e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier } 632e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier 6336888e32a9e0b284c4dcdefcc3158949110699bc2Nikolay Borisov arm_get_current_stackframe(regs, &fr); 6341b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles walk_stackframe(&fr, callchain_trace, entry); 6351b8873a0c6ec511870c106c80b94658f857c47f2Jamie Iles} 636e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier 637e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngierunsigned long perf_instruction_pointer(struct pt_regs *regs) 638e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier{ 639e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 640e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier return perf_guest_cbs->get_guest_ip(); 641e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier 642e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier return instruction_pointer(regs); 643e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier} 644e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier 645e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngierunsigned long perf_misc_flags(struct pt_regs *regs) 646e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier{ 647e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier int misc = 0; 648e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier 649e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 650e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier if (perf_guest_cbs->is_user_mode()) 651e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier misc |= PERF_RECORD_MISC_GUEST_USER; 652e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier else 653e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier misc |= PERF_RECORD_MISC_GUEST_KERNEL; 654e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier } else { 655e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier if (user_mode(regs)) 656e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier misc |= PERF_RECORD_MISC_USER; 657e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier else 658e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier misc |= PERF_RECORD_MISC_KERNEL; 659e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier } 660e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier 661e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier return misc; 662e50c54189f7c6211a99539156e3978474f0b1a0bMarc Zyngier} 663