arm-ccn.c revision 8fb22264350d99793a12867f4e2a43e50150f778
1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright (C) 2014 ARM Limited 12 */ 13 14#include <linux/ctype.h> 15#include <linux/hrtimer.h> 16#include <linux/idr.h> 17#include <linux/interrupt.h> 18#include <linux/io.h> 19#include <linux/module.h> 20#include <linux/perf_event.h> 21#include <linux/platform_device.h> 22#include <linux/slab.h> 23 24#define CCN_NUM_XP_PORTS 2 25#define CCN_NUM_VCS 4 26#define CCN_NUM_REGIONS 256 27#define CCN_REGION_SIZE 0x10000 28 29#define CCN_ALL_OLY_ID 0xff00 30#define CCN_ALL_OLY_ID__OLY_ID__SHIFT 0 31#define CCN_ALL_OLY_ID__OLY_ID__MASK 0x1f 32#define CCN_ALL_OLY_ID__NODE_ID__SHIFT 8 33#define CCN_ALL_OLY_ID__NODE_ID__MASK 0x3f 34 35#define CCN_MN_ERRINT_STATUS 0x0008 36#define CCN_MN_ERRINT_STATUS__INTREQ__DESSERT 0x11 37#define CCN_MN_ERRINT_STATUS__ALL_ERRORS__ENABLE 0x02 38#define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLED 0x20 39#define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE 0x22 40#define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_ENABLE 0x04 41#define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLED 0x40 42#define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLE 0x44 43#define CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE 0x08 44#define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED 0x80 45#define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE 0x88 46#define CCN_MN_OLY_COMP_LIST_63_0 0x01e0 47#define CCN_MN_ERR_SIG_VAL_63_0 0x0300 48#define CCN_MN_ERR_SIG_VAL_63_0__DT (1 << 1) 49 50#define CCN_DT_ACTIVE_DSM 0x0000 51#define CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(n) ((n) * 8) 52#define CCN_DT_ACTIVE_DSM__DSM_ID__MASK 0xff 53#define CCN_DT_CTL 0x0028 54#define CCN_DT_CTL__DT_EN (1 << 0) 55#define CCN_DT_PMEVCNT(n) (0x0100 + (n) * 0x8) 56#define CCN_DT_PMCCNTR 0x0140 57#define CCN_DT_PMCCNTRSR 0x0190 58#define CCN_DT_PMOVSR 0x0198 59#define CCN_DT_PMOVSR_CLR 0x01a0 60#define CCN_DT_PMCR 0x01a8 61#define CCN_DT_PMCR__OVFL_INTR_EN (1 << 6) 62#define CCN_DT_PMCR__PMU_EN (1 << 0) 63#define CCN_DT_PMSR 0x01b0 64#define CCN_DT_PMSR_REQ 0x01b8 65#define CCN_DT_PMSR_CLR 0x01c0 66 67#define CCN_HNF_PMU_EVENT_SEL 0x0600 68#define CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4) 69#define CCN_HNF_PMU_EVENT_SEL__ID__MASK 0xf 70 71#define CCN_XP_DT_CONFIG 0x0300 72#define CCN_XP_DT_CONFIG__DT_CFG__SHIFT(n) ((n) * 4) 73#define CCN_XP_DT_CONFIG__DT_CFG__MASK 0xf 74#define CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH 0x0 75#define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT_0_OR_1 0x1 76#define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(n) (0x2 + (n)) 77#define CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(n) (0x4 + (n)) 78#define CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(d, n) (0x8 + (d) * 4 + (n)) 79#define CCN_XP_DT_INTERFACE_SEL 0x0308 80#define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(n) (0 + (n) * 8) 81#define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK 0x1 82#define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(n) (1 + (n) * 8) 83#define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK 0x1 84#define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(n) (2 + (n) * 8) 85#define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK 0x3 86#define CCN_XP_DT_CMP_VAL_L(n) (0x0310 + (n) * 0x40) 87#define CCN_XP_DT_CMP_VAL_H(n) (0x0318 + (n) * 0x40) 88#define CCN_XP_DT_CMP_MASK_L(n) (0x0320 + (n) * 0x40) 89#define CCN_XP_DT_CMP_MASK_H(n) (0x0328 + (n) * 0x40) 90#define CCN_XP_DT_CONTROL 0x0370 91#define CCN_XP_DT_CONTROL__DT_ENABLE (1 << 0) 92#define CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(n) (12 + (n) * 4) 93#define CCN_XP_DT_CONTROL__WP_ARM_SEL__MASK 0xf 94#define CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS 0xf 95#define CCN_XP_PMU_EVENT_SEL 0x0600 96#define CCN_XP_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 7) 97#define CCN_XP_PMU_EVENT_SEL__ID__MASK 0x3f 98 99#define CCN_SBAS_PMU_EVENT_SEL 0x0600 100#define CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4) 101#define CCN_SBAS_PMU_EVENT_SEL__ID__MASK 0xf 102 103#define CCN_RNI_PMU_EVENT_SEL 0x0600 104#define CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4) 105#define CCN_RNI_PMU_EVENT_SEL__ID__MASK 0xf 106 107#define CCN_TYPE_MN 0x01 108#define CCN_TYPE_DT 0x02 109#define CCN_TYPE_HNF 0x04 110#define CCN_TYPE_HNI 0x05 111#define CCN_TYPE_XP 0x08 112#define CCN_TYPE_SBSX 0x0c 113#define CCN_TYPE_SBAS 0x10 114#define CCN_TYPE_RNI_1P 0x14 115#define CCN_TYPE_RNI_2P 0x15 116#define CCN_TYPE_RNI_3P 0x16 117#define CCN_TYPE_RND_1P 0x18 /* RN-D = RN-I + DVM */ 118#define CCN_TYPE_RND_2P 0x19 119#define CCN_TYPE_RND_3P 0x1a 120#define CCN_TYPE_CYCLES 0xff /* Pseudotype */ 121 122#define CCN_EVENT_WATCHPOINT 0xfe /* Pseudoevent */ 123 124#define CCN_NUM_PMU_EVENTS 4 125#define CCN_NUM_XP_WATCHPOINTS 2 /* See DT.dbg_id.num_watchpoints */ 126#define CCN_NUM_PMU_EVENT_COUNTERS 8 /* See DT.dbg_id.num_pmucntr */ 127#define CCN_IDX_PMU_CYCLE_COUNTER CCN_NUM_PMU_EVENT_COUNTERS 128 129#define CCN_NUM_PREDEFINED_MASKS 4 130#define CCN_IDX_MASK_ANY (CCN_NUM_PMU_EVENT_COUNTERS + 0) 131#define CCN_IDX_MASK_EXACT (CCN_NUM_PMU_EVENT_COUNTERS + 1) 132#define CCN_IDX_MASK_ORDER (CCN_NUM_PMU_EVENT_COUNTERS + 2) 133#define CCN_IDX_MASK_OPCODE (CCN_NUM_PMU_EVENT_COUNTERS + 3) 134 135struct arm_ccn_component { 136 void __iomem *base; 137 u32 type; 138 139 DECLARE_BITMAP(pmu_events_mask, CCN_NUM_PMU_EVENTS); 140 union { 141 struct { 142 DECLARE_BITMAP(dt_cmp_mask, CCN_NUM_XP_WATCHPOINTS); 143 } xp; 144 }; 145}; 146 147#define pmu_to_arm_ccn(_pmu) container_of(container_of(_pmu, \ 148 struct arm_ccn_dt, pmu), struct arm_ccn, dt) 149 150struct arm_ccn_dt { 151 int id; 152 void __iomem *base; 153 154 spinlock_t config_lock; 155 156 DECLARE_BITMAP(pmu_counters_mask, CCN_NUM_PMU_EVENT_COUNTERS + 1); 157 struct { 158 struct arm_ccn_component *source; 159 struct perf_event *event; 160 } pmu_counters[CCN_NUM_PMU_EVENT_COUNTERS + 1]; 161 162 struct { 163 u64 l, h; 164 } cmp_mask[CCN_NUM_PMU_EVENT_COUNTERS + CCN_NUM_PREDEFINED_MASKS]; 165 166 struct hrtimer hrtimer; 167 168 struct pmu pmu; 169}; 170 171struct arm_ccn { 172 struct device *dev; 173 void __iomem *base; 174 unsigned irq_used:1; 175 unsigned sbas_present:1; 176 unsigned sbsx_present:1; 177 178 int num_nodes; 179 struct arm_ccn_component *node; 180 181 int num_xps; 182 struct arm_ccn_component *xp; 183 184 struct arm_ccn_dt dt; 185}; 186 187 188static int arm_ccn_node_to_xp(int node) 189{ 190 return node / CCN_NUM_XP_PORTS; 191} 192 193static int arm_ccn_node_to_xp_port(int node) 194{ 195 return node % CCN_NUM_XP_PORTS; 196} 197 198 199/* 200 * Bit shifts and masks in these defines must be kept in sync with 201 * arm_ccn_pmu_config_set() and CCN_FORMAT_ATTRs below! 202 */ 203#define CCN_CONFIG_NODE(_config) (((_config) >> 0) & 0xff) 204#define CCN_CONFIG_XP(_config) (((_config) >> 0) & 0xff) 205#define CCN_CONFIG_TYPE(_config) (((_config) >> 8) & 0xff) 206#define CCN_CONFIG_EVENT(_config) (((_config) >> 16) & 0xff) 207#define CCN_CONFIG_PORT(_config) (((_config) >> 24) & 0x3) 208#define CCN_CONFIG_VC(_config) (((_config) >> 26) & 0x7) 209#define CCN_CONFIG_DIR(_config) (((_config) >> 29) & 0x1) 210#define CCN_CONFIG_MASK(_config) (((_config) >> 30) & 0xf) 211 212static void arm_ccn_pmu_config_set(u64 *config, u32 node_xp, u32 type, u32 port) 213{ 214 *config &= ~((0xff << 0) | (0xff << 8) | (0xff << 24)); 215 *config |= (node_xp << 0) | (type << 8) | (port << 24); 216} 217 218static ssize_t arm_ccn_pmu_format_show(struct device *dev, 219 struct device_attribute *attr, char *buf) 220{ 221 struct dev_ext_attribute *ea = container_of(attr, 222 struct dev_ext_attribute, attr); 223 224 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)ea->var); 225} 226 227#define CCN_FORMAT_ATTR(_name, _config) \ 228 struct dev_ext_attribute arm_ccn_pmu_format_attr_##_name = \ 229 { __ATTR(_name, S_IRUGO, arm_ccn_pmu_format_show, \ 230 NULL), _config } 231 232static CCN_FORMAT_ATTR(node, "config:0-7"); 233static CCN_FORMAT_ATTR(xp, "config:0-7"); 234static CCN_FORMAT_ATTR(type, "config:8-15"); 235static CCN_FORMAT_ATTR(event, "config:16-23"); 236static CCN_FORMAT_ATTR(port, "config:24-25"); 237static CCN_FORMAT_ATTR(vc, "config:26-28"); 238static CCN_FORMAT_ATTR(dir, "config:29-29"); 239static CCN_FORMAT_ATTR(mask, "config:30-33"); 240static CCN_FORMAT_ATTR(cmp_l, "config1:0-62"); 241static CCN_FORMAT_ATTR(cmp_h, "config2:0-59"); 242 243static struct attribute *arm_ccn_pmu_format_attrs[] = { 244 &arm_ccn_pmu_format_attr_node.attr.attr, 245 &arm_ccn_pmu_format_attr_xp.attr.attr, 246 &arm_ccn_pmu_format_attr_type.attr.attr, 247 &arm_ccn_pmu_format_attr_event.attr.attr, 248 &arm_ccn_pmu_format_attr_port.attr.attr, 249 &arm_ccn_pmu_format_attr_vc.attr.attr, 250 &arm_ccn_pmu_format_attr_dir.attr.attr, 251 &arm_ccn_pmu_format_attr_mask.attr.attr, 252 &arm_ccn_pmu_format_attr_cmp_l.attr.attr, 253 &arm_ccn_pmu_format_attr_cmp_h.attr.attr, 254 NULL 255}; 256 257static struct attribute_group arm_ccn_pmu_format_attr_group = { 258 .name = "format", 259 .attrs = arm_ccn_pmu_format_attrs, 260}; 261 262 263struct arm_ccn_pmu_event { 264 struct device_attribute attr; 265 u32 type; 266 u32 event; 267 int num_ports; 268 int num_vcs; 269 const char *def; 270 int mask; 271}; 272 273#define CCN_EVENT_ATTR(_name) \ 274 __ATTR(_name, S_IRUGO, arm_ccn_pmu_event_show, NULL) 275 276/* 277 * Events defined in TRM for MN, HN-I and SBSX are actually watchpoints set on 278 * their ports in XP they are connected to. For the sake of usability they are 279 * explicitly defined here (and translated into a relevant watchpoint in 280 * arm_ccn_pmu_event_init()) so the user can easily request them without deep 281 * knowledge of the flit format. 282 */ 283 284#define CCN_EVENT_MN(_name, _def, _mask) { .attr = CCN_EVENT_ATTR(mn_##_name), \ 285 .type = CCN_TYPE_MN, .event = CCN_EVENT_WATCHPOINT, \ 286 .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, \ 287 .def = _def, .mask = _mask, } 288 289#define CCN_EVENT_HNI(_name, _def, _mask) { \ 290 .attr = CCN_EVENT_ATTR(hni_##_name), .type = CCN_TYPE_HNI, \ 291 .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \ 292 .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, } 293 294#define CCN_EVENT_SBSX(_name, _def, _mask) { \ 295 .attr = CCN_EVENT_ATTR(sbsx_##_name), .type = CCN_TYPE_SBSX, \ 296 .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \ 297 .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, } 298 299#define CCN_EVENT_HNF(_name, _event) { .attr = CCN_EVENT_ATTR(hnf_##_name), \ 300 .type = CCN_TYPE_HNF, .event = _event, } 301 302#define CCN_EVENT_XP(_name, _event) { .attr = CCN_EVENT_ATTR(xp_##_name), \ 303 .type = CCN_TYPE_XP, .event = _event, \ 304 .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, } 305 306/* 307 * RN-I & RN-D (RN-D = RN-I + DVM) nodes have different type ID depending 308 * on configuration. One of them is picked to represent the whole group, 309 * as they all share the same event types. 310 */ 311#define CCN_EVENT_RNI(_name, _event) { .attr = CCN_EVENT_ATTR(rni_##_name), \ 312 .type = CCN_TYPE_RNI_3P, .event = _event, } 313 314#define CCN_EVENT_SBAS(_name, _event) { .attr = CCN_EVENT_ATTR(sbas_##_name), \ 315 .type = CCN_TYPE_SBAS, .event = _event, } 316 317#define CCN_EVENT_CYCLES(_name) { .attr = CCN_EVENT_ATTR(_name), \ 318 .type = CCN_TYPE_CYCLES } 319 320 321static ssize_t arm_ccn_pmu_event_show(struct device *dev, 322 struct device_attribute *attr, char *buf) 323{ 324 struct arm_ccn_pmu_event *event = container_of(attr, 325 struct arm_ccn_pmu_event, attr); 326 ssize_t res; 327 328 res = snprintf(buf, PAGE_SIZE, "type=0x%x", event->type); 329 if (event->event) 330 res += snprintf(buf + res, PAGE_SIZE - res, ",event=0x%x", 331 event->event); 332 if (event->def) 333 res += snprintf(buf + res, PAGE_SIZE - res, ",%s", 334 event->def); 335 if (event->mask) 336 res += snprintf(buf + res, PAGE_SIZE - res, ",mask=0x%x", 337 event->mask); 338 res += snprintf(buf + res, PAGE_SIZE - res, "\n"); 339 340 return res; 341} 342 343static umode_t arm_ccn_pmu_events_is_visible(struct kobject *kobj, 344 struct attribute *attr, int index) 345{ 346 struct device *dev = kobj_to_dev(kobj); 347 struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); 348 struct device_attribute *dev_attr = container_of(attr, 349 struct device_attribute, attr); 350 struct arm_ccn_pmu_event *event = container_of(dev_attr, 351 struct arm_ccn_pmu_event, attr); 352 353 if (event->type == CCN_TYPE_SBAS && !ccn->sbas_present) 354 return 0; 355 if (event->type == CCN_TYPE_SBSX && !ccn->sbsx_present) 356 return 0; 357 358 return attr->mode; 359} 360 361static struct arm_ccn_pmu_event arm_ccn_pmu_events[] = { 362 CCN_EVENT_MN(eobarrier, "dir=0,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE), 363 CCN_EVENT_MN(ecbarrier, "dir=0,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE), 364 CCN_EVENT_MN(dvmop, "dir=0,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE), 365 CCN_EVENT_HNI(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY), 366 CCN_EVENT_HNI(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY), 367 CCN_EVENT_HNI(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY), 368 CCN_EVENT_HNI(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY), 369 CCN_EVENT_HNI(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000", 370 CCN_IDX_MASK_ORDER), 371 CCN_EVENT_SBSX(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY), 372 CCN_EVENT_SBSX(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY), 373 CCN_EVENT_SBSX(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY), 374 CCN_EVENT_SBSX(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY), 375 CCN_EVENT_SBSX(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000", 376 CCN_IDX_MASK_ORDER), 377 CCN_EVENT_HNF(cache_miss, 0x1), 378 CCN_EVENT_HNF(l3_sf_cache_access, 0x02), 379 CCN_EVENT_HNF(cache_fill, 0x3), 380 CCN_EVENT_HNF(pocq_retry, 0x4), 381 CCN_EVENT_HNF(pocq_reqs_recvd, 0x5), 382 CCN_EVENT_HNF(sf_hit, 0x6), 383 CCN_EVENT_HNF(sf_evictions, 0x7), 384 CCN_EVENT_HNF(snoops_sent, 0x8), 385 CCN_EVENT_HNF(snoops_broadcast, 0x9), 386 CCN_EVENT_HNF(l3_eviction, 0xa), 387 CCN_EVENT_HNF(l3_fill_invalid_way, 0xb), 388 CCN_EVENT_HNF(mc_retries, 0xc), 389 CCN_EVENT_HNF(mc_reqs, 0xd), 390 CCN_EVENT_HNF(qos_hh_retry, 0xe), 391 CCN_EVENT_RNI(rdata_beats_p0, 0x1), 392 CCN_EVENT_RNI(rdata_beats_p1, 0x2), 393 CCN_EVENT_RNI(rdata_beats_p2, 0x3), 394 CCN_EVENT_RNI(rxdat_flits, 0x4), 395 CCN_EVENT_RNI(txdat_flits, 0x5), 396 CCN_EVENT_RNI(txreq_flits, 0x6), 397 CCN_EVENT_RNI(txreq_flits_retried, 0x7), 398 CCN_EVENT_RNI(rrt_full, 0x8), 399 CCN_EVENT_RNI(wrt_full, 0x9), 400 CCN_EVENT_RNI(txreq_flits_replayed, 0xa), 401 CCN_EVENT_XP(upload_starvation, 0x1), 402 CCN_EVENT_XP(download_starvation, 0x2), 403 CCN_EVENT_XP(respin, 0x3), 404 CCN_EVENT_XP(valid_flit, 0x4), 405 CCN_EVENT_XP(watchpoint, CCN_EVENT_WATCHPOINT), 406 CCN_EVENT_SBAS(rdata_beats_p0, 0x1), 407 CCN_EVENT_SBAS(rxdat_flits, 0x4), 408 CCN_EVENT_SBAS(txdat_flits, 0x5), 409 CCN_EVENT_SBAS(txreq_flits, 0x6), 410 CCN_EVENT_SBAS(txreq_flits_retried, 0x7), 411 CCN_EVENT_SBAS(rrt_full, 0x8), 412 CCN_EVENT_SBAS(wrt_full, 0x9), 413 CCN_EVENT_SBAS(txreq_flits_replayed, 0xa), 414 CCN_EVENT_CYCLES(cycles), 415}; 416 417/* Populated in arm_ccn_init() */ 418static struct attribute 419 *arm_ccn_pmu_events_attrs[ARRAY_SIZE(arm_ccn_pmu_events) + 1]; 420 421static struct attribute_group arm_ccn_pmu_events_attr_group = { 422 .name = "events", 423 .is_visible = arm_ccn_pmu_events_is_visible, 424 .attrs = arm_ccn_pmu_events_attrs, 425}; 426 427 428static u64 *arm_ccn_pmu_get_cmp_mask(struct arm_ccn *ccn, const char *name) 429{ 430 unsigned long i; 431 432 if (WARN_ON(!name || !name[0] || !isxdigit(name[0]) || !name[1])) 433 return NULL; 434 i = isdigit(name[0]) ? name[0] - '0' : 0xa + tolower(name[0]) - 'a'; 435 436 switch (name[1]) { 437 case 'l': 438 return &ccn->dt.cmp_mask[i].l; 439 case 'h': 440 return &ccn->dt.cmp_mask[i].h; 441 default: 442 return NULL; 443 } 444} 445 446static ssize_t arm_ccn_pmu_cmp_mask_show(struct device *dev, 447 struct device_attribute *attr, char *buf) 448{ 449 struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); 450 u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name); 451 452 return mask ? snprintf(buf, PAGE_SIZE, "0x%016llx\n", *mask) : -EINVAL; 453} 454 455static ssize_t arm_ccn_pmu_cmp_mask_store(struct device *dev, 456 struct device_attribute *attr, const char *buf, size_t count) 457{ 458 struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); 459 u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name); 460 int err = -EINVAL; 461 462 if (mask) 463 err = kstrtoull(buf, 0, mask); 464 465 return err ? err : count; 466} 467 468#define CCN_CMP_MASK_ATTR(_name) \ 469 struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \ 470 __ATTR(_name, S_IRUGO | S_IWUSR, \ 471 arm_ccn_pmu_cmp_mask_show, arm_ccn_pmu_cmp_mask_store) 472 473#define CCN_CMP_MASK_ATTR_RO(_name) \ 474 struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \ 475 __ATTR(_name, S_IRUGO, arm_ccn_pmu_cmp_mask_show, NULL) 476 477static CCN_CMP_MASK_ATTR(0l); 478static CCN_CMP_MASK_ATTR(0h); 479static CCN_CMP_MASK_ATTR(1l); 480static CCN_CMP_MASK_ATTR(1h); 481static CCN_CMP_MASK_ATTR(2l); 482static CCN_CMP_MASK_ATTR(2h); 483static CCN_CMP_MASK_ATTR(3l); 484static CCN_CMP_MASK_ATTR(3h); 485static CCN_CMP_MASK_ATTR(4l); 486static CCN_CMP_MASK_ATTR(4h); 487static CCN_CMP_MASK_ATTR(5l); 488static CCN_CMP_MASK_ATTR(5h); 489static CCN_CMP_MASK_ATTR(6l); 490static CCN_CMP_MASK_ATTR(6h); 491static CCN_CMP_MASK_ATTR(7l); 492static CCN_CMP_MASK_ATTR(7h); 493static CCN_CMP_MASK_ATTR_RO(8l); 494static CCN_CMP_MASK_ATTR_RO(8h); 495static CCN_CMP_MASK_ATTR_RO(9l); 496static CCN_CMP_MASK_ATTR_RO(9h); 497static CCN_CMP_MASK_ATTR_RO(al); 498static CCN_CMP_MASK_ATTR_RO(ah); 499static CCN_CMP_MASK_ATTR_RO(bl); 500static CCN_CMP_MASK_ATTR_RO(bh); 501 502static struct attribute *arm_ccn_pmu_cmp_mask_attrs[] = { 503 &arm_ccn_pmu_cmp_mask_attr_0l.attr, &arm_ccn_pmu_cmp_mask_attr_0h.attr, 504 &arm_ccn_pmu_cmp_mask_attr_1l.attr, &arm_ccn_pmu_cmp_mask_attr_1h.attr, 505 &arm_ccn_pmu_cmp_mask_attr_2l.attr, &arm_ccn_pmu_cmp_mask_attr_2h.attr, 506 &arm_ccn_pmu_cmp_mask_attr_3l.attr, &arm_ccn_pmu_cmp_mask_attr_3h.attr, 507 &arm_ccn_pmu_cmp_mask_attr_4l.attr, &arm_ccn_pmu_cmp_mask_attr_4h.attr, 508 &arm_ccn_pmu_cmp_mask_attr_5l.attr, &arm_ccn_pmu_cmp_mask_attr_5h.attr, 509 &arm_ccn_pmu_cmp_mask_attr_6l.attr, &arm_ccn_pmu_cmp_mask_attr_6h.attr, 510 &arm_ccn_pmu_cmp_mask_attr_7l.attr, &arm_ccn_pmu_cmp_mask_attr_7h.attr, 511 &arm_ccn_pmu_cmp_mask_attr_8l.attr, &arm_ccn_pmu_cmp_mask_attr_8h.attr, 512 &arm_ccn_pmu_cmp_mask_attr_9l.attr, &arm_ccn_pmu_cmp_mask_attr_9h.attr, 513 &arm_ccn_pmu_cmp_mask_attr_al.attr, &arm_ccn_pmu_cmp_mask_attr_ah.attr, 514 &arm_ccn_pmu_cmp_mask_attr_bl.attr, &arm_ccn_pmu_cmp_mask_attr_bh.attr, 515 NULL 516}; 517 518static struct attribute_group arm_ccn_pmu_cmp_mask_attr_group = { 519 .name = "cmp_mask", 520 .attrs = arm_ccn_pmu_cmp_mask_attrs, 521}; 522 523 524/* 525 * Default poll period is 10ms, which is way over the top anyway, 526 * as in the worst case scenario (an event every cycle), with 1GHz 527 * clocked bus, the smallest, 32 bit counter will overflow in 528 * more than 4s. 529 */ 530static unsigned int arm_ccn_pmu_poll_period_us = 10000; 531module_param_named(pmu_poll_period_us, arm_ccn_pmu_poll_period_us, uint, 532 S_IRUGO | S_IWUSR); 533 534static ktime_t arm_ccn_pmu_timer_period(void) 535{ 536 return ns_to_ktime((u64)arm_ccn_pmu_poll_period_us * 1000); 537} 538 539 540static const struct attribute_group *arm_ccn_pmu_attr_groups[] = { 541 &arm_ccn_pmu_events_attr_group, 542 &arm_ccn_pmu_format_attr_group, 543 &arm_ccn_pmu_cmp_mask_attr_group, 544 NULL 545}; 546 547 548static int arm_ccn_pmu_alloc_bit(unsigned long *bitmap, unsigned long size) 549{ 550 int bit; 551 552 do { 553 bit = find_first_zero_bit(bitmap, size); 554 if (bit >= size) 555 return -EAGAIN; 556 } while (test_and_set_bit(bit, bitmap)); 557 558 return bit; 559} 560 561/* All RN-I and RN-D nodes have identical PMUs */ 562static int arm_ccn_pmu_type_eq(u32 a, u32 b) 563{ 564 if (a == b) 565 return 1; 566 567 switch (a) { 568 case CCN_TYPE_RNI_1P: 569 case CCN_TYPE_RNI_2P: 570 case CCN_TYPE_RNI_3P: 571 case CCN_TYPE_RND_1P: 572 case CCN_TYPE_RND_2P: 573 case CCN_TYPE_RND_3P: 574 switch (b) { 575 case CCN_TYPE_RNI_1P: 576 case CCN_TYPE_RNI_2P: 577 case CCN_TYPE_RNI_3P: 578 case CCN_TYPE_RND_1P: 579 case CCN_TYPE_RND_2P: 580 case CCN_TYPE_RND_3P: 581 return 1; 582 } 583 break; 584 } 585 586 return 0; 587} 588 589static void arm_ccn_pmu_event_destroy(struct perf_event *event) 590{ 591 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 592 struct hw_perf_event *hw = &event->hw; 593 594 if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) { 595 clear_bit(CCN_IDX_PMU_CYCLE_COUNTER, ccn->dt.pmu_counters_mask); 596 } else { 597 struct arm_ccn_component *source = 598 ccn->dt.pmu_counters[hw->idx].source; 599 600 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP && 601 CCN_CONFIG_EVENT(event->attr.config) == 602 CCN_EVENT_WATCHPOINT) 603 clear_bit(hw->config_base, source->xp.dt_cmp_mask); 604 else 605 clear_bit(hw->config_base, source->pmu_events_mask); 606 clear_bit(hw->idx, ccn->dt.pmu_counters_mask); 607 } 608 609 ccn->dt.pmu_counters[hw->idx].source = NULL; 610 ccn->dt.pmu_counters[hw->idx].event = NULL; 611} 612 613static int arm_ccn_pmu_event_init(struct perf_event *event) 614{ 615 struct arm_ccn *ccn; 616 struct hw_perf_event *hw = &event->hw; 617 u32 node_xp, type, event_id; 618 int valid, bit; 619 struct arm_ccn_component *source; 620 int i; 621 622 if (event->attr.type != event->pmu->type) 623 return -ENOENT; 624 625 ccn = pmu_to_arm_ccn(event->pmu); 626 event->destroy = arm_ccn_pmu_event_destroy; 627 628 if (hw->sample_period) { 629 dev_warn(ccn->dev, "Sampling not supported!\n"); 630 return -EOPNOTSUPP; 631 } 632 633 if (has_branch_stack(event) || event->attr.exclude_user || 634 event->attr.exclude_kernel || event->attr.exclude_hv || 635 event->attr.exclude_idle) { 636 dev_warn(ccn->dev, "Can't exclude execution levels!\n"); 637 return -EOPNOTSUPP; 638 } 639 640 if (event->cpu < 0) { 641 dev_warn(ccn->dev, "Can't provide per-task data!\n"); 642 return -EOPNOTSUPP; 643 } 644 645 node_xp = CCN_CONFIG_NODE(event->attr.config); 646 type = CCN_CONFIG_TYPE(event->attr.config); 647 event_id = CCN_CONFIG_EVENT(event->attr.config); 648 649 /* Validate node/xp vs topology */ 650 switch (type) { 651 case CCN_TYPE_XP: 652 if (node_xp >= ccn->num_xps) { 653 dev_warn(ccn->dev, "Invalid XP ID %d!\n", node_xp); 654 return -EINVAL; 655 } 656 break; 657 case CCN_TYPE_CYCLES: 658 break; 659 default: 660 if (node_xp >= ccn->num_nodes) { 661 dev_warn(ccn->dev, "Invalid node ID %d!\n", node_xp); 662 return -EINVAL; 663 } 664 if (!arm_ccn_pmu_type_eq(type, ccn->node[node_xp].type)) { 665 dev_warn(ccn->dev, "Invalid type 0x%x for node %d!\n", 666 type, node_xp); 667 return -EINVAL; 668 } 669 break; 670 } 671 672 /* Validate event ID vs available for the type */ 673 for (i = 0, valid = 0; i < ARRAY_SIZE(arm_ccn_pmu_events) && !valid; 674 i++) { 675 struct arm_ccn_pmu_event *e = &arm_ccn_pmu_events[i]; 676 u32 port = CCN_CONFIG_PORT(event->attr.config); 677 u32 vc = CCN_CONFIG_VC(event->attr.config); 678 679 if (!arm_ccn_pmu_type_eq(type, e->type)) 680 continue; 681 if (event_id != e->event) 682 continue; 683 if (e->num_ports && port >= e->num_ports) { 684 dev_warn(ccn->dev, "Invalid port %d for node/XP %d!\n", 685 port, node_xp); 686 return -EINVAL; 687 } 688 if (e->num_vcs && vc >= e->num_vcs) { 689 dev_warn(ccn->dev, "Invalid vc %d for node/XP %d!\n", 690 vc, node_xp); 691 return -EINVAL; 692 } 693 valid = 1; 694 } 695 if (!valid) { 696 dev_warn(ccn->dev, "Invalid event 0x%x for node/XP %d!\n", 697 event_id, node_xp); 698 return -EINVAL; 699 } 700 701 /* Watchpoint-based event for a node is actually set on XP */ 702 if (event_id == CCN_EVENT_WATCHPOINT && type != CCN_TYPE_XP) { 703 u32 port; 704 705 type = CCN_TYPE_XP; 706 port = arm_ccn_node_to_xp_port(node_xp); 707 node_xp = arm_ccn_node_to_xp(node_xp); 708 709 arm_ccn_pmu_config_set(&event->attr.config, 710 node_xp, type, port); 711 } 712 713 /* Allocate the cycle counter */ 714 if (type == CCN_TYPE_CYCLES) { 715 if (test_and_set_bit(CCN_IDX_PMU_CYCLE_COUNTER, 716 ccn->dt.pmu_counters_mask)) 717 return -EAGAIN; 718 719 hw->idx = CCN_IDX_PMU_CYCLE_COUNTER; 720 ccn->dt.pmu_counters[CCN_IDX_PMU_CYCLE_COUNTER].event = event; 721 722 return 0; 723 } 724 725 /* Allocate an event counter */ 726 hw->idx = arm_ccn_pmu_alloc_bit(ccn->dt.pmu_counters_mask, 727 CCN_NUM_PMU_EVENT_COUNTERS); 728 if (hw->idx < 0) { 729 dev_warn(ccn->dev, "No more counters available!\n"); 730 return -EAGAIN; 731 } 732 733 if (type == CCN_TYPE_XP) 734 source = &ccn->xp[node_xp]; 735 else 736 source = &ccn->node[node_xp]; 737 ccn->dt.pmu_counters[hw->idx].source = source; 738 739 /* Allocate an event source or a watchpoint */ 740 if (type == CCN_TYPE_XP && event_id == CCN_EVENT_WATCHPOINT) 741 bit = arm_ccn_pmu_alloc_bit(source->xp.dt_cmp_mask, 742 CCN_NUM_XP_WATCHPOINTS); 743 else 744 bit = arm_ccn_pmu_alloc_bit(source->pmu_events_mask, 745 CCN_NUM_PMU_EVENTS); 746 if (bit < 0) { 747 dev_warn(ccn->dev, "No more event sources/watchpoints on node/XP %d!\n", 748 node_xp); 749 clear_bit(hw->idx, ccn->dt.pmu_counters_mask); 750 return -EAGAIN; 751 } 752 hw->config_base = bit; 753 754 ccn->dt.pmu_counters[hw->idx].event = event; 755 756 return 0; 757} 758 759static u64 arm_ccn_pmu_read_counter(struct arm_ccn *ccn, int idx) 760{ 761 u64 res; 762 763 if (idx == CCN_IDX_PMU_CYCLE_COUNTER) { 764#ifdef readq 765 res = readq(ccn->dt.base + CCN_DT_PMCCNTR); 766#else 767 /* 40 bit counter, can do snapshot and read in two parts */ 768 writel(0x1, ccn->dt.base + CCN_DT_PMSR_REQ); 769 while (!(readl(ccn->dt.base + CCN_DT_PMSR) & 0x1)) 770 ; 771 writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR); 772 res = readl(ccn->dt.base + CCN_DT_PMCCNTRSR + 4) & 0xff; 773 res <<= 32; 774 res |= readl(ccn->dt.base + CCN_DT_PMCCNTRSR); 775#endif 776 } else { 777 res = readl(ccn->dt.base + CCN_DT_PMEVCNT(idx)); 778 } 779 780 return res; 781} 782 783static void arm_ccn_pmu_event_update(struct perf_event *event) 784{ 785 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 786 struct hw_perf_event *hw = &event->hw; 787 u64 prev_count, new_count, mask; 788 789 do { 790 prev_count = local64_read(&hw->prev_count); 791 new_count = arm_ccn_pmu_read_counter(ccn, hw->idx); 792 } while (local64_xchg(&hw->prev_count, new_count) != prev_count); 793 794 mask = (1LLU << (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER ? 40 : 32)) - 1; 795 796 local64_add((new_count - prev_count) & mask, &event->count); 797} 798 799static void arm_ccn_pmu_xp_dt_config(struct perf_event *event, int enable) 800{ 801 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 802 struct hw_perf_event *hw = &event->hw; 803 struct arm_ccn_component *xp; 804 u32 val, dt_cfg; 805 806 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) 807 xp = &ccn->xp[CCN_CONFIG_XP(event->attr.config)]; 808 else 809 xp = &ccn->xp[arm_ccn_node_to_xp( 810 CCN_CONFIG_NODE(event->attr.config))]; 811 812 if (enable) 813 dt_cfg = hw->event_base; 814 else 815 dt_cfg = CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH; 816 817 spin_lock(&ccn->dt.config_lock); 818 819 val = readl(xp->base + CCN_XP_DT_CONFIG); 820 val &= ~(CCN_XP_DT_CONFIG__DT_CFG__MASK << 821 CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx)); 822 val |= dt_cfg << CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx); 823 writel(val, xp->base + CCN_XP_DT_CONFIG); 824 825 spin_unlock(&ccn->dt.config_lock); 826} 827 828static void arm_ccn_pmu_event_start(struct perf_event *event, int flags) 829{ 830 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 831 struct hw_perf_event *hw = &event->hw; 832 833 local64_set(&event->hw.prev_count, 834 arm_ccn_pmu_read_counter(ccn, hw->idx)); 835 hw->state = 0; 836 837 if (!ccn->irq_used) 838 hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(), 839 HRTIMER_MODE_REL); 840 841 /* Set the DT bus input, engaging the counter */ 842 arm_ccn_pmu_xp_dt_config(event, 1); 843} 844 845static void arm_ccn_pmu_event_stop(struct perf_event *event, int flags) 846{ 847 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 848 struct hw_perf_event *hw = &event->hw; 849 u64 timeout; 850 851 /* Disable counting, setting the DT bus to pass-through mode */ 852 arm_ccn_pmu_xp_dt_config(event, 0); 853 854 if (!ccn->irq_used) 855 hrtimer_cancel(&ccn->dt.hrtimer); 856 857 /* Let the DT bus drain */ 858 timeout = arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) + 859 ccn->num_xps; 860 while (arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) < 861 timeout) 862 cpu_relax(); 863 864 if (flags & PERF_EF_UPDATE) 865 arm_ccn_pmu_event_update(event); 866 867 hw->state |= PERF_HES_STOPPED; 868} 869 870static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event) 871{ 872 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 873 struct hw_perf_event *hw = &event->hw; 874 struct arm_ccn_component *source = 875 ccn->dt.pmu_counters[hw->idx].source; 876 unsigned long wp = hw->config_base; 877 u32 val; 878 u64 cmp_l = event->attr.config1; 879 u64 cmp_h = event->attr.config2; 880 u64 mask_l = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].l; 881 u64 mask_h = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].h; 882 883 hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(wp); 884 885 /* Direction (RX/TX), device (port) & virtual channel */ 886 val = readl(source->base + CCN_XP_DT_INTERFACE_SEL); 887 val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK << 888 CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp)); 889 val |= CCN_CONFIG_DIR(event->attr.config) << 890 CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp); 891 val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK << 892 CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp)); 893 val |= CCN_CONFIG_PORT(event->attr.config) << 894 CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp); 895 val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK << 896 CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp)); 897 val |= CCN_CONFIG_VC(event->attr.config) << 898 CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp); 899 writel(val, source->base + CCN_XP_DT_INTERFACE_SEL); 900 901 /* Comparison values */ 902 writel(cmp_l & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp)); 903 writel((cmp_l >> 32) & 0xefffffff, 904 source->base + CCN_XP_DT_CMP_VAL_L(wp) + 4); 905 writel(cmp_h & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_H(wp)); 906 writel((cmp_h >> 32) & 0x0fffffff, 907 source->base + CCN_XP_DT_CMP_VAL_H(wp) + 4); 908 909 /* Mask */ 910 writel(mask_l & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp)); 911 writel((mask_l >> 32) & 0xefffffff, 912 source->base + CCN_XP_DT_CMP_MASK_L(wp) + 4); 913 writel(mask_h & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_H(wp)); 914 writel((mask_h >> 32) & 0x0fffffff, 915 source->base + CCN_XP_DT_CMP_MASK_H(wp) + 4); 916} 917 918static void arm_ccn_pmu_xp_event_config(struct perf_event *event) 919{ 920 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 921 struct hw_perf_event *hw = &event->hw; 922 struct arm_ccn_component *source = 923 ccn->dt.pmu_counters[hw->idx].source; 924 u32 val, id; 925 926 hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(hw->config_base); 927 928 id = (CCN_CONFIG_VC(event->attr.config) << 4) | 929 (CCN_CONFIG_PORT(event->attr.config) << 3) | 930 (CCN_CONFIG_EVENT(event->attr.config) << 0); 931 932 val = readl(source->base + CCN_XP_PMU_EVENT_SEL); 933 val &= ~(CCN_XP_PMU_EVENT_SEL__ID__MASK << 934 CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base)); 935 val |= id << CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base); 936 writel(val, source->base + CCN_XP_PMU_EVENT_SEL); 937} 938 939static void arm_ccn_pmu_node_event_config(struct perf_event *event) 940{ 941 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 942 struct hw_perf_event *hw = &event->hw; 943 struct arm_ccn_component *source = 944 ccn->dt.pmu_counters[hw->idx].source; 945 u32 type = CCN_CONFIG_TYPE(event->attr.config); 946 u32 val, port; 947 948 port = arm_ccn_node_to_xp_port(CCN_CONFIG_NODE(event->attr.config)); 949 hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(port, 950 hw->config_base); 951 952 /* These *_event_sel regs should be identical, but let's make sure... */ 953 BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL != CCN_SBAS_PMU_EVENT_SEL); 954 BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL != CCN_RNI_PMU_EVENT_SEL); 955 BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(1) != 956 CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1)); 957 BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1) != 958 CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(1)); 959 BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__MASK != 960 CCN_SBAS_PMU_EVENT_SEL__ID__MASK); 961 BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__MASK != 962 CCN_RNI_PMU_EVENT_SEL__ID__MASK); 963 if (WARN_ON(type != CCN_TYPE_HNF && type != CCN_TYPE_SBAS && 964 !arm_ccn_pmu_type_eq(type, CCN_TYPE_RNI_3P))) 965 return; 966 967 /* Set the event id for the pre-allocated counter */ 968 val = readl(source->base + CCN_HNF_PMU_EVENT_SEL); 969 val &= ~(CCN_HNF_PMU_EVENT_SEL__ID__MASK << 970 CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base)); 971 val |= CCN_CONFIG_EVENT(event->attr.config) << 972 CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base); 973 writel(val, source->base + CCN_HNF_PMU_EVENT_SEL); 974} 975 976static void arm_ccn_pmu_event_config(struct perf_event *event) 977{ 978 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 979 struct hw_perf_event *hw = &event->hw; 980 u32 xp, offset, val; 981 982 /* Cycle counter requires no setup */ 983 if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) 984 return; 985 986 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) 987 xp = CCN_CONFIG_XP(event->attr.config); 988 else 989 xp = arm_ccn_node_to_xp(CCN_CONFIG_NODE(event->attr.config)); 990 991 spin_lock(&ccn->dt.config_lock); 992 993 /* Set the DT bus "distance" register */ 994 offset = (hw->idx / 4) * 4; 995 val = readl(ccn->dt.base + CCN_DT_ACTIVE_DSM + offset); 996 val &= ~(CCN_DT_ACTIVE_DSM__DSM_ID__MASK << 997 CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4)); 998 val |= xp << CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4); 999 writel(val, ccn->dt.base + CCN_DT_ACTIVE_DSM + offset); 1000 1001 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) { 1002 if (CCN_CONFIG_EVENT(event->attr.config) == 1003 CCN_EVENT_WATCHPOINT) 1004 arm_ccn_pmu_xp_watchpoint_config(event); 1005 else 1006 arm_ccn_pmu_xp_event_config(event); 1007 } else { 1008 arm_ccn_pmu_node_event_config(event); 1009 } 1010 1011 spin_unlock(&ccn->dt.config_lock); 1012} 1013 1014static int arm_ccn_pmu_event_add(struct perf_event *event, int flags) 1015{ 1016 struct hw_perf_event *hw = &event->hw; 1017 1018 arm_ccn_pmu_event_config(event); 1019 1020 hw->state = PERF_HES_STOPPED; 1021 1022 if (flags & PERF_EF_START) 1023 arm_ccn_pmu_event_start(event, PERF_EF_UPDATE); 1024 1025 return 0; 1026} 1027 1028static void arm_ccn_pmu_event_del(struct perf_event *event, int flags) 1029{ 1030 arm_ccn_pmu_event_stop(event, PERF_EF_UPDATE); 1031} 1032 1033static void arm_ccn_pmu_event_read(struct perf_event *event) 1034{ 1035 arm_ccn_pmu_event_update(event); 1036} 1037 1038static irqreturn_t arm_ccn_pmu_overflow_handler(struct arm_ccn_dt *dt) 1039{ 1040 u32 pmovsr = readl(dt->base + CCN_DT_PMOVSR); 1041 int idx; 1042 1043 if (!pmovsr) 1044 return IRQ_NONE; 1045 1046 writel(pmovsr, dt->base + CCN_DT_PMOVSR_CLR); 1047 1048 BUILD_BUG_ON(CCN_IDX_PMU_CYCLE_COUNTER != CCN_NUM_PMU_EVENT_COUNTERS); 1049 1050 for (idx = 0; idx < CCN_NUM_PMU_EVENT_COUNTERS + 1; idx++) { 1051 struct perf_event *event = dt->pmu_counters[idx].event; 1052 int overflowed = pmovsr & BIT(idx); 1053 1054 WARN_ON_ONCE(overflowed && !event); 1055 1056 if (!event || !overflowed) 1057 continue; 1058 1059 arm_ccn_pmu_event_update(event); 1060 } 1061 1062 return IRQ_HANDLED; 1063} 1064 1065static enum hrtimer_restart arm_ccn_pmu_timer_handler(struct hrtimer *hrtimer) 1066{ 1067 struct arm_ccn_dt *dt = container_of(hrtimer, struct arm_ccn_dt, 1068 hrtimer); 1069 unsigned long flags; 1070 1071 local_irq_save(flags); 1072 arm_ccn_pmu_overflow_handler(dt); 1073 local_irq_restore(flags); 1074 1075 hrtimer_forward_now(hrtimer, arm_ccn_pmu_timer_period()); 1076 return HRTIMER_RESTART; 1077} 1078 1079 1080static DEFINE_IDA(arm_ccn_pmu_ida); 1081 1082static int arm_ccn_pmu_init(struct arm_ccn *ccn) 1083{ 1084 int i; 1085 char *name; 1086 1087 /* Initialize DT subsystem */ 1088 ccn->dt.base = ccn->base + CCN_REGION_SIZE; 1089 spin_lock_init(&ccn->dt.config_lock); 1090 writel(CCN_DT_CTL__DT_EN, ccn->dt.base + CCN_DT_CTL); 1091 writel(CCN_DT_PMCR__OVFL_INTR_EN | CCN_DT_PMCR__PMU_EN, 1092 ccn->dt.base + CCN_DT_PMCR); 1093 writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR); 1094 for (i = 0; i < ccn->num_xps; i++) { 1095 writel(0, ccn->xp[i].base + CCN_XP_DT_CONFIG); 1096 writel((CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS << 1097 CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(0)) | 1098 (CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS << 1099 CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(1)) | 1100 CCN_XP_DT_CONTROL__DT_ENABLE, 1101 ccn->xp[i].base + CCN_XP_DT_CONTROL); 1102 } 1103 ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].l = ~0; 1104 ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].h = ~0; 1105 ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].l = 0; 1106 ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].h = 0; 1107 ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].l = ~0; 1108 ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].h = ~(0x1 << 15); 1109 ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].l = ~0; 1110 ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].h = ~(0x1f << 9); 1111 1112 /* Get a convenient /sys/event_source/devices/ name */ 1113 ccn->dt.id = ida_simple_get(&arm_ccn_pmu_ida, 0, 0, GFP_KERNEL); 1114 if (ccn->dt.id == 0) { 1115 name = "ccn"; 1116 } else { 1117 int len = snprintf(NULL, 0, "ccn_%d", ccn->dt.id); 1118 1119 name = devm_kzalloc(ccn->dev, len + 1, GFP_KERNEL); 1120 snprintf(name, len + 1, "ccn_%d", ccn->dt.id); 1121 } 1122 1123 /* Perf driver registration */ 1124 ccn->dt.pmu = (struct pmu) { 1125 .attr_groups = arm_ccn_pmu_attr_groups, 1126 .task_ctx_nr = perf_invalid_context, 1127 .event_init = arm_ccn_pmu_event_init, 1128 .add = arm_ccn_pmu_event_add, 1129 .del = arm_ccn_pmu_event_del, 1130 .start = arm_ccn_pmu_event_start, 1131 .stop = arm_ccn_pmu_event_stop, 1132 .read = arm_ccn_pmu_event_read, 1133 }; 1134 1135 /* No overflow interrupt? Have to use a timer instead. */ 1136 if (!ccn->irq_used) { 1137 dev_info(ccn->dev, "No access to interrupts, using timer.\n"); 1138 hrtimer_init(&ccn->dt.hrtimer, CLOCK_MONOTONIC, 1139 HRTIMER_MODE_REL); 1140 ccn->dt.hrtimer.function = arm_ccn_pmu_timer_handler; 1141 } 1142 1143 return perf_pmu_register(&ccn->dt.pmu, name, -1); 1144} 1145 1146static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) 1147{ 1148 int i; 1149 1150 for (i = 0; i < ccn->num_xps; i++) 1151 writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL); 1152 writel(0, ccn->dt.base + CCN_DT_PMCR); 1153 perf_pmu_unregister(&ccn->dt.pmu); 1154 ida_simple_remove(&arm_ccn_pmu_ida, ccn->dt.id); 1155} 1156 1157 1158static int arm_ccn_for_each_valid_region(struct arm_ccn *ccn, 1159 int (*callback)(struct arm_ccn *ccn, int region, 1160 void __iomem *base, u32 type, u32 id)) 1161{ 1162 int region; 1163 1164 for (region = 0; region < CCN_NUM_REGIONS; region++) { 1165 u32 val, type, id; 1166 void __iomem *base; 1167 int err; 1168 1169 val = readl(ccn->base + CCN_MN_OLY_COMP_LIST_63_0 + 1170 4 * (region / 32)); 1171 if (!(val & (1 << (region % 32)))) 1172 continue; 1173 1174 base = ccn->base + region * CCN_REGION_SIZE; 1175 val = readl(base + CCN_ALL_OLY_ID); 1176 type = (val >> CCN_ALL_OLY_ID__OLY_ID__SHIFT) & 1177 CCN_ALL_OLY_ID__OLY_ID__MASK; 1178 id = (val >> CCN_ALL_OLY_ID__NODE_ID__SHIFT) & 1179 CCN_ALL_OLY_ID__NODE_ID__MASK; 1180 1181 err = callback(ccn, region, base, type, id); 1182 if (err) 1183 return err; 1184 } 1185 1186 return 0; 1187} 1188 1189static int arm_ccn_get_nodes_num(struct arm_ccn *ccn, int region, 1190 void __iomem *base, u32 type, u32 id) 1191{ 1192 1193 if (type == CCN_TYPE_XP && id >= ccn->num_xps) 1194 ccn->num_xps = id + 1; 1195 else if (id >= ccn->num_nodes) 1196 ccn->num_nodes = id + 1; 1197 1198 return 0; 1199} 1200 1201static int arm_ccn_init_nodes(struct arm_ccn *ccn, int region, 1202 void __iomem *base, u32 type, u32 id) 1203{ 1204 struct arm_ccn_component *component; 1205 1206 dev_dbg(ccn->dev, "Region %d: id=%u, type=0x%02x\n", region, id, type); 1207 1208 switch (type) { 1209 case CCN_TYPE_MN: 1210 case CCN_TYPE_DT: 1211 return 0; 1212 case CCN_TYPE_XP: 1213 component = &ccn->xp[id]; 1214 break; 1215 case CCN_TYPE_SBSX: 1216 ccn->sbsx_present = 1; 1217 component = &ccn->node[id]; 1218 break; 1219 case CCN_TYPE_SBAS: 1220 ccn->sbas_present = 1; 1221 /* Fall-through */ 1222 default: 1223 component = &ccn->node[id]; 1224 break; 1225 } 1226 1227 component->base = base; 1228 component->type = type; 1229 1230 return 0; 1231} 1232 1233 1234static irqreturn_t arm_ccn_error_handler(struct arm_ccn *ccn, 1235 const u32 *err_sig_val) 1236{ 1237 /* This should be really handled by firmware... */ 1238 dev_err(ccn->dev, "Error reported in %08x%08x%08x%08x%08x%08x.\n", 1239 err_sig_val[5], err_sig_val[4], err_sig_val[3], 1240 err_sig_val[2], err_sig_val[1], err_sig_val[0]); 1241 dev_err(ccn->dev, "Disabling interrupt generation for all errors.\n"); 1242 writel(CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE, 1243 ccn->base + CCN_MN_ERRINT_STATUS); 1244 1245 return IRQ_HANDLED; 1246} 1247 1248 1249static irqreturn_t arm_ccn_irq_handler(int irq, void *dev_id) 1250{ 1251 irqreturn_t res = IRQ_NONE; 1252 struct arm_ccn *ccn = dev_id; 1253 u32 err_sig_val[6]; 1254 u32 err_or; 1255 int i; 1256 1257 /* PMU overflow is a special case */ 1258 err_or = err_sig_val[0] = readl(ccn->base + CCN_MN_ERR_SIG_VAL_63_0); 1259 if (err_or & CCN_MN_ERR_SIG_VAL_63_0__DT) { 1260 err_or &= ~CCN_MN_ERR_SIG_VAL_63_0__DT; 1261 res = arm_ccn_pmu_overflow_handler(&ccn->dt); 1262 } 1263 1264 /* Have to read all err_sig_vals to clear them */ 1265 for (i = 1; i < ARRAY_SIZE(err_sig_val); i++) { 1266 err_sig_val[i] = readl(ccn->base + 1267 CCN_MN_ERR_SIG_VAL_63_0 + i * 4); 1268 err_or |= err_sig_val[i]; 1269 } 1270 if (err_or) 1271 res |= arm_ccn_error_handler(ccn, err_sig_val); 1272 1273 if (res != IRQ_NONE) 1274 writel(CCN_MN_ERRINT_STATUS__INTREQ__DESSERT, 1275 ccn->base + CCN_MN_ERRINT_STATUS); 1276 1277 return res; 1278} 1279 1280 1281static int arm_ccn_probe(struct platform_device *pdev) 1282{ 1283 struct arm_ccn *ccn; 1284 struct resource *res; 1285 int err; 1286 1287 ccn = devm_kzalloc(&pdev->dev, sizeof(*ccn), GFP_KERNEL); 1288 if (!ccn) 1289 return -ENOMEM; 1290 ccn->dev = &pdev->dev; 1291 platform_set_drvdata(pdev, ccn); 1292 1293 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1294 if (!res) 1295 return -EINVAL; 1296 1297 if (!devm_request_mem_region(ccn->dev, res->start, 1298 resource_size(res), pdev->name)) 1299 return -EBUSY; 1300 1301 ccn->base = devm_ioremap(ccn->dev, res->start, 1302 resource_size(res)); 1303 if (!ccn->base) 1304 return -EFAULT; 1305 1306 res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1307 if (!res) 1308 return -EINVAL; 1309 1310 /* Check if we can use the interrupt */ 1311 writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE, 1312 ccn->base + CCN_MN_ERRINT_STATUS); 1313 if (readl(ccn->base + CCN_MN_ERRINT_STATUS) & 1314 CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED) { 1315 /* Can set 'disable' bits, so can acknowledge interrupts */ 1316 writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE, 1317 ccn->base + CCN_MN_ERRINT_STATUS); 1318 err = devm_request_irq(ccn->dev, res->start, 1319 arm_ccn_irq_handler, 0, dev_name(ccn->dev), 1320 ccn); 1321 if (err) 1322 return err; 1323 1324 ccn->irq_used = 1; 1325 } 1326 1327 1328 /* Build topology */ 1329 1330 err = arm_ccn_for_each_valid_region(ccn, arm_ccn_get_nodes_num); 1331 if (err) 1332 return err; 1333 1334 ccn->node = devm_kzalloc(ccn->dev, sizeof(*ccn->node) * ccn->num_nodes, 1335 GFP_KERNEL); 1336 ccn->xp = devm_kzalloc(ccn->dev, sizeof(*ccn->node) * ccn->num_xps, 1337 GFP_KERNEL); 1338 if (!ccn->node || !ccn->xp) 1339 return -ENOMEM; 1340 1341 err = arm_ccn_for_each_valid_region(ccn, arm_ccn_init_nodes); 1342 if (err) 1343 return err; 1344 1345 return arm_ccn_pmu_init(ccn); 1346} 1347 1348static int arm_ccn_remove(struct platform_device *pdev) 1349{ 1350 struct arm_ccn *ccn = platform_get_drvdata(pdev); 1351 1352 arm_ccn_pmu_cleanup(ccn); 1353 1354 return 0; 1355} 1356 1357static const struct of_device_id arm_ccn_match[] = { 1358 { .compatible = "arm,ccn-504", }, 1359 {}, 1360}; 1361 1362static struct platform_driver arm_ccn_driver = { 1363 .driver = { 1364 .name = "arm-ccn", 1365 .of_match_table = arm_ccn_match, 1366 }, 1367 .probe = arm_ccn_probe, 1368 .remove = arm_ccn_remove, 1369}; 1370 1371static int __init arm_ccn_init(void) 1372{ 1373 int i; 1374 1375 for (i = 0; i < ARRAY_SIZE(arm_ccn_pmu_events); i++) 1376 arm_ccn_pmu_events_attrs[i] = &arm_ccn_pmu_events[i].attr.attr; 1377 1378 return platform_driver_register(&arm_ccn_driver); 1379} 1380 1381static void __exit arm_ccn_exit(void) 1382{ 1383 platform_driver_unregister(&arm_ccn_driver); 1384} 1385 1386module_init(arm_ccn_init); 1387module_exit(arm_ccn_exit); 1388 1389MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>"); 1390MODULE_LICENSE("GPL"); 1391