hw_breakpoint.c revision f435ab79928e4d54082e2838c4562a165e37999c 
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License version 2 as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
14 *
15 * Copyright (C) 2009, 2010 ARM Limited
16 *
17 * Author: Will Deacon <will.deacon@arm.com>
18 */
19
20/*
21 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
22 * using the CPU's debug registers.
23 */
24#define pr_fmt(fmt) "hw-breakpoint: " fmt
25
26#include <linux/errno.h>
27#include <linux/hardirq.h>
28#include <linux/perf_event.h>
29#include <linux/hw_breakpoint.h>
30#include <linux/smp.h>
31
32#include <asm/cacheflush.h>
33#include <asm/cputype.h>
34#include <asm/current.h>
35#include <asm/hw_breakpoint.h>
36#include <asm/kdebug.h>
37#include <asm/traps.h>
38
39/* Breakpoint currently in use for each BRP. */
40static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
41
42/* Watchpoint currently in use for each WRP. */
43static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
44
45/* Number of BRP/WRP registers on this CPU. */
46static int core_num_brps;
47static int core_num_wrps;
48
49/* Debug architecture version. */
50static u8 debug_arch;
51
52/* Maximum supported watchpoint length. */
53static u8 max_watchpoint_len;
54
55#define READ_WB_REG_CASE(OP2, M, VAL)			\
56	case ((OP2 << 4) + M):				\
57		ARM_DBG_READ(c0, c ## M, OP2, VAL);	\
58		break
59
60#define WRITE_WB_REG_CASE(OP2, M, VAL)			\
61	case ((OP2 << 4) + M):				\
62		ARM_DBG_WRITE(c0, c ## M, OP2, VAL);	\
63		break
64
65#define GEN_READ_WB_REG_CASES(OP2, VAL)		\
66	READ_WB_REG_CASE(OP2, 0, VAL);		\
67	READ_WB_REG_CASE(OP2, 1, VAL);		\
68	READ_WB_REG_CASE(OP2, 2, VAL);		\
69	READ_WB_REG_CASE(OP2, 3, VAL);		\
70	READ_WB_REG_CASE(OP2, 4, VAL);		\
71	READ_WB_REG_CASE(OP2, 5, VAL);		\
72	READ_WB_REG_CASE(OP2, 6, VAL);		\
73	READ_WB_REG_CASE(OP2, 7, VAL);		\
74	READ_WB_REG_CASE(OP2, 8, VAL);		\
75	READ_WB_REG_CASE(OP2, 9, VAL);		\
76	READ_WB_REG_CASE(OP2, 10, VAL);		\
77	READ_WB_REG_CASE(OP2, 11, VAL);		\
78	READ_WB_REG_CASE(OP2, 12, VAL);		\
79	READ_WB_REG_CASE(OP2, 13, VAL);		\
80	READ_WB_REG_CASE(OP2, 14, VAL);		\
81	READ_WB_REG_CASE(OP2, 15, VAL)
82
83#define GEN_WRITE_WB_REG_CASES(OP2, VAL)	\
84	WRITE_WB_REG_CASE(OP2, 0, VAL);		\
85	WRITE_WB_REG_CASE(OP2, 1, VAL);		\
86	WRITE_WB_REG_CASE(OP2, 2, VAL);		\
87	WRITE_WB_REG_CASE(OP2, 3, VAL);		\
88	WRITE_WB_REG_CASE(OP2, 4, VAL);		\
89	WRITE_WB_REG_CASE(OP2, 5, VAL);		\
90	WRITE_WB_REG_CASE(OP2, 6, VAL);		\
91	WRITE_WB_REG_CASE(OP2, 7, VAL);		\
92	WRITE_WB_REG_CASE(OP2, 8, VAL);		\
93	WRITE_WB_REG_CASE(OP2, 9, VAL);		\
94	WRITE_WB_REG_CASE(OP2, 10, VAL);	\
95	WRITE_WB_REG_CASE(OP2, 11, VAL);	\
96	WRITE_WB_REG_CASE(OP2, 12, VAL);	\
97	WRITE_WB_REG_CASE(OP2, 13, VAL);	\
98	WRITE_WB_REG_CASE(OP2, 14, VAL);	\
99	WRITE_WB_REG_CASE(OP2, 15, VAL)
100
101static u32 read_wb_reg(int n)
102{
103	u32 val = 0;
104
105	switch (n) {
106	GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
107	GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
108	GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
109	GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
110	default:
111		pr_warning("attempt to read from unknown breakpoint "
112				"register %d\n", n);
113	}
114
115	return val;
116}
117
118static void write_wb_reg(int n, u32 val)
119{
120	switch (n) {
121	GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
122	GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
123	GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
124	GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
125	default:
126		pr_warning("attempt to write to unknown breakpoint "
127				"register %d\n", n);
128	}
129	isb();
130}
131
132/* Determine debug architecture. */
133static u8 get_debug_arch(void)
134{
135	u32 didr;
136
137	/* Do we implement the extended CPUID interface? */
138	if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
139		pr_warn_once("CPUID feature registers not supported. "
140			     "Assuming v6 debug is present.\n");
141		return ARM_DEBUG_ARCH_V6;
142	}
143
144	ARM_DBG_READ(c0, c0, 0, didr);
145	return (didr >> 16) & 0xf;
146}
147
148u8 arch_get_debug_arch(void)
149{
150	return debug_arch;
151}
152
153static int debug_arch_supported(void)
154{
155	u8 arch = get_debug_arch();
156
157	/* We don't support the memory-mapped interface. */
158	return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) ||
159		arch >= ARM_DEBUG_ARCH_V7_1;
160}
161
162/* Can we determine the watchpoint access type from the fsr? */
163static int debug_exception_updates_fsr(void)
164{
165	return 0;
166}
167
168/* Determine number of WRP registers available. */
169static int get_num_wrp_resources(void)
170{
171	u32 didr;
172	ARM_DBG_READ(c0, c0, 0, didr);
173	return ((didr >> 28) & 0xf) + 1;
174}
175
176/* Determine number of BRP registers available. */
177static int get_num_brp_resources(void)
178{
179	u32 didr;
180	ARM_DBG_READ(c0, c0, 0, didr);
181	return ((didr >> 24) & 0xf) + 1;
182}
183
184/* Does this core support mismatch breakpoints? */
185static int core_has_mismatch_brps(void)
186{
187	return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
188		get_num_brp_resources() > 1);
189}
190
191/* Determine number of usable WRPs available. */
192static int get_num_wrps(void)
193{
194	/*
195	 * On debug architectures prior to 7.1, when a watchpoint fires, the
196	 * only way to work out which watchpoint it was is by disassembling
197	 * the faulting instruction and working out the address of the memory
198	 * access.
199	 *
200	 * Furthermore, we can only do this if the watchpoint was precise
201	 * since imprecise watchpoints prevent us from calculating register
202	 * based addresses.
203	 *
204	 * Providing we have more than 1 breakpoint register, we only report
205	 * a single watchpoint register for the time being. This way, we always
206	 * know which watchpoint fired. In the future we can either add a
207	 * disassembler and address generation emulator, or we can insert a
208	 * check to see if the DFAR is set on watchpoint exception entry
209	 * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
210	 * that it is set on some implementations].
211	 */
212	if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1)
213		return 1;
214
215	return get_num_wrp_resources();
216}
217
218/* Determine number of usable BRPs available. */
219static int get_num_brps(void)
220{
221	int brps = get_num_brp_resources();
222	return core_has_mismatch_brps() ? brps - 1 : brps;
223}
224
225/*
226 * In order to access the breakpoint/watchpoint control registers,
227 * we must be running in debug monitor mode. Unfortunately, we can
228 * be put into halting debug mode at any time by an external debugger
229 * but there is nothing we can do to prevent that.
230 */
231static int monitor_mode_enabled(void)
232{
233	u32 dscr;
234	ARM_DBG_READ(c0, c1, 0, dscr);
235	return !!(dscr & ARM_DSCR_MDBGEN);
236}
237
238static int enable_monitor_mode(void)
239{
240	u32 dscr;
241	ARM_DBG_READ(c0, c1, 0, dscr);
242
243	/* If monitor mode is already enabled, just return. */
244	if (dscr & ARM_DSCR_MDBGEN)
245		goto out;
246
247	/* Write to the corresponding DSCR. */
248	switch (get_debug_arch()) {
249	case ARM_DEBUG_ARCH_V6:
250	case ARM_DEBUG_ARCH_V6_1:
251		ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN));
252		break;
253	case ARM_DEBUG_ARCH_V7_ECP14:
254	case ARM_DEBUG_ARCH_V7_1:
255		ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN));
256		isb();
257		break;
258	default:
259		return -ENODEV;
260	}
261
262	/* Check that the write made it through. */
263	ARM_DBG_READ(c0, c1, 0, dscr);
264	if (!(dscr & ARM_DSCR_MDBGEN)) {
265		pr_warn_once("Failed to enable monitor mode on CPU %d.\n",
266				smp_processor_id());
267		return -EPERM;
268	}
269
270out:
271	return 0;
272}
273
274int hw_breakpoint_slots(int type)
275{
276	if (!debug_arch_supported())
277		return 0;
278
279	/*
280	 * We can be called early, so don't rely on
281	 * our static variables being initialised.
282	 */
283	switch (type) {
284	case TYPE_INST:
285		return get_num_brps();
286	case TYPE_DATA:
287		return get_num_wrps();
288	default:
289		pr_warning("unknown slot type: %d\n", type);
290		return 0;
291	}
292}
293
294/*
295 * Check if 8-bit byte-address select is available.
296 * This clobbers WRP 0.
297 */
298static u8 get_max_wp_len(void)
299{
300	u32 ctrl_reg;
301	struct arch_hw_breakpoint_ctrl ctrl;
302	u8 size = 4;
303
304	if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
305		goto out;
306
307	memset(&ctrl, 0, sizeof(ctrl));
308	ctrl.len = ARM_BREAKPOINT_LEN_8;
309	ctrl_reg = encode_ctrl_reg(ctrl);
310
311	write_wb_reg(ARM_BASE_WVR, 0);
312	write_wb_reg(ARM_BASE_WCR, ctrl_reg);
313	if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
314		size = 8;
315
316out:
317	return size;
318}
319
320u8 arch_get_max_wp_len(void)
321{
322	return max_watchpoint_len;
323}
324
325/*
326 * Install a perf counter breakpoint.
327 */
328int arch_install_hw_breakpoint(struct perf_event *bp)
329{
330	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
331	struct perf_event **slot, **slots;
332	int i, max_slots, ctrl_base, val_base;
333	u32 addr, ctrl;
334
335	addr = info->address;
336	ctrl = encode_ctrl_reg(info->ctrl) | 0x1;
337
338	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
339		/* Breakpoint */
340		ctrl_base = ARM_BASE_BCR;
341		val_base = ARM_BASE_BVR;
342		slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
343		max_slots = core_num_brps;
344	} else {
345		/* Watchpoint */
346		ctrl_base = ARM_BASE_WCR;
347		val_base = ARM_BASE_WVR;
348		slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
349		max_slots = core_num_wrps;
350	}
351
352	for (i = 0; i < max_slots; ++i) {
353		slot = &slots[i];
354
355		if (!*slot) {
356			*slot = bp;
357			break;
358		}
359	}
360
361	if (i == max_slots) {
362		pr_warning("Can't find any breakpoint slot\n");
363		return -EBUSY;
364	}
365
366	/* Override the breakpoint data with the step data. */
367	if (info->step_ctrl.enabled) {
368		addr = info->trigger & ~0x3;
369		ctrl = encode_ctrl_reg(info->step_ctrl);
370		if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) {
371			i = 0;
372			ctrl_base = ARM_BASE_BCR + core_num_brps;
373			val_base = ARM_BASE_BVR + core_num_brps;
374		}
375	}
376
377	/* Setup the address register. */
378	write_wb_reg(val_base + i, addr);
379
380	/* Setup the control register. */
381	write_wb_reg(ctrl_base + i, ctrl);
382	return 0;
383}
384
385void arch_uninstall_hw_breakpoint(struct perf_event *bp)
386{
387	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
388	struct perf_event **slot, **slots;
389	int i, max_slots, base;
390
391	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
392		/* Breakpoint */
393		base = ARM_BASE_BCR;
394		slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
395		max_slots = core_num_brps;
396	} else {
397		/* Watchpoint */
398		base = ARM_BASE_WCR;
399		slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
400		max_slots = core_num_wrps;
401	}
402
403	/* Remove the breakpoint. */
404	for (i = 0; i < max_slots; ++i) {
405		slot = &slots[i];
406
407		if (*slot == bp) {
408			*slot = NULL;
409			break;
410		}
411	}
412
413	if (i == max_slots) {
414		pr_warning("Can't find any breakpoint slot\n");
415		return;
416	}
417
418	/* Ensure that we disable the mismatch breakpoint. */
419	if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE &&
420	    info->step_ctrl.enabled) {
421		i = 0;
422		base = ARM_BASE_BCR + core_num_brps;
423	}
424
425	/* Reset the control register. */
426	write_wb_reg(base + i, 0);
427}
428
429static int get_hbp_len(u8 hbp_len)
430{
431	unsigned int len_in_bytes = 0;
432
433	switch (hbp_len) {
434	case ARM_BREAKPOINT_LEN_1:
435		len_in_bytes = 1;
436		break;
437	case ARM_BREAKPOINT_LEN_2:
438		len_in_bytes = 2;
439		break;
440	case ARM_BREAKPOINT_LEN_4:
441		len_in_bytes = 4;
442		break;
443	case ARM_BREAKPOINT_LEN_8:
444		len_in_bytes = 8;
445		break;
446	}
447
448	return len_in_bytes;
449}
450
451/*
452 * Check whether bp virtual address is in kernel space.
453 */
454int arch_check_bp_in_kernelspace(struct perf_event *bp)
455{
456	unsigned int len;
457	unsigned long va;
458	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
459
460	va = info->address;
461	len = get_hbp_len(info->ctrl.len);
462
463	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
464}
465
466/*
467 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
468 * Hopefully this will disappear when ptrace can bypass the conversion
469 * to generic breakpoint descriptions.
470 */
471int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
472			   int *gen_len, int *gen_type)
473{
474	/* Type */
475	switch (ctrl.type) {
476	case ARM_BREAKPOINT_EXECUTE:
477		*gen_type = HW_BREAKPOINT_X;
478		break;
479	case ARM_BREAKPOINT_LOAD:
480		*gen_type = HW_BREAKPOINT_R;
481		break;
482	case ARM_BREAKPOINT_STORE:
483		*gen_type = HW_BREAKPOINT_W;
484		break;
485	case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
486		*gen_type = HW_BREAKPOINT_RW;
487		break;
488	default:
489		return -EINVAL;
490	}
491
492	/* Len */
493	switch (ctrl.len) {
494	case ARM_BREAKPOINT_LEN_1:
495		*gen_len = HW_BREAKPOINT_LEN_1;
496		break;
497	case ARM_BREAKPOINT_LEN_2:
498		*gen_len = HW_BREAKPOINT_LEN_2;
499		break;
500	case ARM_BREAKPOINT_LEN_4:
501		*gen_len = HW_BREAKPOINT_LEN_4;
502		break;
503	case ARM_BREAKPOINT_LEN_8:
504		*gen_len = HW_BREAKPOINT_LEN_8;
505		break;
506	default:
507		return -EINVAL;
508	}
509
510	return 0;
511}
512
513/*
514 * Construct an arch_hw_breakpoint from a perf_event.
515 */
516static int arch_build_bp_info(struct perf_event *bp)
517{
518	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
519
520	/* Type */
521	switch (bp->attr.bp_type) {
522	case HW_BREAKPOINT_X:
523		info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
524		break;
525	case HW_BREAKPOINT_R:
526		info->ctrl.type = ARM_BREAKPOINT_LOAD;
527		break;
528	case HW_BREAKPOINT_W:
529		info->ctrl.type = ARM_BREAKPOINT_STORE;
530		break;
531	case HW_BREAKPOINT_RW:
532		info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
533		break;
534	default:
535		return -EINVAL;
536	}
537
538	/* Len */
539	switch (bp->attr.bp_len) {
540	case HW_BREAKPOINT_LEN_1:
541		info->ctrl.len = ARM_BREAKPOINT_LEN_1;
542		break;
543	case HW_BREAKPOINT_LEN_2:
544		info->ctrl.len = ARM_BREAKPOINT_LEN_2;
545		break;
546	case HW_BREAKPOINT_LEN_4:
547		info->ctrl.len = ARM_BREAKPOINT_LEN_4;
548		break;
549	case HW_BREAKPOINT_LEN_8:
550		info->ctrl.len = ARM_BREAKPOINT_LEN_8;
551		if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE)
552			&& max_watchpoint_len >= 8)
553			break;
554	default:
555		return -EINVAL;
556	}
557
558	/*
559	 * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
560	 * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
561	 * by the hardware and must be aligned to the appropriate number of
562	 * bytes.
563	 */
564	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
565	    info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
566	    info->ctrl.len != ARM_BREAKPOINT_LEN_4)
567		return -EINVAL;
568
569	/* Address */
570	info->address = bp->attr.bp_addr;
571
572	/* Privilege */
573	info->ctrl.privilege = ARM_BREAKPOINT_USER;
574	if (arch_check_bp_in_kernelspace(bp))
575		info->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
576
577	/* Enabled? */
578	info->ctrl.enabled = !bp->attr.disabled;
579
580	/* Mismatch */
581	info->ctrl.mismatch = 0;
582
583	return 0;
584}
585
586/*
587 * Validate the arch-specific HW Breakpoint register settings.
588 */
589int arch_validate_hwbkpt_settings(struct perf_event *bp)
590{
591	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
592	int ret = 0;
593	u32 offset, alignment_mask = 0x3;
594
595	/* Ensure that we are in monitor debug mode. */
596	if (!monitor_mode_enabled())
597		return -ENODEV;
598
599	/* Build the arch_hw_breakpoint. */
600	ret = arch_build_bp_info(bp);
601	if (ret)
602		goto out;
603
604	/* Check address alignment. */
605	if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
606		alignment_mask = 0x7;
607	offset = info->address & alignment_mask;
608	switch (offset) {
609	case 0:
610		/* Aligned */
611		break;
612	case 1:
613	case 2:
614		/* Allow halfword watchpoints and breakpoints. */
615		if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
616			break;
617	case 3:
618		/* Allow single byte watchpoint. */
619		if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
620			break;
621	default:
622		ret = -EINVAL;
623		goto out;
624	}
625
626	info->address &= ~alignment_mask;
627	info->ctrl.len <<= offset;
628
629	if (!bp->overflow_handler) {
630		/*
631		 * Mismatch breakpoints are required for single-stepping
632		 * breakpoints.
633		 */
634		if (!core_has_mismatch_brps())
635			return -EINVAL;
636
637		/* We don't allow mismatch breakpoints in kernel space. */
638		if (arch_check_bp_in_kernelspace(bp))
639			return -EPERM;
640
641		/*
642		 * Per-cpu breakpoints are not supported by our stepping
643		 * mechanism.
644		 */
645		if (!bp->hw.bp_target)
646			return -EINVAL;
647
648		/*
649		 * We only support specific access types if the fsr
650		 * reports them.
651		 */
652		if (!debug_exception_updates_fsr() &&
653		    (info->ctrl.type == ARM_BREAKPOINT_LOAD ||
654		     info->ctrl.type == ARM_BREAKPOINT_STORE))
655			return -EINVAL;
656	}
657
658out:
659	return ret;
660}
661
662/*
663 * Enable/disable single-stepping over the breakpoint bp at address addr.
664 */
665static void enable_single_step(struct perf_event *bp, u32 addr)
666{
667	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
668
669	arch_uninstall_hw_breakpoint(bp);
670	info->step_ctrl.mismatch  = 1;
671	info->step_ctrl.len	  = ARM_BREAKPOINT_LEN_4;
672	info->step_ctrl.type	  = ARM_BREAKPOINT_EXECUTE;
673	info->step_ctrl.privilege = info->ctrl.privilege;
674	info->step_ctrl.enabled	  = 1;
675	info->trigger		  = addr;
676	arch_install_hw_breakpoint(bp);
677}
678
679static void disable_single_step(struct perf_event *bp)
680{
681	arch_uninstall_hw_breakpoint(bp);
682	counter_arch_bp(bp)->step_ctrl.enabled = 0;
683	arch_install_hw_breakpoint(bp);
684}
685
686static void watchpoint_handler(unsigned long addr, unsigned int fsr,
687			       struct pt_regs *regs)
688{
689	int i, access;
690	u32 val, ctrl_reg, alignment_mask;
691	struct perf_event *wp, **slots;
692	struct arch_hw_breakpoint *info;
693	struct arch_hw_breakpoint_ctrl ctrl;
694
695	slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
696
697	for (i = 0; i < core_num_wrps; ++i) {
698		rcu_read_lock();
699
700		wp = slots[i];
701
702		if (wp == NULL)
703			goto unlock;
704
705		info = counter_arch_bp(wp);
706		/*
707		 * The DFAR is an unknown value on debug architectures prior
708		 * to 7.1. Since we only allow a single watchpoint on these
709		 * older CPUs, we can set the trigger to the lowest possible
710		 * faulting address.
711		 */
712		if (debug_arch < ARM_DEBUG_ARCH_V7_1) {
713			BUG_ON(i > 0);
714			info->trigger = wp->attr.bp_addr;
715		} else {
716			if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
717				alignment_mask = 0x7;
718			else
719				alignment_mask = 0x3;
720
721			/* Check if the watchpoint value matches. */
722			val = read_wb_reg(ARM_BASE_WVR + i);
723			if (val != (addr & ~alignment_mask))
724				goto unlock;
725
726			/* Possible match, check the byte address select. */
727			ctrl_reg = read_wb_reg(ARM_BASE_WCR + i);
728			decode_ctrl_reg(ctrl_reg, &ctrl);
729			if (!((1 << (addr & alignment_mask)) & ctrl.len))
730				goto unlock;
731
732			/* Check that the access type matches. */
733			if (debug_exception_updates_fsr()) {
734				access = (fsr & ARM_FSR_ACCESS_MASK) ?
735					  HW_BREAKPOINT_W : HW_BREAKPOINT_R;
736				if (!(access & hw_breakpoint_type(wp)))
737					goto unlock;
738			}
739
740			/* We have a winner. */
741			info->trigger = addr;
742		}
743
744		pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
745		perf_bp_event(wp, regs);
746
747		/*
748		 * If no overflow handler is present, insert a temporary
749		 * mismatch breakpoint so we can single-step over the
750		 * watchpoint trigger.
751		 */
752		if (!wp->overflow_handler)
753			enable_single_step(wp, instruction_pointer(regs));
754
755unlock:
756		rcu_read_unlock();
757	}
758}
759
760static void watchpoint_single_step_handler(unsigned long pc)
761{
762	int i;
763	struct perf_event *wp, **slots;
764	struct arch_hw_breakpoint *info;
765
766	slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
767
768	for (i = 0; i < core_num_wrps; ++i) {
769		rcu_read_lock();
770
771		wp = slots[i];
772
773		if (wp == NULL)
774			goto unlock;
775
776		info = counter_arch_bp(wp);
777		if (!info->step_ctrl.enabled)
778			goto unlock;
779
780		/*
781		 * Restore the original watchpoint if we've completed the
782		 * single-step.
783		 */
784		if (info->trigger != pc)
785			disable_single_step(wp);
786
787unlock:
788		rcu_read_unlock();
789	}
790}
791
792static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
793{
794	int i;
795	u32 ctrl_reg, val, addr;
796	struct perf_event *bp, **slots;
797	struct arch_hw_breakpoint *info;
798	struct arch_hw_breakpoint_ctrl ctrl;
799
800	slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
801
802	/* The exception entry code places the amended lr in the PC. */
803	addr = regs->ARM_pc;
804
805	/* Check the currently installed breakpoints first. */
806	for (i = 0; i < core_num_brps; ++i) {
807		rcu_read_lock();
808
809		bp = slots[i];
810
811		if (bp == NULL)
812			goto unlock;
813
814		info = counter_arch_bp(bp);
815
816		/* Check if the breakpoint value matches. */
817		val = read_wb_reg(ARM_BASE_BVR + i);
818		if (val != (addr & ~0x3))
819			goto mismatch;
820
821		/* Possible match, check the byte address select to confirm. */
822		ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
823		decode_ctrl_reg(ctrl_reg, &ctrl);
824		if ((1 << (addr & 0x3)) & ctrl.len) {
825			info->trigger = addr;
826			pr_debug("breakpoint fired: address = 0x%x\n", addr);
827			perf_bp_event(bp, regs);
828			if (!bp->overflow_handler)
829				enable_single_step(bp, addr);
830			goto unlock;
831		}
832
833mismatch:
834		/* If we're stepping a breakpoint, it can now be restored. */
835		if (info->step_ctrl.enabled)
836			disable_single_step(bp);
837unlock:
838		rcu_read_unlock();
839	}
840
841	/* Handle any pending watchpoint single-step breakpoints. */
842	watchpoint_single_step_handler(addr);
843}
844
845/*
846 * Called from either the Data Abort Handler [watchpoint] or the
847 * Prefetch Abort Handler [breakpoint] with interrupts disabled.
848 */
849static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
850				 struct pt_regs *regs)
851{
852	int ret = 0;
853	u32 dscr;
854
855	preempt_disable();
856
857	if (interrupts_enabled(regs))
858		local_irq_enable();
859
860	/* We only handle watchpoints and hardware breakpoints. */
861	ARM_DBG_READ(c0, c1, 0, dscr);
862
863	/* Perform perf callbacks. */
864	switch (ARM_DSCR_MOE(dscr)) {
865	case ARM_ENTRY_BREAKPOINT:
866		breakpoint_handler(addr, regs);
867		break;
868	case ARM_ENTRY_ASYNC_WATCHPOINT:
869		WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
870	case ARM_ENTRY_SYNC_WATCHPOINT:
871		watchpoint_handler(addr, fsr, regs);
872		break;
873	default:
874		ret = 1; /* Unhandled fault. */
875	}
876
877	preempt_enable();
878
879	return ret;
880}
881
882/*
883 * One-time initialisation.
884 */
885static cpumask_t debug_err_mask;
886
887static int debug_reg_trap(struct pt_regs *regs, unsigned int instr)
888{
889	int cpu = smp_processor_id();
890
891	pr_warning("Debug register access (0x%x) caused undefined instruction on CPU %d\n",
892		   instr, cpu);
893
894	/* Set the error flag for this CPU and skip the faulting instruction. */
895	cpumask_set_cpu(cpu, &debug_err_mask);
896	instruction_pointer(regs) += 4;
897	return 0;
898}
899
900static struct undef_hook debug_reg_hook = {
901	.instr_mask	= 0x0fe80f10,
902	.instr_val	= 0x0e000e10,
903	.fn		= debug_reg_trap,
904};
905
906static void reset_ctrl_regs(void *unused)
907{
908	int i, raw_num_brps, err = 0, cpu = smp_processor_id();
909	u32 val;
910
911	/*
912	 * v7 debug contains save and restore registers so that debug state
913	 * can be maintained across low-power modes without leaving the debug
914	 * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
915	 * the debug registers out of reset, so we must unlock the OS Lock
916	 * Access Register to avoid taking undefined instruction exceptions
917	 * later on.
918	 */
919	switch (debug_arch) {
920	case ARM_DEBUG_ARCH_V6:
921	case ARM_DEBUG_ARCH_V6_1:
922		/* ARMv6 cores clear the registers out of reset. */
923		goto out_mdbgen;
924	case ARM_DEBUG_ARCH_V7_ECP14:
925		/*
926		 * Ensure sticky power-down is clear (i.e. debug logic is
927		 * powered up).
928		 */
929		ARM_DBG_READ(c1, c5, 4, val);
930		if ((val & 0x1) == 0)
931			err = -EPERM;
932
933		/*
934		 * Check whether we implement OS save and restore.
935		 */
936		ARM_DBG_READ(c1, c1, 4, val);
937		if ((val & 0x9) == 0)
938			goto clear_vcr;
939		break;
940	case ARM_DEBUG_ARCH_V7_1:
941		/*
942		 * Ensure the OS double lock is clear.
943		 */
944		ARM_DBG_READ(c1, c3, 4, val);
945		if ((val & 0x1) == 1)
946			err = -EPERM;
947		break;
948	}
949
950	if (err) {
951		pr_warning("CPU %d debug is powered down!\n", cpu);
952		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
953		return;
954	}
955
956	/*
957	 * Unconditionally clear the OS lock by writing a value
958	 * other than 0xC5ACCE55 to the access register.
959	 */
960	ARM_DBG_WRITE(c1, c0, 4, 0);
961	isb();
962
963	/*
964	 * Clear any configured vector-catch events before
965	 * enabling monitor mode.
966	 */
967clear_vcr:
968	ARM_DBG_WRITE(c0, c7, 0, 0);
969	isb();
970
971	if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
972		pr_warning("CPU %d failed to disable vector catch\n", cpu);
973		return;
974	}
975
976	/*
977	 * The control/value register pairs are UNKNOWN out of reset so
978	 * clear them to avoid spurious debug events.
979	 */
980	raw_num_brps = get_num_brp_resources();
981	for (i = 0; i < raw_num_brps; ++i) {
982		write_wb_reg(ARM_BASE_BCR + i, 0UL);
983		write_wb_reg(ARM_BASE_BVR + i, 0UL);
984	}
985
986	for (i = 0; i < core_num_wrps; ++i) {
987		write_wb_reg(ARM_BASE_WCR + i, 0UL);
988		write_wb_reg(ARM_BASE_WVR + i, 0UL);
989	}
990
991	if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
992		pr_warning("CPU %d failed to clear debug register pairs\n", cpu);
993		return;
994	}
995
996	/*
997	 * Have a crack at enabling monitor mode. We don't actually need
998	 * it yet, but reporting an error early is useful if it fails.
999	 */
1000out_mdbgen:
1001	if (enable_monitor_mode())
1002		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
1003}
1004
1005static int __cpuinit dbg_reset_notify(struct notifier_block *self,
1006				      unsigned long action, void *cpu)
1007{
1008	if (action == CPU_ONLINE)
1009		smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1);
1010
1011	return NOTIFY_OK;
1012}
1013
1014static struct notifier_block __cpuinitdata dbg_reset_nb = {
1015	.notifier_call = dbg_reset_notify,
1016};
1017
1018static int __init arch_hw_breakpoint_init(void)
1019{
1020	debug_arch = get_debug_arch();
1021
1022	if (!debug_arch_supported()) {
1023		pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
1024		return 0;
1025	}
1026
1027	/* Determine how many BRPs/WRPs are available. */
1028	core_num_brps = get_num_brps();
1029	core_num_wrps = get_num_wrps();
1030
1031	/*
1032	 * We need to tread carefully here because DBGSWENABLE may be
1033	 * driven low on this core and there isn't an architected way to
1034	 * determine that.
1035	 */
1036	register_undef_hook(&debug_reg_hook);
1037
1038	/*
1039	 * Reset the breakpoint resources. We assume that a halting
1040	 * debugger will leave the world in a nice state for us.
1041	 */
1042	on_each_cpu(reset_ctrl_regs, NULL, 1);
1043	unregister_undef_hook(&debug_reg_hook);
1044	if (!cpumask_empty(&debug_err_mask)) {
1045		core_num_brps = 0;
1046		core_num_wrps = 0;
1047		return 0;
1048	}
1049
1050	pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n",
1051		core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " :
1052		"", core_num_wrps);
1053
1054	/* Work out the maximum supported watchpoint length. */
1055	max_watchpoint_len = get_max_wp_len();
1056	pr_info("maximum watchpoint size is %u bytes.\n",
1057			max_watchpoint_len);
1058
1059	/* Register debug fault handler. */
1060	hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1061			TRAP_HWBKPT, "watchpoint debug exception");
1062	hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1063			TRAP_HWBKPT, "breakpoint debug exception");
1064
1065	/* Register hotplug notifier. */
1066	register_cpu_notifier(&dbg_reset_nb);
1067	return 0;
1068}
1069arch_initcall(arch_hw_breakpoint_init);
1070
1071void hw_breakpoint_pmu_read(struct perf_event *bp)
1072{
1073}
1074
1075/*
1076 * Dummy function to register with die_notifier.
1077 */
1078int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1079					unsigned long val, void *data)
1080{
1081	return NOTIFY_DONE;
1082}
1083