armv8_deprecated.c revision 203077a7db3bc692ff8177e822110400868efab4 
1/*
2 *  Copyright (C) 2014 ARM Limited
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#include <linux/cpu.h>
10#include <linux/init.h>
11#include <linux/list.h>
12#include <linux/perf_event.h>
13#include <linux/sched.h>
14#include <linux/slab.h>
15#include <linux/sysctl.h>
16
17#include <asm/insn.h>
18#include <asm/opcodes.h>
19#include <asm/system_misc.h>
20#include <asm/traps.h>
21#include <asm/uaccess.h>
22
23#define CREATE_TRACE_POINTS
24#include "trace-events-emulation.h"
25
26/*
27 * The runtime support for deprecated instruction support can be in one of
28 * following three states -
29 *
30 * 0 = undef
31 * 1 = emulate (software emulation)
32 * 2 = hw (supported in hardware)
33 */
34enum insn_emulation_mode {
35	INSN_UNDEF,
36	INSN_EMULATE,
37	INSN_HW,
38};
39
40enum legacy_insn_status {
41	INSN_DEPRECATED,
42	INSN_OBSOLETE,
43};
44
45struct insn_emulation_ops {
46	const char		*name;
47	enum legacy_insn_status	status;
48	struct undef_hook	*hooks;
49	int			(*set_hw_mode)(bool enable);
50};
51
52struct insn_emulation {
53	struct list_head node;
54	struct insn_emulation_ops *ops;
55	int current_mode;
56	int min;
57	int max;
58};
59
60static LIST_HEAD(insn_emulation);
61static int nr_insn_emulated;
62static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
63
64static void register_emulation_hooks(struct insn_emulation_ops *ops)
65{
66	struct undef_hook *hook;
67
68	BUG_ON(!ops->hooks);
69
70	for (hook = ops->hooks; hook->instr_mask; hook++)
71		register_undef_hook(hook);
72
73	pr_notice("Registered %s emulation handler\n", ops->name);
74}
75
76static void remove_emulation_hooks(struct insn_emulation_ops *ops)
77{
78	struct undef_hook *hook;
79
80	BUG_ON(!ops->hooks);
81
82	for (hook = ops->hooks; hook->instr_mask; hook++)
83		unregister_undef_hook(hook);
84
85	pr_notice("Removed %s emulation handler\n", ops->name);
86}
87
88static int update_insn_emulation_mode(struct insn_emulation *insn,
89				       enum insn_emulation_mode prev)
90{
91	int ret = 0;
92
93	switch (prev) {
94	case INSN_UNDEF: /* Nothing to be done */
95		break;
96	case INSN_EMULATE:
97		remove_emulation_hooks(insn->ops);
98		break;
99	case INSN_HW:
100		if (insn->ops->set_hw_mode) {
101			insn->ops->set_hw_mode(false);
102			pr_notice("Disabled %s support\n", insn->ops->name);
103		}
104		break;
105	}
106
107	switch (insn->current_mode) {
108	case INSN_UNDEF:
109		break;
110	case INSN_EMULATE:
111		register_emulation_hooks(insn->ops);
112		break;
113	case INSN_HW:
114		if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true))
115			pr_notice("Enabled %s support\n", insn->ops->name);
116		else
117			ret = -EINVAL;
118		break;
119	}
120
121	return ret;
122}
123
124static void register_insn_emulation(struct insn_emulation_ops *ops)
125{
126	unsigned long flags;
127	struct insn_emulation *insn;
128
129	insn = kzalloc(sizeof(*insn), GFP_KERNEL);
130	insn->ops = ops;
131	insn->min = INSN_UNDEF;
132
133	switch (ops->status) {
134	case INSN_DEPRECATED:
135		insn->current_mode = INSN_EMULATE;
136		insn->max = INSN_HW;
137		break;
138	case INSN_OBSOLETE:
139		insn->current_mode = INSN_UNDEF;
140		insn->max = INSN_EMULATE;
141		break;
142	}
143
144	raw_spin_lock_irqsave(&insn_emulation_lock, flags);
145	list_add(&insn->node, &insn_emulation);
146	nr_insn_emulated++;
147	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
148
149	/* Register any handlers if required */
150	update_insn_emulation_mode(insn, INSN_UNDEF);
151}
152
153static int emulation_proc_handler(struct ctl_table *table, int write,
154				  void __user *buffer, size_t *lenp,
155				  loff_t *ppos)
156{
157	int ret = 0;
158	struct insn_emulation *insn = (struct insn_emulation *) table->data;
159	enum insn_emulation_mode prev_mode = insn->current_mode;
160
161	table->data = &insn->current_mode;
162	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
163
164	if (ret || !write || prev_mode == insn->current_mode)
165		goto ret;
166
167	ret = update_insn_emulation_mode(insn, prev_mode);
168	if (!ret) {
169		/* Mode change failed, revert to previous mode. */
170		insn->current_mode = prev_mode;
171		update_insn_emulation_mode(insn, INSN_UNDEF);
172	}
173ret:
174	table->data = insn;
175	return ret;
176}
177
178static struct ctl_table ctl_abi[] = {
179	{
180		.procname = "abi",
181		.mode = 0555,
182	},
183	{ }
184};
185
186static void register_insn_emulation_sysctl(struct ctl_table *table)
187{
188	unsigned long flags;
189	int i = 0;
190	struct insn_emulation *insn;
191	struct ctl_table *insns_sysctl, *sysctl;
192
193	insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1),
194			      GFP_KERNEL);
195
196	raw_spin_lock_irqsave(&insn_emulation_lock, flags);
197	list_for_each_entry(insn, &insn_emulation, node) {
198		sysctl = &insns_sysctl[i];
199
200		sysctl->mode = 0644;
201		sysctl->maxlen = sizeof(int);
202
203		sysctl->procname = insn->ops->name;
204		sysctl->data = insn;
205		sysctl->extra1 = &insn->min;
206		sysctl->extra2 = &insn->max;
207		sysctl->proc_handler = emulation_proc_handler;
208		i++;
209	}
210	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
211
212	table->child = insns_sysctl;
213	register_sysctl_table(table);
214}
215
216/*
217 *  Implement emulation of the SWP/SWPB instructions using load-exclusive and
218 *  store-exclusive.
219 *
220 *  Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
221 *  Where: Rt  = destination
222 *	   Rt2 = source
223 *	   Rn  = address
224 */
225
226/*
227 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
228 */
229#define __user_swpX_asm(data, addr, res, temp, B)		\
230	__asm__ __volatile__(					\
231	"	mov		%w2, %w1\n"			\
232	"0:	ldxr"B"		%w1, [%3]\n"			\
233	"1:	stxr"B"		%w0, %w2, [%3]\n"		\
234	"	cbz		%w0, 2f\n"			\
235	"	mov		%w0, %w4\n"			\
236	"2:\n"							\
237	"	.pushsection	 .fixup,\"ax\"\n"		\
238	"	.align		2\n"				\
239	"3:	mov		%w0, %w5\n"			\
240	"	b		2b\n"				\
241	"	.popsection"					\
242	"	.pushsection	 __ex_table,\"a\"\n"		\
243	"	.align		3\n"				\
244	"	.quad		0b, 3b\n"			\
245	"	.quad		1b, 3b\n"			\
246	"	.popsection"					\
247	: "=&r" (res), "+r" (data), "=&r" (temp)		\
248	: "r" (addr), "i" (-EAGAIN), "i" (-EFAULT)		\
249	: "memory")
250
251#define __user_swp_asm(data, addr, res, temp) \
252	__user_swpX_asm(data, addr, res, temp, "")
253#define __user_swpb_asm(data, addr, res, temp) \
254	__user_swpX_asm(data, addr, res, temp, "b")
255
256/*
257 * Bit 22 of the instruction encoding distinguishes between
258 * the SWP and SWPB variants (bit set means SWPB).
259 */
260#define TYPE_SWPB (1 << 22)
261
262/*
263 * Set up process info to signal segmentation fault - called on access error.
264 */
265static void set_segfault(struct pt_regs *regs, unsigned long addr)
266{
267	siginfo_t info;
268
269	down_read(&current->mm->mmap_sem);
270	if (find_vma(current->mm, addr) == NULL)
271		info.si_code = SEGV_MAPERR;
272	else
273		info.si_code = SEGV_ACCERR;
274	up_read(&current->mm->mmap_sem);
275
276	info.si_signo = SIGSEGV;
277	info.si_errno = 0;
278	info.si_addr  = (void *) instruction_pointer(regs);
279
280	pr_debug("SWP{B} emulation: access caused memory abort!\n");
281	arm64_notify_die("Illegal memory access", regs, &info, 0);
282}
283
284static int emulate_swpX(unsigned int address, unsigned int *data,
285			unsigned int type)
286{
287	unsigned int res = 0;
288
289	if ((type != TYPE_SWPB) && (address & 0x3)) {
290		/* SWP to unaligned address not permitted */
291		pr_debug("SWP instruction on unaligned pointer!\n");
292		return -EFAULT;
293	}
294
295	while (1) {
296		unsigned long temp;
297
298		if (type == TYPE_SWPB)
299			__user_swpb_asm(*data, address, res, temp);
300		else
301			__user_swp_asm(*data, address, res, temp);
302
303		if (likely(res != -EAGAIN) || signal_pending(current))
304			break;
305
306		cond_resched();
307	}
308
309	return res;
310}
311
312/*
313 * swp_handler logs the id of calling process, dissects the instruction, sanity
314 * checks the memory location, calls emulate_swpX for the actual operation and
315 * deals with fixup/error handling before returning
316 */
317static int swp_handler(struct pt_regs *regs, u32 instr)
318{
319	u32 destreg, data, type, address = 0;
320	int rn, rt2, res = 0;
321
322	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
323
324	type = instr & TYPE_SWPB;
325
326	switch (arm_check_condition(instr, regs->pstate)) {
327	case ARM_OPCODE_CONDTEST_PASS:
328		break;
329	case ARM_OPCODE_CONDTEST_FAIL:
330		/* Condition failed - return to next instruction */
331		goto ret;
332	case ARM_OPCODE_CONDTEST_UNCOND:
333		/* If unconditional encoding - not a SWP, undef */
334		return -EFAULT;
335	default:
336		return -EINVAL;
337	}
338
339	rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
340	rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
341
342	address = (u32)regs->user_regs.regs[rn];
343	data	= (u32)regs->user_regs.regs[rt2];
344	destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
345
346	pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
347		rn, address, destreg,
348		aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
349
350	/* Check access in reasonable access range for both SWP and SWPB */
351	if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) {
352		pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
353			address);
354		goto fault;
355	}
356
357	res = emulate_swpX(address, &data, type);
358	if (res == -EFAULT)
359		goto fault;
360	else if (res == 0)
361		regs->user_regs.regs[destreg] = data;
362
363ret:
364	if (type == TYPE_SWPB)
365		trace_instruction_emulation("swpb", regs->pc);
366	else
367		trace_instruction_emulation("swp", regs->pc);
368
369	pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
370			current->comm, (unsigned long)current->pid, regs->pc);
371
372	regs->pc += 4;
373	return 0;
374
375fault:
376	set_segfault(regs, address);
377
378	return 0;
379}
380
381/*
382 * Only emulate SWP/SWPB executed in ARM state/User mode.
383 * The kernel must be SWP free and SWP{B} does not exist in Thumb.
384 */
385static struct undef_hook swp_hooks[] = {
386	{
387		.instr_mask	= 0x0fb00ff0,
388		.instr_val	= 0x01000090,
389		.pstate_mask	= COMPAT_PSR_MODE_MASK,
390		.pstate_val	= COMPAT_PSR_MODE_USR,
391		.fn		= swp_handler
392	},
393	{ }
394};
395
396static struct insn_emulation_ops swp_ops = {
397	.name = "swp",
398	.status = INSN_OBSOLETE,
399	.hooks = swp_hooks,
400	.set_hw_mode = NULL,
401};
402
403static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
404{
405	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
406
407	switch (arm_check_condition(instr, regs->pstate)) {
408	case ARM_OPCODE_CONDTEST_PASS:
409		break;
410	case ARM_OPCODE_CONDTEST_FAIL:
411		/* Condition failed - return to next instruction */
412		goto ret;
413	case ARM_OPCODE_CONDTEST_UNCOND:
414		/* If unconditional encoding - not a barrier instruction */
415		return -EFAULT;
416	default:
417		return -EINVAL;
418	}
419
420	switch (aarch32_insn_mcr_extract_crm(instr)) {
421	case 10:
422		/*
423		 * dmb - mcr p15, 0, Rt, c7, c10, 5
424		 * dsb - mcr p15, 0, Rt, c7, c10, 4
425		 */
426		if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
427			dmb(sy);
428			trace_instruction_emulation(
429				"mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
430		} else {
431			dsb(sy);
432			trace_instruction_emulation(
433				"mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
434		}
435		break;
436	case 5:
437		/*
438		 * isb - mcr p15, 0, Rt, c7, c5, 4
439		 *
440		 * Taking an exception or returning from one acts as an
441		 * instruction barrier. So no explicit barrier needed here.
442		 */
443		trace_instruction_emulation(
444			"mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
445		break;
446	}
447
448ret:
449	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
450			current->comm, (unsigned long)current->pid, regs->pc);
451
452	regs->pc += 4;
453	return 0;
454}
455
456#define SCTLR_EL1_CP15BEN (1 << 5)
457
458static inline void config_sctlr_el1(u32 clear, u32 set)
459{
460	u32 val;
461
462	asm volatile("mrs %0, sctlr_el1" : "=r" (val));
463	val &= ~clear;
464	val |= set;
465	asm volatile("msr sctlr_el1, %0" : : "r" (val));
466}
467
468static void enable_cp15_ben(void *info)
469{
470	config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
471}
472
473static void disable_cp15_ben(void *info)
474{
475	config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
476}
477
478static int cpu_hotplug_notify(struct notifier_block *b,
479			      unsigned long action, void *hcpu)
480{
481	switch (action) {
482	case CPU_STARTING:
483	case CPU_STARTING_FROZEN:
484		enable_cp15_ben(NULL);
485		return NOTIFY_DONE;
486	case CPU_DYING:
487	case CPU_DYING_FROZEN:
488		disable_cp15_ben(NULL);
489		return NOTIFY_DONE;
490	}
491
492	return NOTIFY_OK;
493}
494
495static struct notifier_block cpu_hotplug_notifier = {
496	.notifier_call = cpu_hotplug_notify,
497};
498
499static int cp15_barrier_set_hw_mode(bool enable)
500{
501	if (enable) {
502		register_cpu_notifier(&cpu_hotplug_notifier);
503		on_each_cpu(enable_cp15_ben, NULL, true);
504	} else {
505		unregister_cpu_notifier(&cpu_hotplug_notifier);
506		on_each_cpu(disable_cp15_ben, NULL, true);
507	}
508
509	return true;
510}
511
512static struct undef_hook cp15_barrier_hooks[] = {
513	{
514		.instr_mask	= 0x0fff0fdf,
515		.instr_val	= 0x0e070f9a,
516		.pstate_mask	= COMPAT_PSR_MODE_MASK,
517		.pstate_val	= COMPAT_PSR_MODE_USR,
518		.fn		= cp15barrier_handler,
519	},
520	{
521		.instr_mask	= 0x0fff0fff,
522		.instr_val	= 0x0e070f95,
523		.pstate_mask	= COMPAT_PSR_MODE_MASK,
524		.pstate_val	= COMPAT_PSR_MODE_USR,
525		.fn		= cp15barrier_handler,
526	},
527	{ }
528};
529
530static struct insn_emulation_ops cp15_barrier_ops = {
531	.name = "cp15_barrier",
532	.status = INSN_DEPRECATED,
533	.hooks = cp15_barrier_hooks,
534	.set_hw_mode = cp15_barrier_set_hw_mode,
535};
536
537/*
538 * Invoked as late_initcall, since not needed before init spawned.
539 */
540static int __init armv8_deprecated_init(void)
541{
542	if (IS_ENABLED(CONFIG_SWP_EMULATION))
543		register_insn_emulation(&swp_ops);
544
545	if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
546		register_insn_emulation(&cp15_barrier_ops);
547
548	register_insn_emulation_sysctl(ctl_abi);
549
550	return 0;
551}
552
553late_initcall(armv8_deprecated_init);
554