armv8_deprecated.c revision cbefb97b6d7961359f2af89d77b8d5ab77f45528
1/* 2 * Copyright (C) 2014 ARM Limited 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9#include <linux/cpu.h> 10#include <linux/init.h> 11#include <linux/list.h> 12#include <linux/perf_event.h> 13#include <linux/sched.h> 14#include <linux/slab.h> 15#include <linux/sysctl.h> 16 17#include <asm/insn.h> 18#include <asm/opcodes.h> 19#include <asm/system_misc.h> 20#include <asm/traps.h> 21#include <asm/uaccess.h> 22 23/* 24 * The runtime support for deprecated instruction support can be in one of 25 * following three states - 26 * 27 * 0 = undef 28 * 1 = emulate (software emulation) 29 * 2 = hw (supported in hardware) 30 */ 31enum insn_emulation_mode { 32 INSN_UNDEF, 33 INSN_EMULATE, 34 INSN_HW, 35}; 36 37enum legacy_insn_status { 38 INSN_DEPRECATED, 39 INSN_OBSOLETE, 40}; 41 42struct insn_emulation_ops { 43 const char *name; 44 enum legacy_insn_status status; 45 struct undef_hook *hooks; 46 int (*set_hw_mode)(bool enable); 47}; 48 49struct insn_emulation { 50 struct list_head node; 51 struct insn_emulation_ops *ops; 52 int current_mode; 53 int min; 54 int max; 55}; 56 57static LIST_HEAD(insn_emulation); 58static int nr_insn_emulated; 59static DEFINE_RAW_SPINLOCK(insn_emulation_lock); 60 61static void register_emulation_hooks(struct insn_emulation_ops *ops) 62{ 63 struct undef_hook *hook; 64 65 BUG_ON(!ops->hooks); 66 67 for (hook = ops->hooks; hook->instr_mask; hook++) 68 register_undef_hook(hook); 69 70 pr_notice("Registered %s emulation handler\n", ops->name); 71} 72 73static void remove_emulation_hooks(struct insn_emulation_ops *ops) 74{ 75 struct undef_hook *hook; 76 77 BUG_ON(!ops->hooks); 78 79 for (hook = ops->hooks; hook->instr_mask; hook++) 80 unregister_undef_hook(hook); 81 82 pr_notice("Removed %s emulation handler\n", ops->name); 83} 84 85static int update_insn_emulation_mode(struct insn_emulation *insn, 86 enum insn_emulation_mode prev) 87{ 88 int ret = 0; 89 90 switch (prev) { 91 case INSN_UNDEF: /* Nothing to be done */ 92 break; 93 case INSN_EMULATE: 94 remove_emulation_hooks(insn->ops); 95 break; 96 case INSN_HW: 97 if (insn->ops->set_hw_mode) { 98 insn->ops->set_hw_mode(false); 99 pr_notice("Disabled %s support\n", insn->ops->name); 100 } 101 break; 102 } 103 104 switch (insn->current_mode) { 105 case INSN_UNDEF: 106 break; 107 case INSN_EMULATE: 108 register_emulation_hooks(insn->ops); 109 break; 110 case INSN_HW: 111 if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) 112 pr_notice("Enabled %s support\n", insn->ops->name); 113 else 114 ret = -EINVAL; 115 break; 116 } 117 118 return ret; 119} 120 121static void register_insn_emulation(struct insn_emulation_ops *ops) 122{ 123 unsigned long flags; 124 struct insn_emulation *insn; 125 126 insn = kzalloc(sizeof(*insn), GFP_KERNEL); 127 insn->ops = ops; 128 insn->min = INSN_UNDEF; 129 130 switch (ops->status) { 131 case INSN_DEPRECATED: 132 insn->current_mode = INSN_EMULATE; 133 insn->max = INSN_HW; 134 break; 135 case INSN_OBSOLETE: 136 insn->current_mode = INSN_UNDEF; 137 insn->max = INSN_EMULATE; 138 break; 139 } 140 141 raw_spin_lock_irqsave(&insn_emulation_lock, flags); 142 list_add(&insn->node, &insn_emulation); 143 nr_insn_emulated++; 144 raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); 145 146 /* Register any handlers if required */ 147 update_insn_emulation_mode(insn, INSN_UNDEF); 148} 149 150static int emulation_proc_handler(struct ctl_table *table, int write, 151 void __user *buffer, size_t *lenp, 152 loff_t *ppos) 153{ 154 int ret = 0; 155 struct insn_emulation *insn = (struct insn_emulation *) table->data; 156 enum insn_emulation_mode prev_mode = insn->current_mode; 157 158 table->data = &insn->current_mode; 159 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 160 161 if (ret || !write || prev_mode == insn->current_mode) 162 goto ret; 163 164 ret = update_insn_emulation_mode(insn, prev_mode); 165 if (!ret) { 166 /* Mode change failed, revert to previous mode. */ 167 insn->current_mode = prev_mode; 168 update_insn_emulation_mode(insn, INSN_UNDEF); 169 } 170ret: 171 table->data = insn; 172 return ret; 173} 174 175static struct ctl_table ctl_abi[] = { 176 { 177 .procname = "abi", 178 .mode = 0555, 179 }, 180 { } 181}; 182 183static void register_insn_emulation_sysctl(struct ctl_table *table) 184{ 185 unsigned long flags; 186 int i = 0; 187 struct insn_emulation *insn; 188 struct ctl_table *insns_sysctl, *sysctl; 189 190 insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1), 191 GFP_KERNEL); 192 193 raw_spin_lock_irqsave(&insn_emulation_lock, flags); 194 list_for_each_entry(insn, &insn_emulation, node) { 195 sysctl = &insns_sysctl[i]; 196 197 sysctl->mode = 0644; 198 sysctl->maxlen = sizeof(int); 199 200 sysctl->procname = insn->ops->name; 201 sysctl->data = insn; 202 sysctl->extra1 = &insn->min; 203 sysctl->extra2 = &insn->max; 204 sysctl->proc_handler = emulation_proc_handler; 205 i++; 206 } 207 raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); 208 209 table->child = insns_sysctl; 210 register_sysctl_table(table); 211} 212 213/* 214 * Implement emulation of the SWP/SWPB instructions using load-exclusive and 215 * store-exclusive. 216 * 217 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] 218 * Where: Rt = destination 219 * Rt2 = source 220 * Rn = address 221 */ 222 223/* 224 * Error-checking SWP macros implemented using ldxr{b}/stxr{b} 225 */ 226#define __user_swpX_asm(data, addr, res, temp, B) \ 227 __asm__ __volatile__( \ 228 " mov %w2, %w1\n" \ 229 "0: ldxr"B" %w1, [%3]\n" \ 230 "1: stxr"B" %w0, %w2, [%3]\n" \ 231 " cbz %w0, 2f\n" \ 232 " mov %w0, %w4\n" \ 233 "2:\n" \ 234 " .pushsection .fixup,\"ax\"\n" \ 235 " .align 2\n" \ 236 "3: mov %w0, %w5\n" \ 237 " b 2b\n" \ 238 " .popsection" \ 239 " .pushsection __ex_table,\"a\"\n" \ 240 " .align 3\n" \ 241 " .quad 0b, 3b\n" \ 242 " .quad 1b, 3b\n" \ 243 " .popsection" \ 244 : "=&r" (res), "+r" (data), "=&r" (temp) \ 245 : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ 246 : "memory") 247 248#define __user_swp_asm(data, addr, res, temp) \ 249 __user_swpX_asm(data, addr, res, temp, "") 250#define __user_swpb_asm(data, addr, res, temp) \ 251 __user_swpX_asm(data, addr, res, temp, "b") 252 253/* 254 * Bit 22 of the instruction encoding distinguishes between 255 * the SWP and SWPB variants (bit set means SWPB). 256 */ 257#define TYPE_SWPB (1 << 22) 258 259/* 260 * Set up process info to signal segmentation fault - called on access error. 261 */ 262static void set_segfault(struct pt_regs *regs, unsigned long addr) 263{ 264 siginfo_t info; 265 266 down_read(¤t->mm->mmap_sem); 267 if (find_vma(current->mm, addr) == NULL) 268 info.si_code = SEGV_MAPERR; 269 else 270 info.si_code = SEGV_ACCERR; 271 up_read(¤t->mm->mmap_sem); 272 273 info.si_signo = SIGSEGV; 274 info.si_errno = 0; 275 info.si_addr = (void *) instruction_pointer(regs); 276 277 pr_debug("SWP{B} emulation: access caused memory abort!\n"); 278 arm64_notify_die("Illegal memory access", regs, &info, 0); 279} 280 281static int emulate_swpX(unsigned int address, unsigned int *data, 282 unsigned int type) 283{ 284 unsigned int res = 0; 285 286 if ((type != TYPE_SWPB) && (address & 0x3)) { 287 /* SWP to unaligned address not permitted */ 288 pr_debug("SWP instruction on unaligned pointer!\n"); 289 return -EFAULT; 290 } 291 292 while (1) { 293 unsigned long temp; 294 295 if (type == TYPE_SWPB) 296 __user_swpb_asm(*data, address, res, temp); 297 else 298 __user_swp_asm(*data, address, res, temp); 299 300 if (likely(res != -EAGAIN) || signal_pending(current)) 301 break; 302 303 cond_resched(); 304 } 305 306 return res; 307} 308 309/* 310 * swp_handler logs the id of calling process, dissects the instruction, sanity 311 * checks the memory location, calls emulate_swpX for the actual operation and 312 * deals with fixup/error handling before returning 313 */ 314static int swp_handler(struct pt_regs *regs, u32 instr) 315{ 316 u32 destreg, data, type, address = 0; 317 int rn, rt2, res = 0; 318 319 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); 320 321 type = instr & TYPE_SWPB; 322 323 switch (arm_check_condition(instr, regs->pstate)) { 324 case ARM_OPCODE_CONDTEST_PASS: 325 break; 326 case ARM_OPCODE_CONDTEST_FAIL: 327 /* Condition failed - return to next instruction */ 328 goto ret; 329 case ARM_OPCODE_CONDTEST_UNCOND: 330 /* If unconditional encoding - not a SWP, undef */ 331 return -EFAULT; 332 default: 333 return -EINVAL; 334 } 335 336 rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET); 337 rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET); 338 339 address = (u32)regs->user_regs.regs[rn]; 340 data = (u32)regs->user_regs.regs[rt2]; 341 destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET); 342 343 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", 344 rn, address, destreg, 345 aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data); 346 347 /* Check access in reasonable access range for both SWP and SWPB */ 348 if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { 349 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", 350 address); 351 goto fault; 352 } 353 354 res = emulate_swpX(address, &data, type); 355 if (res == -EFAULT) 356 goto fault; 357 else if (res == 0) 358 regs->user_regs.regs[destreg] = data; 359 360ret: 361 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n", 362 current->comm, (unsigned long)current->pid, regs->pc); 363 364 regs->pc += 4; 365 return 0; 366 367fault: 368 set_segfault(regs, address); 369 370 return 0; 371} 372 373/* 374 * Only emulate SWP/SWPB executed in ARM state/User mode. 375 * The kernel must be SWP free and SWP{B} does not exist in Thumb. 376 */ 377static struct undef_hook swp_hooks[] = { 378 { 379 .instr_mask = 0x0fb00ff0, 380 .instr_val = 0x01000090, 381 .pstate_mask = COMPAT_PSR_MODE_MASK, 382 .pstate_val = COMPAT_PSR_MODE_USR, 383 .fn = swp_handler 384 }, 385 { } 386}; 387 388static struct insn_emulation_ops swp_ops = { 389 .name = "swp", 390 .status = INSN_OBSOLETE, 391 .hooks = swp_hooks, 392 .set_hw_mode = NULL, 393}; 394 395static int cp15barrier_handler(struct pt_regs *regs, u32 instr) 396{ 397 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); 398 399 switch (arm_check_condition(instr, regs->pstate)) { 400 case ARM_OPCODE_CONDTEST_PASS: 401 break; 402 case ARM_OPCODE_CONDTEST_FAIL: 403 /* Condition failed - return to next instruction */ 404 goto ret; 405 case ARM_OPCODE_CONDTEST_UNCOND: 406 /* If unconditional encoding - not a barrier instruction */ 407 return -EFAULT; 408 default: 409 return -EINVAL; 410 } 411 412 switch (aarch32_insn_mcr_extract_crm(instr)) { 413 case 10: 414 /* 415 * dmb - mcr p15, 0, Rt, c7, c10, 5 416 * dsb - mcr p15, 0, Rt, c7, c10, 4 417 */ 418 if (aarch32_insn_mcr_extract_opc2(instr) == 5) 419 dmb(sy); 420 else 421 dsb(sy); 422 break; 423 case 5: 424 /* 425 * isb - mcr p15, 0, Rt, c7, c5, 4 426 * 427 * Taking an exception or returning from one acts as an 428 * instruction barrier. So no explicit barrier needed here. 429 */ 430 break; 431 } 432 433ret: 434 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n", 435 current->comm, (unsigned long)current->pid, regs->pc); 436 437 regs->pc += 4; 438 return 0; 439} 440 441#define SCTLR_EL1_CP15BEN (1 << 5) 442 443static inline void config_sctlr_el1(u32 clear, u32 set) 444{ 445 u32 val; 446 447 asm volatile("mrs %0, sctlr_el1" : "=r" (val)); 448 val &= ~clear; 449 val |= set; 450 asm volatile("msr sctlr_el1, %0" : : "r" (val)); 451} 452 453static void enable_cp15_ben(void *info) 454{ 455 config_sctlr_el1(0, SCTLR_EL1_CP15BEN); 456} 457 458static void disable_cp15_ben(void *info) 459{ 460 config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); 461} 462 463static int cpu_hotplug_notify(struct notifier_block *b, 464 unsigned long action, void *hcpu) 465{ 466 switch (action) { 467 case CPU_STARTING: 468 case CPU_STARTING_FROZEN: 469 enable_cp15_ben(NULL); 470 return NOTIFY_DONE; 471 case CPU_DYING: 472 case CPU_DYING_FROZEN: 473 disable_cp15_ben(NULL); 474 return NOTIFY_DONE; 475 } 476 477 return NOTIFY_OK; 478} 479 480static struct notifier_block cpu_hotplug_notifier = { 481 .notifier_call = cpu_hotplug_notify, 482}; 483 484static int cp15_barrier_set_hw_mode(bool enable) 485{ 486 if (enable) { 487 register_cpu_notifier(&cpu_hotplug_notifier); 488 on_each_cpu(enable_cp15_ben, NULL, true); 489 } else { 490 unregister_cpu_notifier(&cpu_hotplug_notifier); 491 on_each_cpu(disable_cp15_ben, NULL, true); 492 } 493 494 return true; 495} 496 497static struct undef_hook cp15_barrier_hooks[] = { 498 { 499 .instr_mask = 0x0fff0fdf, 500 .instr_val = 0x0e070f9a, 501 .pstate_mask = COMPAT_PSR_MODE_MASK, 502 .pstate_val = COMPAT_PSR_MODE_USR, 503 .fn = cp15barrier_handler, 504 }, 505 { 506 .instr_mask = 0x0fff0fff, 507 .instr_val = 0x0e070f95, 508 .pstate_mask = COMPAT_PSR_MODE_MASK, 509 .pstate_val = COMPAT_PSR_MODE_USR, 510 .fn = cp15barrier_handler, 511 }, 512 { } 513}; 514 515static struct insn_emulation_ops cp15_barrier_ops = { 516 .name = "cp15_barrier", 517 .status = INSN_DEPRECATED, 518 .hooks = cp15_barrier_hooks, 519 .set_hw_mode = cp15_barrier_set_hw_mode, 520}; 521 522/* 523 * Invoked as late_initcall, since not needed before init spawned. 524 */ 525static int __init armv8_deprecated_init(void) 526{ 527 if (IS_ENABLED(CONFIG_SWP_EMULATION)) 528 register_insn_emulation(&swp_ops); 529 530 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) 531 register_insn_emulation(&cp15_barrier_ops); 532 533 register_insn_emulation_sysctl(ctl_abi); 534 535 return 0; 536} 537 538late_initcall(armv8_deprecated_init); 539