1 /* 2 * Copyright (C) 2014 ARM Limited 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/cpu.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/perf_event.h> 13 #include <linux/sched.h> 14 #include <linux/slab.h> 15 #include <linux/sysctl.h> 16 17 #include <asm/insn.h> 18 #include <asm/opcodes.h> 19 #include <asm/system_misc.h> 20 #include <asm/traps.h> 21 #include <asm/uaccess.h> 22 23 #define CREATE_TRACE_POINTS 24 #include "trace-events-emulation.h" 25 26 /* 27 * The runtime support for deprecated instruction support can be in one of 28 * following three states - 29 * 30 * 0 = undef 31 * 1 = emulate (software emulation) 32 * 2 = hw (supported in hardware) 33 */ 34 enum insn_emulation_mode { 35 INSN_UNDEF, 36 INSN_EMULATE, 37 INSN_HW, 38 }; 39 40 enum legacy_insn_status { 41 INSN_DEPRECATED, 42 INSN_OBSOLETE, 43 }; 44 45 struct insn_emulation_ops { 46 const char *name; 47 enum legacy_insn_status status; 48 struct undef_hook *hooks; 49 int (*set_hw_mode)(bool enable); 50 }; 51 52 struct insn_emulation { 53 struct list_head node; 54 struct insn_emulation_ops *ops; 55 int current_mode; 56 int min; 57 int max; 58 }; 59 60 static LIST_HEAD(insn_emulation); 61 static int nr_insn_emulated; 62 static DEFINE_RAW_SPINLOCK(insn_emulation_lock); 63 64 static void register_emulation_hooks(struct insn_emulation_ops *ops) 65 { 66 struct undef_hook *hook; 67 68 BUG_ON(!ops->hooks); 69 70 for (hook = ops->hooks; hook->instr_mask; hook++) 71 register_undef_hook(hook); 72 73 pr_notice("Registered %s emulation handler\n", ops->name); 74 } 75 76 static void remove_emulation_hooks(struct insn_emulation_ops *ops) 77 { 78 struct undef_hook *hook; 79 80 BUG_ON(!ops->hooks); 81 82 for (hook = ops->hooks; hook->instr_mask; hook++) 83 unregister_undef_hook(hook); 84 85 pr_notice("Removed %s emulation handler\n", ops->name); 86 } 87 88 static int update_insn_emulation_mode(struct insn_emulation *insn, 89 enum insn_emulation_mode prev) 90 { 91 int ret = 0; 92 93 switch (prev) { 94 case INSN_UNDEF: /* Nothing to be done */ 95 break; 96 case INSN_EMULATE: 97 remove_emulation_hooks(insn->ops); 98 break; 99 case INSN_HW: 100 if (insn->ops->set_hw_mode) { 101 insn->ops->set_hw_mode(false); 102 pr_notice("Disabled %s support\n", insn->ops->name); 103 } 104 break; 105 } 106 107 switch (insn->current_mode) { 108 case INSN_UNDEF: 109 break; 110 case INSN_EMULATE: 111 register_emulation_hooks(insn->ops); 112 break; 113 case INSN_HW: 114 if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) 115 pr_notice("Enabled %s support\n", insn->ops->name); 116 else 117 ret = -EINVAL; 118 break; 119 } 120 121 return ret; 122 } 123 124 static void register_insn_emulation(struct insn_emulation_ops *ops) 125 { 126 unsigned long flags; 127 struct insn_emulation *insn; 128 129 insn = kzalloc(sizeof(*insn), GFP_KERNEL); 130 insn->ops = ops; 131 insn->min = INSN_UNDEF; 132 133 switch (ops->status) { 134 case INSN_DEPRECATED: 135 insn->current_mode = INSN_EMULATE; 136 insn->max = INSN_HW; 137 break; 138 case INSN_OBSOLETE: 139 insn->current_mode = INSN_UNDEF; 140 insn->max = INSN_EMULATE; 141 break; 142 } 143 144 raw_spin_lock_irqsave(&insn_emulation_lock, flags); 145 list_add(&insn->node, &insn_emulation); 146 nr_insn_emulated++; 147 raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); 148 149 /* Register any handlers if required */ 150 update_insn_emulation_mode(insn, INSN_UNDEF); 151 } 152 153 static int emulation_proc_handler(struct ctl_table *table, int write, 154 void __user *buffer, size_t *lenp, 155 loff_t *ppos) 156 { 157 int ret = 0; 158 struct insn_emulation *insn = (struct insn_emulation *) table->data; 159 enum insn_emulation_mode prev_mode = insn->current_mode; 160 161 table->data = &insn->current_mode; 162 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 163 164 if (ret || !write || prev_mode == insn->current_mode) 165 goto ret; 166 167 ret = update_insn_emulation_mode(insn, prev_mode); 168 if (ret) { 169 /* Mode change failed, revert to previous mode. */ 170 insn->current_mode = prev_mode; 171 update_insn_emulation_mode(insn, INSN_UNDEF); 172 } 173 ret: 174 table->data = insn; 175 return ret; 176 } 177 178 static struct ctl_table ctl_abi[] = { 179 { 180 .procname = "abi", 181 .mode = 0555, 182 }, 183 { } 184 }; 185 186 static void register_insn_emulation_sysctl(struct ctl_table *table) 187 { 188 unsigned long flags; 189 int i = 0; 190 struct insn_emulation *insn; 191 struct ctl_table *insns_sysctl, *sysctl; 192 193 insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1), 194 GFP_KERNEL); 195 196 raw_spin_lock_irqsave(&insn_emulation_lock, flags); 197 list_for_each_entry(insn, &insn_emulation, node) { 198 sysctl = &insns_sysctl[i]; 199 200 sysctl->mode = 0644; 201 sysctl->maxlen = sizeof(int); 202 203 sysctl->procname = insn->ops->name; 204 sysctl->data = insn; 205 sysctl->extra1 = &insn->min; 206 sysctl->extra2 = &insn->max; 207 sysctl->proc_handler = emulation_proc_handler; 208 i++; 209 } 210 raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); 211 212 table->child = insns_sysctl; 213 register_sysctl_table(table); 214 } 215 216 /* 217 * Implement emulation of the SWP/SWPB instructions using load-exclusive and 218 * store-exclusive. 219 * 220 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] 221 * Where: Rt = destination 222 * Rt2 = source 223 * Rn = address 224 */ 225 226 /* 227 * Error-checking SWP macros implemented using ldxr{b}/stxr{b} 228 */ 229 #define __user_swpX_asm(data, addr, res, temp, B) \ 230 __asm__ __volatile__( \ 231 " mov %w2, %w1\n" \ 232 "0: ldxr"B" %w1, [%3]\n" \ 233 "1: stxr"B" %w0, %w2, [%3]\n" \ 234 " cbz %w0, 2f\n" \ 235 " mov %w0, %w4\n" \ 236 "2:\n" \ 237 " .pushsection .fixup,\"ax\"\n" \ 238 " .align 2\n" \ 239 "3: mov %w0, %w5\n" \ 240 " b 2b\n" \ 241 " .popsection" \ 242 " .pushsection __ex_table,\"a\"\n" \ 243 " .align 3\n" \ 244 " .quad 0b, 3b\n" \ 245 " .quad 1b, 3b\n" \ 246 " .popsection" \ 247 : "=&r" (res), "+r" (data), "=&r" (temp) \ 248 : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ 249 : "memory") 250 251 #define __user_swp_asm(data, addr, res, temp) \ 252 __user_swpX_asm(data, addr, res, temp, "") 253 #define __user_swpb_asm(data, addr, res, temp) \ 254 __user_swpX_asm(data, addr, res, temp, "b") 255 256 /* 257 * Bit 22 of the instruction encoding distinguishes between 258 * the SWP and SWPB variants (bit set means SWPB). 259 */ 260 #define TYPE_SWPB (1 << 22) 261 262 /* 263 * Set up process info to signal segmentation fault - called on access error. 264 */ 265 static void set_segfault(struct pt_regs *regs, unsigned long addr) 266 { 267 siginfo_t info; 268 269 down_read(¤t->mm->mmap_sem); 270 if (find_vma(current->mm, addr) == NULL) 271 info.si_code = SEGV_MAPERR; 272 else 273 info.si_code = SEGV_ACCERR; 274 up_read(¤t->mm->mmap_sem); 275 276 info.si_signo = SIGSEGV; 277 info.si_errno = 0; 278 info.si_addr = (void *) instruction_pointer(regs); 279 280 pr_debug("SWP{B} emulation: access caused memory abort!\n"); 281 arm64_notify_die("Illegal memory access", regs, &info, 0); 282 } 283 284 static int emulate_swpX(unsigned int address, unsigned int *data, 285 unsigned int type) 286 { 287 unsigned int res = 0; 288 289 if ((type != TYPE_SWPB) && (address & 0x3)) { 290 /* SWP to unaligned address not permitted */ 291 pr_debug("SWP instruction on unaligned pointer!\n"); 292 return -EFAULT; 293 } 294 295 while (1) { 296 unsigned long temp; 297 298 if (type == TYPE_SWPB) 299 __user_swpb_asm(*data, address, res, temp); 300 else 301 __user_swp_asm(*data, address, res, temp); 302 303 if (likely(res != -EAGAIN) || signal_pending(current)) 304 break; 305 306 cond_resched(); 307 } 308 309 return res; 310 } 311 312 /* 313 * swp_handler logs the id of calling process, dissects the instruction, sanity 314 * checks the memory location, calls emulate_swpX for the actual operation and 315 * deals with fixup/error handling before returning 316 */ 317 static int swp_handler(struct pt_regs *regs, u32 instr) 318 { 319 u32 destreg, data, type, address = 0; 320 int rn, rt2, res = 0; 321 322 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); 323 324 type = instr & TYPE_SWPB; 325 326 switch (arm_check_condition(instr, regs->pstate)) { 327 case ARM_OPCODE_CONDTEST_PASS: 328 break; 329 case ARM_OPCODE_CONDTEST_FAIL: 330 /* Condition failed - return to next instruction */ 331 goto ret; 332 case ARM_OPCODE_CONDTEST_UNCOND: 333 /* If unconditional encoding - not a SWP, undef */ 334 return -EFAULT; 335 default: 336 return -EINVAL; 337 } 338 339 rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET); 340 rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET); 341 342 address = (u32)regs->user_regs.regs[rn]; 343 data = (u32)regs->user_regs.regs[rt2]; 344 destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET); 345 346 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", 347 rn, address, destreg, 348 aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data); 349 350 /* Check access in reasonable access range for both SWP and SWPB */ 351 if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { 352 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", 353 address); 354 goto fault; 355 } 356 357 res = emulate_swpX(address, &data, type); 358 if (res == -EFAULT) 359 goto fault; 360 else if (res == 0) 361 regs->user_regs.regs[destreg] = data; 362 363 ret: 364 if (type == TYPE_SWPB) 365 trace_instruction_emulation("swpb", regs->pc); 366 else 367 trace_instruction_emulation("swp", regs->pc); 368 369 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n", 370 current->comm, (unsigned long)current->pid, regs->pc); 371 372 regs->pc += 4; 373 return 0; 374 375 fault: 376 set_segfault(regs, address); 377 378 return 0; 379 } 380 381 /* 382 * Only emulate SWP/SWPB executed in ARM state/User mode. 383 * The kernel must be SWP free and SWP{B} does not exist in Thumb. 384 */ 385 static struct undef_hook swp_hooks[] = { 386 { 387 .instr_mask = 0x0fb00ff0, 388 .instr_val = 0x01000090, 389 .pstate_mask = COMPAT_PSR_MODE_MASK, 390 .pstate_val = COMPAT_PSR_MODE_USR, 391 .fn = swp_handler 392 }, 393 { } 394 }; 395 396 static struct insn_emulation_ops swp_ops = { 397 .name = "swp", 398 .status = INSN_OBSOLETE, 399 .hooks = swp_hooks, 400 .set_hw_mode = NULL, 401 }; 402 403 static int cp15barrier_handler(struct pt_regs *regs, u32 instr) 404 { 405 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); 406 407 switch (arm_check_condition(instr, regs->pstate)) { 408 case ARM_OPCODE_CONDTEST_PASS: 409 break; 410 case ARM_OPCODE_CONDTEST_FAIL: 411 /* Condition failed - return to next instruction */ 412 goto ret; 413 case ARM_OPCODE_CONDTEST_UNCOND: 414 /* If unconditional encoding - not a barrier instruction */ 415 return -EFAULT; 416 default: 417 return -EINVAL; 418 } 419 420 switch (aarch32_insn_mcr_extract_crm(instr)) { 421 case 10: 422 /* 423 * dmb - mcr p15, 0, Rt, c7, c10, 5 424 * dsb - mcr p15, 0, Rt, c7, c10, 4 425 */ 426 if (aarch32_insn_mcr_extract_opc2(instr) == 5) { 427 dmb(sy); 428 trace_instruction_emulation( 429 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc); 430 } else { 431 dsb(sy); 432 trace_instruction_emulation( 433 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc); 434 } 435 break; 436 case 5: 437 /* 438 * isb - mcr p15, 0, Rt, c7, c5, 4 439 * 440 * Taking an exception or returning from one acts as an 441 * instruction barrier. So no explicit barrier needed here. 442 */ 443 trace_instruction_emulation( 444 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc); 445 break; 446 } 447 448 ret: 449 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n", 450 current->comm, (unsigned long)current->pid, regs->pc); 451 452 regs->pc += 4; 453 return 0; 454 } 455 456 #define SCTLR_EL1_CP15BEN (1 << 5) 457 458 static inline void config_sctlr_el1(u32 clear, u32 set) 459 { 460 u32 val; 461 462 asm volatile("mrs %0, sctlr_el1" : "=r" (val)); 463 val &= ~clear; 464 val |= set; 465 asm volatile("msr sctlr_el1, %0" : : "r" (val)); 466 } 467 468 static void enable_cp15_ben(void *info) 469 { 470 config_sctlr_el1(0, SCTLR_EL1_CP15BEN); 471 } 472 473 static void disable_cp15_ben(void *info) 474 { 475 config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); 476 } 477 478 static int cpu_hotplug_notify(struct notifier_block *b, 479 unsigned long action, void *hcpu) 480 { 481 switch (action) { 482 case CPU_STARTING: 483 case CPU_STARTING_FROZEN: 484 enable_cp15_ben(NULL); 485 return NOTIFY_DONE; 486 case CPU_DYING: 487 case CPU_DYING_FROZEN: 488 disable_cp15_ben(NULL); 489 return NOTIFY_DONE; 490 } 491 492 return NOTIFY_OK; 493 } 494 495 static struct notifier_block cpu_hotplug_notifier = { 496 .notifier_call = cpu_hotplug_notify, 497 }; 498 499 static int cp15_barrier_set_hw_mode(bool enable) 500 { 501 if (enable) { 502 register_cpu_notifier(&cpu_hotplug_notifier); 503 on_each_cpu(enable_cp15_ben, NULL, true); 504 } else { 505 unregister_cpu_notifier(&cpu_hotplug_notifier); 506 on_each_cpu(disable_cp15_ben, NULL, true); 507 } 508 509 return true; 510 } 511 512 static struct undef_hook cp15_barrier_hooks[] = { 513 { 514 .instr_mask = 0x0fff0fdf, 515 .instr_val = 0x0e070f9a, 516 .pstate_mask = COMPAT_PSR_MODE_MASK, 517 .pstate_val = COMPAT_PSR_MODE_USR, 518 .fn = cp15barrier_handler, 519 }, 520 { 521 .instr_mask = 0x0fff0fff, 522 .instr_val = 0x0e070f95, 523 .pstate_mask = COMPAT_PSR_MODE_MASK, 524 .pstate_val = COMPAT_PSR_MODE_USR, 525 .fn = cp15barrier_handler, 526 }, 527 { } 528 }; 529 530 static struct insn_emulation_ops cp15_barrier_ops = { 531 .name = "cp15_barrier", 532 .status = INSN_DEPRECATED, 533 .hooks = cp15_barrier_hooks, 534 .set_hw_mode = cp15_barrier_set_hw_mode, 535 }; 536 537 /* 538 * Invoked as late_initcall, since not needed before init spawned. 539 */ 540 static int __init armv8_deprecated_init(void) 541 { 542 if (IS_ENABLED(CONFIG_SWP_EMULATION)) 543 register_insn_emulation(&swp_ops); 544 545 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) 546 register_insn_emulation(&cp15_barrier_ops); 547 548 register_insn_emulation_sysctl(ctl_abi); 549 550 return 0; 551 } 552 553 late_initcall(armv8_deprecated_init); 554