1 /* 2 * linux/arch/arm/vfp/vfpmodule.c 3 * 4 * Copyright (C) 2004 ARM Limited. 5 * Written by Deep Blue Solutions Limited. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/types.h> 12 #include <linux/cpu.h> 13 #include <linux/cpu_pm.h> 14 #include <linux/hardirq.h> 15 #include <linux/kernel.h> 16 #include <linux/notifier.h> 17 #include <linux/signal.h> 18 #include <linux/sched/signal.h> 19 #include <linux/smp.h> 20 #include <linux/init.h> 21 #include <linux/uaccess.h> 22 #include <linux/user.h> 23 #include <linux/export.h> 24 25 #include <asm/cp15.h> 26 #include <asm/cputype.h> 27 #include <asm/system_info.h> 28 #include <asm/thread_notify.h> 29 #include <asm/vfp.h> 30 31 #include "vfpinstr.h" 32 #include "vfp.h" 33 34 /* 35 * Our undef handlers (in entry.S) 36 */ 37 asmlinkage void vfp_testing_entry(void); 38 asmlinkage void vfp_support_entry(void); 39 asmlinkage void vfp_null_entry(void); 40 41 asmlinkage void (*vfp_vector)(void) = vfp_null_entry; 42 43 /* 44 * Dual-use variable. 45 * Used in startup: set to non-zero if VFP checks fail 46 * After startup, holds VFP architecture 47 */ 48 unsigned int VFP_arch; 49 50 /* 51 * The pointer to the vfpstate structure of the thread which currently 52 * owns the context held in the VFP hardware, or NULL if the hardware 53 * context is invalid. 54 * 55 * For UP, this is sufficient to tell which thread owns the VFP context. 56 * However, for SMP, we also need to check the CPU number stored in the 57 * saved state too to catch migrations. 58 */ 59 union vfp_state *vfp_current_hw_state[NR_CPUS]; 60 61 /* 62 * Is 'thread's most up to date state stored in this CPUs hardware? 63 * Must be called from non-preemptible context. 64 */ 65 static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread) 66 { 67 #ifdef CONFIG_SMP 68 if (thread->vfpstate.hard.cpu != cpu) 69 return false; 70 #endif 71 return vfp_current_hw_state[cpu] == &thread->vfpstate; 72 } 73 74 /* 75 * Force a reload of the VFP context from the thread structure. We do 76 * this by ensuring that access to the VFP hardware is disabled, and 77 * clear vfp_current_hw_state. Must be called from non-preemptible context. 78 */ 79 static void vfp_force_reload(unsigned int cpu, struct thread_info *thread) 80 { 81 if (vfp_state_in_hw(cpu, thread)) { 82 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 83 vfp_current_hw_state[cpu] = NULL; 84 } 85 #ifdef CONFIG_SMP 86 thread->vfpstate.hard.cpu = NR_CPUS; 87 #endif 88 } 89 90 /* 91 * Per-thread VFP initialization. 92 */ 93 static void vfp_thread_flush(struct thread_info *thread) 94 { 95 union vfp_state *vfp = &thread->vfpstate; 96 unsigned int cpu; 97 98 /* 99 * Disable VFP to ensure we initialize it first. We must ensure 100 * that the modification of vfp_current_hw_state[] and hardware 101 * disable are done for the same CPU and without preemption. 102 * 103 * Do this first to ensure that preemption won't overwrite our 104 * state saving should access to the VFP be enabled at this point. 105 */ 106 cpu = get_cpu(); 107 if (vfp_current_hw_state[cpu] == vfp) 108 vfp_current_hw_state[cpu] = NULL; 109 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 110 put_cpu(); 111 112 memset(vfp, 0, sizeof(union vfp_state)); 113 114 vfp->hard.fpexc = FPEXC_EN; 115 vfp->hard.fpscr = FPSCR_ROUND_NEAREST; 116 #ifdef CONFIG_SMP 117 vfp->hard.cpu = NR_CPUS; 118 #endif 119 } 120 121 static void vfp_thread_exit(struct thread_info *thread) 122 { 123 /* release case: Per-thread VFP cleanup. */ 124 union vfp_state *vfp = &thread->vfpstate; 125 unsigned int cpu = get_cpu(); 126 127 if (vfp_current_hw_state[cpu] == vfp) 128 vfp_current_hw_state[cpu] = NULL; 129 put_cpu(); 130 } 131 132 static void vfp_thread_copy(struct thread_info *thread) 133 { 134 struct thread_info *parent = current_thread_info(); 135 136 vfp_sync_hwstate(parent); 137 thread->vfpstate = parent->vfpstate; 138 #ifdef CONFIG_SMP 139 thread->vfpstate.hard.cpu = NR_CPUS; 140 #endif 141 } 142 143 /* 144 * When this function is called with the following 'cmd's, the following 145 * is true while this function is being run: 146 * THREAD_NOFTIFY_SWTICH: 147 * - the previously running thread will not be scheduled onto another CPU. 148 * - the next thread to be run (v) will not be running on another CPU. 149 * - thread->cpu is the local CPU number 150 * - not preemptible as we're called in the middle of a thread switch 151 * THREAD_NOTIFY_FLUSH: 152 * - the thread (v) will be running on the local CPU, so 153 * v === current_thread_info() 154 * - thread->cpu is the local CPU number at the time it is accessed, 155 * but may change at any time. 156 * - we could be preempted if tree preempt rcu is enabled, so 157 * it is unsafe to use thread->cpu. 158 * THREAD_NOTIFY_EXIT 159 * - we could be preempted if tree preempt rcu is enabled, so 160 * it is unsafe to use thread->cpu. 161 */ 162 static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) 163 { 164 struct thread_info *thread = v; 165 u32 fpexc; 166 #ifdef CONFIG_SMP 167 unsigned int cpu; 168 #endif 169 170 switch (cmd) { 171 case THREAD_NOTIFY_SWITCH: 172 fpexc = fmrx(FPEXC); 173 174 #ifdef CONFIG_SMP 175 cpu = thread->cpu; 176 177 /* 178 * On SMP, if VFP is enabled, save the old state in 179 * case the thread migrates to a different CPU. The 180 * restoring is done lazily. 181 */ 182 if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) 183 vfp_save_state(vfp_current_hw_state[cpu], fpexc); 184 #endif 185 186 /* 187 * Always disable VFP so we can lazily save/restore the 188 * old state. 189 */ 190 fmxr(FPEXC, fpexc & ~FPEXC_EN); 191 break; 192 193 case THREAD_NOTIFY_FLUSH: 194 vfp_thread_flush(thread); 195 break; 196 197 case THREAD_NOTIFY_EXIT: 198 vfp_thread_exit(thread); 199 break; 200 201 case THREAD_NOTIFY_COPY: 202 vfp_thread_copy(thread); 203 break; 204 } 205 206 return NOTIFY_DONE; 207 } 208 209 static struct notifier_block vfp_notifier_block = { 210 .notifier_call = vfp_notifier, 211 }; 212 213 /* 214 * Raise a SIGFPE for the current process. 215 * sicode describes the signal being raised. 216 */ 217 static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs) 218 { 219 /* 220 * This is the same as NWFPE, because it's not clear what 221 * this is used for 222 */ 223 current->thread.error_code = 0; 224 current->thread.trap_no = 6; 225 226 send_sig_fault(SIGFPE, sicode, 227 (void __user *)(instruction_pointer(regs) - 4), 228 current); 229 } 230 231 static void vfp_panic(char *reason, u32 inst) 232 { 233 int i; 234 235 pr_err("VFP: Error: %s\n", reason); 236 pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n", 237 fmrx(FPEXC), fmrx(FPSCR), inst); 238 for (i = 0; i < 32; i += 2) 239 pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n", 240 i, vfp_get_float(i), i+1, vfp_get_float(i+1)); 241 } 242 243 /* 244 * Process bitmask of exception conditions. 245 */ 246 static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs) 247 { 248 int si_code = 0; 249 250 pr_debug("VFP: raising exceptions %08x\n", exceptions); 251 252 if (exceptions == VFP_EXCEPTION_ERROR) { 253 vfp_panic("unhandled bounce", inst); 254 vfp_raise_sigfpe(FPE_FLTINV, regs); 255 return; 256 } 257 258 /* 259 * If any of the status flags are set, update the FPSCR. 260 * Comparison instructions always return at least one of 261 * these flags set. 262 */ 263 if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V)) 264 fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V); 265 266 fpscr |= exceptions; 267 268 fmxr(FPSCR, fpscr); 269 270 #define RAISE(stat,en,sig) \ 271 if (exceptions & stat && fpscr & en) \ 272 si_code = sig; 273 274 /* 275 * These are arranged in priority order, least to highest. 276 */ 277 RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV); 278 RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES); 279 RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND); 280 RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF); 281 RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV); 282 283 if (si_code) 284 vfp_raise_sigfpe(si_code, regs); 285 } 286 287 /* 288 * Emulate a VFP instruction. 289 */ 290 static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) 291 { 292 u32 exceptions = VFP_EXCEPTION_ERROR; 293 294 pr_debug("VFP: emulate: INST=0x%08x SCR=0x%08x\n", inst, fpscr); 295 296 if (INST_CPRTDO(inst)) { 297 if (!INST_CPRT(inst)) { 298 /* 299 * CPDO 300 */ 301 if (vfp_single(inst)) { 302 exceptions = vfp_single_cpdo(inst, fpscr); 303 } else { 304 exceptions = vfp_double_cpdo(inst, fpscr); 305 } 306 } else { 307 /* 308 * A CPRT instruction can not appear in FPINST2, nor 309 * can it cause an exception. Therefore, we do not 310 * have to emulate it. 311 */ 312 } 313 } else { 314 /* 315 * A CPDT instruction can not appear in FPINST2, nor can 316 * it cause an exception. Therefore, we do not have to 317 * emulate it. 318 */ 319 } 320 return exceptions & ~VFP_NAN_FLAG; 321 } 322 323 /* 324 * Package up a bounce condition. 325 */ 326 void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) 327 { 328 u32 fpscr, orig_fpscr, fpsid, exceptions; 329 330 pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc); 331 332 /* 333 * At this point, FPEXC can have the following configuration: 334 * 335 * EX DEX IXE 336 * 0 1 x - synchronous exception 337 * 1 x 0 - asynchronous exception 338 * 1 x 1 - sychronous on VFP subarch 1 and asynchronous on later 339 * 0 0 1 - synchronous on VFP9 (non-standard subarch 1 340 * implementation), undefined otherwise 341 * 342 * Clear various bits and enable access to the VFP so we can 343 * handle the bounce. 344 */ 345 fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK)); 346 347 fpsid = fmrx(FPSID); 348 orig_fpscr = fpscr = fmrx(FPSCR); 349 350 /* 351 * Check for the special VFP subarch 1 and FPSCR.IXE bit case 352 */ 353 if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT) 354 && (fpscr & FPSCR_IXE)) { 355 /* 356 * Synchronous exception, emulate the trigger instruction 357 */ 358 goto emulate; 359 } 360 361 if (fpexc & FPEXC_EX) { 362 #ifndef CONFIG_CPU_FEROCEON 363 /* 364 * Asynchronous exception. The instruction is read from FPINST 365 * and the interrupted instruction has to be restarted. 366 */ 367 trigger = fmrx(FPINST); 368 regs->ARM_pc -= 4; 369 #endif 370 } else if (!(fpexc & FPEXC_DEX)) { 371 /* 372 * Illegal combination of bits. It can be caused by an 373 * unallocated VFP instruction but with FPSCR.IXE set and not 374 * on VFP subarch 1. 375 */ 376 vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs); 377 goto exit; 378 } 379 380 /* 381 * Modify fpscr to indicate the number of iterations remaining. 382 * If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates 383 * whether FPEXC.VECITR or FPSCR.LEN is used. 384 */ 385 if (fpexc & (FPEXC_EX | FPEXC_VV)) { 386 u32 len; 387 388 len = fpexc + (1 << FPEXC_LENGTH_BIT); 389 390 fpscr &= ~FPSCR_LENGTH_MASK; 391 fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT); 392 } 393 394 /* 395 * Handle the first FP instruction. We used to take note of the 396 * FPEXC bounce reason, but this appears to be unreliable. 397 * Emulate the bounced instruction instead. 398 */ 399 exceptions = vfp_emulate_instruction(trigger, fpscr, regs); 400 if (exceptions) 401 vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); 402 403 /* 404 * If there isn't a second FP instruction, exit now. Note that 405 * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1. 406 */ 407 if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V)) 408 goto exit; 409 410 /* 411 * The barrier() here prevents fpinst2 being read 412 * before the condition above. 413 */ 414 barrier(); 415 trigger = fmrx(FPINST2); 416 417 emulate: 418 exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs); 419 if (exceptions) 420 vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); 421 exit: 422 preempt_enable(); 423 } 424 425 static void vfp_enable(void *unused) 426 { 427 u32 access; 428 429 BUG_ON(preemptible()); 430 access = get_copro_access(); 431 432 /* 433 * Enable full access to VFP (cp10 and cp11) 434 */ 435 set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11)); 436 } 437 438 /* Called by platforms on which we want to disable VFP because it may not be 439 * present on all CPUs within a SMP complex. Needs to be called prior to 440 * vfp_init(). 441 */ 442 void vfp_disable(void) 443 { 444 if (VFP_arch) { 445 pr_debug("%s: should be called prior to vfp_init\n", __func__); 446 return; 447 } 448 VFP_arch = 1; 449 } 450 451 #ifdef CONFIG_CPU_PM 452 static int vfp_pm_suspend(void) 453 { 454 struct thread_info *ti = current_thread_info(); 455 u32 fpexc = fmrx(FPEXC); 456 457 /* if vfp is on, then save state for resumption */ 458 if (fpexc & FPEXC_EN) { 459 pr_debug("%s: saving vfp state\n", __func__); 460 vfp_save_state(&ti->vfpstate, fpexc); 461 462 /* disable, just in case */ 463 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 464 } else if (vfp_current_hw_state[ti->cpu]) { 465 #ifndef CONFIG_SMP 466 fmxr(FPEXC, fpexc | FPEXC_EN); 467 vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc); 468 fmxr(FPEXC, fpexc); 469 #endif 470 } 471 472 /* clear any information we had about last context state */ 473 vfp_current_hw_state[ti->cpu] = NULL; 474 475 return 0; 476 } 477 478 static void vfp_pm_resume(void) 479 { 480 /* ensure we have access to the vfp */ 481 vfp_enable(NULL); 482 483 /* and disable it to ensure the next usage restores the state */ 484 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 485 } 486 487 static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, 488 void *v) 489 { 490 switch (cmd) { 491 case CPU_PM_ENTER: 492 vfp_pm_suspend(); 493 break; 494 case CPU_PM_ENTER_FAILED: 495 case CPU_PM_EXIT: 496 vfp_pm_resume(); 497 break; 498 } 499 return NOTIFY_OK; 500 } 501 502 static struct notifier_block vfp_cpu_pm_notifier_block = { 503 .notifier_call = vfp_cpu_pm_notifier, 504 }; 505 506 static void vfp_pm_init(void) 507 { 508 cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block); 509 } 510 511 #else 512 static inline void vfp_pm_init(void) { } 513 #endif /* CONFIG_CPU_PM */ 514 515 /* 516 * Ensure that the VFP state stored in 'thread->vfpstate' is up to date 517 * with the hardware state. 518 */ 519 void vfp_sync_hwstate(struct thread_info *thread) 520 { 521 unsigned int cpu = get_cpu(); 522 523 if (vfp_state_in_hw(cpu, thread)) { 524 u32 fpexc = fmrx(FPEXC); 525 526 /* 527 * Save the last VFP state on this CPU. 528 */ 529 fmxr(FPEXC, fpexc | FPEXC_EN); 530 vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN); 531 fmxr(FPEXC, fpexc); 532 } 533 534 put_cpu(); 535 } 536 537 /* Ensure that the thread reloads the hardware VFP state on the next use. */ 538 void vfp_flush_hwstate(struct thread_info *thread) 539 { 540 unsigned int cpu = get_cpu(); 541 542 vfp_force_reload(cpu, thread); 543 544 put_cpu(); 545 } 546 547 /* 548 * Save the current VFP state into the provided structures and prepare 549 * for entry into a new function (signal handler). 550 */ 551 int vfp_preserve_user_clear_hwstate(struct user_vfp *ufp, 552 struct user_vfp_exc *ufp_exc) 553 { 554 struct thread_info *thread = current_thread_info(); 555 struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; 556 557 /* Ensure that the saved hwstate is up-to-date. */ 558 vfp_sync_hwstate(thread); 559 560 /* 561 * Copy the floating point registers. There can be unused 562 * registers see asm/hwcap.h for details. 563 */ 564 memcpy(&ufp->fpregs, &hwstate->fpregs, sizeof(hwstate->fpregs)); 565 566 /* 567 * Copy the status and control register. 568 */ 569 ufp->fpscr = hwstate->fpscr; 570 571 /* 572 * Copy the exception registers. 573 */ 574 ufp_exc->fpexc = hwstate->fpexc; 575 ufp_exc->fpinst = hwstate->fpinst; 576 ufp_exc->fpinst2 = hwstate->fpinst2; 577 578 /* Ensure that VFP is disabled. */ 579 vfp_flush_hwstate(thread); 580 581 /* 582 * As per the PCS, clear the length and stride bits for function 583 * entry. 584 */ 585 hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK); 586 return 0; 587 } 588 589 /* Sanitise and restore the current VFP state from the provided structures. */ 590 int vfp_restore_user_hwstate(struct user_vfp *ufp, struct user_vfp_exc *ufp_exc) 591 { 592 struct thread_info *thread = current_thread_info(); 593 struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; 594 unsigned long fpexc; 595 596 /* Disable VFP to avoid corrupting the new thread state. */ 597 vfp_flush_hwstate(thread); 598 599 /* 600 * Copy the floating point registers. There can be unused 601 * registers see asm/hwcap.h for details. 602 */ 603 memcpy(&hwstate->fpregs, &ufp->fpregs, sizeof(hwstate->fpregs)); 604 /* 605 * Copy the status and control register. 606 */ 607 hwstate->fpscr = ufp->fpscr; 608 609 /* 610 * Sanitise and restore the exception registers. 611 */ 612 fpexc = ufp_exc->fpexc; 613 614 /* Ensure the VFP is enabled. */ 615 fpexc |= FPEXC_EN; 616 617 /* Ensure FPINST2 is invalid and the exception flag is cleared. */ 618 fpexc &= ~(FPEXC_EX | FPEXC_FP2V); 619 hwstate->fpexc = fpexc; 620 621 hwstate->fpinst = ufp_exc->fpinst; 622 hwstate->fpinst2 = ufp_exc->fpinst2; 623 624 return 0; 625 } 626 627 /* 628 * VFP hardware can lose all context when a CPU goes offline. 629 * As we will be running in SMP mode with CPU hotplug, we will save the 630 * hardware state at every thread switch. We clear our held state when 631 * a CPU has been killed, indicating that the VFP hardware doesn't contain 632 * a threads VFP state. When a CPU starts up, we re-enable access to the 633 * VFP hardware. The callbacks below are called on the CPU which 634 * is being offlined/onlined. 635 */ 636 static int vfp_dying_cpu(unsigned int cpu) 637 { 638 vfp_current_hw_state[cpu] = NULL; 639 return 0; 640 } 641 642 static int vfp_starting_cpu(unsigned int unused) 643 { 644 vfp_enable(NULL); 645 return 0; 646 } 647 648 void vfp_kmode_exception(void) 649 { 650 /* 651 * If we reach this point, a floating point exception has been raised 652 * while running in kernel mode. If the NEON/VFP unit was enabled at the 653 * time, it means a VFP instruction has been issued that requires 654 * software assistance to complete, something which is not currently 655 * supported in kernel mode. 656 * If the NEON/VFP unit was disabled, and the location pointed to below 657 * is properly preceded by a call to kernel_neon_begin(), something has 658 * caused the task to be scheduled out and back in again. In this case, 659 * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should 660 * be helpful in localizing the problem. 661 */ 662 if (fmrx(FPEXC) & FPEXC_EN) 663 pr_crit("BUG: unsupported FP instruction in kernel mode\n"); 664 else 665 pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n"); 666 } 667 668 #ifdef CONFIG_KERNEL_MODE_NEON 669 670 /* 671 * Kernel-side NEON support functions 672 */ 673 void kernel_neon_begin(void) 674 { 675 struct thread_info *thread = current_thread_info(); 676 unsigned int cpu; 677 u32 fpexc; 678 679 /* 680 * Kernel mode NEON is only allowed outside of interrupt context 681 * with preemption disabled. This will make sure that the kernel 682 * mode NEON register contents never need to be preserved. 683 */ 684 BUG_ON(in_interrupt()); 685 cpu = get_cpu(); 686 687 fpexc = fmrx(FPEXC) | FPEXC_EN; 688 fmxr(FPEXC, fpexc); 689 690 /* 691 * Save the userland NEON/VFP state. Under UP, 692 * the owner could be a task other than 'current' 693 */ 694 if (vfp_state_in_hw(cpu, thread)) 695 vfp_save_state(&thread->vfpstate, fpexc); 696 #ifndef CONFIG_SMP 697 else if (vfp_current_hw_state[cpu] != NULL) 698 vfp_save_state(vfp_current_hw_state[cpu], fpexc); 699 #endif 700 vfp_current_hw_state[cpu] = NULL; 701 } 702 EXPORT_SYMBOL(kernel_neon_begin); 703 704 void kernel_neon_end(void) 705 { 706 /* Disable the NEON/VFP unit. */ 707 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); 708 put_cpu(); 709 } 710 EXPORT_SYMBOL(kernel_neon_end); 711 712 #endif /* CONFIG_KERNEL_MODE_NEON */ 713 714 /* 715 * VFP support code initialisation. 716 */ 717 static int __init vfp_init(void) 718 { 719 unsigned int vfpsid; 720 unsigned int cpu_arch = cpu_architecture(); 721 722 /* 723 * Enable the access to the VFP on all online CPUs so the 724 * following test on FPSID will succeed. 725 */ 726 if (cpu_arch >= CPU_ARCH_ARMv6) 727 on_each_cpu(vfp_enable, NULL, 1); 728 729 /* 730 * First check that there is a VFP that we can use. 731 * The handler is already setup to just log calls, so 732 * we just need to read the VFPSID register. 733 */ 734 vfp_vector = vfp_testing_entry; 735 barrier(); 736 vfpsid = fmrx(FPSID); 737 barrier(); 738 vfp_vector = vfp_null_entry; 739 740 pr_info("VFP support v0.3: "); 741 if (VFP_arch) { 742 pr_cont("not present\n"); 743 return 0; 744 /* Extract the architecture on CPUID scheme */ 745 } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) { 746 VFP_arch = vfpsid & FPSID_CPUID_ARCH_MASK; 747 VFP_arch >>= FPSID_ARCH_BIT; 748 /* 749 * Check for the presence of the Advanced SIMD 750 * load/store instructions, integer and single 751 * precision floating point operations. Only check 752 * for NEON if the hardware has the MVFR registers. 753 */ 754 if (IS_ENABLED(CONFIG_NEON) && 755 (fmrx(MVFR1) & 0x000fff00) == 0x00011100) 756 elf_hwcap |= HWCAP_NEON; 757 758 if (IS_ENABLED(CONFIG_VFPv3)) { 759 u32 mvfr0 = fmrx(MVFR0); 760 if (((mvfr0 & MVFR0_DP_MASK) >> MVFR0_DP_BIT) == 0x2 || 761 ((mvfr0 & MVFR0_SP_MASK) >> MVFR0_SP_BIT) == 0x2) { 762 elf_hwcap |= HWCAP_VFPv3; 763 /* 764 * Check for VFPv3 D16 and VFPv4 D16. CPUs in 765 * this configuration only have 16 x 64bit 766 * registers. 767 */ 768 if ((mvfr0 & MVFR0_A_SIMD_MASK) == 1) 769 /* also v4-D16 */ 770 elf_hwcap |= HWCAP_VFPv3D16; 771 else 772 elf_hwcap |= HWCAP_VFPD32; 773 } 774 775 if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000) 776 elf_hwcap |= HWCAP_VFPv4; 777 } 778 /* Extract the architecture version on pre-cpuid scheme */ 779 } else { 780 if (vfpsid & FPSID_NODOUBLE) { 781 pr_cont("no double precision support\n"); 782 return 0; 783 } 784 785 VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT; 786 } 787 788 cpuhp_setup_state_nocalls(CPUHP_AP_ARM_VFP_STARTING, 789 "arm/vfp:starting", vfp_starting_cpu, 790 vfp_dying_cpu); 791 792 vfp_vector = vfp_support_entry; 793 794 thread_register_notifier(&vfp_notifier_block); 795 vfp_pm_init(); 796 797 /* 798 * We detected VFP, and the support code is 799 * in place; report VFP support to userspace. 800 */ 801 elf_hwcap |= HWCAP_VFP; 802 803 pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n", 804 (vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT, 805 VFP_arch, 806 (vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT, 807 (vfpsid & FPSID_VARIANT_MASK) >> FPSID_VARIANT_BIT, 808 (vfpsid & FPSID_REV_MASK) >> FPSID_REV_BIT); 809 810 return 0; 811 } 812 813 core_initcall(vfp_init); 814