1 /* 2 * FP/SIMD context switching and fault handling 3 * 4 * Copyright (C) 2012 ARM Ltd. 5 * Author: Catalin Marinas <catalin.marinas@arm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include <linux/bitmap.h> 21 #include <linux/bitops.h> 22 #include <linux/bottom_half.h> 23 #include <linux/bug.h> 24 #include <linux/cache.h> 25 #include <linux/compat.h> 26 #include <linux/cpu.h> 27 #include <linux/cpu_pm.h> 28 #include <linux/kernel.h> 29 #include <linux/linkage.h> 30 #include <linux/irqflags.h> 31 #include <linux/init.h> 32 #include <linux/percpu.h> 33 #include <linux/prctl.h> 34 #include <linux/preempt.h> 35 #include <linux/ptrace.h> 36 #include <linux/sched/signal.h> 37 #include <linux/sched/task_stack.h> 38 #include <linux/signal.h> 39 #include <linux/slab.h> 40 #include <linux/stddef.h> 41 #include <linux/sysctl.h> 42 #include <linux/swab.h> 43 44 #include <asm/esr.h> 45 #include <asm/fpsimd.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cputype.h> 48 #include <asm/processor.h> 49 #include <asm/simd.h> 50 #include <asm/sigcontext.h> 51 #include <asm/sysreg.h> 52 #include <asm/traps.h> 53 #include <asm/virt.h> 54 55 #define FPEXC_IOF (1 << 0) 56 #define FPEXC_DZF (1 << 1) 57 #define FPEXC_OFF (1 << 2) 58 #define FPEXC_UFF (1 << 3) 59 #define FPEXC_IXF (1 << 4) 60 #define FPEXC_IDF (1 << 7) 61 62 /* 63 * (Note: in this discussion, statements about FPSIMD apply equally to SVE.) 64 * 65 * In order to reduce the number of times the FPSIMD state is needlessly saved 66 * and restored, we need to keep track of two things: 67 * (a) for each task, we need to remember which CPU was the last one to have 68 * the task's FPSIMD state loaded into its FPSIMD registers; 69 * (b) for each CPU, we need to remember which task's userland FPSIMD state has 70 * been loaded into its FPSIMD registers most recently, or whether it has 71 * been used to perform kernel mode NEON in the meantime. 72 * 73 * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to 74 * the id of the current CPU every time the state is loaded onto a CPU. For (b), 75 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the 76 * address of the userland FPSIMD state of the task that was loaded onto the CPU 77 * the most recently, or NULL if kernel mode NEON has been performed after that. 78 * 79 * With this in place, we no longer have to restore the next FPSIMD state right 80 * when switching between tasks. Instead, we can defer this check to userland 81 * resume, at which time we verify whether the CPU's fpsimd_last_state and the 82 * task's fpsimd_cpu are still mutually in sync. If this is the case, we 83 * can omit the FPSIMD restore. 84 * 85 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to 86 * indicate whether or not the userland FPSIMD state of the current task is 87 * present in the registers. The flag is set unless the FPSIMD registers of this 88 * CPU currently contain the most recent userland FPSIMD state of the current 89 * task. 90 * 91 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may 92 * save the task's FPSIMD context back to task_struct from softirq context. 93 * To prevent this from racing with the manipulation of the task's FPSIMD state 94 * from task context and thereby corrupting the state, it is necessary to 95 * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE 96 * flag with local_bh_disable() unless softirqs are already masked. 97 * 98 * For a certain task, the sequence may look something like this: 99 * - the task gets scheduled in; if both the task's fpsimd_cpu field 100 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu 101 * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is 102 * cleared, otherwise it is set; 103 * 104 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's 105 * userland FPSIMD state is copied from memory to the registers, the task's 106 * fpsimd_cpu field is set to the id of the current CPU, the current 107 * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the 108 * TIF_FOREIGN_FPSTATE flag is cleared; 109 * 110 * - the task executes an ordinary syscall; upon return to userland, the 111 * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is 112 * restored; 113 * 114 * - the task executes a syscall which executes some NEON instructions; this is 115 * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD 116 * register contents to memory, clears the fpsimd_last_state per-cpu variable 117 * and sets the TIF_FOREIGN_FPSTATE flag; 118 * 119 * - the task gets preempted after kernel_neon_end() is called; as we have not 120 * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so 121 * whatever is in the FPSIMD registers is not saved to memory, but discarded. 122 */ 123 struct fpsimd_last_state_struct { 124 struct user_fpsimd_state *st; 125 void *sve_state; 126 unsigned int sve_vl; 127 }; 128 129 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state); 130 131 /* Default VL for tasks that don't set it explicitly: */ 132 static int sve_default_vl = -1; 133 134 #ifdef CONFIG_ARM64_SVE 135 136 /* Maximum supported vector length across all CPUs (initially poisoned) */ 137 int __ro_after_init sve_max_vl = SVE_VL_MIN; 138 int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN; 139 140 /* 141 * Set of available vector lengths, 142 * where length vq encoded as bit __vq_to_bit(vq): 143 */ 144 __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); 145 /* Set of vector lengths present on at least one cpu: */ 146 static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX); 147 148 static void __percpu *efi_sve_state; 149 150 #else /* ! CONFIG_ARM64_SVE */ 151 152 /* Dummy declaration for code that will be optimised out: */ 153 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); 154 extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX); 155 extern void __percpu *efi_sve_state; 156 157 #endif /* ! CONFIG_ARM64_SVE */ 158 159 /* 160 * Call __sve_free() directly only if you know task can't be scheduled 161 * or preempted. 162 */ 163 static void __sve_free(struct task_struct *task) 164 { 165 kfree(task->thread.sve_state); 166 task->thread.sve_state = NULL; 167 } 168 169 static void sve_free(struct task_struct *task) 170 { 171 WARN_ON(test_tsk_thread_flag(task, TIF_SVE)); 172 173 __sve_free(task); 174 } 175 176 /* 177 * TIF_SVE controls whether a task can use SVE without trapping while 178 * in userspace, and also the way a task's FPSIMD/SVE state is stored 179 * in thread_struct. 180 * 181 * The kernel uses this flag to track whether a user task is actively 182 * using SVE, and therefore whether full SVE register state needs to 183 * be tracked. If not, the cheaper FPSIMD context handling code can 184 * be used instead of the more costly SVE equivalents. 185 * 186 * * TIF_SVE set: 187 * 188 * The task can execute SVE instructions while in userspace without 189 * trapping to the kernel. 190 * 191 * When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the 192 * corresponding Zn), P0-P15 and FFR are encoded in in 193 * task->thread.sve_state, formatted appropriately for vector 194 * length task->thread.sve_vl. 195 * 196 * task->thread.sve_state must point to a valid buffer at least 197 * sve_state_size(task) bytes in size. 198 * 199 * During any syscall, the kernel may optionally clear TIF_SVE and 200 * discard the vector state except for the FPSIMD subset. 201 * 202 * * TIF_SVE clear: 203 * 204 * An attempt by the user task to execute an SVE instruction causes 205 * do_sve_acc() to be called, which does some preparation and then 206 * sets TIF_SVE. 207 * 208 * When stored, FPSIMD registers V0-V31 are encoded in 209 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are 210 * logically zero but not stored anywhere; P0-P15 and FFR are not 211 * stored and have unspecified values from userspace's point of 212 * view. For hygiene purposes, the kernel zeroes them on next use, 213 * but userspace is discouraged from relying on this. 214 * 215 * task->thread.sve_state does not need to be non-NULL, valid or any 216 * particular size: it must not be dereferenced. 217 * 218 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state 219 * irrespective of whether TIF_SVE is clear or set, since these are 220 * not vector length dependent. 221 */ 222 223 /* 224 * Update current's FPSIMD/SVE registers from thread_struct. 225 * 226 * This function should be called only when the FPSIMD/SVE state in 227 * thread_struct is known to be up to date, when preparing to enter 228 * userspace. 229 * 230 * Softirqs (and preemption) must be disabled. 231 */ 232 static void task_fpsimd_load(void) 233 { 234 WARN_ON(!in_softirq() && !irqs_disabled()); 235 236 if (system_supports_sve() && test_thread_flag(TIF_SVE)) 237 sve_load_state(sve_pffr(¤t->thread), 238 ¤t->thread.uw.fpsimd_state.fpsr, 239 sve_vq_from_vl(current->thread.sve_vl) - 1); 240 else 241 fpsimd_load_state(¤t->thread.uw.fpsimd_state); 242 } 243 244 /* 245 * Ensure FPSIMD/SVE storage in memory for the loaded context is up to 246 * date with respect to the CPU registers. 247 * 248 * Softirqs (and preemption) must be disabled. 249 */ 250 void fpsimd_save(void) 251 { 252 struct fpsimd_last_state_struct const *last = 253 this_cpu_ptr(&fpsimd_last_state); 254 /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */ 255 256 WARN_ON(!in_softirq() && !irqs_disabled()); 257 258 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { 259 if (system_supports_sve() && test_thread_flag(TIF_SVE)) { 260 if (WARN_ON(sve_get_vl() != last->sve_vl)) { 261 /* 262 * Can't save the user regs, so current would 263 * re-enter user with corrupt state. 264 * There's no way to recover, so kill it: 265 */ 266 force_signal_inject(SIGKILL, SI_KERNEL, 0); 267 return; 268 } 269 270 sve_save_state((char *)last->sve_state + 271 sve_ffr_offset(last->sve_vl), 272 &last->st->fpsr); 273 } else 274 fpsimd_save_state(last->st); 275 } 276 } 277 278 /* 279 * All vector length selection from userspace comes through here. 280 * We're on a slow path, so some sanity-checks are included. 281 * If things go wrong there's a bug somewhere, but try to fall back to a 282 * safe choice. 283 */ 284 static unsigned int find_supported_vector_length(unsigned int vl) 285 { 286 int bit; 287 int max_vl = sve_max_vl; 288 289 if (WARN_ON(!sve_vl_valid(vl))) 290 vl = SVE_VL_MIN; 291 292 if (WARN_ON(!sve_vl_valid(max_vl))) 293 max_vl = SVE_VL_MIN; 294 295 if (vl > max_vl) 296 vl = max_vl; 297 298 bit = find_next_bit(sve_vq_map, SVE_VQ_MAX, 299 __vq_to_bit(sve_vq_from_vl(vl))); 300 return sve_vl_from_vq(__bit_to_vq(bit)); 301 } 302 303 #ifdef CONFIG_SYSCTL 304 305 static int sve_proc_do_default_vl(struct ctl_table *table, int write, 306 void __user *buffer, size_t *lenp, 307 loff_t *ppos) 308 { 309 int ret; 310 int vl = sve_default_vl; 311 struct ctl_table tmp_table = { 312 .data = &vl, 313 .maxlen = sizeof(vl), 314 }; 315 316 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); 317 if (ret || !write) 318 return ret; 319 320 /* Writing -1 has the special meaning "set to max": */ 321 if (vl == -1) 322 vl = sve_max_vl; 323 324 if (!sve_vl_valid(vl)) 325 return -EINVAL; 326 327 sve_default_vl = find_supported_vector_length(vl); 328 return 0; 329 } 330 331 static struct ctl_table sve_default_vl_table[] = { 332 { 333 .procname = "sve_default_vector_length", 334 .mode = 0644, 335 .proc_handler = sve_proc_do_default_vl, 336 }, 337 { } 338 }; 339 340 static int __init sve_sysctl_init(void) 341 { 342 if (system_supports_sve()) 343 if (!register_sysctl("abi", sve_default_vl_table)) 344 return -EINVAL; 345 346 return 0; 347 } 348 349 #else /* ! CONFIG_SYSCTL */ 350 static int __init sve_sysctl_init(void) { return 0; } 351 #endif /* ! CONFIG_SYSCTL */ 352 353 #define ZREG(sve_state, vq, n) ((char *)(sve_state) + \ 354 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET)) 355 356 #ifdef CONFIG_CPU_BIG_ENDIAN 357 static __uint128_t arm64_cpu_to_le128(__uint128_t x) 358 { 359 u64 a = swab64(x); 360 u64 b = swab64(x >> 64); 361 362 return ((__uint128_t)a << 64) | b; 363 } 364 #else 365 static __uint128_t arm64_cpu_to_le128(__uint128_t x) 366 { 367 return x; 368 } 369 #endif 370 371 #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) 372 373 /* 374 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to 375 * task->thread.sve_state. 376 * 377 * Task can be a non-runnable task, or current. In the latter case, 378 * softirqs (and preemption) must be disabled. 379 * task->thread.sve_state must point to at least sve_state_size(task) 380 * bytes of allocated kernel memory. 381 * task->thread.uw.fpsimd_state must be up to date before calling this 382 * function. 383 */ 384 static void fpsimd_to_sve(struct task_struct *task) 385 { 386 unsigned int vq; 387 void *sst = task->thread.sve_state; 388 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; 389 unsigned int i; 390 __uint128_t *p; 391 392 if (!system_supports_sve()) 393 return; 394 395 vq = sve_vq_from_vl(task->thread.sve_vl); 396 for (i = 0; i < 32; ++i) { 397 p = (__uint128_t *)ZREG(sst, vq, i); 398 *p = arm64_cpu_to_le128(fst->vregs[i]); 399 } 400 } 401 402 /* 403 * Transfer the SVE state in task->thread.sve_state to 404 * task->thread.uw.fpsimd_state. 405 * 406 * Task can be a non-runnable task, or current. In the latter case, 407 * softirqs (and preemption) must be disabled. 408 * task->thread.sve_state must point to at least sve_state_size(task) 409 * bytes of allocated kernel memory. 410 * task->thread.sve_state must be up to date before calling this function. 411 */ 412 static void sve_to_fpsimd(struct task_struct *task) 413 { 414 unsigned int vq; 415 void const *sst = task->thread.sve_state; 416 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; 417 unsigned int i; 418 __uint128_t const *p; 419 420 if (!system_supports_sve()) 421 return; 422 423 vq = sve_vq_from_vl(task->thread.sve_vl); 424 for (i = 0; i < 32; ++i) { 425 p = (__uint128_t const *)ZREG(sst, vq, i); 426 fst->vregs[i] = arm64_le128_to_cpu(*p); 427 } 428 } 429 430 #ifdef CONFIG_ARM64_SVE 431 432 /* 433 * Return how many bytes of memory are required to store the full SVE 434 * state for task, given task's currently configured vector length. 435 */ 436 size_t sve_state_size(struct task_struct const *task) 437 { 438 return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl)); 439 } 440 441 /* 442 * Ensure that task->thread.sve_state is allocated and sufficiently large. 443 * 444 * This function should be used only in preparation for replacing 445 * task->thread.sve_state with new data. The memory is always zeroed 446 * here to prevent stale data from showing through: this is done in 447 * the interest of testability and predictability: except in the 448 * do_sve_acc() case, there is no ABI requirement to hide stale data 449 * written previously be task. 450 */ 451 void sve_alloc(struct task_struct *task) 452 { 453 if (task->thread.sve_state) { 454 memset(task->thread.sve_state, 0, sve_state_size(current)); 455 return; 456 } 457 458 /* This is a small allocation (maximum ~8KB) and Should Not Fail. */ 459 task->thread.sve_state = 460 kzalloc(sve_state_size(task), GFP_KERNEL); 461 462 /* 463 * If future SVE revisions can have larger vectors though, 464 * this may cease to be true: 465 */ 466 BUG_ON(!task->thread.sve_state); 467 } 468 469 470 /* 471 * Ensure that task->thread.sve_state is up to date with respect to 472 * the user task, irrespective of when SVE is in use or not. 473 * 474 * This should only be called by ptrace. task must be non-runnable. 475 * task->thread.sve_state must point to at least sve_state_size(task) 476 * bytes of allocated kernel memory. 477 */ 478 void fpsimd_sync_to_sve(struct task_struct *task) 479 { 480 if (!test_tsk_thread_flag(task, TIF_SVE)) 481 fpsimd_to_sve(task); 482 } 483 484 /* 485 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to 486 * the user task, irrespective of whether SVE is in use or not. 487 * 488 * This should only be called by ptrace. task must be non-runnable. 489 * task->thread.sve_state must point to at least sve_state_size(task) 490 * bytes of allocated kernel memory. 491 */ 492 void sve_sync_to_fpsimd(struct task_struct *task) 493 { 494 if (test_tsk_thread_flag(task, TIF_SVE)) 495 sve_to_fpsimd(task); 496 } 497 498 /* 499 * Ensure that task->thread.sve_state is up to date with respect to 500 * the task->thread.uw.fpsimd_state. 501 * 502 * This should only be called by ptrace to merge new FPSIMD register 503 * values into a task for which SVE is currently active. 504 * task must be non-runnable. 505 * task->thread.sve_state must point to at least sve_state_size(task) 506 * bytes of allocated kernel memory. 507 * task->thread.uw.fpsimd_state must already have been initialised with 508 * the new FPSIMD register values to be merged in. 509 */ 510 void sve_sync_from_fpsimd_zeropad(struct task_struct *task) 511 { 512 unsigned int vq; 513 void *sst = task->thread.sve_state; 514 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; 515 unsigned int i; 516 __uint128_t *p; 517 518 if (!test_tsk_thread_flag(task, TIF_SVE)) 519 return; 520 521 vq = sve_vq_from_vl(task->thread.sve_vl); 522 523 memset(sst, 0, SVE_SIG_REGS_SIZE(vq)); 524 525 for (i = 0; i < 32; ++i) { 526 p = (__uint128_t *)ZREG(sst, vq, i); 527 *p = arm64_cpu_to_le128(fst->vregs[i]); 528 } 529 } 530 531 int sve_set_vector_length(struct task_struct *task, 532 unsigned long vl, unsigned long flags) 533 { 534 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT | 535 PR_SVE_SET_VL_ONEXEC)) 536 return -EINVAL; 537 538 if (!sve_vl_valid(vl)) 539 return -EINVAL; 540 541 /* 542 * Clamp to the maximum vector length that VL-agnostic SVE code can 543 * work with. A flag may be assigned in the future to allow setting 544 * of larger vector lengths without confusing older software. 545 */ 546 if (vl > SVE_VL_ARCH_MAX) 547 vl = SVE_VL_ARCH_MAX; 548 549 vl = find_supported_vector_length(vl); 550 551 if (flags & (PR_SVE_VL_INHERIT | 552 PR_SVE_SET_VL_ONEXEC)) 553 task->thread.sve_vl_onexec = vl; 554 else 555 /* Reset VL to system default on next exec: */ 556 task->thread.sve_vl_onexec = 0; 557 558 /* Only actually set the VL if not deferred: */ 559 if (flags & PR_SVE_SET_VL_ONEXEC) 560 goto out; 561 562 if (vl == task->thread.sve_vl) 563 goto out; 564 565 /* 566 * To ensure the FPSIMD bits of the SVE vector registers are preserved, 567 * write any live register state back to task_struct, and convert to a 568 * non-SVE thread. 569 */ 570 if (task == current) { 571 local_bh_disable(); 572 573 fpsimd_save(); 574 } 575 576 fpsimd_flush_task_state(task); 577 if (test_and_clear_tsk_thread_flag(task, TIF_SVE)) 578 sve_to_fpsimd(task); 579 580 if (task == current) 581 local_bh_enable(); 582 583 /* 584 * Force reallocation of task SVE state to the correct size 585 * on next use: 586 */ 587 sve_free(task); 588 589 task->thread.sve_vl = vl; 590 591 out: 592 update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT, 593 flags & PR_SVE_VL_INHERIT); 594 595 return 0; 596 } 597 598 /* 599 * Encode the current vector length and flags for return. 600 * This is only required for prctl(): ptrace has separate fields 601 * 602 * flags are as for sve_set_vector_length(). 603 */ 604 static int sve_prctl_status(unsigned long flags) 605 { 606 int ret; 607 608 if (flags & PR_SVE_SET_VL_ONEXEC) 609 ret = current->thread.sve_vl_onexec; 610 else 611 ret = current->thread.sve_vl; 612 613 if (test_thread_flag(TIF_SVE_VL_INHERIT)) 614 ret |= PR_SVE_VL_INHERIT; 615 616 return ret; 617 } 618 619 /* PR_SVE_SET_VL */ 620 int sve_set_current_vl(unsigned long arg) 621 { 622 unsigned long vl, flags; 623 int ret; 624 625 vl = arg & PR_SVE_VL_LEN_MASK; 626 flags = arg & ~vl; 627 628 if (!system_supports_sve()) 629 return -EINVAL; 630 631 ret = sve_set_vector_length(current, vl, flags); 632 if (ret) 633 return ret; 634 635 return sve_prctl_status(flags); 636 } 637 638 /* PR_SVE_GET_VL */ 639 int sve_get_current_vl(void) 640 { 641 if (!system_supports_sve()) 642 return -EINVAL; 643 644 return sve_prctl_status(0); 645 } 646 647 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX)) 648 { 649 unsigned int vq, vl; 650 unsigned long zcr; 651 652 bitmap_zero(map, SVE_VQ_MAX); 653 654 zcr = ZCR_ELx_LEN_MASK; 655 zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr; 656 657 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) { 658 write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */ 659 vl = sve_get_vl(); 660 vq = sve_vq_from_vl(vl); /* skip intervening lengths */ 661 set_bit(__vq_to_bit(vq), map); 662 } 663 } 664 665 /* 666 * Initialise the set of known supported VQs for the boot CPU. 667 * This is called during kernel boot, before secondary CPUs are brought up. 668 */ 669 void __init sve_init_vq_map(void) 670 { 671 sve_probe_vqs(sve_vq_map); 672 bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX); 673 } 674 675 /* 676 * If we haven't committed to the set of supported VQs yet, filter out 677 * those not supported by the current CPU. 678 * This function is called during the bring-up of early secondary CPUs only. 679 */ 680 void sve_update_vq_map(void) 681 { 682 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 683 684 sve_probe_vqs(tmp_map); 685 bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX); 686 bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX); 687 } 688 689 /* 690 * Check whether the current CPU supports all VQs in the committed set. 691 * This function is called during the bring-up of late secondary CPUs only. 692 */ 693 int sve_verify_vq_map(void) 694 { 695 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 696 unsigned long b; 697 698 sve_probe_vqs(tmp_map); 699 700 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); 701 if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) { 702 pr_warn("SVE: cpu%d: Required vector length(s) missing\n", 703 smp_processor_id()); 704 return -EINVAL; 705 } 706 707 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available()) 708 return 0; 709 710 /* 711 * For KVM, it is necessary to ensure that this CPU doesn't 712 * support any vector length that guests may have probed as 713 * unsupported. 714 */ 715 716 /* Recover the set of supported VQs: */ 717 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); 718 /* Find VQs supported that are not globally supported: */ 719 bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX); 720 721 /* Find the lowest such VQ, if any: */ 722 b = find_last_bit(tmp_map, SVE_VQ_MAX); 723 if (b >= SVE_VQ_MAX) 724 return 0; /* no mismatches */ 725 726 /* 727 * Mismatches above sve_max_virtualisable_vl are fine, since 728 * no guest is allowed to configure ZCR_EL2.LEN to exceed this: 729 */ 730 if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) { 731 pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n", 732 smp_processor_id()); 733 return -EINVAL; 734 } 735 736 return 0; 737 } 738 739 static void __init sve_efi_setup(void) 740 { 741 if (!IS_ENABLED(CONFIG_EFI)) 742 return; 743 744 /* 745 * alloc_percpu() warns and prints a backtrace if this goes wrong. 746 * This is evidence of a crippled system and we are returning void, 747 * so no attempt is made to handle this situation here. 748 */ 749 if (!sve_vl_valid(sve_max_vl)) 750 goto fail; 751 752 efi_sve_state = __alloc_percpu( 753 SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES); 754 if (!efi_sve_state) 755 goto fail; 756 757 return; 758 759 fail: 760 panic("Cannot allocate percpu memory for EFI SVE save/restore"); 761 } 762 763 /* 764 * Enable SVE for EL1. 765 * Intended for use by the cpufeatures code during CPU boot. 766 */ 767 void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) 768 { 769 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1); 770 isb(); 771 } 772 773 /* 774 * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE 775 * vector length. 776 * 777 * Use only if SVE is present. 778 * This function clobbers the SVE vector length. 779 */ 780 u64 read_zcr_features(void) 781 { 782 u64 zcr; 783 unsigned int vq_max; 784 785 /* 786 * Set the maximum possible VL, and write zeroes to all other 787 * bits to see if they stick. 788 */ 789 sve_kernel_enable(NULL); 790 write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1); 791 792 zcr = read_sysreg_s(SYS_ZCR_EL1); 793 zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */ 794 vq_max = sve_vq_from_vl(sve_get_vl()); 795 zcr |= vq_max - 1; /* set LEN field to maximum effective value */ 796 797 return zcr; 798 } 799 800 void __init sve_setup(void) 801 { 802 u64 zcr; 803 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 804 unsigned long b; 805 806 if (!system_supports_sve()) 807 return; 808 809 /* 810 * The SVE architecture mandates support for 128-bit vectors, 811 * so sve_vq_map must have at least SVE_VQ_MIN set. 812 * If something went wrong, at least try to patch it up: 813 */ 814 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map))) 815 set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map); 816 817 zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1); 818 sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1); 819 820 /* 821 * Sanity-check that the max VL we determined through CPU features 822 * corresponds properly to sve_vq_map. If not, do our best: 823 */ 824 if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl))) 825 sve_max_vl = find_supported_vector_length(sve_max_vl); 826 827 /* 828 * For the default VL, pick the maximum supported value <= 64. 829 * VL == 64 is guaranteed not to grow the signal frame. 830 */ 831 sve_default_vl = find_supported_vector_length(64); 832 833 bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map, 834 SVE_VQ_MAX); 835 836 b = find_last_bit(tmp_map, SVE_VQ_MAX); 837 if (b >= SVE_VQ_MAX) 838 /* No non-virtualisable VLs found */ 839 sve_max_virtualisable_vl = SVE_VQ_MAX; 840 else if (WARN_ON(b == SVE_VQ_MAX - 1)) 841 /* No virtualisable VLs? This is architecturally forbidden. */ 842 sve_max_virtualisable_vl = SVE_VQ_MIN; 843 else /* b + 1 < SVE_VQ_MAX */ 844 sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); 845 846 if (sve_max_virtualisable_vl > sve_max_vl) 847 sve_max_virtualisable_vl = sve_max_vl; 848 849 pr_info("SVE: maximum available vector length %u bytes per vector\n", 850 sve_max_vl); 851 pr_info("SVE: default vector length %u bytes per vector\n", 852 sve_default_vl); 853 854 /* KVM decides whether to support mismatched systems. Just warn here: */ 855 if (sve_max_virtualisable_vl < sve_max_vl) 856 pr_warn("SVE: unvirtualisable vector lengths present\n"); 857 858 sve_efi_setup(); 859 } 860 861 /* 862 * Called from the put_task_struct() path, which cannot get here 863 * unless dead_task is really dead and not schedulable. 864 */ 865 void fpsimd_release_task(struct task_struct *dead_task) 866 { 867 __sve_free(dead_task); 868 } 869 870 #endif /* CONFIG_ARM64_SVE */ 871 872 /* 873 * Trapped SVE access 874 * 875 * Storage is allocated for the full SVE state, the current FPSIMD 876 * register contents are migrated across, and TIF_SVE is set so that 877 * the SVE access trap will be disabled the next time this task 878 * reaches ret_to_user. 879 * 880 * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load() 881 * would have disabled the SVE access trap for userspace during 882 * ret_to_user, making an SVE access trap impossible in that case. 883 */ 884 asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs) 885 { 886 /* Even if we chose not to use SVE, the hardware could still trap: */ 887 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) { 888 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); 889 return; 890 } 891 892 sve_alloc(current); 893 894 local_bh_disable(); 895 896 fpsimd_save(); 897 898 /* Force ret_to_user to reload the registers: */ 899 fpsimd_flush_task_state(current); 900 901 fpsimd_to_sve(current); 902 if (test_and_set_thread_flag(TIF_SVE)) 903 WARN_ON(1); /* SVE access shouldn't have trapped */ 904 905 local_bh_enable(); 906 } 907 908 /* 909 * Trapped FP/ASIMD access. 910 */ 911 asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) 912 { 913 /* TODO: implement lazy context saving/restoring */ 914 WARN_ON(1); 915 } 916 917 /* 918 * Raise a SIGFPE for the current process. 919 */ 920 asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) 921 { 922 unsigned int si_code = FPE_FLTUNK; 923 924 if (esr & ESR_ELx_FP_EXC_TFV) { 925 if (esr & FPEXC_IOF) 926 si_code = FPE_FLTINV; 927 else if (esr & FPEXC_DZF) 928 si_code = FPE_FLTDIV; 929 else if (esr & FPEXC_OFF) 930 si_code = FPE_FLTOVF; 931 else if (esr & FPEXC_UFF) 932 si_code = FPE_FLTUND; 933 else if (esr & FPEXC_IXF) 934 si_code = FPE_FLTRES; 935 } 936 937 send_sig_fault(SIGFPE, si_code, 938 (void __user *)instruction_pointer(regs), 939 current); 940 } 941 942 void fpsimd_thread_switch(struct task_struct *next) 943 { 944 bool wrong_task, wrong_cpu; 945 946 if (!system_supports_fpsimd()) 947 return; 948 949 /* Save unsaved fpsimd state, if any: */ 950 fpsimd_save(); 951 952 /* 953 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's 954 * state. For kernel threads, FPSIMD registers are never loaded 955 * and wrong_task and wrong_cpu will always be true. 956 */ 957 wrong_task = __this_cpu_read(fpsimd_last_state.st) != 958 &next->thread.uw.fpsimd_state; 959 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); 960 961 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, 962 wrong_task || wrong_cpu); 963 } 964 965 void fpsimd_flush_thread(void) 966 { 967 int vl, supported_vl; 968 969 if (!system_supports_fpsimd()) 970 return; 971 972 local_bh_disable(); 973 974 fpsimd_flush_task_state(current); 975 memset(¤t->thread.uw.fpsimd_state, 0, 976 sizeof(current->thread.uw.fpsimd_state)); 977 978 if (system_supports_sve()) { 979 clear_thread_flag(TIF_SVE); 980 sve_free(current); 981 982 /* 983 * Reset the task vector length as required. 984 * This is where we ensure that all user tasks have a valid 985 * vector length configured: no kernel task can become a user 986 * task without an exec and hence a call to this function. 987 * By the time the first call to this function is made, all 988 * early hardware probing is complete, so sve_default_vl 989 * should be valid. 990 * If a bug causes this to go wrong, we make some noise and 991 * try to fudge thread.sve_vl to a safe value here. 992 */ 993 vl = current->thread.sve_vl_onexec ? 994 current->thread.sve_vl_onexec : sve_default_vl; 995 996 if (WARN_ON(!sve_vl_valid(vl))) 997 vl = SVE_VL_MIN; 998 999 supported_vl = find_supported_vector_length(vl); 1000 if (WARN_ON(supported_vl != vl)) 1001 vl = supported_vl; 1002 1003 current->thread.sve_vl = vl; 1004 1005 /* 1006 * If the task is not set to inherit, ensure that the vector 1007 * length will be reset by a subsequent exec: 1008 */ 1009 if (!test_thread_flag(TIF_SVE_VL_INHERIT)) 1010 current->thread.sve_vl_onexec = 0; 1011 } 1012 1013 local_bh_enable(); 1014 } 1015 1016 /* 1017 * Save the userland FPSIMD state of 'current' to memory, but only if the state 1018 * currently held in the registers does in fact belong to 'current' 1019 */ 1020 void fpsimd_preserve_current_state(void) 1021 { 1022 if (!system_supports_fpsimd()) 1023 return; 1024 1025 local_bh_disable(); 1026 fpsimd_save(); 1027 local_bh_enable(); 1028 } 1029 1030 /* 1031 * Like fpsimd_preserve_current_state(), but ensure that 1032 * current->thread.uw.fpsimd_state is updated so that it can be copied to 1033 * the signal frame. 1034 */ 1035 void fpsimd_signal_preserve_current_state(void) 1036 { 1037 fpsimd_preserve_current_state(); 1038 if (system_supports_sve() && test_thread_flag(TIF_SVE)) 1039 sve_to_fpsimd(current); 1040 } 1041 1042 /* 1043 * Associate current's FPSIMD context with this cpu 1044 * Preemption must be disabled when calling this function. 1045 */ 1046 void fpsimd_bind_task_to_cpu(void) 1047 { 1048 struct fpsimd_last_state_struct *last = 1049 this_cpu_ptr(&fpsimd_last_state); 1050 1051 last->st = ¤t->thread.uw.fpsimd_state; 1052 last->sve_state = current->thread.sve_state; 1053 last->sve_vl = current->thread.sve_vl; 1054 current->thread.fpsimd_cpu = smp_processor_id(); 1055 1056 if (system_supports_sve()) { 1057 /* Toggle SVE trapping for userspace if needed */ 1058 if (test_thread_flag(TIF_SVE)) 1059 sve_user_enable(); 1060 else 1061 sve_user_disable(); 1062 1063 /* Serialised by exception return to user */ 1064 } 1065 } 1066 1067 void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state, 1068 unsigned int sve_vl) 1069 { 1070 struct fpsimd_last_state_struct *last = 1071 this_cpu_ptr(&fpsimd_last_state); 1072 1073 WARN_ON(!in_softirq() && !irqs_disabled()); 1074 1075 last->st = st; 1076 last->sve_state = sve_state; 1077 last->sve_vl = sve_vl; 1078 } 1079 1080 /* 1081 * Load the userland FPSIMD state of 'current' from memory, but only if the 1082 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD 1083 * state of 'current' 1084 */ 1085 void fpsimd_restore_current_state(void) 1086 { 1087 if (!system_supports_fpsimd()) 1088 return; 1089 1090 local_bh_disable(); 1091 1092 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { 1093 task_fpsimd_load(); 1094 fpsimd_bind_task_to_cpu(); 1095 } 1096 1097 local_bh_enable(); 1098 } 1099 1100 /* 1101 * Load an updated userland FPSIMD state for 'current' from memory and set the 1102 * flag that indicates that the FPSIMD register contents are the most recent 1103 * FPSIMD state of 'current' 1104 */ 1105 void fpsimd_update_current_state(struct user_fpsimd_state const *state) 1106 { 1107 if (!system_supports_fpsimd()) 1108 return; 1109 1110 local_bh_disable(); 1111 1112 current->thread.uw.fpsimd_state = *state; 1113 if (system_supports_sve() && test_thread_flag(TIF_SVE)) 1114 fpsimd_to_sve(current); 1115 1116 task_fpsimd_load(); 1117 fpsimd_bind_task_to_cpu(); 1118 1119 clear_thread_flag(TIF_FOREIGN_FPSTATE); 1120 1121 local_bh_enable(); 1122 } 1123 1124 /* 1125 * Invalidate live CPU copies of task t's FPSIMD state 1126 * 1127 * This function may be called with preemption enabled. The barrier() 1128 * ensures that the assignment to fpsimd_cpu is visible to any 1129 * preemption/softirq that could race with set_tsk_thread_flag(), so 1130 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared. 1131 * 1132 * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any 1133 * subsequent code. 1134 */ 1135 void fpsimd_flush_task_state(struct task_struct *t) 1136 { 1137 t->thread.fpsimd_cpu = NR_CPUS; 1138 1139 barrier(); 1140 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE); 1141 1142 barrier(); 1143 } 1144 1145 /* 1146 * Invalidate any task's FPSIMD state that is present on this cpu. 1147 * This function must be called with softirqs disabled. 1148 */ 1149 void fpsimd_flush_cpu_state(void) 1150 { 1151 __this_cpu_write(fpsimd_last_state.st, NULL); 1152 set_thread_flag(TIF_FOREIGN_FPSTATE); 1153 } 1154 1155 #ifdef CONFIG_KERNEL_MODE_NEON 1156 1157 DEFINE_PER_CPU(bool, kernel_neon_busy); 1158 EXPORT_PER_CPU_SYMBOL(kernel_neon_busy); 1159 1160 /* 1161 * Kernel-side NEON support functions 1162 */ 1163 1164 /* 1165 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling 1166 * context 1167 * 1168 * Must not be called unless may_use_simd() returns true. 1169 * Task context in the FPSIMD registers is saved back to memory as necessary. 1170 * 1171 * A matching call to kernel_neon_end() must be made before returning from the 1172 * calling context. 1173 * 1174 * The caller may freely use the FPSIMD registers until kernel_neon_end() is 1175 * called. 1176 */ 1177 void kernel_neon_begin(void) 1178 { 1179 if (WARN_ON(!system_supports_fpsimd())) 1180 return; 1181 1182 BUG_ON(!may_use_simd()); 1183 1184 local_bh_disable(); 1185 1186 __this_cpu_write(kernel_neon_busy, true); 1187 1188 /* Save unsaved fpsimd state, if any: */ 1189 fpsimd_save(); 1190 1191 /* Invalidate any task state remaining in the fpsimd regs: */ 1192 fpsimd_flush_cpu_state(); 1193 1194 preempt_disable(); 1195 1196 local_bh_enable(); 1197 } 1198 EXPORT_SYMBOL(kernel_neon_begin); 1199 1200 /* 1201 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task 1202 * 1203 * Must be called from a context in which kernel_neon_begin() was previously 1204 * called, with no call to kernel_neon_end() in the meantime. 1205 * 1206 * The caller must not use the FPSIMD registers after this function is called, 1207 * unless kernel_neon_begin() is called again in the meantime. 1208 */ 1209 void kernel_neon_end(void) 1210 { 1211 bool busy; 1212 1213 if (!system_supports_fpsimd()) 1214 return; 1215 1216 busy = __this_cpu_xchg(kernel_neon_busy, false); 1217 WARN_ON(!busy); /* No matching kernel_neon_begin()? */ 1218 1219 preempt_enable(); 1220 } 1221 EXPORT_SYMBOL(kernel_neon_end); 1222 1223 #ifdef CONFIG_EFI 1224 1225 static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state); 1226 static DEFINE_PER_CPU(bool, efi_fpsimd_state_used); 1227 static DEFINE_PER_CPU(bool, efi_sve_state_used); 1228 1229 /* 1230 * EFI runtime services support functions 1231 * 1232 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call. 1233 * This means that for EFI (and only for EFI), we have to assume that FPSIMD 1234 * is always used rather than being an optional accelerator. 1235 * 1236 * These functions provide the necessary support for ensuring FPSIMD 1237 * save/restore in the contexts from which EFI is used. 1238 * 1239 * Do not use them for any other purpose -- if tempted to do so, you are 1240 * either doing something wrong or you need to propose some refactoring. 1241 */ 1242 1243 /* 1244 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call 1245 */ 1246 void __efi_fpsimd_begin(void) 1247 { 1248 if (!system_supports_fpsimd()) 1249 return; 1250 1251 WARN_ON(preemptible()); 1252 1253 if (may_use_simd()) { 1254 kernel_neon_begin(); 1255 } else { 1256 /* 1257 * If !efi_sve_state, SVE can't be in use yet and doesn't need 1258 * preserving: 1259 */ 1260 if (system_supports_sve() && likely(efi_sve_state)) { 1261 char *sve_state = this_cpu_ptr(efi_sve_state); 1262 1263 __this_cpu_write(efi_sve_state_used, true); 1264 1265 sve_save_state(sve_state + sve_ffr_offset(sve_max_vl), 1266 &this_cpu_ptr(&efi_fpsimd_state)->fpsr); 1267 } else { 1268 fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); 1269 } 1270 1271 __this_cpu_write(efi_fpsimd_state_used, true); 1272 } 1273 } 1274 1275 /* 1276 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call 1277 */ 1278 void __efi_fpsimd_end(void) 1279 { 1280 if (!system_supports_fpsimd()) 1281 return; 1282 1283 if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) { 1284 kernel_neon_end(); 1285 } else { 1286 if (system_supports_sve() && 1287 likely(__this_cpu_read(efi_sve_state_used))) { 1288 char const *sve_state = this_cpu_ptr(efi_sve_state); 1289 1290 sve_load_state(sve_state + sve_ffr_offset(sve_max_vl), 1291 &this_cpu_ptr(&efi_fpsimd_state)->fpsr, 1292 sve_vq_from_vl(sve_get_vl()) - 1); 1293 1294 __this_cpu_write(efi_sve_state_used, false); 1295 } else { 1296 fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state)); 1297 } 1298 } 1299 } 1300 1301 #endif /* CONFIG_EFI */ 1302 1303 #endif /* CONFIG_KERNEL_MODE_NEON */ 1304 1305 #ifdef CONFIG_CPU_PM 1306 static int fpsimd_cpu_pm_notifier(struct notifier_block *self, 1307 unsigned long cmd, void *v) 1308 { 1309 switch (cmd) { 1310 case CPU_PM_ENTER: 1311 fpsimd_save(); 1312 fpsimd_flush_cpu_state(); 1313 break; 1314 case CPU_PM_EXIT: 1315 break; 1316 case CPU_PM_ENTER_FAILED: 1317 default: 1318 return NOTIFY_DONE; 1319 } 1320 return NOTIFY_OK; 1321 } 1322 1323 static struct notifier_block fpsimd_cpu_pm_notifier_block = { 1324 .notifier_call = fpsimd_cpu_pm_notifier, 1325 }; 1326 1327 static void __init fpsimd_pm_init(void) 1328 { 1329 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); 1330 } 1331 1332 #else 1333 static inline void fpsimd_pm_init(void) { } 1334 #endif /* CONFIG_CPU_PM */ 1335 1336 #ifdef CONFIG_HOTPLUG_CPU 1337 static int fpsimd_cpu_dead(unsigned int cpu) 1338 { 1339 per_cpu(fpsimd_last_state.st, cpu) = NULL; 1340 return 0; 1341 } 1342 1343 static inline void fpsimd_hotplug_init(void) 1344 { 1345 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead", 1346 NULL, fpsimd_cpu_dead); 1347 } 1348 1349 #else 1350 static inline void fpsimd_hotplug_init(void) { } 1351 #endif 1352 1353 /* 1354 * FP/SIMD support code initialisation. 1355 */ 1356 static int __init fpsimd_init(void) 1357 { 1358 if (cpu_have_named_feature(FP)) { 1359 fpsimd_pm_init(); 1360 fpsimd_hotplug_init(); 1361 } else { 1362 pr_notice("Floating-point is not implemented\n"); 1363 } 1364 1365 if (!cpu_have_named_feature(ASIMD)) 1366 pr_notice("Advanced SIMD is not implemented\n"); 1367 1368 return sve_sysctl_init(); 1369 } 1370 core_initcall(fpsimd_init); 1371