xref: /openbmc/linux/arch/arm64/kernel/fpsimd.c (revision a8c5cb99)
1 /*
2  * FP/SIMD context switching and fault handling
3  *
4  * Copyright (C) 2012 ARM Ltd.
5  * Author: Catalin Marinas <catalin.marinas@arm.com>
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include <linux/bitmap.h>
21 #include <linux/bottom_half.h>
22 #include <linux/bug.h>
23 #include <linux/cache.h>
24 #include <linux/compat.h>
25 #include <linux/cpu.h>
26 #include <linux/cpu_pm.h>
27 #include <linux/kernel.h>
28 #include <linux/linkage.h>
29 #include <linux/irqflags.h>
30 #include <linux/init.h>
31 #include <linux/percpu.h>
32 #include <linux/prctl.h>
33 #include <linux/preempt.h>
34 #include <linux/ptrace.h>
35 #include <linux/sched/signal.h>
36 #include <linux/sched/task_stack.h>
37 #include <linux/signal.h>
38 #include <linux/slab.h>
39 #include <linux/stddef.h>
40 #include <linux/sysctl.h>
41 
42 #include <asm/esr.h>
43 #include <asm/fpsimd.h>
44 #include <asm/cpufeature.h>
45 #include <asm/cputype.h>
46 #include <asm/processor.h>
47 #include <asm/simd.h>
48 #include <asm/sigcontext.h>
49 #include <asm/sysreg.h>
50 #include <asm/traps.h>
51 
52 #define FPEXC_IOF	(1 << 0)
53 #define FPEXC_DZF	(1 << 1)
54 #define FPEXC_OFF	(1 << 2)
55 #define FPEXC_UFF	(1 << 3)
56 #define FPEXC_IXF	(1 << 4)
57 #define FPEXC_IDF	(1 << 7)
58 
59 /*
60  * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
61  *
62  * In order to reduce the number of times the FPSIMD state is needlessly saved
63  * and restored, we need to keep track of two things:
64  * (a) for each task, we need to remember which CPU was the last one to have
65  *     the task's FPSIMD state loaded into its FPSIMD registers;
66  * (b) for each CPU, we need to remember which task's userland FPSIMD state has
67  *     been loaded into its FPSIMD registers most recently, or whether it has
68  *     been used to perform kernel mode NEON in the meantime.
69  *
70  * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to
71  * the id of the current CPU every time the state is loaded onto a CPU. For (b),
72  * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
73  * address of the userland FPSIMD state of the task that was loaded onto the CPU
74  * the most recently, or NULL if kernel mode NEON has been performed after that.
75  *
76  * With this in place, we no longer have to restore the next FPSIMD state right
77  * when switching between tasks. Instead, we can defer this check to userland
78  * resume, at which time we verify whether the CPU's fpsimd_last_state and the
79  * task's fpsimd_cpu are still mutually in sync. If this is the case, we
80  * can omit the FPSIMD restore.
81  *
82  * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
83  * indicate whether or not the userland FPSIMD state of the current task is
84  * present in the registers. The flag is set unless the FPSIMD registers of this
85  * CPU currently contain the most recent userland FPSIMD state of the current
86  * task.
87  *
88  * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
89  * save the task's FPSIMD context back to task_struct from softirq context.
90  * To prevent this from racing with the manipulation of the task's FPSIMD state
91  * from task context and thereby corrupting the state, it is necessary to
92  * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
93  * flag with local_bh_disable() unless softirqs are already masked.
94  *
95  * For a certain task, the sequence may look something like this:
96  * - the task gets scheduled in; if both the task's fpsimd_cpu field
97  *   contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
98  *   variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
99  *   cleared, otherwise it is set;
100  *
101  * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
102  *   userland FPSIMD state is copied from memory to the registers, the task's
103  *   fpsimd_cpu field is set to the id of the current CPU, the current
104  *   CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
105  *   TIF_FOREIGN_FPSTATE flag is cleared;
106  *
107  * - the task executes an ordinary syscall; upon return to userland, the
108  *   TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
109  *   restored;
110  *
111  * - the task executes a syscall which executes some NEON instructions; this is
112  *   preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
113  *   register contents to memory, clears the fpsimd_last_state per-cpu variable
114  *   and sets the TIF_FOREIGN_FPSTATE flag;
115  *
116  * - the task gets preempted after kernel_neon_end() is called; as we have not
117  *   returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
118  *   whatever is in the FPSIMD registers is not saved to memory, but discarded.
119  */
120 struct fpsimd_last_state_struct {
121 	struct user_fpsimd_state *st;
122 };
123 
124 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
125 
126 /* Default VL for tasks that don't set it explicitly: */
127 static int sve_default_vl = -1;
128 
129 #ifdef CONFIG_ARM64_SVE
130 
131 /* Maximum supported vector length across all CPUs (initially poisoned) */
132 int __ro_after_init sve_max_vl = SVE_VL_MIN;
133 /* Set of available vector lengths, as vq_to_bit(vq): */
134 static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
135 static void __percpu *efi_sve_state;
136 
137 #else /* ! CONFIG_ARM64_SVE */
138 
139 /* Dummy declaration for code that will be optimised out: */
140 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
141 extern void __percpu *efi_sve_state;
142 
143 #endif /* ! CONFIG_ARM64_SVE */
144 
145 /*
146  * Call __sve_free() directly only if you know task can't be scheduled
147  * or preempted.
148  */
149 static void __sve_free(struct task_struct *task)
150 {
151 	kfree(task->thread.sve_state);
152 	task->thread.sve_state = NULL;
153 }
154 
155 static void sve_free(struct task_struct *task)
156 {
157 	WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
158 
159 	__sve_free(task);
160 }
161 
162 static void change_cpacr(u64 val, u64 mask)
163 {
164 	u64 cpacr = read_sysreg(CPACR_EL1);
165 	u64 new = (cpacr & ~mask) | val;
166 
167 	if (new != cpacr)
168 		write_sysreg(new, CPACR_EL1);
169 }
170 
171 static void sve_user_disable(void)
172 {
173 	change_cpacr(0, CPACR_EL1_ZEN_EL0EN);
174 }
175 
176 static void sve_user_enable(void)
177 {
178 	change_cpacr(CPACR_EL1_ZEN_EL0EN, CPACR_EL1_ZEN_EL0EN);
179 }
180 
181 /*
182  * TIF_SVE controls whether a task can use SVE without trapping while
183  * in userspace, and also the way a task's FPSIMD/SVE state is stored
184  * in thread_struct.
185  *
186  * The kernel uses this flag to track whether a user task is actively
187  * using SVE, and therefore whether full SVE register state needs to
188  * be tracked.  If not, the cheaper FPSIMD context handling code can
189  * be used instead of the more costly SVE equivalents.
190  *
191  *  * TIF_SVE set:
192  *
193  *    The task can execute SVE instructions while in userspace without
194  *    trapping to the kernel.
195  *
196  *    When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
197  *    corresponding Zn), P0-P15 and FFR are encoded in in
198  *    task->thread.sve_state, formatted appropriately for vector
199  *    length task->thread.sve_vl.
200  *
201  *    task->thread.sve_state must point to a valid buffer at least
202  *    sve_state_size(task) bytes in size.
203  *
204  *    During any syscall, the kernel may optionally clear TIF_SVE and
205  *    discard the vector state except for the FPSIMD subset.
206  *
207  *  * TIF_SVE clear:
208  *
209  *    An attempt by the user task to execute an SVE instruction causes
210  *    do_sve_acc() to be called, which does some preparation and then
211  *    sets TIF_SVE.
212  *
213  *    When stored, FPSIMD registers V0-V31 are encoded in
214  *    task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
215  *    logically zero but not stored anywhere; P0-P15 and FFR are not
216  *    stored and have unspecified values from userspace's point of
217  *    view.  For hygiene purposes, the kernel zeroes them on next use,
218  *    but userspace is discouraged from relying on this.
219  *
220  *    task->thread.sve_state does not need to be non-NULL, valid or any
221  *    particular size: it must not be dereferenced.
222  *
223  *  * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
224  *    irrespective of whether TIF_SVE is clear or set, since these are
225  *    not vector length dependent.
226  */
227 
228 /*
229  * Update current's FPSIMD/SVE registers from thread_struct.
230  *
231  * This function should be called only when the FPSIMD/SVE state in
232  * thread_struct is known to be up to date, when preparing to enter
233  * userspace.
234  *
235  * Softirqs (and preemption) must be disabled.
236  */
237 static void task_fpsimd_load(void)
238 {
239 	WARN_ON(!in_softirq() && !irqs_disabled());
240 
241 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
242 		sve_load_state(sve_pffr(&current->thread),
243 			       &current->thread.uw.fpsimd_state.fpsr,
244 			       sve_vq_from_vl(current->thread.sve_vl) - 1);
245 	else
246 		fpsimd_load_state(&current->thread.uw.fpsimd_state);
247 }
248 
249 /*
250  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
251  * date with respect to the CPU registers.
252  *
253  * Softirqs (and preemption) must be disabled.
254  */
255 void fpsimd_save(void)
256 {
257 	struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
258 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
259 
260 	WARN_ON(!in_softirq() && !irqs_disabled());
261 
262 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
263 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
264 			if (WARN_ON(sve_get_vl() != current->thread.sve_vl)) {
265 				/*
266 				 * Can't save the user regs, so current would
267 				 * re-enter user with corrupt state.
268 				 * There's no way to recover, so kill it:
269 				 */
270 				force_signal_inject(SIGKILL, SI_KERNEL, 0);
271 				return;
272 			}
273 
274 			sve_save_state(sve_pffr(&current->thread), &st->fpsr);
275 		} else
276 			fpsimd_save_state(st);
277 	}
278 }
279 
280 /*
281  * Helpers to translate bit indices in sve_vq_map to VQ values (and
282  * vice versa).  This allows find_next_bit() to be used to find the
283  * _maximum_ VQ not exceeding a certain value.
284  */
285 
286 static unsigned int vq_to_bit(unsigned int vq)
287 {
288 	return SVE_VQ_MAX - vq;
289 }
290 
291 static unsigned int bit_to_vq(unsigned int bit)
292 {
293 	if (WARN_ON(bit >= SVE_VQ_MAX))
294 		bit = SVE_VQ_MAX - 1;
295 
296 	return SVE_VQ_MAX - bit;
297 }
298 
299 /*
300  * All vector length selection from userspace comes through here.
301  * We're on a slow path, so some sanity-checks are included.
302  * If things go wrong there's a bug somewhere, but try to fall back to a
303  * safe choice.
304  */
305 static unsigned int find_supported_vector_length(unsigned int vl)
306 {
307 	int bit;
308 	int max_vl = sve_max_vl;
309 
310 	if (WARN_ON(!sve_vl_valid(vl)))
311 		vl = SVE_VL_MIN;
312 
313 	if (WARN_ON(!sve_vl_valid(max_vl)))
314 		max_vl = SVE_VL_MIN;
315 
316 	if (vl > max_vl)
317 		vl = max_vl;
318 
319 	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
320 			    vq_to_bit(sve_vq_from_vl(vl)));
321 	return sve_vl_from_vq(bit_to_vq(bit));
322 }
323 
324 #ifdef CONFIG_SYSCTL
325 
326 static int sve_proc_do_default_vl(struct ctl_table *table, int write,
327 				  void __user *buffer, size_t *lenp,
328 				  loff_t *ppos)
329 {
330 	int ret;
331 	int vl = sve_default_vl;
332 	struct ctl_table tmp_table = {
333 		.data = &vl,
334 		.maxlen = sizeof(vl),
335 	};
336 
337 	ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
338 	if (ret || !write)
339 		return ret;
340 
341 	/* Writing -1 has the special meaning "set to max": */
342 	if (vl == -1)
343 		vl = sve_max_vl;
344 
345 	if (!sve_vl_valid(vl))
346 		return -EINVAL;
347 
348 	sve_default_vl = find_supported_vector_length(vl);
349 	return 0;
350 }
351 
352 static struct ctl_table sve_default_vl_table[] = {
353 	{
354 		.procname	= "sve_default_vector_length",
355 		.mode		= 0644,
356 		.proc_handler	= sve_proc_do_default_vl,
357 	},
358 	{ }
359 };
360 
361 static int __init sve_sysctl_init(void)
362 {
363 	if (system_supports_sve())
364 		if (!register_sysctl("abi", sve_default_vl_table))
365 			return -EINVAL;
366 
367 	return 0;
368 }
369 
370 #else /* ! CONFIG_SYSCTL */
371 static int __init sve_sysctl_init(void) { return 0; }
372 #endif /* ! CONFIG_SYSCTL */
373 
374 #define ZREG(sve_state, vq, n) ((char *)(sve_state) +		\
375 	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
376 
377 /*
378  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
379  * task->thread.sve_state.
380  *
381  * Task can be a non-runnable task, or current.  In the latter case,
382  * softirqs (and preemption) must be disabled.
383  * task->thread.sve_state must point to at least sve_state_size(task)
384  * bytes of allocated kernel memory.
385  * task->thread.uw.fpsimd_state must be up to date before calling this
386  * function.
387  */
388 static void fpsimd_to_sve(struct task_struct *task)
389 {
390 	unsigned int vq;
391 	void *sst = task->thread.sve_state;
392 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
393 	unsigned int i;
394 
395 	if (!system_supports_sve())
396 		return;
397 
398 	vq = sve_vq_from_vl(task->thread.sve_vl);
399 	for (i = 0; i < 32; ++i)
400 		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
401 		       sizeof(fst->vregs[i]));
402 }
403 
404 /*
405  * Transfer the SVE state in task->thread.sve_state to
406  * task->thread.uw.fpsimd_state.
407  *
408  * Task can be a non-runnable task, or current.  In the latter case,
409  * softirqs (and preemption) must be disabled.
410  * task->thread.sve_state must point to at least sve_state_size(task)
411  * bytes of allocated kernel memory.
412  * task->thread.sve_state must be up to date before calling this function.
413  */
414 static void sve_to_fpsimd(struct task_struct *task)
415 {
416 	unsigned int vq;
417 	void const *sst = task->thread.sve_state;
418 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
419 	unsigned int i;
420 
421 	if (!system_supports_sve())
422 		return;
423 
424 	vq = sve_vq_from_vl(task->thread.sve_vl);
425 	for (i = 0; i < 32; ++i)
426 		memcpy(&fst->vregs[i], ZREG(sst, vq, i),
427 		       sizeof(fst->vregs[i]));
428 }
429 
430 #ifdef CONFIG_ARM64_SVE
431 
432 /*
433  * Return how many bytes of memory are required to store the full SVE
434  * state for task, given task's currently configured vector length.
435  */
436 size_t sve_state_size(struct task_struct const *task)
437 {
438 	return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
439 }
440 
441 /*
442  * Ensure that task->thread.sve_state is allocated and sufficiently large.
443  *
444  * This function should be used only in preparation for replacing
445  * task->thread.sve_state with new data.  The memory is always zeroed
446  * here to prevent stale data from showing through: this is done in
447  * the interest of testability and predictability: except in the
448  * do_sve_acc() case, there is no ABI requirement to hide stale data
449  * written previously be task.
450  */
451 void sve_alloc(struct task_struct *task)
452 {
453 	if (task->thread.sve_state) {
454 		memset(task->thread.sve_state, 0, sve_state_size(current));
455 		return;
456 	}
457 
458 	/* This is a small allocation (maximum ~8KB) and Should Not Fail. */
459 	task->thread.sve_state =
460 		kzalloc(sve_state_size(task), GFP_KERNEL);
461 
462 	/*
463 	 * If future SVE revisions can have larger vectors though,
464 	 * this may cease to be true:
465 	 */
466 	BUG_ON(!task->thread.sve_state);
467 }
468 
469 
470 /*
471  * Ensure that task->thread.sve_state is up to date with respect to
472  * the user task, irrespective of when SVE is in use or not.
473  *
474  * This should only be called by ptrace.  task must be non-runnable.
475  * task->thread.sve_state must point to at least sve_state_size(task)
476  * bytes of allocated kernel memory.
477  */
478 void fpsimd_sync_to_sve(struct task_struct *task)
479 {
480 	if (!test_tsk_thread_flag(task, TIF_SVE))
481 		fpsimd_to_sve(task);
482 }
483 
484 /*
485  * Ensure that task->thread.uw.fpsimd_state is up to date with respect to
486  * the user task, irrespective of whether SVE is in use or not.
487  *
488  * This should only be called by ptrace.  task must be non-runnable.
489  * task->thread.sve_state must point to at least sve_state_size(task)
490  * bytes of allocated kernel memory.
491  */
492 void sve_sync_to_fpsimd(struct task_struct *task)
493 {
494 	if (test_tsk_thread_flag(task, TIF_SVE))
495 		sve_to_fpsimd(task);
496 }
497 
498 /*
499  * Ensure that task->thread.sve_state is up to date with respect to
500  * the task->thread.uw.fpsimd_state.
501  *
502  * This should only be called by ptrace to merge new FPSIMD register
503  * values into a task for which SVE is currently active.
504  * task must be non-runnable.
505  * task->thread.sve_state must point to at least sve_state_size(task)
506  * bytes of allocated kernel memory.
507  * task->thread.uw.fpsimd_state must already have been initialised with
508  * the new FPSIMD register values to be merged in.
509  */
510 void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
511 {
512 	unsigned int vq;
513 	void *sst = task->thread.sve_state;
514 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
515 	unsigned int i;
516 
517 	if (!test_tsk_thread_flag(task, TIF_SVE))
518 		return;
519 
520 	vq = sve_vq_from_vl(task->thread.sve_vl);
521 
522 	memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
523 
524 	for (i = 0; i < 32; ++i)
525 		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
526 		       sizeof(fst->vregs[i]));
527 }
528 
529 int sve_set_vector_length(struct task_struct *task,
530 			  unsigned long vl, unsigned long flags)
531 {
532 	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
533 				     PR_SVE_SET_VL_ONEXEC))
534 		return -EINVAL;
535 
536 	if (!sve_vl_valid(vl))
537 		return -EINVAL;
538 
539 	/*
540 	 * Clamp to the maximum vector length that VL-agnostic SVE code can
541 	 * work with.  A flag may be assigned in the future to allow setting
542 	 * of larger vector lengths without confusing older software.
543 	 */
544 	if (vl > SVE_VL_ARCH_MAX)
545 		vl = SVE_VL_ARCH_MAX;
546 
547 	vl = find_supported_vector_length(vl);
548 
549 	if (flags & (PR_SVE_VL_INHERIT |
550 		     PR_SVE_SET_VL_ONEXEC))
551 		task->thread.sve_vl_onexec = vl;
552 	else
553 		/* Reset VL to system default on next exec: */
554 		task->thread.sve_vl_onexec = 0;
555 
556 	/* Only actually set the VL if not deferred: */
557 	if (flags & PR_SVE_SET_VL_ONEXEC)
558 		goto out;
559 
560 	if (vl == task->thread.sve_vl)
561 		goto out;
562 
563 	/*
564 	 * To ensure the FPSIMD bits of the SVE vector registers are preserved,
565 	 * write any live register state back to task_struct, and convert to a
566 	 * non-SVE thread.
567 	 */
568 	if (task == current) {
569 		local_bh_disable();
570 
571 		fpsimd_save();
572 		set_thread_flag(TIF_FOREIGN_FPSTATE);
573 	}
574 
575 	fpsimd_flush_task_state(task);
576 	if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
577 		sve_to_fpsimd(task);
578 
579 	if (task == current)
580 		local_bh_enable();
581 
582 	/*
583 	 * Force reallocation of task SVE state to the correct size
584 	 * on next use:
585 	 */
586 	sve_free(task);
587 
588 	task->thread.sve_vl = vl;
589 
590 out:
591 	update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
592 			       flags & PR_SVE_VL_INHERIT);
593 
594 	return 0;
595 }
596 
597 /*
598  * Encode the current vector length and flags for return.
599  * This is only required for prctl(): ptrace has separate fields
600  *
601  * flags are as for sve_set_vector_length().
602  */
603 static int sve_prctl_status(unsigned long flags)
604 {
605 	int ret;
606 
607 	if (flags & PR_SVE_SET_VL_ONEXEC)
608 		ret = current->thread.sve_vl_onexec;
609 	else
610 		ret = current->thread.sve_vl;
611 
612 	if (test_thread_flag(TIF_SVE_VL_INHERIT))
613 		ret |= PR_SVE_VL_INHERIT;
614 
615 	return ret;
616 }
617 
618 /* PR_SVE_SET_VL */
619 int sve_set_current_vl(unsigned long arg)
620 {
621 	unsigned long vl, flags;
622 	int ret;
623 
624 	vl = arg & PR_SVE_VL_LEN_MASK;
625 	flags = arg & ~vl;
626 
627 	if (!system_supports_sve())
628 		return -EINVAL;
629 
630 	ret = sve_set_vector_length(current, vl, flags);
631 	if (ret)
632 		return ret;
633 
634 	return sve_prctl_status(flags);
635 }
636 
637 /* PR_SVE_GET_VL */
638 int sve_get_current_vl(void)
639 {
640 	if (!system_supports_sve())
641 		return -EINVAL;
642 
643 	return sve_prctl_status(0);
644 }
645 
646 /*
647  * Bitmap for temporary storage of the per-CPU set of supported vector lengths
648  * during secondary boot.
649  */
650 static DECLARE_BITMAP(sve_secondary_vq_map, SVE_VQ_MAX);
651 
652 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
653 {
654 	unsigned int vq, vl;
655 	unsigned long zcr;
656 
657 	bitmap_zero(map, SVE_VQ_MAX);
658 
659 	zcr = ZCR_ELx_LEN_MASK;
660 	zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
661 
662 	for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
663 		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
664 		vl = sve_get_vl();
665 		vq = sve_vq_from_vl(vl); /* skip intervening lengths */
666 		set_bit(vq_to_bit(vq), map);
667 	}
668 }
669 
670 void __init sve_init_vq_map(void)
671 {
672 	sve_probe_vqs(sve_vq_map);
673 }
674 
675 /*
676  * If we haven't committed to the set of supported VQs yet, filter out
677  * those not supported by the current CPU.
678  */
679 void sve_update_vq_map(void)
680 {
681 	sve_probe_vqs(sve_secondary_vq_map);
682 	bitmap_and(sve_vq_map, sve_vq_map, sve_secondary_vq_map, SVE_VQ_MAX);
683 }
684 
685 /* Check whether the current CPU supports all VQs in the committed set */
686 int sve_verify_vq_map(void)
687 {
688 	int ret = 0;
689 
690 	sve_probe_vqs(sve_secondary_vq_map);
691 	bitmap_andnot(sve_secondary_vq_map, sve_vq_map, sve_secondary_vq_map,
692 		      SVE_VQ_MAX);
693 	if (!bitmap_empty(sve_secondary_vq_map, SVE_VQ_MAX)) {
694 		pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
695 			smp_processor_id());
696 		ret = -EINVAL;
697 	}
698 
699 	return ret;
700 }
701 
702 static void __init sve_efi_setup(void)
703 {
704 	if (!IS_ENABLED(CONFIG_EFI))
705 		return;
706 
707 	/*
708 	 * alloc_percpu() warns and prints a backtrace if this goes wrong.
709 	 * This is evidence of a crippled system and we are returning void,
710 	 * so no attempt is made to handle this situation here.
711 	 */
712 	if (!sve_vl_valid(sve_max_vl))
713 		goto fail;
714 
715 	efi_sve_state = __alloc_percpu(
716 		SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
717 	if (!efi_sve_state)
718 		goto fail;
719 
720 	return;
721 
722 fail:
723 	panic("Cannot allocate percpu memory for EFI SVE save/restore");
724 }
725 
726 /*
727  * Enable SVE for EL1.
728  * Intended for use by the cpufeatures code during CPU boot.
729  */
730 void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
731 {
732 	write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
733 	isb();
734 }
735 
736 /*
737  * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
738  * vector length.
739  *
740  * Use only if SVE is present.
741  * This function clobbers the SVE vector length.
742  */
743 u64 read_zcr_features(void)
744 {
745 	u64 zcr;
746 	unsigned int vq_max;
747 
748 	/*
749 	 * Set the maximum possible VL, and write zeroes to all other
750 	 * bits to see if they stick.
751 	 */
752 	sve_kernel_enable(NULL);
753 	write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
754 
755 	zcr = read_sysreg_s(SYS_ZCR_EL1);
756 	zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
757 	vq_max = sve_vq_from_vl(sve_get_vl());
758 	zcr |= vq_max - 1; /* set LEN field to maximum effective value */
759 
760 	return zcr;
761 }
762 
763 void __init sve_setup(void)
764 {
765 	u64 zcr;
766 
767 	if (!system_supports_sve())
768 		return;
769 
770 	/*
771 	 * The SVE architecture mandates support for 128-bit vectors,
772 	 * so sve_vq_map must have at least SVE_VQ_MIN set.
773 	 * If something went wrong, at least try to patch it up:
774 	 */
775 	if (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
776 		set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
777 
778 	zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
779 	sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
780 
781 	/*
782 	 * Sanity-check that the max VL we determined through CPU features
783 	 * corresponds properly to sve_vq_map.  If not, do our best:
784 	 */
785 	if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
786 		sve_max_vl = find_supported_vector_length(sve_max_vl);
787 
788 	/*
789 	 * For the default VL, pick the maximum supported value <= 64.
790 	 * VL == 64 is guaranteed not to grow the signal frame.
791 	 */
792 	sve_default_vl = find_supported_vector_length(64);
793 
794 	pr_info("SVE: maximum available vector length %u bytes per vector\n",
795 		sve_max_vl);
796 	pr_info("SVE: default vector length %u bytes per vector\n",
797 		sve_default_vl);
798 
799 	sve_efi_setup();
800 }
801 
802 /*
803  * Called from the put_task_struct() path, which cannot get here
804  * unless dead_task is really dead and not schedulable.
805  */
806 void fpsimd_release_task(struct task_struct *dead_task)
807 {
808 	__sve_free(dead_task);
809 }
810 
811 #endif /* CONFIG_ARM64_SVE */
812 
813 /*
814  * Trapped SVE access
815  *
816  * Storage is allocated for the full SVE state, the current FPSIMD
817  * register contents are migrated across, and TIF_SVE is set so that
818  * the SVE access trap will be disabled the next time this task
819  * reaches ret_to_user.
820  *
821  * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
822  * would have disabled the SVE access trap for userspace during
823  * ret_to_user, making an SVE access trap impossible in that case.
824  */
825 asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
826 {
827 	/* Even if we chose not to use SVE, the hardware could still trap: */
828 	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
829 		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
830 		return;
831 	}
832 
833 	sve_alloc(current);
834 
835 	local_bh_disable();
836 
837 	fpsimd_save();
838 	fpsimd_to_sve(current);
839 
840 	/* Force ret_to_user to reload the registers: */
841 	fpsimd_flush_task_state(current);
842 	set_thread_flag(TIF_FOREIGN_FPSTATE);
843 
844 	if (test_and_set_thread_flag(TIF_SVE))
845 		WARN_ON(1); /* SVE access shouldn't have trapped */
846 
847 	local_bh_enable();
848 }
849 
850 /*
851  * Trapped FP/ASIMD access.
852  */
853 asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
854 {
855 	/* TODO: implement lazy context saving/restoring */
856 	WARN_ON(1);
857 }
858 
859 /*
860  * Raise a SIGFPE for the current process.
861  */
862 asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
863 {
864 	siginfo_t info;
865 	unsigned int si_code = FPE_FLTUNK;
866 
867 	if (esr & ESR_ELx_FP_EXC_TFV) {
868 		if (esr & FPEXC_IOF)
869 			si_code = FPE_FLTINV;
870 		else if (esr & FPEXC_DZF)
871 			si_code = FPE_FLTDIV;
872 		else if (esr & FPEXC_OFF)
873 			si_code = FPE_FLTOVF;
874 		else if (esr & FPEXC_UFF)
875 			si_code = FPE_FLTUND;
876 		else if (esr & FPEXC_IXF)
877 			si_code = FPE_FLTRES;
878 	}
879 
880 	clear_siginfo(&info);
881 	info.si_signo = SIGFPE;
882 	info.si_code = si_code;
883 	info.si_addr = (void __user *)instruction_pointer(regs);
884 
885 	send_sig_info(SIGFPE, &info, current);
886 }
887 
888 void fpsimd_thread_switch(struct task_struct *next)
889 {
890 	bool wrong_task, wrong_cpu;
891 
892 	if (!system_supports_fpsimd())
893 		return;
894 
895 	/* Save unsaved fpsimd state, if any: */
896 	fpsimd_save();
897 
898 	/*
899 	 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
900 	 * state.  For kernel threads, FPSIMD registers are never loaded
901 	 * and wrong_task and wrong_cpu will always be true.
902 	 */
903 	wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
904 					&next->thread.uw.fpsimd_state;
905 	wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
906 
907 	update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
908 			       wrong_task || wrong_cpu);
909 }
910 
911 void fpsimd_flush_thread(void)
912 {
913 	int vl, supported_vl;
914 
915 	if (!system_supports_fpsimd())
916 		return;
917 
918 	local_bh_disable();
919 
920 	memset(&current->thread.uw.fpsimd_state, 0,
921 	       sizeof(current->thread.uw.fpsimd_state));
922 	fpsimd_flush_task_state(current);
923 
924 	if (system_supports_sve()) {
925 		clear_thread_flag(TIF_SVE);
926 		sve_free(current);
927 
928 		/*
929 		 * Reset the task vector length as required.
930 		 * This is where we ensure that all user tasks have a valid
931 		 * vector length configured: no kernel task can become a user
932 		 * task without an exec and hence a call to this function.
933 		 * By the time the first call to this function is made, all
934 		 * early hardware probing is complete, so sve_default_vl
935 		 * should be valid.
936 		 * If a bug causes this to go wrong, we make some noise and
937 		 * try to fudge thread.sve_vl to a safe value here.
938 		 */
939 		vl = current->thread.sve_vl_onexec ?
940 			current->thread.sve_vl_onexec : sve_default_vl;
941 
942 		if (WARN_ON(!sve_vl_valid(vl)))
943 			vl = SVE_VL_MIN;
944 
945 		supported_vl = find_supported_vector_length(vl);
946 		if (WARN_ON(supported_vl != vl))
947 			vl = supported_vl;
948 
949 		current->thread.sve_vl = vl;
950 
951 		/*
952 		 * If the task is not set to inherit, ensure that the vector
953 		 * length will be reset by a subsequent exec:
954 		 */
955 		if (!test_thread_flag(TIF_SVE_VL_INHERIT))
956 			current->thread.sve_vl_onexec = 0;
957 	}
958 
959 	set_thread_flag(TIF_FOREIGN_FPSTATE);
960 
961 	local_bh_enable();
962 }
963 
964 /*
965  * Save the userland FPSIMD state of 'current' to memory, but only if the state
966  * currently held in the registers does in fact belong to 'current'
967  */
968 void fpsimd_preserve_current_state(void)
969 {
970 	if (!system_supports_fpsimd())
971 		return;
972 
973 	local_bh_disable();
974 	fpsimd_save();
975 	local_bh_enable();
976 }
977 
978 /*
979  * Like fpsimd_preserve_current_state(), but ensure that
980  * current->thread.uw.fpsimd_state is updated so that it can be copied to
981  * the signal frame.
982  */
983 void fpsimd_signal_preserve_current_state(void)
984 {
985 	fpsimd_preserve_current_state();
986 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
987 		sve_to_fpsimd(current);
988 }
989 
990 /*
991  * Associate current's FPSIMD context with this cpu
992  * Preemption must be disabled when calling this function.
993  */
994 void fpsimd_bind_task_to_cpu(void)
995 {
996 	struct fpsimd_last_state_struct *last =
997 		this_cpu_ptr(&fpsimd_last_state);
998 
999 	last->st = &current->thread.uw.fpsimd_state;
1000 	current->thread.fpsimd_cpu = smp_processor_id();
1001 
1002 	if (system_supports_sve()) {
1003 		/* Toggle SVE trapping for userspace if needed */
1004 		if (test_thread_flag(TIF_SVE))
1005 			sve_user_enable();
1006 		else
1007 			sve_user_disable();
1008 
1009 		/* Serialised by exception return to user */
1010 	}
1011 }
1012 
1013 void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
1014 {
1015 	struct fpsimd_last_state_struct *last =
1016 		this_cpu_ptr(&fpsimd_last_state);
1017 
1018 	WARN_ON(!in_softirq() && !irqs_disabled());
1019 
1020 	last->st = st;
1021 }
1022 
1023 /*
1024  * Load the userland FPSIMD state of 'current' from memory, but only if the
1025  * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
1026  * state of 'current'
1027  */
1028 void fpsimd_restore_current_state(void)
1029 {
1030 	if (!system_supports_fpsimd())
1031 		return;
1032 
1033 	local_bh_disable();
1034 
1035 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1036 		task_fpsimd_load();
1037 		fpsimd_bind_task_to_cpu();
1038 	}
1039 
1040 	local_bh_enable();
1041 }
1042 
1043 /*
1044  * Load an updated userland FPSIMD state for 'current' from memory and set the
1045  * flag that indicates that the FPSIMD register contents are the most recent
1046  * FPSIMD state of 'current'
1047  */
1048 void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1049 {
1050 	if (!system_supports_fpsimd())
1051 		return;
1052 
1053 	local_bh_disable();
1054 
1055 	current->thread.uw.fpsimd_state = *state;
1056 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
1057 		fpsimd_to_sve(current);
1058 
1059 	task_fpsimd_load();
1060 	fpsimd_bind_task_to_cpu();
1061 
1062 	clear_thread_flag(TIF_FOREIGN_FPSTATE);
1063 
1064 	local_bh_enable();
1065 }
1066 
1067 /*
1068  * Invalidate live CPU copies of task t's FPSIMD state
1069  */
1070 void fpsimd_flush_task_state(struct task_struct *t)
1071 {
1072 	t->thread.fpsimd_cpu = NR_CPUS;
1073 }
1074 
1075 void fpsimd_flush_cpu_state(void)
1076 {
1077 	__this_cpu_write(fpsimd_last_state.st, NULL);
1078 	set_thread_flag(TIF_FOREIGN_FPSTATE);
1079 }
1080 
1081 #ifdef CONFIG_KERNEL_MODE_NEON
1082 
1083 DEFINE_PER_CPU(bool, kernel_neon_busy);
1084 EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
1085 
1086 /*
1087  * Kernel-side NEON support functions
1088  */
1089 
1090 /*
1091  * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
1092  * context
1093  *
1094  * Must not be called unless may_use_simd() returns true.
1095  * Task context in the FPSIMD registers is saved back to memory as necessary.
1096  *
1097  * A matching call to kernel_neon_end() must be made before returning from the
1098  * calling context.
1099  *
1100  * The caller may freely use the FPSIMD registers until kernel_neon_end() is
1101  * called.
1102  */
1103 void kernel_neon_begin(void)
1104 {
1105 	if (WARN_ON(!system_supports_fpsimd()))
1106 		return;
1107 
1108 	BUG_ON(!may_use_simd());
1109 
1110 	local_bh_disable();
1111 
1112 	__this_cpu_write(kernel_neon_busy, true);
1113 
1114 	/* Save unsaved fpsimd state, if any: */
1115 	fpsimd_save();
1116 
1117 	/* Invalidate any task state remaining in the fpsimd regs: */
1118 	fpsimd_flush_cpu_state();
1119 
1120 	preempt_disable();
1121 
1122 	local_bh_enable();
1123 }
1124 EXPORT_SYMBOL(kernel_neon_begin);
1125 
1126 /*
1127  * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1128  *
1129  * Must be called from a context in which kernel_neon_begin() was previously
1130  * called, with no call to kernel_neon_end() in the meantime.
1131  *
1132  * The caller must not use the FPSIMD registers after this function is called,
1133  * unless kernel_neon_begin() is called again in the meantime.
1134  */
1135 void kernel_neon_end(void)
1136 {
1137 	bool busy;
1138 
1139 	if (!system_supports_fpsimd())
1140 		return;
1141 
1142 	busy = __this_cpu_xchg(kernel_neon_busy, false);
1143 	WARN_ON(!busy);	/* No matching kernel_neon_begin()? */
1144 
1145 	preempt_enable();
1146 }
1147 EXPORT_SYMBOL(kernel_neon_end);
1148 
1149 #ifdef CONFIG_EFI
1150 
1151 static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
1152 static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
1153 static DEFINE_PER_CPU(bool, efi_sve_state_used);
1154 
1155 /*
1156  * EFI runtime services support functions
1157  *
1158  * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
1159  * This means that for EFI (and only for EFI), we have to assume that FPSIMD
1160  * is always used rather than being an optional accelerator.
1161  *
1162  * These functions provide the necessary support for ensuring FPSIMD
1163  * save/restore in the contexts from which EFI is used.
1164  *
1165  * Do not use them for any other purpose -- if tempted to do so, you are
1166  * either doing something wrong or you need to propose some refactoring.
1167  */
1168 
1169 /*
1170  * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
1171  */
1172 void __efi_fpsimd_begin(void)
1173 {
1174 	if (!system_supports_fpsimd())
1175 		return;
1176 
1177 	WARN_ON(preemptible());
1178 
1179 	if (may_use_simd()) {
1180 		kernel_neon_begin();
1181 	} else {
1182 		/*
1183 		 * If !efi_sve_state, SVE can't be in use yet and doesn't need
1184 		 * preserving:
1185 		 */
1186 		if (system_supports_sve() && likely(efi_sve_state)) {
1187 			char *sve_state = this_cpu_ptr(efi_sve_state);
1188 
1189 			__this_cpu_write(efi_sve_state_used, true);
1190 
1191 			sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
1192 				       &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
1193 		} else {
1194 			fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
1195 		}
1196 
1197 		__this_cpu_write(efi_fpsimd_state_used, true);
1198 	}
1199 }
1200 
1201 /*
1202  * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
1203  */
1204 void __efi_fpsimd_end(void)
1205 {
1206 	if (!system_supports_fpsimd())
1207 		return;
1208 
1209 	if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
1210 		kernel_neon_end();
1211 	} else {
1212 		if (system_supports_sve() &&
1213 		    likely(__this_cpu_read(efi_sve_state_used))) {
1214 			char const *sve_state = this_cpu_ptr(efi_sve_state);
1215 
1216 			sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
1217 				       &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
1218 				       sve_vq_from_vl(sve_get_vl()) - 1);
1219 
1220 			__this_cpu_write(efi_sve_state_used, false);
1221 		} else {
1222 			fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
1223 		}
1224 	}
1225 }
1226 
1227 #endif /* CONFIG_EFI */
1228 
1229 #endif /* CONFIG_KERNEL_MODE_NEON */
1230 
1231 #ifdef CONFIG_CPU_PM
1232 static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
1233 				  unsigned long cmd, void *v)
1234 {
1235 	switch (cmd) {
1236 	case CPU_PM_ENTER:
1237 		fpsimd_save();
1238 		fpsimd_flush_cpu_state();
1239 		break;
1240 	case CPU_PM_EXIT:
1241 		break;
1242 	case CPU_PM_ENTER_FAILED:
1243 	default:
1244 		return NOTIFY_DONE;
1245 	}
1246 	return NOTIFY_OK;
1247 }
1248 
1249 static struct notifier_block fpsimd_cpu_pm_notifier_block = {
1250 	.notifier_call = fpsimd_cpu_pm_notifier,
1251 };
1252 
1253 static void __init fpsimd_pm_init(void)
1254 {
1255 	cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
1256 }
1257 
1258 #else
1259 static inline void fpsimd_pm_init(void) { }
1260 #endif /* CONFIG_CPU_PM */
1261 
1262 #ifdef CONFIG_HOTPLUG_CPU
1263 static int fpsimd_cpu_dead(unsigned int cpu)
1264 {
1265 	per_cpu(fpsimd_last_state.st, cpu) = NULL;
1266 	return 0;
1267 }
1268 
1269 static inline void fpsimd_hotplug_init(void)
1270 {
1271 	cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
1272 				  NULL, fpsimd_cpu_dead);
1273 }
1274 
1275 #else
1276 static inline void fpsimd_hotplug_init(void) { }
1277 #endif
1278 
1279 /*
1280  * FP/SIMD support code initialisation.
1281  */
1282 static int __init fpsimd_init(void)
1283 {
1284 	if (elf_hwcap & HWCAP_FP) {
1285 		fpsimd_pm_init();
1286 		fpsimd_hotplug_init();
1287 	} else {
1288 		pr_notice("Floating-point is not implemented\n");
1289 	}
1290 
1291 	if (!(elf_hwcap & HWCAP_ASIMD))
1292 		pr_notice("Advanced SIMD is not implemented\n");
1293 
1294 	return sve_sysctl_init();
1295 }
1296 core_initcall(fpsimd_init);
1297