xref: /openbmc/linux/arch/riscv/kernel/vector.c (revision 75b59f2a)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2023 SiFive
4  * Author: Andy Chiu <andy.chiu@sifive.com>
5  */
6 #include <linux/export.h>
7 #include <linux/sched/signal.h>
8 #include <linux/types.h>
9 #include <linux/slab.h>
10 #include <linux/sched.h>
11 #include <linux/uaccess.h>
12 #include <linux/prctl.h>
13 
14 #include <asm/thread_info.h>
15 #include <asm/processor.h>
16 #include <asm/insn.h>
17 #include <asm/vector.h>
18 #include <asm/csr.h>
19 #include <asm/elf.h>
20 #include <asm/ptrace.h>
21 #include <asm/bug.h>
22 
23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
24 
25 unsigned long riscv_v_vsize __read_mostly;
26 EXPORT_SYMBOL_GPL(riscv_v_vsize);
27 
riscv_v_setup_vsize(void)28 int riscv_v_setup_vsize(void)
29 {
30 	unsigned long this_vsize;
31 
32 	/* There are 32 vector registers with vlenb length. */
33 	riscv_v_enable();
34 	this_vsize = csr_read(CSR_VLENB) * 32;
35 	riscv_v_disable();
36 
37 	if (!riscv_v_vsize) {
38 		riscv_v_vsize = this_vsize;
39 		return 0;
40 	}
41 
42 	if (riscv_v_vsize != this_vsize) {
43 		WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
44 		return -EOPNOTSUPP;
45 	}
46 
47 	return 0;
48 }
49 
insn_is_vector(u32 insn_buf)50 static bool insn_is_vector(u32 insn_buf)
51 {
52 	u32 opcode = insn_buf & __INSN_OPCODE_MASK;
53 	u32 width, csr;
54 
55 	/*
56 	 * All V-related instructions, including CSR operations are 4-Byte. So,
57 	 * do not handle if the instruction length is not 4-Byte.
58 	 */
59 	if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
60 		return false;
61 
62 	switch (opcode) {
63 	case RVV_OPCODE_VECTOR:
64 		return true;
65 	case RVV_OPCODE_VL:
66 	case RVV_OPCODE_VS:
67 		width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
68 		if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
69 		    width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
70 			return true;
71 
72 		break;
73 	case RVG_OPCODE_SYSTEM:
74 		csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
75 		if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
76 		    (csr >= CSR_VL && csr <= CSR_VLENB))
77 			return true;
78 	}
79 
80 	return false;
81 }
82 
riscv_v_thread_zalloc(void)83 static int riscv_v_thread_zalloc(void)
84 {
85 	void *datap;
86 
87 	datap = kzalloc(riscv_v_vsize, GFP_KERNEL);
88 	if (!datap)
89 		return -ENOMEM;
90 
91 	current->thread.vstate.datap = datap;
92 	memset(&current->thread.vstate, 0, offsetof(struct __riscv_v_ext_state,
93 						    datap));
94 	return 0;
95 }
96 
97 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
98 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
99 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
100 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
riscv_v_ctrl_get_cur(struct task_struct * tsk)101 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
102 {
103 	return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
104 }
105 
riscv_v_ctrl_get_next(struct task_struct * tsk)106 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
107 {
108 	return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
109 }
110 
riscv_v_ctrl_test_inherit(struct task_struct * tsk)111 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
112 {
113 	return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
114 }
115 
riscv_v_ctrl_set(struct task_struct * tsk,int cur,int nxt,bool inherit)116 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
117 				    bool inherit)
118 {
119 	unsigned long ctrl;
120 
121 	ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
122 	ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
123 	if (inherit)
124 		ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
125 	tsk->thread.vstate_ctrl = ctrl;
126 }
127 
riscv_v_vstate_ctrl_user_allowed(void)128 bool riscv_v_vstate_ctrl_user_allowed(void)
129 {
130 	return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
131 }
132 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
133 
riscv_v_first_use_handler(struct pt_regs * regs)134 bool riscv_v_first_use_handler(struct pt_regs *regs)
135 {
136 	u32 __user *epc = (u32 __user *)regs->epc;
137 	u32 insn = (u32)regs->badaddr;
138 
139 	/* Do not handle if V is not supported, or disabled */
140 	if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
141 		return false;
142 
143 	/* If V has been enabled then it is not the first-use trap */
144 	if (riscv_v_vstate_query(regs))
145 		return false;
146 
147 	/* Get the instruction */
148 	if (!insn) {
149 		if (__get_user(insn, epc))
150 			return false;
151 	}
152 
153 	/* Filter out non-V instructions */
154 	if (!insn_is_vector(insn))
155 		return false;
156 
157 	/* Sanity check. datap should be null by the time of the first-use trap */
158 	WARN_ON(current->thread.vstate.datap);
159 
160 	/*
161 	 * Now we sure that this is a V instruction. And it executes in the
162 	 * context where VS has been off. So, try to allocate the user's V
163 	 * context and resume execution.
164 	 */
165 	if (riscv_v_thread_zalloc()) {
166 		force_sig(SIGBUS);
167 		return true;
168 	}
169 	riscv_v_vstate_on(regs);
170 	riscv_v_vstate_restore(current, regs);
171 	return true;
172 }
173 
riscv_v_vstate_ctrl_init(struct task_struct * tsk)174 void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
175 {
176 	bool inherit;
177 	int cur, next;
178 
179 	if (!has_vector())
180 		return;
181 
182 	next = riscv_v_ctrl_get_next(tsk);
183 	if (!next) {
184 		if (READ_ONCE(riscv_v_implicit_uacc))
185 			cur = PR_RISCV_V_VSTATE_CTRL_ON;
186 		else
187 			cur = PR_RISCV_V_VSTATE_CTRL_OFF;
188 	} else {
189 		cur = next;
190 	}
191 	/* Clear next mask if inherit-bit is not set */
192 	inherit = riscv_v_ctrl_test_inherit(tsk);
193 	if (!inherit)
194 		next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
195 
196 	riscv_v_ctrl_set(tsk, cur, next, inherit);
197 }
198 
riscv_v_vstate_ctrl_get_current(void)199 long riscv_v_vstate_ctrl_get_current(void)
200 {
201 	if (!has_vector())
202 		return -EINVAL;
203 
204 	return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
205 }
206 
riscv_v_vstate_ctrl_set_current(unsigned long arg)207 long riscv_v_vstate_ctrl_set_current(unsigned long arg)
208 {
209 	bool inherit;
210 	int cur, next;
211 
212 	if (!has_vector())
213 		return -EINVAL;
214 
215 	if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
216 		return -EINVAL;
217 
218 	cur = VSTATE_CTRL_GET_CUR(arg);
219 	switch (cur) {
220 	case PR_RISCV_V_VSTATE_CTRL_OFF:
221 		/* Do not allow user to turn off V if current is not off */
222 		if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
223 			return -EPERM;
224 
225 		break;
226 	case PR_RISCV_V_VSTATE_CTRL_ON:
227 		break;
228 	case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
229 		cur = riscv_v_ctrl_get_cur(current);
230 		break;
231 	default:
232 		return -EINVAL;
233 	}
234 
235 	next = VSTATE_CTRL_GET_NEXT(arg);
236 	inherit = VSTATE_CTRL_GET_INHERIT(arg);
237 	switch (next) {
238 	case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
239 	case PR_RISCV_V_VSTATE_CTRL_OFF:
240 	case PR_RISCV_V_VSTATE_CTRL_ON:
241 		riscv_v_ctrl_set(current, cur, next, inherit);
242 		return 0;
243 	}
244 
245 	return -EINVAL;
246 }
247 
248 #ifdef CONFIG_SYSCTL
249 
250 static struct ctl_table riscv_v_default_vstate_table[] = {
251 	{
252 		.procname	= "riscv_v_default_allow",
253 		.data		= &riscv_v_implicit_uacc,
254 		.maxlen		= sizeof(riscv_v_implicit_uacc),
255 		.mode		= 0644,
256 		.proc_handler	= proc_dobool,
257 	},
258 	{ }
259 };
260 
riscv_v_sysctl_init(void)261 static int __init riscv_v_sysctl_init(void)
262 {
263 	if (has_vector())
264 		if (!register_sysctl("abi", riscv_v_default_vstate_table))
265 			return -EINVAL;
266 	return 0;
267 }
268 
269 #else /* ! CONFIG_SYSCTL */
riscv_v_sysctl_init(void)270 static int __init riscv_v_sysctl_init(void) { return 0; }
271 #endif /* ! CONFIG_SYSCTL */
272 
riscv_v_init(void)273 static int riscv_v_init(void)
274 {
275 	return riscv_v_sysctl_init();
276 }
277 core_initcall(riscv_v_init);
278