1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2023 SiFive
4 * Author: Andy Chiu <andy.chiu@sifive.com>
5 */
6 #include <linux/export.h>
7 #include <linux/sched/signal.h>
8 #include <linux/types.h>
9 #include <linux/slab.h>
10 #include <linux/sched.h>
11 #include <linux/uaccess.h>
12 #include <linux/prctl.h>
13
14 #include <asm/thread_info.h>
15 #include <asm/processor.h>
16 #include <asm/insn.h>
17 #include <asm/vector.h>
18 #include <asm/csr.h>
19 #include <asm/elf.h>
20 #include <asm/ptrace.h>
21 #include <asm/bug.h>
22
23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
24
25 unsigned long riscv_v_vsize __read_mostly;
26 EXPORT_SYMBOL_GPL(riscv_v_vsize);
27
riscv_v_setup_vsize(void)28 int riscv_v_setup_vsize(void)
29 {
30 unsigned long this_vsize;
31
32 /* There are 32 vector registers with vlenb length. */
33 riscv_v_enable();
34 this_vsize = csr_read(CSR_VLENB) * 32;
35 riscv_v_disable();
36
37 if (!riscv_v_vsize) {
38 riscv_v_vsize = this_vsize;
39 return 0;
40 }
41
42 if (riscv_v_vsize != this_vsize) {
43 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
44 return -EOPNOTSUPP;
45 }
46
47 return 0;
48 }
49
insn_is_vector(u32 insn_buf)50 static bool insn_is_vector(u32 insn_buf)
51 {
52 u32 opcode = insn_buf & __INSN_OPCODE_MASK;
53 u32 width, csr;
54
55 /*
56 * All V-related instructions, including CSR operations are 4-Byte. So,
57 * do not handle if the instruction length is not 4-Byte.
58 */
59 if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
60 return false;
61
62 switch (opcode) {
63 case RVV_OPCODE_VECTOR:
64 return true;
65 case RVV_OPCODE_VL:
66 case RVV_OPCODE_VS:
67 width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
68 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
69 width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
70 return true;
71
72 break;
73 case RVG_OPCODE_SYSTEM:
74 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
75 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
76 (csr >= CSR_VL && csr <= CSR_VLENB))
77 return true;
78 }
79
80 return false;
81 }
82
riscv_v_thread_zalloc(void)83 static int riscv_v_thread_zalloc(void)
84 {
85 void *datap;
86
87 datap = kzalloc(riscv_v_vsize, GFP_KERNEL);
88 if (!datap)
89 return -ENOMEM;
90
91 current->thread.vstate.datap = datap;
92 memset(¤t->thread.vstate, 0, offsetof(struct __riscv_v_ext_state,
93 datap));
94 return 0;
95 }
96
97 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
98 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
99 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
100 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
riscv_v_ctrl_get_cur(struct task_struct * tsk)101 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
102 {
103 return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
104 }
105
riscv_v_ctrl_get_next(struct task_struct * tsk)106 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
107 {
108 return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
109 }
110
riscv_v_ctrl_test_inherit(struct task_struct * tsk)111 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
112 {
113 return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
114 }
115
riscv_v_ctrl_set(struct task_struct * tsk,int cur,int nxt,bool inherit)116 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
117 bool inherit)
118 {
119 unsigned long ctrl;
120
121 ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
122 ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
123 if (inherit)
124 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
125 tsk->thread.vstate_ctrl = ctrl;
126 }
127
riscv_v_vstate_ctrl_user_allowed(void)128 bool riscv_v_vstate_ctrl_user_allowed(void)
129 {
130 return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
131 }
132 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
133
riscv_v_first_use_handler(struct pt_regs * regs)134 bool riscv_v_first_use_handler(struct pt_regs *regs)
135 {
136 u32 __user *epc = (u32 __user *)regs->epc;
137 u32 insn = (u32)regs->badaddr;
138
139 /* Do not handle if V is not supported, or disabled */
140 if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
141 return false;
142
143 /* If V has been enabled then it is not the first-use trap */
144 if (riscv_v_vstate_query(regs))
145 return false;
146
147 /* Get the instruction */
148 if (!insn) {
149 if (__get_user(insn, epc))
150 return false;
151 }
152
153 /* Filter out non-V instructions */
154 if (!insn_is_vector(insn))
155 return false;
156
157 /* Sanity check. datap should be null by the time of the first-use trap */
158 WARN_ON(current->thread.vstate.datap);
159
160 /*
161 * Now we sure that this is a V instruction. And it executes in the
162 * context where VS has been off. So, try to allocate the user's V
163 * context and resume execution.
164 */
165 if (riscv_v_thread_zalloc()) {
166 force_sig(SIGBUS);
167 return true;
168 }
169 riscv_v_vstate_on(regs);
170 riscv_v_vstate_restore(current, regs);
171 return true;
172 }
173
riscv_v_vstate_ctrl_init(struct task_struct * tsk)174 void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
175 {
176 bool inherit;
177 int cur, next;
178
179 if (!has_vector())
180 return;
181
182 next = riscv_v_ctrl_get_next(tsk);
183 if (!next) {
184 if (READ_ONCE(riscv_v_implicit_uacc))
185 cur = PR_RISCV_V_VSTATE_CTRL_ON;
186 else
187 cur = PR_RISCV_V_VSTATE_CTRL_OFF;
188 } else {
189 cur = next;
190 }
191 /* Clear next mask if inherit-bit is not set */
192 inherit = riscv_v_ctrl_test_inherit(tsk);
193 if (!inherit)
194 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
195
196 riscv_v_ctrl_set(tsk, cur, next, inherit);
197 }
198
riscv_v_vstate_ctrl_get_current(void)199 long riscv_v_vstate_ctrl_get_current(void)
200 {
201 if (!has_vector())
202 return -EINVAL;
203
204 return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
205 }
206
riscv_v_vstate_ctrl_set_current(unsigned long arg)207 long riscv_v_vstate_ctrl_set_current(unsigned long arg)
208 {
209 bool inherit;
210 int cur, next;
211
212 if (!has_vector())
213 return -EINVAL;
214
215 if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
216 return -EINVAL;
217
218 cur = VSTATE_CTRL_GET_CUR(arg);
219 switch (cur) {
220 case PR_RISCV_V_VSTATE_CTRL_OFF:
221 /* Do not allow user to turn off V if current is not off */
222 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
223 return -EPERM;
224
225 break;
226 case PR_RISCV_V_VSTATE_CTRL_ON:
227 break;
228 case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
229 cur = riscv_v_ctrl_get_cur(current);
230 break;
231 default:
232 return -EINVAL;
233 }
234
235 next = VSTATE_CTRL_GET_NEXT(arg);
236 inherit = VSTATE_CTRL_GET_INHERIT(arg);
237 switch (next) {
238 case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
239 case PR_RISCV_V_VSTATE_CTRL_OFF:
240 case PR_RISCV_V_VSTATE_CTRL_ON:
241 riscv_v_ctrl_set(current, cur, next, inherit);
242 return 0;
243 }
244
245 return -EINVAL;
246 }
247
248 #ifdef CONFIG_SYSCTL
249
250 static struct ctl_table riscv_v_default_vstate_table[] = {
251 {
252 .procname = "riscv_v_default_allow",
253 .data = &riscv_v_implicit_uacc,
254 .maxlen = sizeof(riscv_v_implicit_uacc),
255 .mode = 0644,
256 .proc_handler = proc_dobool,
257 },
258 { }
259 };
260
riscv_v_sysctl_init(void)261 static int __init riscv_v_sysctl_init(void)
262 {
263 if (has_vector())
264 if (!register_sysctl("abi", riscv_v_default_vstate_table))
265 return -EINVAL;
266 return 0;
267 }
268
269 #else /* ! CONFIG_SYSCTL */
riscv_v_sysctl_init(void)270 static int __init riscv_v_sysctl_init(void) { return 0; }
271 #endif /* ! CONFIG_SYSCTL */
272
riscv_v_init(void)273 static int riscv_v_init(void)
274 {
275 return riscv_v_sysctl_init();
276 }
277 core_initcall(riscv_v_init);
278