1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2023 SiFive 4 * Author: Andy Chiu <andy.chiu@sifive.com> 5 */ 6 #include <linux/export.h> 7 #include <linux/sched/signal.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 #include <linux/uaccess.h> 12 #include <linux/prctl.h> 13 14 #include <asm/thread_info.h> 15 #include <asm/processor.h> 16 #include <asm/insn.h> 17 #include <asm/vector.h> 18 #include <asm/csr.h> 19 #include <asm/elf.h> 20 #include <asm/ptrace.h> 21 #include <asm/bug.h> 22 23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE); 24 25 unsigned long riscv_v_vsize __read_mostly; 26 EXPORT_SYMBOL_GPL(riscv_v_vsize); 27 28 int riscv_v_setup_vsize(void) 29 { 30 unsigned long this_vsize; 31 32 /* There are 32 vector registers with vlenb length. */ 33 riscv_v_enable(); 34 this_vsize = csr_read(CSR_VLENB) * 32; 35 riscv_v_disable(); 36 37 if (!riscv_v_vsize) { 38 riscv_v_vsize = this_vsize; 39 return 0; 40 } 41 42 if (riscv_v_vsize != this_vsize) { 43 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems"); 44 return -EOPNOTSUPP; 45 } 46 47 return 0; 48 } 49 50 static bool insn_is_vector(u32 insn_buf) 51 { 52 u32 opcode = insn_buf & __INSN_OPCODE_MASK; 53 u32 width, csr; 54 55 /* 56 * All V-related instructions, including CSR operations are 4-Byte. So, 57 * do not handle if the instruction length is not 4-Byte. 58 */ 59 if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) 60 return false; 61 62 switch (opcode) { 63 case RVV_OPCODE_VECTOR: 64 return true; 65 case RVV_OPCODE_VL: 66 case RVV_OPCODE_VS: 67 width = RVV_EXRACT_VL_VS_WIDTH(insn_buf); 68 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || 69 width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) 70 return true; 71 72 break; 73 case RVG_OPCODE_SYSTEM: 74 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); 75 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || 76 (csr >= CSR_VL && csr <= CSR_VLENB)) 77 return true; 78 } 79 80 return false; 81 } 82 83 static int riscv_v_thread_zalloc(void) 84 { 85 void *datap; 86 87 datap = kzalloc(riscv_v_vsize, GFP_KERNEL); 88 if (!datap) 89 return -ENOMEM; 90 91 current->thread.vstate.datap = datap; 92 memset(¤t->thread.vstate, 0, offsetof(struct __riscv_v_ext_state, 93 datap)); 94 return 0; 95 } 96 97 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) 98 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2) 99 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) 100 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT)) 101 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk) 102 { 103 return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl); 104 } 105 106 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk) 107 { 108 return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl); 109 } 110 111 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk) 112 { 113 return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl); 114 } 115 116 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, 117 bool inherit) 118 { 119 unsigned long ctrl; 120 121 ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; 122 ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); 123 if (inherit) 124 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; 125 tsk->thread.vstate_ctrl = ctrl; 126 } 127 128 bool riscv_v_vstate_ctrl_user_allowed(void) 129 { 130 return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON; 131 } 132 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed); 133 134 bool riscv_v_first_use_handler(struct pt_regs *regs) 135 { 136 u32 __user *epc = (u32 __user *)regs->epc; 137 u32 insn = (u32)regs->badaddr; 138 139 /* Do not handle if V is not supported, or disabled */ 140 if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V)) 141 return false; 142 143 /* If V has been enabled then it is not the first-use trap */ 144 if (riscv_v_vstate_query(regs)) 145 return false; 146 147 /* Get the instruction */ 148 if (!insn) { 149 if (__get_user(insn, epc)) 150 return false; 151 } 152 153 /* Filter out non-V instructions */ 154 if (!insn_is_vector(insn)) 155 return false; 156 157 /* Sanity check. datap should be null by the time of the first-use trap */ 158 WARN_ON(current->thread.vstate.datap); 159 160 /* 161 * Now we sure that this is a V instruction. And it executes in the 162 * context where VS has been off. So, try to allocate the user's V 163 * context and resume execution. 164 */ 165 if (riscv_v_thread_zalloc()) { 166 force_sig(SIGBUS); 167 return true; 168 } 169 riscv_v_vstate_on(regs); 170 riscv_v_vstate_restore(current, regs); 171 return true; 172 } 173 174 void riscv_v_vstate_ctrl_init(struct task_struct *tsk) 175 { 176 bool inherit; 177 int cur, next; 178 179 if (!has_vector()) 180 return; 181 182 next = riscv_v_ctrl_get_next(tsk); 183 if (!next) { 184 if (READ_ONCE(riscv_v_implicit_uacc)) 185 cur = PR_RISCV_V_VSTATE_CTRL_ON; 186 else 187 cur = PR_RISCV_V_VSTATE_CTRL_OFF; 188 } else { 189 cur = next; 190 } 191 /* Clear next mask if inherit-bit is not set */ 192 inherit = riscv_v_ctrl_test_inherit(tsk); 193 if (!inherit) 194 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT; 195 196 riscv_v_ctrl_set(tsk, cur, next, inherit); 197 } 198 199 long riscv_v_vstate_ctrl_get_current(void) 200 { 201 if (!has_vector()) 202 return -EINVAL; 203 204 return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK; 205 } 206 207 long riscv_v_vstate_ctrl_set_current(unsigned long arg) 208 { 209 bool inherit; 210 int cur, next; 211 212 if (!has_vector()) 213 return -EINVAL; 214 215 if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) 216 return -EINVAL; 217 218 cur = VSTATE_CTRL_GET_CUR(arg); 219 switch (cur) { 220 case PR_RISCV_V_VSTATE_CTRL_OFF: 221 /* Do not allow user to turn off V if current is not off */ 222 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) 223 return -EPERM; 224 225 break; 226 case PR_RISCV_V_VSTATE_CTRL_ON: 227 break; 228 case PR_RISCV_V_VSTATE_CTRL_DEFAULT: 229 cur = riscv_v_ctrl_get_cur(current); 230 break; 231 default: 232 return -EINVAL; 233 } 234 235 next = VSTATE_CTRL_GET_NEXT(arg); 236 inherit = VSTATE_CTRL_GET_INHERIT(arg); 237 switch (next) { 238 case PR_RISCV_V_VSTATE_CTRL_DEFAULT: 239 case PR_RISCV_V_VSTATE_CTRL_OFF: 240 case PR_RISCV_V_VSTATE_CTRL_ON: 241 riscv_v_ctrl_set(current, cur, next, inherit); 242 return 0; 243 } 244 245 return -EINVAL; 246 } 247 248 #ifdef CONFIG_SYSCTL 249 250 static struct ctl_table riscv_v_default_vstate_table[] = { 251 { 252 .procname = "riscv_v_default_allow", 253 .data = &riscv_v_implicit_uacc, 254 .maxlen = sizeof(riscv_v_implicit_uacc), 255 .mode = 0644, 256 .proc_handler = proc_dobool, 257 }, 258 { } 259 }; 260 261 static int __init riscv_v_sysctl_init(void) 262 { 263 if (has_vector()) 264 if (!register_sysctl("abi", riscv_v_default_vstate_table)) 265 return -EINVAL; 266 return 0; 267 } 268 269 #else /* ! CONFIG_SYSCTL */ 270 static int __init riscv_v_sysctl_init(void) { return 0; } 271 #endif /* ! CONFIG_SYSCTL */ 272 273 static int riscv_v_init(void) 274 { 275 return riscv_v_sysctl_init(); 276 } 277 core_initcall(riscv_v_init); 278