1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #ifndef __ASM_RISCV_VECTOR_H 7 #define __ASM_RISCV_VECTOR_H 8 9 #include <linux/types.h> 10 #include <uapi/asm-generic/errno.h> 11 12 #ifdef CONFIG_RISCV_ISA_V 13 14 #include <linux/stringify.h> 15 #include <linux/sched.h> 16 #include <linux/sched/task_stack.h> 17 #include <asm/ptrace.h> 18 #include <asm/hwcap.h> 19 #include <asm/csr.h> 20 #include <asm/asm.h> 21 22 extern unsigned long riscv_v_vsize; 23 int riscv_v_setup_vsize(void); 24 bool riscv_v_first_use_handler(struct pt_regs *regs); 25 26 static __always_inline bool has_vector(void) 27 { 28 return riscv_has_extension_unlikely(RISCV_ISA_EXT_v); 29 } 30 31 static inline void __riscv_v_vstate_clean(struct pt_regs *regs) 32 { 33 regs->status = (regs->status & ~SR_VS) | SR_VS_CLEAN; 34 } 35 36 static inline void __riscv_v_vstate_dirty(struct pt_regs *regs) 37 { 38 regs->status = (regs->status & ~SR_VS) | SR_VS_DIRTY; 39 } 40 41 static inline void riscv_v_vstate_off(struct pt_regs *regs) 42 { 43 regs->status = (regs->status & ~SR_VS) | SR_VS_OFF; 44 } 45 46 static inline void riscv_v_vstate_on(struct pt_regs *regs) 47 { 48 regs->status = (regs->status & ~SR_VS) | SR_VS_INITIAL; 49 } 50 51 static inline bool riscv_v_vstate_query(struct pt_regs *regs) 52 { 53 return (regs->status & SR_VS) != 0; 54 } 55 56 static __always_inline void riscv_v_enable(void) 57 { 58 csr_set(CSR_SSTATUS, SR_VS); 59 } 60 61 static __always_inline void riscv_v_disable(void) 62 { 63 csr_clear(CSR_SSTATUS, SR_VS); 64 } 65 66 static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) 67 { 68 asm volatile ( 69 "csrr %0, " __stringify(CSR_VSTART) "\n\t" 70 "csrr %1, " __stringify(CSR_VTYPE) "\n\t" 71 "csrr %2, " __stringify(CSR_VL) "\n\t" 72 "csrr %3, " __stringify(CSR_VCSR) "\n\t" 73 : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl), 74 "=r" (dest->vcsr) : :); 75 } 76 77 static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src) 78 { 79 asm volatile ( 80 ".option push\n\t" 81 ".option arch, +v\n\t" 82 "vsetvl x0, %2, %1\n\t" 83 ".option pop\n\t" 84 "csrw " __stringify(CSR_VSTART) ", %0\n\t" 85 "csrw " __stringify(CSR_VCSR) ", %3\n\t" 86 : : "r" (src->vstart), "r" (src->vtype), "r" (src->vl), 87 "r" (src->vcsr) :); 88 } 89 90 static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, 91 void *datap) 92 { 93 unsigned long vl; 94 95 riscv_v_enable(); 96 __vstate_csr_save(save_to); 97 asm volatile ( 98 ".option push\n\t" 99 ".option arch, +v\n\t" 100 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 101 "vse8.v v0, (%1)\n\t" 102 "add %1, %1, %0\n\t" 103 "vse8.v v8, (%1)\n\t" 104 "add %1, %1, %0\n\t" 105 "vse8.v v16, (%1)\n\t" 106 "add %1, %1, %0\n\t" 107 "vse8.v v24, (%1)\n\t" 108 ".option pop\n\t" 109 : "=&r" (vl) : "r" (datap) : "memory"); 110 riscv_v_disable(); 111 } 112 113 static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from, 114 void *datap) 115 { 116 unsigned long vl; 117 118 riscv_v_enable(); 119 asm volatile ( 120 ".option push\n\t" 121 ".option arch, +v\n\t" 122 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 123 "vle8.v v0, (%1)\n\t" 124 "add %1, %1, %0\n\t" 125 "vle8.v v8, (%1)\n\t" 126 "add %1, %1, %0\n\t" 127 "vle8.v v16, (%1)\n\t" 128 "add %1, %1, %0\n\t" 129 "vle8.v v24, (%1)\n\t" 130 ".option pop\n\t" 131 : "=&r" (vl) : "r" (datap) : "memory"); 132 __vstate_csr_restore(restore_from); 133 riscv_v_disable(); 134 } 135 136 static inline void __riscv_v_vstate_discard(void) 137 { 138 unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1); 139 140 riscv_v_enable(); 141 asm volatile ( 142 ".option push\n\t" 143 ".option arch, +v\n\t" 144 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 145 "vmv.v.i v0, -1\n\t" 146 "vmv.v.i v8, -1\n\t" 147 "vmv.v.i v16, -1\n\t" 148 "vmv.v.i v24, -1\n\t" 149 "vsetvl %0, x0, %1\n\t" 150 ".option pop\n\t" 151 : "=&r" (vl) : "r" (vtype_inval) : "memory"); 152 riscv_v_disable(); 153 } 154 155 static inline void riscv_v_vstate_discard(struct pt_regs *regs) 156 { 157 if ((regs->status & SR_VS) == SR_VS_OFF) 158 return; 159 160 __riscv_v_vstate_discard(); 161 __riscv_v_vstate_dirty(regs); 162 } 163 164 static inline void riscv_v_vstate_save(struct task_struct *task, 165 struct pt_regs *regs) 166 { 167 if ((regs->status & SR_VS) == SR_VS_DIRTY) { 168 struct __riscv_v_ext_state *vstate = &task->thread.vstate; 169 170 __riscv_v_vstate_save(vstate, vstate->datap); 171 __riscv_v_vstate_clean(regs); 172 } 173 } 174 175 static inline void riscv_v_vstate_restore(struct task_struct *task, 176 struct pt_regs *regs) 177 { 178 if ((regs->status & SR_VS) != SR_VS_OFF) { 179 struct __riscv_v_ext_state *vstate = &task->thread.vstate; 180 181 __riscv_v_vstate_restore(vstate, vstate->datap); 182 __riscv_v_vstate_clean(regs); 183 } 184 } 185 186 static inline void __switch_to_vector(struct task_struct *prev, 187 struct task_struct *next) 188 { 189 struct pt_regs *regs; 190 191 regs = task_pt_regs(prev); 192 riscv_v_vstate_save(prev, regs); 193 riscv_v_vstate_restore(next, task_pt_regs(next)); 194 } 195 196 void riscv_v_vstate_ctrl_init(struct task_struct *tsk); 197 bool riscv_v_vstate_ctrl_user_allowed(void); 198 199 #else /* ! CONFIG_RISCV_ISA_V */ 200 201 struct pt_regs; 202 203 static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; } 204 static __always_inline bool has_vector(void) { return false; } 205 static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; } 206 static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; } 207 static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; } 208 #define riscv_v_vsize (0) 209 #define riscv_v_vstate_discard(regs) do {} while (0) 210 #define riscv_v_vstate_save(task, regs) do {} while (0) 211 #define riscv_v_vstate_restore(task, regs) do {} while (0) 212 #define __switch_to_vector(__prev, __next) do {} while (0) 213 #define riscv_v_vstate_off(regs) do {} while (0) 214 #define riscv_v_vstate_on(regs) do {} while (0) 215 216 #endif /* CONFIG_RISCV_ISA_V */ 217 218 #endif /* ! __ASM_RISCV_VECTOR_H */ 219