1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 ARM Ltd. 4 */ 5 #ifndef __ASM_FP_H 6 #define __ASM_FP_H 7 8 #include <asm/errno.h> 9 #include <asm/ptrace.h> 10 #include <asm/processor.h> 11 #include <asm/sigcontext.h> 12 #include <asm/sysreg.h> 13 14 #ifndef __ASSEMBLY__ 15 16 #include <linux/bitmap.h> 17 #include <linux/build_bug.h> 18 #include <linux/bug.h> 19 #include <linux/cache.h> 20 #include <linux/init.h> 21 #include <linux/stddef.h> 22 #include <linux/types.h> 23 24 #ifdef CONFIG_COMPAT 25 /* Masks for extracting the FPSR and FPCR from the FPSCR */ 26 #define VFP_FPSCR_STAT_MASK 0xf800009f 27 #define VFP_FPSCR_CTRL_MASK 0x07f79f00 28 /* 29 * The VFP state has 32x64-bit registers and a single 32-bit 30 * control/status register. 31 */ 32 #define VFP_STATE_SIZE ((32 * 8) + 4) 33 #endif 34 35 struct task_struct; 36 37 extern void fpsimd_save_state(struct user_fpsimd_state *state); 38 extern void fpsimd_load_state(struct user_fpsimd_state *state); 39 40 extern void fpsimd_thread_switch(struct task_struct *next); 41 extern void fpsimd_flush_thread(void); 42 43 extern void fpsimd_signal_preserve_current_state(void); 44 extern void fpsimd_preserve_current_state(void); 45 extern void fpsimd_restore_current_state(void); 46 extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); 47 48 extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state, 49 void *sve_state, unsigned int sve_vl, 50 u64 *svcr); 51 52 extern void fpsimd_flush_task_state(struct task_struct *target); 53 extern void fpsimd_save_and_flush_cpu_state(void); 54 55 /* Maximum VL that SVE/SME VL-agnostic software can transparently support */ 56 #define VL_ARCH_MAX 0x100 57 58 /* Offset of FFR in the SVE register dump */ 59 static inline size_t sve_ffr_offset(int vl) 60 { 61 return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET; 62 } 63 64 static inline void *sve_pffr(struct thread_struct *thread) 65 { 66 return (char *)thread->sve_state + sve_ffr_offset(thread_get_sve_vl(thread)); 67 } 68 69 extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); 70 extern void sve_load_state(void const *state, u32 const *pfpsr, 71 int restore_ffr); 72 extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); 73 extern unsigned int sve_get_vl(void); 74 extern void sve_set_vq(unsigned long vq_minus_1); 75 76 struct arm64_cpu_capabilities; 77 extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused); 78 extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused); 79 extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused); 80 81 extern u64 read_zcr_features(void); 82 extern u64 read_smcr_features(void); 83 84 /* 85 * Helpers to translate bit indices in sve_vq_map to VQ values (and 86 * vice versa). This allows find_next_bit() to be used to find the 87 * _maximum_ VQ not exceeding a certain value. 88 */ 89 static inline unsigned int __vq_to_bit(unsigned int vq) 90 { 91 return SVE_VQ_MAX - vq; 92 } 93 94 static inline unsigned int __bit_to_vq(unsigned int bit) 95 { 96 return SVE_VQ_MAX - bit; 97 } 98 99 100 struct vl_info { 101 enum vec_type type; 102 const char *name; /* For display purposes */ 103 104 /* Minimum supported vector length across all CPUs */ 105 int min_vl; 106 107 /* Maximum supported vector length across all CPUs */ 108 int max_vl; 109 int max_virtualisable_vl; 110 111 /* 112 * Set of available vector lengths, 113 * where length vq encoded as bit __vq_to_bit(vq): 114 */ 115 DECLARE_BITMAP(vq_map, SVE_VQ_MAX); 116 117 /* Set of vector lengths present on at least one cpu: */ 118 DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX); 119 }; 120 121 #ifdef CONFIG_ARM64_SVE 122 123 extern void sve_alloc(struct task_struct *task); 124 extern void fpsimd_release_task(struct task_struct *task); 125 extern void fpsimd_sync_to_sve(struct task_struct *task); 126 extern void sve_sync_to_fpsimd(struct task_struct *task); 127 extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task); 128 129 extern int vec_set_vector_length(struct task_struct *task, enum vec_type type, 130 unsigned long vl, unsigned long flags); 131 132 extern int sve_set_current_vl(unsigned long arg); 133 extern int sve_get_current_vl(void); 134 135 static inline void sve_user_disable(void) 136 { 137 sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0); 138 } 139 140 static inline void sve_user_enable(void) 141 { 142 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN); 143 } 144 145 #define sve_cond_update_zcr_vq(val, reg) \ 146 do { \ 147 u64 __zcr = read_sysreg_s((reg)); \ 148 u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \ 149 __new |= (val) & ZCR_ELx_LEN_MASK; \ 150 if (__zcr != __new) \ 151 write_sysreg_s(__new, (reg)); \ 152 } while (0) 153 154 /* 155 * Probing and setup functions. 156 * Calls to these functions must be serialised with one another. 157 */ 158 enum vec_type; 159 160 extern void __init vec_init_vq_map(enum vec_type type); 161 extern void vec_update_vq_map(enum vec_type type); 162 extern int vec_verify_vq_map(enum vec_type type); 163 extern void __init sve_setup(void); 164 165 extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX]; 166 167 static inline void write_vl(enum vec_type type, u64 val) 168 { 169 u64 tmp; 170 171 switch (type) { 172 #ifdef CONFIG_ARM64_SVE 173 case ARM64_VEC_SVE: 174 tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK; 175 write_sysreg_s(tmp | val, SYS_ZCR_EL1); 176 break; 177 #endif 178 #ifdef CONFIG_ARM64_SME 179 case ARM64_VEC_SME: 180 tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK; 181 write_sysreg_s(tmp | val, SYS_SMCR_EL1); 182 break; 183 #endif 184 default: 185 WARN_ON_ONCE(1); 186 break; 187 } 188 } 189 190 static inline int vec_max_vl(enum vec_type type) 191 { 192 return vl_info[type].max_vl; 193 } 194 195 static inline int vec_max_virtualisable_vl(enum vec_type type) 196 { 197 return vl_info[type].max_virtualisable_vl; 198 } 199 200 static inline int sve_max_vl(void) 201 { 202 return vec_max_vl(ARM64_VEC_SVE); 203 } 204 205 static inline int sve_max_virtualisable_vl(void) 206 { 207 return vec_max_virtualisable_vl(ARM64_VEC_SVE); 208 } 209 210 /* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */ 211 static inline bool vq_available(enum vec_type type, unsigned int vq) 212 { 213 return test_bit(__vq_to_bit(vq), vl_info[type].vq_map); 214 } 215 216 static inline bool sve_vq_available(unsigned int vq) 217 { 218 return vq_available(ARM64_VEC_SVE, vq); 219 } 220 221 #else /* ! CONFIG_ARM64_SVE */ 222 223 static inline void sve_alloc(struct task_struct *task) { } 224 static inline void fpsimd_release_task(struct task_struct *task) { } 225 static inline void sve_sync_to_fpsimd(struct task_struct *task) { } 226 static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { } 227 228 static inline int sve_max_virtualisable_vl(void) 229 { 230 return 0; 231 } 232 233 static inline int sve_set_current_vl(unsigned long arg) 234 { 235 return -EINVAL; 236 } 237 238 static inline int sve_get_current_vl(void) 239 { 240 return -EINVAL; 241 } 242 243 static inline int sve_max_vl(void) 244 { 245 return -EINVAL; 246 } 247 248 static inline bool sve_vq_available(unsigned int vq) { return false; } 249 250 static inline void sve_user_disable(void) { BUILD_BUG(); } 251 static inline void sve_user_enable(void) { BUILD_BUG(); } 252 253 #define sve_cond_update_zcr_vq(val, reg) do { } while (0) 254 255 static inline void vec_init_vq_map(enum vec_type t) { } 256 static inline void vec_update_vq_map(enum vec_type t) { } 257 static inline int vec_verify_vq_map(enum vec_type t) { return 0; } 258 static inline void sve_setup(void) { } 259 260 #endif /* ! CONFIG_ARM64_SVE */ 261 262 #ifdef CONFIG_ARM64_SME 263 264 static inline void sme_smstart_sm(void) 265 { 266 asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr")); 267 } 268 269 static inline void sme_smstop_sm(void) 270 { 271 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr")); 272 } 273 274 static inline void sme_smstop(void) 275 { 276 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr")); 277 } 278 279 extern void __init sme_setup(void); 280 281 static inline int sme_max_vl(void) 282 { 283 return vec_max_vl(ARM64_VEC_SME); 284 } 285 286 static inline int sme_max_virtualisable_vl(void) 287 { 288 return vec_max_virtualisable_vl(ARM64_VEC_SME); 289 } 290 291 extern unsigned int sme_get_vl(void); 292 extern int sme_set_current_vl(unsigned long arg); 293 extern int sme_get_current_vl(void); 294 295 #else 296 297 static inline void sme_smstart_sm(void) { } 298 static inline void sme_smstop_sm(void) { } 299 static inline void sme_smstop(void) { } 300 301 static inline void sme_setup(void) { } 302 static inline unsigned int sme_get_vl(void) { return 0; } 303 static inline int sme_max_vl(void) { return 0; } 304 static inline int sme_max_virtualisable_vl(void) { return 0; } 305 static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; } 306 static inline int sme_get_current_vl(void) { return -EINVAL; } 307 308 #endif /* ! CONFIG_ARM64_SME */ 309 310 /* For use by EFI runtime services calls only */ 311 extern void __efi_fpsimd_begin(void); 312 extern void __efi_fpsimd_end(void); 313 314 #endif 315 316 #endif 317