1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 ARM Ltd. 4 */ 5 #ifndef __ASM_FP_H 6 #define __ASM_FP_H 7 8 #include <asm/errno.h> 9 #include <asm/ptrace.h> 10 #include <asm/processor.h> 11 #include <asm/sigcontext.h> 12 #include <asm/sysreg.h> 13 14 #ifndef __ASSEMBLY__ 15 16 #include <linux/bitmap.h> 17 #include <linux/build_bug.h> 18 #include <linux/bug.h> 19 #include <linux/cache.h> 20 #include <linux/init.h> 21 #include <linux/stddef.h> 22 #include <linux/types.h> 23 24 #ifdef CONFIG_COMPAT 25 /* Masks for extracting the FPSR and FPCR from the FPSCR */ 26 #define VFP_FPSCR_STAT_MASK 0xf800009f 27 #define VFP_FPSCR_CTRL_MASK 0x07f79f00 28 /* 29 * The VFP state has 32x64-bit registers and a single 32-bit 30 * control/status register. 31 */ 32 #define VFP_STATE_SIZE ((32 * 8) + 4) 33 #endif 34 35 struct task_struct; 36 37 extern void fpsimd_save_state(struct user_fpsimd_state *state); 38 extern void fpsimd_load_state(struct user_fpsimd_state *state); 39 40 extern void fpsimd_thread_switch(struct task_struct *next); 41 extern void fpsimd_flush_thread(void); 42 43 extern void fpsimd_signal_preserve_current_state(void); 44 extern void fpsimd_preserve_current_state(void); 45 extern void fpsimd_restore_current_state(void); 46 extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); 47 48 extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state, 49 void *sve_state, unsigned int sve_vl, 50 void *za_state, unsigned int sme_vl, 51 u64 *svcr); 52 53 extern void fpsimd_flush_task_state(struct task_struct *target); 54 extern void fpsimd_save_and_flush_cpu_state(void); 55 56 static inline bool thread_sm_enabled(struct thread_struct *thread) 57 { 58 return system_supports_sme() && (thread->svcr & SYS_SVCR_EL0_SM_MASK); 59 } 60 61 static inline bool thread_za_enabled(struct thread_struct *thread) 62 { 63 return system_supports_sme() && (thread->svcr & SYS_SVCR_EL0_ZA_MASK); 64 } 65 66 /* Maximum VL that SVE/SME VL-agnostic software can transparently support */ 67 #define VL_ARCH_MAX 0x100 68 69 /* Offset of FFR in the SVE register dump */ 70 static inline size_t sve_ffr_offset(int vl) 71 { 72 return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET; 73 } 74 75 static inline void *sve_pffr(struct thread_struct *thread) 76 { 77 unsigned int vl; 78 79 if (system_supports_sme() && thread_sm_enabled(thread)) 80 vl = thread_get_sme_vl(thread); 81 else 82 vl = thread_get_sve_vl(thread); 83 84 return (char *)thread->sve_state + sve_ffr_offset(vl); 85 } 86 87 extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); 88 extern void sve_load_state(void const *state, u32 const *pfpsr, 89 int restore_ffr); 90 extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); 91 extern unsigned int sve_get_vl(void); 92 extern void sve_set_vq(unsigned long vq_minus_1); 93 extern void sme_set_vq(unsigned long vq_minus_1); 94 extern void za_save_state(void *state); 95 extern void za_load_state(void const *state); 96 97 struct arm64_cpu_capabilities; 98 extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused); 99 extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused); 100 extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused); 101 102 extern u64 read_zcr_features(void); 103 extern u64 read_smcr_features(void); 104 105 /* 106 * Helpers to translate bit indices in sve_vq_map to VQ values (and 107 * vice versa). This allows find_next_bit() to be used to find the 108 * _maximum_ VQ not exceeding a certain value. 109 */ 110 static inline unsigned int __vq_to_bit(unsigned int vq) 111 { 112 return SVE_VQ_MAX - vq; 113 } 114 115 static inline unsigned int __bit_to_vq(unsigned int bit) 116 { 117 return SVE_VQ_MAX - bit; 118 } 119 120 121 struct vl_info { 122 enum vec_type type; 123 const char *name; /* For display purposes */ 124 125 /* Minimum supported vector length across all CPUs */ 126 int min_vl; 127 128 /* Maximum supported vector length across all CPUs */ 129 int max_vl; 130 int max_virtualisable_vl; 131 132 /* 133 * Set of available vector lengths, 134 * where length vq encoded as bit __vq_to_bit(vq): 135 */ 136 DECLARE_BITMAP(vq_map, SVE_VQ_MAX); 137 138 /* Set of vector lengths present on at least one cpu: */ 139 DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX); 140 }; 141 142 #ifdef CONFIG_ARM64_SVE 143 144 extern void sve_alloc(struct task_struct *task); 145 extern void fpsimd_release_task(struct task_struct *task); 146 extern void fpsimd_sync_to_sve(struct task_struct *task); 147 extern void fpsimd_force_sync_to_sve(struct task_struct *task); 148 extern void sve_sync_to_fpsimd(struct task_struct *task); 149 extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task); 150 151 extern int vec_set_vector_length(struct task_struct *task, enum vec_type type, 152 unsigned long vl, unsigned long flags); 153 154 extern int sve_set_current_vl(unsigned long arg); 155 extern int sve_get_current_vl(void); 156 157 static inline void sve_user_disable(void) 158 { 159 sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0); 160 } 161 162 static inline void sve_user_enable(void) 163 { 164 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN); 165 } 166 167 #define sve_cond_update_zcr_vq(val, reg) \ 168 do { \ 169 u64 __zcr = read_sysreg_s((reg)); \ 170 u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \ 171 __new |= (val) & ZCR_ELx_LEN_MASK; \ 172 if (__zcr != __new) \ 173 write_sysreg_s(__new, (reg)); \ 174 } while (0) 175 176 /* 177 * Probing and setup functions. 178 * Calls to these functions must be serialised with one another. 179 */ 180 enum vec_type; 181 182 extern void __init vec_init_vq_map(enum vec_type type); 183 extern void vec_update_vq_map(enum vec_type type); 184 extern int vec_verify_vq_map(enum vec_type type); 185 extern void __init sve_setup(void); 186 187 extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX]; 188 189 static inline void write_vl(enum vec_type type, u64 val) 190 { 191 u64 tmp; 192 193 switch (type) { 194 #ifdef CONFIG_ARM64_SVE 195 case ARM64_VEC_SVE: 196 tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK; 197 write_sysreg_s(tmp | val, SYS_ZCR_EL1); 198 break; 199 #endif 200 #ifdef CONFIG_ARM64_SME 201 case ARM64_VEC_SME: 202 tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK; 203 write_sysreg_s(tmp | val, SYS_SMCR_EL1); 204 break; 205 #endif 206 default: 207 WARN_ON_ONCE(1); 208 break; 209 } 210 } 211 212 static inline int vec_max_vl(enum vec_type type) 213 { 214 return vl_info[type].max_vl; 215 } 216 217 static inline int vec_max_virtualisable_vl(enum vec_type type) 218 { 219 return vl_info[type].max_virtualisable_vl; 220 } 221 222 static inline int sve_max_vl(void) 223 { 224 return vec_max_vl(ARM64_VEC_SVE); 225 } 226 227 static inline int sve_max_virtualisable_vl(void) 228 { 229 return vec_max_virtualisable_vl(ARM64_VEC_SVE); 230 } 231 232 /* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */ 233 static inline bool vq_available(enum vec_type type, unsigned int vq) 234 { 235 return test_bit(__vq_to_bit(vq), vl_info[type].vq_map); 236 } 237 238 static inline bool sve_vq_available(unsigned int vq) 239 { 240 return vq_available(ARM64_VEC_SVE, vq); 241 } 242 243 size_t sve_state_size(struct task_struct const *task); 244 245 #else /* ! CONFIG_ARM64_SVE */ 246 247 static inline void sve_alloc(struct task_struct *task) { } 248 static inline void fpsimd_release_task(struct task_struct *task) { } 249 static inline void sve_sync_to_fpsimd(struct task_struct *task) { } 250 static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { } 251 252 static inline int sve_max_virtualisable_vl(void) 253 { 254 return 0; 255 } 256 257 static inline int sve_set_current_vl(unsigned long arg) 258 { 259 return -EINVAL; 260 } 261 262 static inline int sve_get_current_vl(void) 263 { 264 return -EINVAL; 265 } 266 267 static inline int sve_max_vl(void) 268 { 269 return -EINVAL; 270 } 271 272 static inline bool sve_vq_available(unsigned int vq) { return false; } 273 274 static inline void sve_user_disable(void) { BUILD_BUG(); } 275 static inline void sve_user_enable(void) { BUILD_BUG(); } 276 277 #define sve_cond_update_zcr_vq(val, reg) do { } while (0) 278 279 static inline void vec_init_vq_map(enum vec_type t) { } 280 static inline void vec_update_vq_map(enum vec_type t) { } 281 static inline int vec_verify_vq_map(enum vec_type t) { return 0; } 282 static inline void sve_setup(void) { } 283 284 static inline size_t sve_state_size(struct task_struct const *task) 285 { 286 return 0; 287 } 288 289 #endif /* ! CONFIG_ARM64_SVE */ 290 291 #ifdef CONFIG_ARM64_SME 292 293 static inline void sme_user_disable(void) 294 { 295 sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0); 296 } 297 298 static inline void sme_user_enable(void) 299 { 300 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN); 301 } 302 303 static inline void sme_smstart_sm(void) 304 { 305 asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr")); 306 } 307 308 static inline void sme_smstop_sm(void) 309 { 310 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr")); 311 } 312 313 static inline void sme_smstop(void) 314 { 315 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr")); 316 } 317 318 extern void __init sme_setup(void); 319 320 static inline int sme_max_vl(void) 321 { 322 return vec_max_vl(ARM64_VEC_SME); 323 } 324 325 static inline int sme_max_virtualisable_vl(void) 326 { 327 return vec_max_virtualisable_vl(ARM64_VEC_SME); 328 } 329 330 extern void sme_alloc(struct task_struct *task); 331 extern unsigned int sme_get_vl(void); 332 extern int sme_set_current_vl(unsigned long arg); 333 extern int sme_get_current_vl(void); 334 335 /* 336 * Return how many bytes of memory are required to store the full SME 337 * specific state (currently just ZA) for task, given task's currently 338 * configured vector length. 339 */ 340 static inline size_t za_state_size(struct task_struct const *task) 341 { 342 unsigned int vl = task_get_sme_vl(task); 343 344 return ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl)); 345 } 346 347 #else 348 349 static inline void sme_user_disable(void) { BUILD_BUG(); } 350 static inline void sme_user_enable(void) { BUILD_BUG(); } 351 352 static inline void sme_smstart_sm(void) { } 353 static inline void sme_smstop_sm(void) { } 354 static inline void sme_smstop(void) { } 355 356 static inline void sme_alloc(struct task_struct *task) { } 357 static inline void sme_setup(void) { } 358 static inline unsigned int sme_get_vl(void) { return 0; } 359 static inline int sme_max_vl(void) { return 0; } 360 static inline int sme_max_virtualisable_vl(void) { return 0; } 361 static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; } 362 static inline int sme_get_current_vl(void) { return -EINVAL; } 363 364 static inline size_t za_state_size(struct task_struct const *task) 365 { 366 return 0; 367 } 368 369 #endif /* ! CONFIG_ARM64_SME */ 370 371 /* For use by EFI runtime services calls only */ 372 extern void __efi_fpsimd_begin(void); 373 extern void __efi_fpsimd_end(void); 374 375 #endif 376 377 #endif 378