1#include <asm/processor.h> 2#include <asm/ppc_asm.h> 3#include <asm/reg.h> 4#include <asm/asm-offsets.h> 5#include <asm/cputable.h> 6#include <asm/thread_info.h> 7#include <asm/page.h> 8#include <asm/ptrace.h> 9#include <asm/export.h> 10 11/* 12 * Load state from memory into VMX registers including VSCR. 13 * Assumes the caller has enabled VMX in the MSR. 14 */ 15_GLOBAL(load_vr_state) 16 li r4,VRSTATE_VSCR 17 lvx v0,r4,r3 18 mtvscr v0 19 REST_32VRS(0,r4,r3) 20 blr 21EXPORT_SYMBOL(load_vr_state) 22 23/* 24 * Store VMX state into memory, including VSCR. 25 * Assumes the caller has enabled VMX in the MSR. 26 */ 27_GLOBAL(store_vr_state) 28 SAVE_32VRS(0, r4, r3) 29 mfvscr v0 30 li r4, VRSTATE_VSCR 31 stvx v0, r4, r3 32 blr 33EXPORT_SYMBOL(store_vr_state) 34 35/* 36 * Disable VMX for the task which had it previously, 37 * and save its vector registers in its thread_struct. 38 * Enables the VMX for use in the kernel on return. 39 * On SMP we know the VMX is free, since we give it up every 40 * switch (ie, no lazy save of the vector registers). 41 * 42 * Note that on 32-bit this can only use registers that will be 43 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11. 44 */ 45_GLOBAL(load_up_altivec) 46 mfmsr r5 /* grab the current MSR */ 47 oris r5,r5,MSR_VEC@h 48 MTMSRD(r5) /* enable use of AltiVec now */ 49 isync 50 51 /* 52 * While userspace in general ignores VRSAVE, glibc uses it as a boolean 53 * to optimise userspace context save/restore. Whenever we take an 54 * altivec unavailable exception we must set VRSAVE to something non 55 * zero. Set it to all 1s. See also the programming note in the ISA. 56 */ 57 mfspr r4,SPRN_VRSAVE 58 cmpwi 0,r4,0 59 bne+ 1f 60 li r4,-1 61 mtspr SPRN_VRSAVE,r4 621: 63 /* enable use of VMX after return */ 64#ifdef CONFIG_PPC32 65 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ 66 oris r9,r9,MSR_VEC@h 67#else 68 ld r4,PACACURRENT(r13) 69 addi r5,r4,THREAD /* Get THREAD */ 70 oris r12,r12,MSR_VEC@h 71 std r12,_MSR(r1) 72#endif 73 /* Don't care if r4 overflows, this is desired behaviour */ 74 lbz r4,THREAD_LOAD_VEC(r5) 75 addi r4,r4,1 76 stb r4,THREAD_LOAD_VEC(r5) 77 addi r6,r5,THREAD_VRSTATE 78 li r4,1 79 li r10,VRSTATE_VSCR 80 stw r4,THREAD_USED_VR(r5) 81 lvx v0,r10,r6 82 mtvscr v0 83 REST_32VRS(0,r4,r6) 84 /* restore registers and return */ 85 blr 86 87/* 88 * save_altivec(tsk) 89 * Save the vector registers to its thread_struct 90 */ 91_GLOBAL(save_altivec) 92 addi r3,r3,THREAD /* want THREAD of task */ 93 PPC_LL r7,THREAD_VRSAVEAREA(r3) 94 PPC_LL r5,PT_REGS(r3) 95 PPC_LCMPI 0,r7,0 96 bne 2f 97 addi r7,r3,THREAD_VRSTATE 982: SAVE_32VRS(0,r4,r7) 99 mfvscr v0 100 li r4,VRSTATE_VSCR 101 stvx v0,r4,r7 102 blr 103 104#ifdef CONFIG_VSX 105 106#ifdef CONFIG_PPC32 107#error This asm code isn't ready for 32-bit kernels 108#endif 109 110/* 111 * load_up_vsx(unused, unused, tsk) 112 * Disable VSX for the task which had it previously, 113 * and save its vector registers in its thread_struct. 114 * Reuse the fp and vsx saves, but first check to see if they have 115 * been saved already. 116 */ 117_GLOBAL(load_up_vsx) 118/* Load FP and VSX registers if they haven't been done yet */ 119 andi. r5,r12,MSR_FP 120 beql+ load_up_fpu /* skip if already loaded */ 121 andis. r5,r12,MSR_VEC@h 122 beql+ load_up_altivec /* skip if already loaded */ 123 124 ld r4,PACACURRENT(r13) 125 addi r4,r4,THREAD /* Get THREAD */ 126 li r6,1 127 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */ 128 /* enable use of VSX after return */ 129 oris r12,r12,MSR_VSX@h 130 std r12,_MSR(r1) 131 b fast_exception_return 132 133#endif /* CONFIG_VSX */ 134 135 136/* 137 * The routines below are in assembler so we can closely control the 138 * usage of floating-point registers. These routines must be called 139 * with preempt disabled. 140 */ 141#ifdef CONFIG_PPC32 142 .data 143fpzero: 144 .long 0 145fpone: 146 .long 0x3f800000 /* 1.0 in single-precision FP */ 147fphalf: 148 .long 0x3f000000 /* 0.5 in single-precision FP */ 149 150#define LDCONST(fr, name) \ 151 lis r11,name@ha; \ 152 lfs fr,name@l(r11) 153#else 154 155 .section ".toc","aw" 156fpzero: 157 .tc FD_0_0[TC],0 158fpone: 159 .tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */ 160fphalf: 161 .tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */ 162 163#define LDCONST(fr, name) \ 164 lfd fr,name@toc(r2) 165#endif 166 167 .text 168/* 169 * Internal routine to enable floating point and set FPSCR to 0. 170 * Don't call it from C; it doesn't use the normal calling convention. 171 */ 172fpenable: 173#ifdef CONFIG_PPC32 174 stwu r1,-64(r1) 175#else 176 stdu r1,-64(r1) 177#endif 178 mfmsr r10 179 ori r11,r10,MSR_FP 180 mtmsr r11 181 isync 182 stfd fr0,24(r1) 183 stfd fr1,16(r1) 184 stfd fr31,8(r1) 185 LDCONST(fr1, fpzero) 186 mffs fr31 187 MTFSF_L(fr1) 188 blr 189 190fpdisable: 191 mtlr r12 192 MTFSF_L(fr31) 193 lfd fr31,8(r1) 194 lfd fr1,16(r1) 195 lfd fr0,24(r1) 196 mtmsr r10 197 isync 198 addi r1,r1,64 199 blr 200 201/* 202 * Vector add, floating point. 203 */ 204_GLOBAL(vaddfp) 205 mflr r12 206 bl fpenable 207 li r0,4 208 mtctr r0 209 li r6,0 2101: lfsx fr0,r4,r6 211 lfsx fr1,r5,r6 212 fadds fr0,fr0,fr1 213 stfsx fr0,r3,r6 214 addi r6,r6,4 215 bdnz 1b 216 b fpdisable 217 218/* 219 * Vector subtract, floating point. 220 */ 221_GLOBAL(vsubfp) 222 mflr r12 223 bl fpenable 224 li r0,4 225 mtctr r0 226 li r6,0 2271: lfsx fr0,r4,r6 228 lfsx fr1,r5,r6 229 fsubs fr0,fr0,fr1 230 stfsx fr0,r3,r6 231 addi r6,r6,4 232 bdnz 1b 233 b fpdisable 234 235/* 236 * Vector multiply and add, floating point. 237 */ 238_GLOBAL(vmaddfp) 239 mflr r12 240 bl fpenable 241 stfd fr2,32(r1) 242 li r0,4 243 mtctr r0 244 li r7,0 2451: lfsx fr0,r4,r7 246 lfsx fr1,r5,r7 247 lfsx fr2,r6,r7 248 fmadds fr0,fr0,fr2,fr1 249 stfsx fr0,r3,r7 250 addi r7,r7,4 251 bdnz 1b 252 lfd fr2,32(r1) 253 b fpdisable 254 255/* 256 * Vector negative multiply and subtract, floating point. 257 */ 258_GLOBAL(vnmsubfp) 259 mflr r12 260 bl fpenable 261 stfd fr2,32(r1) 262 li r0,4 263 mtctr r0 264 li r7,0 2651: lfsx fr0,r4,r7 266 lfsx fr1,r5,r7 267 lfsx fr2,r6,r7 268 fnmsubs fr0,fr0,fr2,fr1 269 stfsx fr0,r3,r7 270 addi r7,r7,4 271 bdnz 1b 272 lfd fr2,32(r1) 273 b fpdisable 274 275/* 276 * Vector reciprocal estimate. We just compute 1.0/x. 277 * r3 -> destination, r4 -> source. 278 */ 279_GLOBAL(vrefp) 280 mflr r12 281 bl fpenable 282 li r0,4 283 LDCONST(fr1, fpone) 284 mtctr r0 285 li r6,0 2861: lfsx fr0,r4,r6 287 fdivs fr0,fr1,fr0 288 stfsx fr0,r3,r6 289 addi r6,r6,4 290 bdnz 1b 291 b fpdisable 292 293/* 294 * Vector reciprocal square-root estimate, floating point. 295 * We use the frsqrte instruction for the initial estimate followed 296 * by 2 iterations of Newton-Raphson to get sufficient accuracy. 297 * r3 -> destination, r4 -> source. 298 */ 299_GLOBAL(vrsqrtefp) 300 mflr r12 301 bl fpenable 302 stfd fr2,32(r1) 303 stfd fr3,40(r1) 304 stfd fr4,48(r1) 305 stfd fr5,56(r1) 306 li r0,4 307 LDCONST(fr4, fpone) 308 LDCONST(fr5, fphalf) 309 mtctr r0 310 li r6,0 3111: lfsx fr0,r4,r6 312 frsqrte fr1,fr0 /* r = frsqrte(s) */ 313 fmuls fr3,fr1,fr0 /* r * s */ 314 fmuls fr2,fr1,fr5 /* r * 0.5 */ 315 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */ 316 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */ 317 fmuls fr3,fr1,fr0 /* r * s */ 318 fmuls fr2,fr1,fr5 /* r * 0.5 */ 319 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */ 320 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */ 321 stfsx fr1,r3,r6 322 addi r6,r6,4 323 bdnz 1b 324 lfd fr5,56(r1) 325 lfd fr4,48(r1) 326 lfd fr3,40(r1) 327 lfd fr2,32(r1) 328 b fpdisable 329