1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License as published by 4 * the Free Software Foundation; either version 2 of the License, or 5 * (at your option) any later version. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 15 * 16 * Copyright (C) IBM Corporation, 2012 17 * 18 * Author: Anton Blanchard <anton@au.ibm.com> 19 */ 20 21#include <asm/ppc_asm.h> 22#include <asm/asm-offsets.h> 23 24 .section ".toc","aw" 25PPC64_CACHES: 26 .tc ppc64_caches[TC],ppc64_caches 27 .section ".text" 28 29/** 30 * __clear_user: - Zero a block of memory in user space, with less checking. 31 * @to: Destination address, in user space. 32 * @n: Number of bytes to zero. 33 * 34 * Zero a block of memory in user space. Caller must check 35 * the specified block with access_ok() before calling this function. 36 * 37 * Returns number of bytes that could not be cleared. 38 * On success, this will be zero. 39 */ 40 41 .macro err1 42100: 43 .section __ex_table,"a" 44 .align 3 45 .llong 100b,.Ldo_err1 46 .previous 47 .endm 48 49 .macro err2 50200: 51 .section __ex_table,"a" 52 .align 3 53 .llong 200b,.Ldo_err2 54 .previous 55 .endm 56 57 .macro err3 58300: 59 .section __ex_table,"a" 60 .align 3 61 .llong 300b,.Ldo_err3 62 .previous 63 .endm 64 65.Ldo_err1: 66 mr r3,r8 67 68.Ldo_err2: 69 mtctr r4 701: 71err3; stb r0,0(r3) 72 addi r3,r3,1 73 addi r4,r4,-1 74 bdnz 1b 75 76.Ldo_err3: 77 mr r3,r4 78 blr 79 80_GLOBAL_TOC(__clear_user) 81 cmpdi r4,32 82 neg r6,r3 83 li r0,0 84 blt .Lshort_clear 85 mr r8,r3 86 mtocrf 0x01,r6 87 clrldi r6,r6,(64-3) 88 89 /* Get the destination 8 byte aligned */ 90 bf cr7*4+3,1f 91err1; stb r0,0(r3) 92 addi r3,r3,1 93 941: bf cr7*4+2,2f 95err1; sth r0,0(r3) 96 addi r3,r3,2 97 982: bf cr7*4+1,3f 99err1; stw r0,0(r3) 100 addi r3,r3,4 101 1023: sub r4,r4,r6 103 104 cmpdi r4,32 105 cmpdi cr1,r4,512 106 blt .Lshort_clear 107 bgt cr1,.Llong_clear 108 109.Lmedium_clear: 110 srdi r6,r4,5 111 mtctr r6 112 113 /* Do 32 byte chunks */ 1144: 115err2; std r0,0(r3) 116err2; std r0,8(r3) 117err2; std r0,16(r3) 118err2; std r0,24(r3) 119 addi r3,r3,32 120 addi r4,r4,-32 121 bdnz 4b 122 123.Lshort_clear: 124 /* up to 31 bytes to go */ 125 cmpdi r4,16 126 blt 6f 127err2; std r0,0(r3) 128err2; std r0,8(r3) 129 addi r3,r3,16 130 addi r4,r4,-16 131 132 /* Up to 15 bytes to go */ 1336: mr r8,r3 134 clrldi r4,r4,(64-4) 135 mtocrf 0x01,r4 136 bf cr7*4+0,7f 137err1; std r0,0(r3) 138 addi r3,r3,8 139 1407: bf cr7*4+1,8f 141err1; stw r0,0(r3) 142 addi r3,r3,4 143 1448: bf cr7*4+2,9f 145err1; sth r0,0(r3) 146 addi r3,r3,2 147 1489: bf cr7*4+3,10f 149err1; stb r0,0(r3) 150 15110: li r3,0 152 blr 153 154.Llong_clear: 155 ld r5,PPC64_CACHES@toc(r2) 156 157 bf cr7*4+0,11f 158err2; std r0,0(r3) 159 addi r3,r3,8 160 addi r4,r4,-8 161 162 /* Destination is 16 byte aligned, need to get it cacheline aligned */ 16311: lwz r7,DCACHEL1LOGLINESIZE(r5) 164 lwz r9,DCACHEL1LINESIZE(r5) 165 166 /* 167 * With worst case alignment the long clear loop takes a minimum 168 * of 1 byte less than 2 cachelines. 169 */ 170 sldi r10,r9,2 171 cmpd r4,r10 172 blt .Lmedium_clear 173 174 neg r6,r3 175 addi r10,r9,-1 176 and. r5,r6,r10 177 beq 13f 178 179 srdi r6,r5,4 180 mtctr r6 181 mr r8,r3 18212: 183err1; std r0,0(r3) 184err1; std r0,8(r3) 185 addi r3,r3,16 186 bdnz 12b 187 188 sub r4,r4,r5 189 19013: srd r6,r4,r7 191 mtctr r6 192 mr r8,r3 19314: 194err1; dcbz r0,r3 195 add r3,r3,r9 196 bdnz 14b 197 198 and r4,r4,r10 199 200 cmpdi r4,32 201 blt .Lshort_clear 202 b .Lmedium_clear 203