1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License as published by 4 * the Free Software Foundation; either version 2 of the License, or 5 * (at your option) any later version. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 15 * 16 * Copyright (C) IBM Corporation, 2012 17 * 18 * Author: Anton Blanchard <anton@au.ibm.com> 19 */ 20 21#include <asm/ppc_asm.h> 22#include <asm/linkage.h> 23#include <asm/asm-offsets.h> 24#include <asm/export.h> 25 26 .section ".toc","aw" 27PPC64_CACHES: 28 .tc ppc64_caches[TC],ppc64_caches 29 .section ".text" 30 31/** 32 * __clear_user: - Zero a block of memory in user space, with less checking. 33 * @to: Destination address, in user space. 34 * @n: Number of bytes to zero. 35 * 36 * Zero a block of memory in user space. Caller must check 37 * the specified block with access_ok() before calling this function. 38 * 39 * Returns number of bytes that could not be cleared. 40 * On success, this will be zero. 41 */ 42 43 .macro err1 44100: 45 EX_TABLE(100b,.Ldo_err1) 46 .endm 47 48 .macro err2 49200: 50 EX_TABLE(200b,.Ldo_err2) 51 .endm 52 53 .macro err3 54300: 55 EX_TABLE(300b,.Ldo_err3) 56 .endm 57 58.Ldo_err1: 59 mr r3,r8 60 61.Ldo_err2: 62 mtctr r4 631: 64err3; stb r0,0(r3) 65 addi r3,r3,1 66 addi r4,r4,-1 67 bdnz 1b 68 69.Ldo_err3: 70 mr r3,r4 71 blr 72 73_GLOBAL_TOC(__clear_user) 74 cmpdi r4,32 75 neg r6,r3 76 li r0,0 77 blt .Lshort_clear 78 mr r8,r3 79 mtocrf 0x01,r6 80 clrldi r6,r6,(64-3) 81 82 /* Get the destination 8 byte aligned */ 83 bf cr7*4+3,1f 84err1; stb r0,0(r3) 85 addi r3,r3,1 86 871: bf cr7*4+2,2f 88err1; sth r0,0(r3) 89 addi r3,r3,2 90 912: bf cr7*4+1,3f 92err1; stw r0,0(r3) 93 addi r3,r3,4 94 953: sub r4,r4,r6 96 97 cmpdi r4,32 98 cmpdi cr1,r4,512 99 blt .Lshort_clear 100 bgt cr1,.Llong_clear 101 102.Lmedium_clear: 103 srdi r6,r4,5 104 mtctr r6 105 106 /* Do 32 byte chunks */ 1074: 108err2; std r0,0(r3) 109err2; std r0,8(r3) 110err2; std r0,16(r3) 111err2; std r0,24(r3) 112 addi r3,r3,32 113 addi r4,r4,-32 114 bdnz 4b 115 116.Lshort_clear: 117 /* up to 31 bytes to go */ 118 cmpdi r4,16 119 blt 6f 120err2; std r0,0(r3) 121err2; std r0,8(r3) 122 addi r3,r3,16 123 addi r4,r4,-16 124 125 /* Up to 15 bytes to go */ 1266: mr r8,r3 127 clrldi r4,r4,(64-4) 128 mtocrf 0x01,r4 129 bf cr7*4+0,7f 130err1; std r0,0(r3) 131 addi r3,r3,8 132 1337: bf cr7*4+1,8f 134err1; stw r0,0(r3) 135 addi r3,r3,4 136 1378: bf cr7*4+2,9f 138err1; sth r0,0(r3) 139 addi r3,r3,2 140 1419: bf cr7*4+3,10f 142err1; stb r0,0(r3) 143 14410: li r3,0 145 blr 146 147.Llong_clear: 148 ld r5,PPC64_CACHES@toc(r2) 149 150 bf cr7*4+0,11f 151err2; std r0,0(r3) 152 addi r3,r3,8 153 addi r4,r4,-8 154 155 /* Destination is 16 byte aligned, need to get it cache block aligned */ 15611: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) 157 lwz r9,DCACHEL1BLOCKSIZE(r5) 158 159 /* 160 * With worst case alignment the long clear loop takes a minimum 161 * of 1 byte less than 2 cachelines. 162 */ 163 sldi r10,r9,2 164 cmpd r4,r10 165 blt .Lmedium_clear 166 167 neg r6,r3 168 addi r10,r9,-1 169 and. r5,r6,r10 170 beq 13f 171 172 srdi r6,r5,4 173 mtctr r6 174 mr r8,r3 17512: 176err1; std r0,0(r3) 177err1; std r0,8(r3) 178 addi r3,r3,16 179 bdnz 12b 180 181 sub r4,r4,r5 182 18313: srd r6,r4,r7 184 mtctr r6 185 mr r8,r3 18614: 187err1; dcbz 0,r3 188 add r3,r3,r9 189 bdnz 14b 190 191 and r4,r4,r10 192 193 cmpdi r4,32 194 blt .Lshort_clear 195 b .Lmedium_clear 196EXPORT_SYMBOL(__clear_user) 197