1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * 4 * Copyright (C) IBM Corporation, 2012 5 * 6 * Author: Anton Blanchard <anton@au.ibm.com> 7 */ 8 9#include <linux/export.h> 10#include <asm/ppc_asm.h> 11#include <asm/linkage.h> 12#include <asm/asm-offsets.h> 13 14/** 15 * __arch_clear_user: - Zero a block of memory in user space, with less checking. 16 * @to: Destination address, in user space. 17 * @n: Number of bytes to zero. 18 * 19 * Zero a block of memory in user space. Caller must check 20 * the specified block with access_ok() before calling this function. 21 * 22 * Returns number of bytes that could not be cleared. 23 * On success, this will be zero. 24 */ 25 26 .macro err1 27100: 28 EX_TABLE(100b,.Ldo_err1) 29 .endm 30 31 .macro err2 32200: 33 EX_TABLE(200b,.Ldo_err2) 34 .endm 35 36 .macro err3 37300: 38 EX_TABLE(300b,.Ldo_err3) 39 .endm 40 41.Ldo_err1: 42 mr r3,r8 43 44.Ldo_err2: 45 mtctr r4 461: 47err3; stb r0,0(r3) 48 addi r3,r3,1 49 addi r4,r4,-1 50 bdnz 1b 51 52.Ldo_err3: 53 mr r3,r4 54 blr 55 56_GLOBAL_TOC(__arch_clear_user) 57 cmpdi r4,32 58 neg r6,r3 59 li r0,0 60 blt .Lshort_clear 61 mr r8,r3 62 mtocrf 0x01,r6 63 clrldi r6,r6,(64-3) 64 65 /* Get the destination 8 byte aligned */ 66 bf cr7*4+3,1f 67err1; stb r0,0(r3) 68 addi r3,r3,1 69 701: bf cr7*4+2,2f 71err1; sth r0,0(r3) 72 addi r3,r3,2 73 742: bf cr7*4+1,3f 75err1; stw r0,0(r3) 76 addi r3,r3,4 77 783: sub r4,r4,r6 79 80 cmpdi r4,32 81 cmpdi cr1,r4,512 82 blt .Lshort_clear 83 bgt cr1,.Llong_clear 84 85.Lmedium_clear: 86 srdi r6,r4,5 87 mtctr r6 88 89 /* Do 32 byte chunks */ 904: 91err2; std r0,0(r3) 92err2; std r0,8(r3) 93err2; std r0,16(r3) 94err2; std r0,24(r3) 95 addi r3,r3,32 96 addi r4,r4,-32 97 bdnz 4b 98 99.Lshort_clear: 100 /* up to 31 bytes to go */ 101 cmpdi r4,16 102 blt 6f 103err2; std r0,0(r3) 104err2; std r0,8(r3) 105 addi r3,r3,16 106 addi r4,r4,-16 107 108 /* Up to 15 bytes to go */ 1096: mr r8,r3 110 clrldi r4,r4,(64-4) 111 mtocrf 0x01,r4 112 bf cr7*4+0,7f 113err1; std r0,0(r3) 114 addi r3,r3,8 115 1167: bf cr7*4+1,8f 117err1; stw r0,0(r3) 118 addi r3,r3,4 119 1208: bf cr7*4+2,9f 121err1; sth r0,0(r3) 122 addi r3,r3,2 123 1249: bf cr7*4+3,10f 125err1; stb r0,0(r3) 126 12710: li r3,0 128 blr 129 130.Llong_clear: 131 LOAD_REG_ADDR(r5, ppc64_caches) 132 133 bf cr7*4+0,11f 134err2; std r0,0(r3) 135 addi r3,r3,8 136 addi r4,r4,-8 137 138 /* Destination is 16 byte aligned, need to get it cache block aligned */ 13911: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) 140 lwz r9,DCACHEL1BLOCKSIZE(r5) 141 142 /* 143 * With worst case alignment the long clear loop takes a minimum 144 * of 1 byte less than 2 cachelines. 145 */ 146 sldi r10,r9,2 147 cmpd r4,r10 148 blt .Lmedium_clear 149 150 neg r6,r3 151 addi r10,r9,-1 152 and. r5,r6,r10 153 beq 13f 154 155 srdi r6,r5,4 156 mtctr r6 157 mr r8,r3 15812: 159err1; std r0,0(r3) 160err1; std r0,8(r3) 161 addi r3,r3,16 162 bdnz 12b 163 164 sub r4,r4,r5 165 16613: srd r6,r4,r7 167 mtctr r6 168 mr r8,r3 16914: 170err1; dcbz 0,r3 171 add r3,r3,r9 172 bdnz 14b 173 174 and r4,r4,r10 175 176 cmpdi r4,32 177 blt .Lshort_clear 178 b .Lmedium_clear 179EXPORT_SYMBOL(__arch_clear_user) 180