117968fbbSAnton Blanchard/* 217968fbbSAnton Blanchard * This program is free software; you can redistribute it and/or modify 317968fbbSAnton Blanchard * it under the terms of the GNU General Public License as published by 417968fbbSAnton Blanchard * the Free Software Foundation; either version 2 of the License, or 517968fbbSAnton Blanchard * (at your option) any later version. 617968fbbSAnton Blanchard * 717968fbbSAnton Blanchard * This program is distributed in the hope that it will be useful, 817968fbbSAnton Blanchard * but WITHOUT ANY WARRANTY; without even the implied warranty of 917968fbbSAnton Blanchard * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1017968fbbSAnton Blanchard * GNU General Public License for more details. 1117968fbbSAnton Blanchard * 1217968fbbSAnton Blanchard * You should have received a copy of the GNU General Public License 1317968fbbSAnton Blanchard * along with this program; if not, write to the Free Software 1417968fbbSAnton Blanchard * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 1517968fbbSAnton Blanchard * 1617968fbbSAnton Blanchard * Copyright (C) IBM Corporation, 2012 1717968fbbSAnton Blanchard * 1817968fbbSAnton Blanchard * Author: Anton Blanchard <anton@au.ibm.com> 1917968fbbSAnton Blanchard */ 2017968fbbSAnton Blanchard 2117968fbbSAnton Blanchard#include <asm/ppc_asm.h> 2224bfa6a9SNicholas Piggin#include <asm/linkage.h> 23cf8fb553SAnton Blanchard#include <asm/asm-offsets.h> 249445aa1aSAl Viro#include <asm/export.h> 25cf8fb553SAnton Blanchard 26cf8fb553SAnton Blanchard .section ".toc","aw" 27cf8fb553SAnton BlanchardPPC64_CACHES: 28cf8fb553SAnton Blanchard .tc ppc64_caches[TC],ppc64_caches 29cf8fb553SAnton Blanchard .section ".text" 3017968fbbSAnton Blanchard 3117968fbbSAnton Blanchard/** 3217968fbbSAnton Blanchard * __clear_user: - Zero a block of memory in user space, with less checking. 3317968fbbSAnton Blanchard * @to: Destination address, in user space. 3417968fbbSAnton Blanchard * @n: Number of bytes to zero. 3517968fbbSAnton Blanchard * 3617968fbbSAnton Blanchard * Zero a block of memory in user space. Caller must check 3717968fbbSAnton Blanchard * the specified block with access_ok() before calling this function. 3817968fbbSAnton Blanchard * 3917968fbbSAnton Blanchard * Returns number of bytes that could not be cleared. 4017968fbbSAnton Blanchard * On success, this will be zero. 4117968fbbSAnton Blanchard */ 4217968fbbSAnton Blanchard 4317968fbbSAnton Blanchard .macro err1 4417968fbbSAnton Blanchard100: 4524bfa6a9SNicholas Piggin EX_TABLE(100b,.Ldo_err1) 4617968fbbSAnton Blanchard .endm 4717968fbbSAnton Blanchard 4817968fbbSAnton Blanchard .macro err2 4917968fbbSAnton Blanchard200: 5024bfa6a9SNicholas Piggin EX_TABLE(200b,.Ldo_err2) 5117968fbbSAnton Blanchard .endm 5217968fbbSAnton Blanchard 5317968fbbSAnton Blanchard .macro err3 5417968fbbSAnton Blanchard300: 5524bfa6a9SNicholas Piggin EX_TABLE(300b,.Ldo_err3) 5617968fbbSAnton Blanchard .endm 5717968fbbSAnton Blanchard 5817968fbbSAnton Blanchard.Ldo_err1: 5917968fbbSAnton Blanchard mr r3,r8 6017968fbbSAnton Blanchard 6117968fbbSAnton Blanchard.Ldo_err2: 6217968fbbSAnton Blanchard mtctr r4 6317968fbbSAnton Blanchard1: 6417968fbbSAnton Blancharderr3; stb r0,0(r3) 6517968fbbSAnton Blanchard addi r3,r3,1 6617968fbbSAnton Blanchard addi r4,r4,-1 6717968fbbSAnton Blanchard bdnz 1b 6817968fbbSAnton Blanchard 6917968fbbSAnton Blanchard.Ldo_err3: 7017968fbbSAnton Blanchard mr r3,r4 7117968fbbSAnton Blanchard blr 7217968fbbSAnton Blanchard 732ac7b016SAnton Blanchard_GLOBAL_TOC(__clear_user) 7417968fbbSAnton Blanchard cmpdi r4,32 7517968fbbSAnton Blanchard neg r6,r3 7617968fbbSAnton Blanchard li r0,0 7717968fbbSAnton Blanchard blt .Lshort_clear 7817968fbbSAnton Blanchard mr r8,r3 7917968fbbSAnton Blanchard mtocrf 0x01,r6 8017968fbbSAnton Blanchard clrldi r6,r6,(64-3) 8117968fbbSAnton Blanchard 8217968fbbSAnton Blanchard /* Get the destination 8 byte aligned */ 8317968fbbSAnton Blanchard bf cr7*4+3,1f 8417968fbbSAnton Blancharderr1; stb r0,0(r3) 8517968fbbSAnton Blanchard addi r3,r3,1 8617968fbbSAnton Blanchard 8717968fbbSAnton Blanchard1: bf cr7*4+2,2f 8817968fbbSAnton Blancharderr1; sth r0,0(r3) 8917968fbbSAnton Blanchard addi r3,r3,2 9017968fbbSAnton Blanchard 9117968fbbSAnton Blanchard2: bf cr7*4+1,3f 9217968fbbSAnton Blancharderr1; stw r0,0(r3) 9317968fbbSAnton Blanchard addi r3,r3,4 9417968fbbSAnton Blanchard 9517968fbbSAnton Blanchard3: sub r4,r4,r6 96cf8fb553SAnton Blanchard 9717968fbbSAnton Blanchard cmpdi r4,32 98cf8fb553SAnton Blanchard cmpdi cr1,r4,512 9917968fbbSAnton Blanchard blt .Lshort_clear 100cf8fb553SAnton Blanchard bgt cr1,.Llong_clear 101cf8fb553SAnton Blanchard 102cf8fb553SAnton Blanchard.Lmedium_clear: 103cf8fb553SAnton Blanchard srdi r6,r4,5 10417968fbbSAnton Blanchard mtctr r6 10517968fbbSAnton Blanchard 10617968fbbSAnton Blanchard /* Do 32 byte chunks */ 10717968fbbSAnton Blanchard4: 10817968fbbSAnton Blancharderr2; std r0,0(r3) 10917968fbbSAnton Blancharderr2; std r0,8(r3) 11017968fbbSAnton Blancharderr2; std r0,16(r3) 11117968fbbSAnton Blancharderr2; std r0,24(r3) 11217968fbbSAnton Blanchard addi r3,r3,32 11317968fbbSAnton Blanchard addi r4,r4,-32 11417968fbbSAnton Blanchard bdnz 4b 11517968fbbSAnton Blanchard 11617968fbbSAnton Blanchard.Lshort_clear: 11717968fbbSAnton Blanchard /* up to 31 bytes to go */ 11817968fbbSAnton Blanchard cmpdi r4,16 11917968fbbSAnton Blanchard blt 6f 12017968fbbSAnton Blancharderr2; std r0,0(r3) 12117968fbbSAnton Blancharderr2; std r0,8(r3) 12217968fbbSAnton Blanchard addi r3,r3,16 12317968fbbSAnton Blanchard addi r4,r4,-16 12417968fbbSAnton Blanchard 12517968fbbSAnton Blanchard /* Up to 15 bytes to go */ 12617968fbbSAnton Blanchard6: mr r8,r3 12717968fbbSAnton Blanchard clrldi r4,r4,(64-4) 12817968fbbSAnton Blanchard mtocrf 0x01,r4 12917968fbbSAnton Blanchard bf cr7*4+0,7f 13017968fbbSAnton Blancharderr1; std r0,0(r3) 13117968fbbSAnton Blanchard addi r3,r3,8 13217968fbbSAnton Blanchard 13317968fbbSAnton Blanchard7: bf cr7*4+1,8f 13417968fbbSAnton Blancharderr1; stw r0,0(r3) 13517968fbbSAnton Blanchard addi r3,r3,4 13617968fbbSAnton Blanchard 13717968fbbSAnton Blanchard8: bf cr7*4+2,9f 13817968fbbSAnton Blancharderr1; sth r0,0(r3) 13917968fbbSAnton Blanchard addi r3,r3,2 14017968fbbSAnton Blanchard 14117968fbbSAnton Blanchard9: bf cr7*4+3,10f 14217968fbbSAnton Blancharderr1; stb r0,0(r3) 14317968fbbSAnton Blanchard 14417968fbbSAnton Blanchard10: li r3,0 14517968fbbSAnton Blanchard blr 146cf8fb553SAnton Blanchard 147cf8fb553SAnton Blanchard.Llong_clear: 148cf8fb553SAnton Blanchard ld r5,PPC64_CACHES@toc(r2) 149cf8fb553SAnton Blanchard 150cf8fb553SAnton Blanchard bf cr7*4+0,11f 151cf8fb553SAnton Blancharderr2; std r0,0(r3) 152cf8fb553SAnton Blanchard addi r3,r3,8 153cf8fb553SAnton Blanchard addi r4,r4,-8 154cf8fb553SAnton Blanchard 155bd067f83SBenjamin Herrenschmidt /* Destination is 16 byte aligned, need to get it cache block aligned */ 156bd067f83SBenjamin Herrenschmidt11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) 157bd067f83SBenjamin Herrenschmidt lwz r9,DCACHEL1BLOCKSIZE(r5) 158cf8fb553SAnton Blanchard 159cf8fb553SAnton Blanchard /* 160cf8fb553SAnton Blanchard * With worst case alignment the long clear loop takes a minimum 161cf8fb553SAnton Blanchard * of 1 byte less than 2 cachelines. 162cf8fb553SAnton Blanchard */ 163cf8fb553SAnton Blanchard sldi r10,r9,2 164cf8fb553SAnton Blanchard cmpd r4,r10 165cf8fb553SAnton Blanchard blt .Lmedium_clear 166cf8fb553SAnton Blanchard 167cf8fb553SAnton Blanchard neg r6,r3 168cf8fb553SAnton Blanchard addi r10,r9,-1 169cf8fb553SAnton Blanchard and. r5,r6,r10 170cf8fb553SAnton Blanchard beq 13f 171cf8fb553SAnton Blanchard 172cf8fb553SAnton Blanchard srdi r6,r5,4 173cf8fb553SAnton Blanchard mtctr r6 174cf8fb553SAnton Blanchard mr r8,r3 175cf8fb553SAnton Blanchard12: 176cf8fb553SAnton Blancharderr1; std r0,0(r3) 177cf8fb553SAnton Blancharderr1; std r0,8(r3) 178cf8fb553SAnton Blanchard addi r3,r3,16 179cf8fb553SAnton Blanchard bdnz 12b 180cf8fb553SAnton Blanchard 181cf8fb553SAnton Blanchard sub r4,r4,r5 182cf8fb553SAnton Blanchard 183cf8fb553SAnton Blanchard13: srd r6,r4,r7 184cf8fb553SAnton Blanchard mtctr r6 185cf8fb553SAnton Blanchard mr r8,r3 186cf8fb553SAnton Blanchard14: 187cf8fb553SAnton Blancharderr1; dcbz r0,r3 188cf8fb553SAnton Blanchard add r3,r3,r9 189cf8fb553SAnton Blanchard bdnz 14b 190cf8fb553SAnton Blanchard 191cf8fb553SAnton Blanchard and r4,r4,r10 192cf8fb553SAnton Blanchard 193cf8fb553SAnton Blanchard cmpdi r4,32 194cf8fb553SAnton Blanchard blt .Lshort_clear 195cf8fb553SAnton Blanchard b .Lmedium_clear 1969445aa1aSAl ViroEXPORT_SYMBOL(__clear_user) 197