11a59d1b8SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-or-later */ 217968fbbSAnton Blanchard/* 317968fbbSAnton Blanchard * 417968fbbSAnton Blanchard * Copyright (C) IBM Corporation, 2012 517968fbbSAnton Blanchard * 617968fbbSAnton Blanchard * Author: Anton Blanchard <anton@au.ibm.com> 717968fbbSAnton Blanchard */ 817968fbbSAnton Blanchard 9*39326182SMasahiro Yamada#include <linux/export.h> 1017968fbbSAnton Blanchard#include <asm/ppc_asm.h> 1124bfa6a9SNicholas Piggin#include <asm/linkage.h> 12cf8fb553SAnton Blanchard#include <asm/asm-offsets.h> 13cf8fb553SAnton Blanchard 1417968fbbSAnton Blanchard/** 1561e3acd8SAndrew Donnellan * __arch_clear_user: - Zero a block of memory in user space, with less checking. 1617968fbbSAnton Blanchard * @to: Destination address, in user space. 1717968fbbSAnton Blanchard * @n: Number of bytes to zero. 1817968fbbSAnton Blanchard * 1917968fbbSAnton Blanchard * Zero a block of memory in user space. Caller must check 2017968fbbSAnton Blanchard * the specified block with access_ok() before calling this function. 2117968fbbSAnton Blanchard * 2217968fbbSAnton Blanchard * Returns number of bytes that could not be cleared. 2317968fbbSAnton Blanchard * On success, this will be zero. 2417968fbbSAnton Blanchard */ 2517968fbbSAnton Blanchard 2617968fbbSAnton Blanchard .macro err1 2717968fbbSAnton Blanchard100: 2824bfa6a9SNicholas Piggin EX_TABLE(100b,.Ldo_err1) 2917968fbbSAnton Blanchard .endm 3017968fbbSAnton Blanchard 3117968fbbSAnton Blanchard .macro err2 3217968fbbSAnton Blanchard200: 3324bfa6a9SNicholas Piggin EX_TABLE(200b,.Ldo_err2) 3417968fbbSAnton Blanchard .endm 3517968fbbSAnton Blanchard 3617968fbbSAnton Blanchard .macro err3 3717968fbbSAnton Blanchard300: 3824bfa6a9SNicholas Piggin EX_TABLE(300b,.Ldo_err3) 3917968fbbSAnton Blanchard .endm 4017968fbbSAnton Blanchard 4117968fbbSAnton Blanchard.Ldo_err1: 4217968fbbSAnton Blanchard mr r3,r8 4317968fbbSAnton Blanchard 4417968fbbSAnton Blanchard.Ldo_err2: 4517968fbbSAnton Blanchard mtctr r4 4617968fbbSAnton Blanchard1: 4717968fbbSAnton Blancharderr3; stb r0,0(r3) 4817968fbbSAnton Blanchard addi r3,r3,1 4917968fbbSAnton Blanchard addi r4,r4,-1 5017968fbbSAnton Blanchard bdnz 1b 5117968fbbSAnton Blanchard 5217968fbbSAnton Blanchard.Ldo_err3: 5317968fbbSAnton Blanchard mr r3,r4 5417968fbbSAnton Blanchard blr 5517968fbbSAnton Blanchard 5661e3acd8SAndrew Donnellan_GLOBAL_TOC(__arch_clear_user) 5717968fbbSAnton Blanchard cmpdi r4,32 5817968fbbSAnton Blanchard neg r6,r3 5917968fbbSAnton Blanchard li r0,0 6017968fbbSAnton Blanchard blt .Lshort_clear 6117968fbbSAnton Blanchard mr r8,r3 6217968fbbSAnton Blanchard mtocrf 0x01,r6 6317968fbbSAnton Blanchard clrldi r6,r6,(64-3) 6417968fbbSAnton Blanchard 6517968fbbSAnton Blanchard /* Get the destination 8 byte aligned */ 6617968fbbSAnton Blanchard bf cr7*4+3,1f 6717968fbbSAnton Blancharderr1; stb r0,0(r3) 6817968fbbSAnton Blanchard addi r3,r3,1 6917968fbbSAnton Blanchard 7017968fbbSAnton Blanchard1: bf cr7*4+2,2f 7117968fbbSAnton Blancharderr1; sth r0,0(r3) 7217968fbbSAnton Blanchard addi r3,r3,2 7317968fbbSAnton Blanchard 7417968fbbSAnton Blanchard2: bf cr7*4+1,3f 7517968fbbSAnton Blancharderr1; stw r0,0(r3) 7617968fbbSAnton Blanchard addi r3,r3,4 7717968fbbSAnton Blanchard 7817968fbbSAnton Blanchard3: sub r4,r4,r6 79cf8fb553SAnton Blanchard 8017968fbbSAnton Blanchard cmpdi r4,32 81cf8fb553SAnton Blanchard cmpdi cr1,r4,512 8217968fbbSAnton Blanchard blt .Lshort_clear 83cf8fb553SAnton Blanchard bgt cr1,.Llong_clear 84cf8fb553SAnton Blanchard 85cf8fb553SAnton Blanchard.Lmedium_clear: 86cf8fb553SAnton Blanchard srdi r6,r4,5 8717968fbbSAnton Blanchard mtctr r6 8817968fbbSAnton Blanchard 8917968fbbSAnton Blanchard /* Do 32 byte chunks */ 9017968fbbSAnton Blanchard4: 9117968fbbSAnton Blancharderr2; std r0,0(r3) 9217968fbbSAnton Blancharderr2; std r0,8(r3) 9317968fbbSAnton Blancharderr2; std r0,16(r3) 9417968fbbSAnton Blancharderr2; std r0,24(r3) 9517968fbbSAnton Blanchard addi r3,r3,32 9617968fbbSAnton Blanchard addi r4,r4,-32 9717968fbbSAnton Blanchard bdnz 4b 9817968fbbSAnton Blanchard 9917968fbbSAnton Blanchard.Lshort_clear: 10017968fbbSAnton Blanchard /* up to 31 bytes to go */ 10117968fbbSAnton Blanchard cmpdi r4,16 10217968fbbSAnton Blanchard blt 6f 10317968fbbSAnton Blancharderr2; std r0,0(r3) 10417968fbbSAnton Blancharderr2; std r0,8(r3) 10517968fbbSAnton Blanchard addi r3,r3,16 10617968fbbSAnton Blanchard addi r4,r4,-16 10717968fbbSAnton Blanchard 10817968fbbSAnton Blanchard /* Up to 15 bytes to go */ 10917968fbbSAnton Blanchard6: mr r8,r3 11017968fbbSAnton Blanchard clrldi r4,r4,(64-4) 11117968fbbSAnton Blanchard mtocrf 0x01,r4 11217968fbbSAnton Blanchard bf cr7*4+0,7f 11317968fbbSAnton Blancharderr1; std r0,0(r3) 11417968fbbSAnton Blanchard addi r3,r3,8 11517968fbbSAnton Blanchard 11617968fbbSAnton Blanchard7: bf cr7*4+1,8f 11717968fbbSAnton Blancharderr1; stw r0,0(r3) 11817968fbbSAnton Blanchard addi r3,r3,4 11917968fbbSAnton Blanchard 12017968fbbSAnton Blanchard8: bf cr7*4+2,9f 12117968fbbSAnton Blancharderr1; sth r0,0(r3) 12217968fbbSAnton Blanchard addi r3,r3,2 12317968fbbSAnton Blanchard 12417968fbbSAnton Blanchard9: bf cr7*4+3,10f 12517968fbbSAnton Blancharderr1; stb r0,0(r3) 12617968fbbSAnton Blanchard 12717968fbbSAnton Blanchard10: li r3,0 12817968fbbSAnton Blanchard blr 129cf8fb553SAnton Blanchard 130cf8fb553SAnton Blanchard.Llong_clear: 131dab3b8f4SNicholas Piggin LOAD_REG_ADDR(r5, ppc64_caches) 132cf8fb553SAnton Blanchard 133cf8fb553SAnton Blanchard bf cr7*4+0,11f 134cf8fb553SAnton Blancharderr2; std r0,0(r3) 135cf8fb553SAnton Blanchard addi r3,r3,8 136cf8fb553SAnton Blanchard addi r4,r4,-8 137cf8fb553SAnton Blanchard 138bd067f83SBenjamin Herrenschmidt /* Destination is 16 byte aligned, need to get it cache block aligned */ 139bd067f83SBenjamin Herrenschmidt11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) 140bd067f83SBenjamin Herrenschmidt lwz r9,DCACHEL1BLOCKSIZE(r5) 141cf8fb553SAnton Blanchard 142cf8fb553SAnton Blanchard /* 143cf8fb553SAnton Blanchard * With worst case alignment the long clear loop takes a minimum 144cf8fb553SAnton Blanchard * of 1 byte less than 2 cachelines. 145cf8fb553SAnton Blanchard */ 146cf8fb553SAnton Blanchard sldi r10,r9,2 147cf8fb553SAnton Blanchard cmpd r4,r10 148cf8fb553SAnton Blanchard blt .Lmedium_clear 149cf8fb553SAnton Blanchard 150cf8fb553SAnton Blanchard neg r6,r3 151cf8fb553SAnton Blanchard addi r10,r9,-1 152cf8fb553SAnton Blanchard and. r5,r6,r10 153cf8fb553SAnton Blanchard beq 13f 154cf8fb553SAnton Blanchard 155cf8fb553SAnton Blanchard srdi r6,r5,4 156cf8fb553SAnton Blanchard mtctr r6 157cf8fb553SAnton Blanchard mr r8,r3 158cf8fb553SAnton Blanchard12: 159cf8fb553SAnton Blancharderr1; std r0,0(r3) 160cf8fb553SAnton Blancharderr1; std r0,8(r3) 161cf8fb553SAnton Blanchard addi r3,r3,16 162cf8fb553SAnton Blanchard bdnz 12b 163cf8fb553SAnton Blanchard 164cf8fb553SAnton Blanchard sub r4,r4,r5 165cf8fb553SAnton Blanchard 166cf8fb553SAnton Blanchard13: srd r6,r4,r7 167cf8fb553SAnton Blanchard mtctr r6 168cf8fb553SAnton Blanchard mr r8,r3 169cf8fb553SAnton Blanchard14: 1708a583c0aSAndreas Schwaberr1; dcbz 0,r3 171cf8fb553SAnton Blanchard add r3,r3,r9 172cf8fb553SAnton Blanchard bdnz 14b 173cf8fb553SAnton Blanchard 174cf8fb553SAnton Blanchard and r4,r4,r10 175cf8fb553SAnton Blanchard 176cf8fb553SAnton Blanchard cmpdi r4,32 177cf8fb553SAnton Blanchard blt .Lshort_clear 178cf8fb553SAnton Blanchard b .Lmedium_clear 17961e3acd8SAndrew DonnellanEXPORT_SYMBOL(__arch_clear_user) 180