1/* 2 * arch/alpha/lib/ev6-clear_user.S 3 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com> 4 * 5 * Zero user space, handling exceptions as we go. 6 * 7 * We have to make sure that $0 is always up-to-date and contains the 8 * right "bytes left to zero" value (and that it is updated only _after_ 9 * a successful copy). There is also some rather minor exception setup 10 * stuff. 11 * 12 * Much of the information about 21264 scheduling/coding comes from: 13 * Compiler Writer's Guide for the Alpha 21264 14 * abbreviated as 'CWG' in other comments here 15 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html 16 * Scheduling notation: 17 * E - either cluster 18 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 19 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 20 * Try not to change the actual algorithm if possible for consistency. 21 * Determining actual stalls (other than slotting) doesn't appear to be easy to do. 22 * From perusing the source code context where this routine is called, it is 23 * a fair assumption that significant fractions of entire pages are zeroed, so 24 * it's going to be worth the effort to hand-unroll a big loop, and use wh64. 25 * ASSUMPTION: 26 * The believed purpose of only updating $0 after a store is that a signal 27 * may come along during the execution of this chunk of code, and we don't 28 * want to leave a hole (and we also want to avoid repeating lots of work) 29 */ 30 31#include <asm/export.h> 32/* Allow an exception for an insn; exit if we get one. */ 33#define EX(x,y...) \ 34 99: x,##y; \ 35 .section __ex_table,"a"; \ 36 .long 99b - .; \ 37 lda $31, $exception-99b($31); \ 38 .previous 39 40 .set noat 41 .set noreorder 42 .align 4 43 44 .globl __clear_user 45 .ent __clear_user 46 .frame $30, 0, $26 47 .prologue 0 48 49 # Pipeline info : Slotting & Comments 50__clear_user: 51 and $17, $17, $0 52 and $16, 7, $4 # .. E .. .. : find dest head misalignment 53 beq $0, $zerolength # U .. .. .. : U L U L 54 55 addq $0, $4, $1 # .. .. .. E : bias counter 56 and $1, 7, $2 # .. .. E .. : number of misaligned bytes in tail 57# Note - we never actually use $2, so this is a moot computation 58# and we can rewrite this later... 59 srl $1, 3, $1 # .. E .. .. : number of quadwords to clear 60 beq $4, $headalign # U .. .. .. : U L U L 61 62/* 63 * Head is not aligned. Write (8 - $4) bytes to head of destination 64 * This means $16 is known to be misaligned 65 */ 66 EX( ldq_u $5, 0($16) ) # .. .. .. L : load dst word to mask back in 67 beq $1, $onebyte # .. .. U .. : sub-word store? 68 mskql $5, $16, $5 # .. U .. .. : take care of misaligned head 69 addq $16, 8, $16 # E .. .. .. : L U U L 70 71 EX( stq_u $5, -8($16) ) # .. .. .. L : 72 subq $1, 1, $1 # .. .. E .. : 73 addq $0, $4, $0 # .. E .. .. : bytes left -= 8 - misalignment 74 subq $0, 8, $0 # E .. .. .. : U L U L 75 76 .align 4 77/* 78 * (The .align directive ought to be a moot point) 79 * values upon initial entry to the loop 80 * $1 is number of quadwords to clear (zero is a valid value) 81 * $2 is number of trailing bytes (0..7) ($2 never used...) 82 * $16 is known to be aligned 0mod8 83 */ 84$headalign: 85 subq $1, 16, $4 # .. .. .. E : If < 16, we can not use the huge loop 86 and $16, 0x3f, $2 # .. .. E .. : Forward work for huge loop 87 subq $2, 0x40, $3 # .. E .. .. : bias counter (huge loop) 88 blt $4, $trailquad # U .. .. .. : U L U L 89 90/* 91 * We know that we're going to do at least 16 quads, which means we are 92 * going to be able to use the large block clear loop at least once. 93 * Figure out how many quads we need to clear before we are 0mod64 aligned 94 * so we can use the wh64 instruction. 95 */ 96 97 nop # .. .. .. E 98 nop # .. .. E .. 99 nop # .. E .. .. 100 beq $3, $bigalign # U .. .. .. : U L U L : Aligned 0mod64 101 102$alignmod64: 103 EX( stq_u $31, 0($16) ) # .. .. .. L 104 addq $3, 8, $3 # .. .. E .. 105 subq $0, 8, $0 # .. E .. .. 106 nop # E .. .. .. : U L U L 107 108 nop # .. .. .. E 109 subq $1, 1, $1 # .. .. E .. 110 addq $16, 8, $16 # .. E .. .. 111 blt $3, $alignmod64 # U .. .. .. : U L U L 112 113$bigalign: 114/* 115 * $0 is the number of bytes left 116 * $1 is the number of quads left 117 * $16 is aligned 0mod64 118 * we know that we'll be taking a minimum of one trip through 119 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle 120 * We are _not_ going to update $0 after every single store. That 121 * would be silly, because there will be cross-cluster dependencies 122 * no matter how the code is scheduled. By doing it in slightly 123 * staggered fashion, we can still do this loop in 5 fetches 124 * The worse case will be doing two extra quads in some future execution, 125 * in the event of an interrupted clear. 126 * Assumes the wh64 needs to be for 2 trips through the loop in the future 127 * The wh64 is issued on for the starting destination address for trip +2 128 * through the loop, and if there are less than two trips left, the target 129 * address will be for the current trip. 130 */ 131 nop # E : 132 nop # E : 133 nop # E : 134 bis $16,$16,$3 # E : U L U L : Initial wh64 address is dest 135 /* This might actually help for the current trip... */ 136 137$do_wh64: 138 wh64 ($3) # .. .. .. L1 : memory subsystem hint 139 subq $1, 16, $4 # .. .. E .. : Forward calculation - repeat the loop? 140 EX( stq_u $31, 0($16) ) # .. L .. .. 141 subq $0, 8, $0 # E .. .. .. : U L U L 142 143 addq $16, 128, $3 # E : Target address of wh64 144 EX( stq_u $31, 8($16) ) # L : 145 EX( stq_u $31, 16($16) ) # L : 146 subq $0, 16, $0 # E : U L L U 147 148 nop # E : 149 EX( stq_u $31, 24($16) ) # L : 150 EX( stq_u $31, 32($16) ) # L : 151 subq $0, 168, $5 # E : U L L U : two trips through the loop left? 152 /* 168 = 192 - 24, since we've already completed some stores */ 153 154 subq $0, 16, $0 # E : 155 EX( stq_u $31, 40($16) ) # L : 156 EX( stq_u $31, 48($16) ) # L : 157 cmovlt $5, $16, $3 # E : U L L U : Latency 2, extra mapping cycle 158 159 subq $1, 8, $1 # E : 160 subq $0, 16, $0 # E : 161 EX( stq_u $31, 56($16) ) # L : 162 nop # E : U L U L 163 164 nop # E : 165 subq $0, 8, $0 # E : 166 addq $16, 64, $16 # E : 167 bge $4, $do_wh64 # U : U L U L 168 169$trailquad: 170 # zero to 16 quadwords left to store, plus any trailing bytes 171 # $1 is the number of quadwords left to go. 172 # 173 nop # .. .. .. E 174 nop # .. .. E .. 175 nop # .. E .. .. 176 beq $1, $trailbytes # U .. .. .. : U L U L : Only 0..7 bytes to go 177 178$onequad: 179 EX( stq_u $31, 0($16) ) # .. .. .. L 180 subq $1, 1, $1 # .. .. E .. 181 subq $0, 8, $0 # .. E .. .. 182 nop # E .. .. .. : U L U L 183 184 nop # .. .. .. E 185 nop # .. .. E .. 186 addq $16, 8, $16 # .. E .. .. 187 bgt $1, $onequad # U .. .. .. : U L U L 188 189 # We have an unknown number of bytes left to go. 190$trailbytes: 191 nop # .. .. .. E 192 nop # .. .. E .. 193 nop # .. E .. .. 194 beq $0, $zerolength # U .. .. .. : U L U L 195 196 # $0 contains the number of bytes left to copy (0..31) 197 # so we will use $0 as the loop counter 198 # We know for a fact that $0 > 0 zero due to previous context 199$onebyte: 200 EX( stb $31, 0($16) ) # .. .. .. L 201 subq $0, 1, $0 # .. .. E .. : 202 addq $16, 1, $16 # .. E .. .. : 203 bgt $0, $onebyte # U .. .. .. : U L U L 204 205$zerolength: 206$exception: # Destination for exception recovery(?) 207 nop # .. .. .. E : 208 nop # .. .. E .. : 209 nop # .. E .. .. : 210 ret $31, ($26), 1 # L0 .. .. .. : L U L U 211 .end __clear_user 212 EXPORT_SYMBOL(__clear_user) 213