1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * arch/alpha/lib/ev6-copy_page.S 4 * 5 * Copy an entire page. 6 */ 7 8/* The following comparison of this routine vs the normal copy_page.S 9 was written by an unnamed ev6 hardware designer and forwarded to me 10 via Steven Hobbs <hobbs@steven.zko.dec.com>. 11 12 First Problem: STQ overflows. 13 ----------------------------- 14 15 It would be nice if EV6 handled every resource overflow efficiently, 16 but for some it doesn't. Including store queue overflows. It causes 17 a trap and a restart of the pipe. 18 19 To get around this we sometimes use (to borrow a term from a VSSAD 20 researcher) "aeration". The idea is to slow the rate at which the 21 processor receives valid instructions by inserting nops in the fetch 22 path. In doing so, you can prevent the overflow and actually make 23 the code run faster. You can, of course, take advantage of the fact 24 that the processor can fetch at most 4 aligned instructions per cycle. 25 26 I inserted enough nops to force it to take 10 cycles to fetch the 27 loop code. In theory, EV6 should be able to execute this loop in 28 9 cycles but I was not able to get it to run that fast -- the initial 29 conditions were such that I could not reach this optimum rate on 30 (chaotic) EV6. I wrote the code such that everything would issue 31 in order. 32 33 Second Problem: Dcache index matches. 34 ------------------------------------- 35 36 If you are going to use this routine on random aligned pages, there 37 is a 25% chance that the pages will be at the same dcache indices. 38 This results in many nasty memory traps without care. 39 40 The solution is to schedule the prefetches to avoid the memory 41 conflicts. I schedule the wh64 prefetches farther ahead of the 42 read prefetches to avoid this problem. 43 44 Third Problem: Needs more prefetching. 45 -------------------------------------- 46 47 In order to improve the code I added deeper prefetching to take the 48 most advantage of EV6's bandwidth. 49 50 I also prefetched the read stream. Note that adding the read prefetch 51 forced me to add another cycle to the inner-most kernel - up to 11 52 from the original 8 cycles per iteration. We could improve performance 53 further by unrolling the loop and doing multiple prefetches per cycle. 54 55 I think that the code below will be very robust and fast code for the 56 purposes of copying aligned pages. It is slower when both source and 57 destination pages are in the dcache, but it is my guess that this is 58 less important than the dcache miss case. */ 59 60#include <linux/export.h> 61 .text 62 .align 4 63 .global copy_page 64 .ent copy_page 65copy_page: 66 .prologue 0 67 68 /* Prefetch 5 read cachelines; write-hint 10 cache lines. */ 69 wh64 ($16) 70 ldl $31,0($17) 71 ldl $31,64($17) 72 lda $1,1*64($16) 73 74 wh64 ($1) 75 ldl $31,128($17) 76 ldl $31,192($17) 77 lda $1,2*64($16) 78 79 wh64 ($1) 80 ldl $31,256($17) 81 lda $18,118 82 lda $1,3*64($16) 83 84 wh64 ($1) 85 nop 86 lda $1,4*64($16) 87 lda $2,5*64($16) 88 89 wh64 ($1) 90 wh64 ($2) 91 lda $1,6*64($16) 92 lda $2,7*64($16) 93 94 wh64 ($1) 95 wh64 ($2) 96 lda $1,8*64($16) 97 lda $2,9*64($16) 98 99 wh64 ($1) 100 wh64 ($2) 101 lda $19,10*64($16) 102 nop 103 104 /* Main prefetching/write-hinting loop. */ 1051: ldq $0,0($17) 106 ldq $1,8($17) 107 unop 108 unop 109 110 unop 111 unop 112 ldq $2,16($17) 113 ldq $3,24($17) 114 115 ldq $4,32($17) 116 ldq $5,40($17) 117 unop 118 unop 119 120 unop 121 unop 122 ldq $6,48($17) 123 ldq $7,56($17) 124 125 ldl $31,320($17) 126 unop 127 unop 128 unop 129 130 /* This gives the extra cycle of aeration above the minimum. */ 131 unop 132 unop 133 unop 134 unop 135 136 wh64 ($19) 137 unop 138 unop 139 unop 140 141 stq $0,0($16) 142 subq $18,1,$18 143 stq $1,8($16) 144 unop 145 146 unop 147 stq $2,16($16) 148 addq $17,64,$17 149 stq $3,24($16) 150 151 stq $4,32($16) 152 stq $5,40($16) 153 addq $19,64,$19 154 unop 155 156 stq $6,48($16) 157 stq $7,56($16) 158 addq $16,64,$16 159 bne $18, 1b 160 161 /* Prefetch the final 5 cache lines of the read stream. */ 162 lda $18,10 163 ldl $31,320($17) 164 ldl $31,384($17) 165 ldl $31,448($17) 166 167 ldl $31,512($17) 168 ldl $31,576($17) 169 nop 170 nop 171 172 /* Non-prefetching, non-write-hinting cleanup loop for the 173 final 10 cache lines. */ 1742: ldq $0,0($17) 175 ldq $1,8($17) 176 ldq $2,16($17) 177 ldq $3,24($17) 178 179 ldq $4,32($17) 180 ldq $5,40($17) 181 ldq $6,48($17) 182 ldq $7,56($17) 183 184 stq $0,0($16) 185 subq $18,1,$18 186 stq $1,8($16) 187 addq $17,64,$17 188 189 stq $2,16($16) 190 stq $3,24($16) 191 stq $4,32($16) 192 stq $5,40($16) 193 194 stq $6,48($16) 195 stq $7,56($16) 196 addq $16,64,$16 197 bne $18, 2b 198 199 ret 200 nop 201 unop 202 nop 203 204 .end copy_page 205 EXPORT_SYMBOL(copy_page) 206