1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * arch/alpha/lib/ev6-memcpy.S 4 * 21264 version by Rick Gorton <rick.gorton@alpha-processor.com> 5 * 6 * Reasonably optimized memcpy() routine for the Alpha 21264 7 * 8 * - memory accessed as aligned quadwords only 9 * - uses bcmpge to compare 8 bytes in parallel 10 * 11 * Much of the information about 21264 scheduling/coding comes from: 12 * Compiler Writer's Guide for the Alpha 21264 13 * abbreviated as 'CWG' in other comments here 14 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html 15 * Scheduling notation: 16 * E - either cluster 17 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 18 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 19 * 20 * Temp usage notes: 21 * $1,$2, - scratch 22 */ 23#include <linux/export.h> 24 .set noreorder 25 .set noat 26 27 .align 4 28 .globl memcpy 29 .ent memcpy 30memcpy: 31 .frame $30,0,$26,0 32 .prologue 0 33 34 mov $16, $0 # E : copy dest to return 35 ble $18, $nomoredata # U : done with the copy? 36 xor $16, $17, $1 # E : are source and dest alignments the same? 37 and $1, 7, $1 # E : are they the same mod 8? 38 39 bne $1, $misaligned # U : Nope - gotta do this the slow way 40 /* source and dest are same mod 8 address */ 41 and $16, 7, $1 # E : Are both 0mod8? 42 beq $1, $both_0mod8 # U : Yes 43 nop # E : 44 45 /* 46 * source and dest are same misalignment. move a byte at a time 47 * until a 0mod8 alignment for both is reached. 48 * At least one byte more to move 49 */ 50 51$head_align: 52 ldbu $1, 0($17) # L : grab a byte 53 subq $18, 1, $18 # E : count-- 54 addq $17, 1, $17 # E : src++ 55 stb $1, 0($16) # L : 56 addq $16, 1, $16 # E : dest++ 57 and $16, 7, $1 # E : Are we at 0mod8 yet? 58 ble $18, $nomoredata # U : done with the copy? 59 bne $1, $head_align # U : 60 61$both_0mod8: 62 cmple $18, 127, $1 # E : Can we unroll the loop? 63 bne $1, $no_unroll # U : 64 and $16, 63, $1 # E : get mod64 alignment 65 beq $1, $do_unroll # U : no single quads to fiddle 66 67$single_head_quad: 68 ldq $1, 0($17) # L : get 8 bytes 69 subq $18, 8, $18 # E : count -= 8 70 addq $17, 8, $17 # E : src += 8 71 nop # E : 72 73 stq $1, 0($16) # L : store 74 addq $16, 8, $16 # E : dest += 8 75 and $16, 63, $1 # E : get mod64 alignment 76 bne $1, $single_head_quad # U : still not fully aligned 77 78$do_unroll: 79 addq $16, 64, $7 # E : Initial (+1 trip) wh64 address 80 cmple $18, 127, $1 # E : Can we go through the unrolled loop? 81 bne $1, $tail_quads # U : Nope 82 nop # E : 83 84$unroll_body: 85 wh64 ($7) # L1 : memory subsystem hint: 64 bytes at 86 # ($7) are about to be over-written 87 ldq $6, 0($17) # L0 : bytes 0..7 88 nop # E : 89 nop # E : 90 91 ldq $4, 8($17) # L : bytes 8..15 92 ldq $5, 16($17) # L : bytes 16..23 93 addq $7, 64, $7 # E : Update next wh64 address 94 nop # E : 95 96 ldq $3, 24($17) # L : bytes 24..31 97 addq $16, 64, $1 # E : fallback value for wh64 98 nop # E : 99 nop # E : 100 101 addq $17, 32, $17 # E : src += 32 bytes 102 stq $6, 0($16) # L : bytes 0..7 103 nop # E : 104 nop # E : 105 106 stq $4, 8($16) # L : bytes 8..15 107 stq $5, 16($16) # L : bytes 16..23 108 subq $18, 192, $2 # E : At least two more trips to go? 109 nop # E : 110 111 stq $3, 24($16) # L : bytes 24..31 112 addq $16, 32, $16 # E : dest += 32 bytes 113 nop # E : 114 nop # E : 115 116 ldq $6, 0($17) # L : bytes 0..7 117 ldq $4, 8($17) # L : bytes 8..15 118 cmovlt $2, $1, $7 # E : Latency 2, extra map slot - Use 119 # fallback wh64 address if < 2 more trips 120 nop # E : 121 122 ldq $5, 16($17) # L : bytes 16..23 123 ldq $3, 24($17) # L : bytes 24..31 124 addq $16, 32, $16 # E : dest += 32 125 subq $18, 64, $18 # E : count -= 64 126 127 addq $17, 32, $17 # E : src += 32 128 stq $6, -32($16) # L : bytes 0..7 129 stq $4, -24($16) # L : bytes 8..15 130 cmple $18, 63, $1 # E : At least one more trip? 131 132 stq $5, -16($16) # L : bytes 16..23 133 stq $3, -8($16) # L : bytes 24..31 134 nop # E : 135 beq $1, $unroll_body 136 137$tail_quads: 138$no_unroll: 139 .align 4 140 subq $18, 8, $18 # E : At least a quad left? 141 blt $18, $less_than_8 # U : Nope 142 nop # E : 143 nop # E : 144 145$move_a_quad: 146 ldq $1, 0($17) # L : fetch 8 147 subq $18, 8, $18 # E : count -= 8 148 addq $17, 8, $17 # E : src += 8 149 nop # E : 150 151 stq $1, 0($16) # L : store 8 152 addq $16, 8, $16 # E : dest += 8 153 bge $18, $move_a_quad # U : 154 nop # E : 155 156$less_than_8: 157 .align 4 158 addq $18, 8, $18 # E : add back for trailing bytes 159 ble $18, $nomoredata # U : All-done 160 nop # E : 161 nop # E : 162 163 /* Trailing bytes */ 164$tail_bytes: 165 subq $18, 1, $18 # E : count-- 166 ldbu $1, 0($17) # L : fetch a byte 167 addq $17, 1, $17 # E : src++ 168 nop # E : 169 170 stb $1, 0($16) # L : store a byte 171 addq $16, 1, $16 # E : dest++ 172 bgt $18, $tail_bytes # U : more to be done? 173 nop # E : 174 175 /* branching to exit takes 3 extra cycles, so replicate exit here */ 176 ret $31, ($26), 1 # L0 : 177 nop # E : 178 nop # E : 179 nop # E : 180 181$misaligned: 182 mov $0, $4 # E : dest temp 183 and $0, 7, $1 # E : dest alignment mod8 184 beq $1, $dest_0mod8 # U : life doesnt totally suck 185 nop 186 187$aligndest: 188 ble $18, $nomoredata # U : 189 ldbu $1, 0($17) # L : fetch a byte 190 subq $18, 1, $18 # E : count-- 191 addq $17, 1, $17 # E : src++ 192 193 stb $1, 0($4) # L : store it 194 addq $4, 1, $4 # E : dest++ 195 and $4, 7, $1 # E : dest 0mod8 yet? 196 bne $1, $aligndest # U : go until we are aligned. 197 198 /* Source has unknown alignment, but dest is known to be 0mod8 */ 199$dest_0mod8: 200 subq $18, 8, $18 # E : At least a quad left? 201 blt $18, $misalign_tail # U : Nope 202 ldq_u $3, 0($17) # L : seed (rotating load) of 8 bytes 203 nop # E : 204 205$mis_quad: 206 ldq_u $16, 8($17) # L : Fetch next 8 207 extql $3, $17, $3 # U : masking 208 extqh $16, $17, $1 # U : masking 209 bis $3, $1, $1 # E : merged bytes to store 210 211 subq $18, 8, $18 # E : count -= 8 212 addq $17, 8, $17 # E : src += 8 213 stq $1, 0($4) # L : store 8 (aligned) 214 mov $16, $3 # E : "rotate" source data 215 216 addq $4, 8, $4 # E : dest += 8 217 bge $18, $mis_quad # U : More quads to move 218 nop 219 nop 220 221$misalign_tail: 222 addq $18, 8, $18 # E : account for tail stuff 223 ble $18, $nomoredata # U : 224 nop 225 nop 226 227$misalign_byte: 228 ldbu $1, 0($17) # L : fetch 1 229 subq $18, 1, $18 # E : count-- 230 addq $17, 1, $17 # E : src++ 231 nop # E : 232 233 stb $1, 0($4) # L : store 234 addq $4, 1, $4 # E : dest++ 235 bgt $18, $misalign_byte # U : more to go? 236 nop 237 238 239$nomoredata: 240 ret $31, ($26), 1 # L0 : 241 nop # E : 242 nop # E : 243 nop # E : 244 245 .end memcpy 246 EXPORT_SYMBOL(memcpy) 247 248/* For backwards module compatibility. */ 249__memcpy = memcpy 250.globl __memcpy 251