1/* Optimized version of the standard memset() function. 2 3 Copyright (c) 2002 Hewlett-Packard Co/CERN 4 Sverre Jarp <Sverre.Jarp@cern.ch> 5 6 Return: dest 7 8 Inputs: 9 in0: dest 10 in1: value 11 in2: count 12 13 The algorithm is fairly straightforward: set byte by byte until we 14 we get to a 16B-aligned address, then loop on 128 B chunks using an 15 early store as prefetching, then loop on 32B chucks, then clear remaining 16 words, finally clear remaining bytes. 17 Since a stf.spill f0 can store 16B in one go, we use this instruction 18 to get peak speed when value = 0. */ 19 20#include <asm/asmmacro.h> 21#include <asm/export.h> 22#undef ret 23 24#define dest in0 25#define value in1 26#define cnt in2 27 28#define tmp r31 29#define save_lc r30 30#define ptr0 r29 31#define ptr1 r28 32#define ptr2 r27 33#define ptr3 r26 34#define ptr9 r24 35#define loopcnt r23 36#define linecnt r22 37#define bytecnt r21 38 39#define fvalue f6 40 41// This routine uses only scratch predicate registers (p6 - p15) 42#define p_scr p6 // default register for same-cycle branches 43#define p_nz p7 44#define p_zr p8 45#define p_unalgn p9 46#define p_y p11 47#define p_n p12 48#define p_yy p13 49#define p_nn p14 50 51#define MIN1 15 52#define MIN1P1HALF 8 53#define LINE_SIZE 128 54#define LSIZE_SH 7 // shift amount 55#define PREF_AHEAD 8 56 57GLOBAL_ENTRY(memset) 58{ .mmi 59 .prologue 60 alloc tmp = ar.pfs, 3, 0, 0, 0 61 lfetch.nt1 [dest] // 62 .save ar.lc, save_lc 63 mov.i save_lc = ar.lc 64 .body 65} { .mmi 66 mov ret0 = dest // return value 67 cmp.ne p_nz, p_zr = value, r0 // use stf.spill if value is zero 68 cmp.eq p_scr, p0 = cnt, r0 69;; } 70{ .mmi 71 and ptr2 = -(MIN1+1), dest // aligned address 72 and tmp = MIN1, dest // prepare to check for correct alignment 73 tbit.nz p_y, p_n = dest, 0 // Do we have an odd address? (M_B_U) 74} { .mib 75 mov ptr1 = dest 76 mux1 value = value, @brcst // create 8 identical bytes in word 77(p_scr) br.ret.dpnt.many rp // return immediately if count = 0 78;; } 79{ .mib 80 cmp.ne p_unalgn, p0 = tmp, r0 // 81} { .mib 82 sub bytecnt = (MIN1+1), tmp // NB: # of bytes to move is 1 higher than loopcnt 83 cmp.gt p_scr, p0 = 16, cnt // is it a minimalistic task? 84(p_scr) br.cond.dptk.many .move_bytes_unaligned // go move just a few (M_B_U) 85;; } 86{ .mmi 87(p_unalgn) add ptr1 = (MIN1+1), ptr2 // after alignment 88(p_unalgn) add ptr2 = MIN1P1HALF, ptr2 // after alignment 89(p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 3 // should we do a st8 ? 90;; } 91{ .mib 92(p_y) add cnt = -8, cnt // 93(p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 2 // should we do a st4 ? 94} { .mib 95(p_y) st8 [ptr2] = value,-4 // 96(p_n) add ptr2 = 4, ptr2 // 97;; } 98{ .mib 99(p_yy) add cnt = -4, cnt // 100(p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 1 // should we do a st2 ? 101} { .mib 102(p_yy) st4 [ptr2] = value,-2 // 103(p_nn) add ptr2 = 2, ptr2 // 104;; } 105{ .mmi 106 mov tmp = LINE_SIZE+1 // for compare 107(p_y) add cnt = -2, cnt // 108(p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 0 // should we do a st1 ? 109} { .mmi 110 setf.sig fvalue=value // transfer value to FLP side 111(p_y) st2 [ptr2] = value,-1 // 112(p_n) add ptr2 = 1, ptr2 // 113;; } 114 115{ .mmi 116(p_yy) st1 [ptr2] = value // 117 cmp.gt p_scr, p0 = tmp, cnt // is it a minimalistic task? 118} { .mbb 119(p_yy) add cnt = -1, cnt // 120(p_scr) br.cond.dpnt.many .fraction_of_line // go move just a few 121;; } 122 123{ .mib 124 nop.m 0 125 shr.u linecnt = cnt, LSIZE_SH 126(p_zr) br.cond.dptk.many .l1b // Jump to use stf.spill 127;; } 128 129 TEXT_ALIGN(32) // --------------------- // L1A: store ahead into cache lines; fill later 130{ .mmi 131 and tmp = -(LINE_SIZE), cnt // compute end of range 132 mov ptr9 = ptr1 // used for prefetching 133 and cnt = (LINE_SIZE-1), cnt // remainder 134} { .mmi 135 mov loopcnt = PREF_AHEAD-1 // default prefetch loop 136 cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value 137;; } 138{ .mmi 139(p_scr) add loopcnt = -1, linecnt // 140 add ptr2 = 8, ptr1 // start of stores (beyond prefetch stores) 141 add ptr1 = tmp, ptr1 // first address beyond total range 142;; } 143{ .mmi 144 add tmp = -1, linecnt // next loop count 145 mov.i ar.lc = loopcnt // 146;; } 147.pref_l1a: 148{ .mib 149 stf8 [ptr9] = fvalue, 128 // Do stores one cache line apart 150 nop.i 0 151 br.cloop.dptk.few .pref_l1a 152;; } 153{ .mmi 154 add ptr0 = 16, ptr2 // Two stores in parallel 155 mov.i ar.lc = tmp // 156;; } 157.l1ax: 158 { .mmi 159 stf8 [ptr2] = fvalue, 8 160 stf8 [ptr0] = fvalue, 8 161 ;; } 162 { .mmi 163 stf8 [ptr2] = fvalue, 24 164 stf8 [ptr0] = fvalue, 24 165 ;; } 166 { .mmi 167 stf8 [ptr2] = fvalue, 8 168 stf8 [ptr0] = fvalue, 8 169 ;; } 170 { .mmi 171 stf8 [ptr2] = fvalue, 24 172 stf8 [ptr0] = fvalue, 24 173 ;; } 174 { .mmi 175 stf8 [ptr2] = fvalue, 8 176 stf8 [ptr0] = fvalue, 8 177 ;; } 178 { .mmi 179 stf8 [ptr2] = fvalue, 24 180 stf8 [ptr0] = fvalue, 24 181 ;; } 182 { .mmi 183 stf8 [ptr2] = fvalue, 8 184 stf8 [ptr0] = fvalue, 32 185 cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching? 186 ;; } 187{ .mmb 188 stf8 [ptr2] = fvalue, 24 189(p_scr) stf8 [ptr9] = fvalue, 128 190 br.cloop.dptk.few .l1ax 191;; } 192{ .mbb 193 cmp.le p_scr, p0 = 8, cnt // just a few bytes left ? 194(p_scr) br.cond.dpnt.many .fraction_of_line // Branch no. 2 195 br.cond.dpnt.many .move_bytes_from_alignment // Branch no. 3 196;; } 197 198 TEXT_ALIGN(32) 199.l1b: // ------------------------------------ // L1B: store ahead into cache lines; fill later 200{ .mmi 201 and tmp = -(LINE_SIZE), cnt // compute end of range 202 mov ptr9 = ptr1 // used for prefetching 203 and cnt = (LINE_SIZE-1), cnt // remainder 204} { .mmi 205 mov loopcnt = PREF_AHEAD-1 // default prefetch loop 206 cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value 207;; } 208{ .mmi 209(p_scr) add loopcnt = -1, linecnt 210 add ptr2 = 16, ptr1 // start of stores (beyond prefetch stores) 211 add ptr1 = tmp, ptr1 // first address beyond total range 212;; } 213{ .mmi 214 add tmp = -1, linecnt // next loop count 215 mov.i ar.lc = loopcnt 216;; } 217.pref_l1b: 218{ .mib 219 stf.spill [ptr9] = f0, 128 // Do stores one cache line apart 220 nop.i 0 221 br.cloop.dptk.few .pref_l1b 222;; } 223{ .mmi 224 add ptr0 = 16, ptr2 // Two stores in parallel 225 mov.i ar.lc = tmp 226;; } 227.l1bx: 228 { .mmi 229 stf.spill [ptr2] = f0, 32 230 stf.spill [ptr0] = f0, 32 231 ;; } 232 { .mmi 233 stf.spill [ptr2] = f0, 32 234 stf.spill [ptr0] = f0, 32 235 ;; } 236 { .mmi 237 stf.spill [ptr2] = f0, 32 238 stf.spill [ptr0] = f0, 64 239 cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching? 240 ;; } 241{ .mmb 242 stf.spill [ptr2] = f0, 32 243(p_scr) stf.spill [ptr9] = f0, 128 244 br.cloop.dptk.few .l1bx 245;; } 246{ .mib 247 cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ? 248(p_scr) br.cond.dpnt.many .move_bytes_from_alignment // 249;; } 250 251.fraction_of_line: 252{ .mib 253 add ptr2 = 16, ptr1 254 shr.u loopcnt = cnt, 5 // loopcnt = cnt / 32 255;; } 256{ .mib 257 cmp.eq p_scr, p0 = loopcnt, r0 258 add loopcnt = -1, loopcnt 259(p_scr) br.cond.dpnt.many .store_words 260;; } 261{ .mib 262 and cnt = 0x1f, cnt // compute the remaining cnt 263 mov.i ar.lc = loopcnt 264;; } 265 TEXT_ALIGN(32) 266.l2: // ------------------------------------ // L2A: store 32B in 2 cycles 267{ .mmb 268 stf8 [ptr1] = fvalue, 8 269 stf8 [ptr2] = fvalue, 8 270;; } { .mmb 271 stf8 [ptr1] = fvalue, 24 272 stf8 [ptr2] = fvalue, 24 273 br.cloop.dptk.many .l2 274;; } 275.store_words: 276{ .mib 277 cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ? 278(p_scr) br.cond.dpnt.many .move_bytes_from_alignment // Branch 279;; } 280 281{ .mmi 282 stf8 [ptr1] = fvalue, 8 // store 283 cmp.le p_y, p_n = 16, cnt 284 add cnt = -8, cnt // subtract 285;; } 286{ .mmi 287(p_y) stf8 [ptr1] = fvalue, 8 // store 288(p_y) cmp.le.unc p_yy, p_nn = 16, cnt 289(p_y) add cnt = -8, cnt // subtract 290;; } 291{ .mmi // store 292(p_yy) stf8 [ptr1] = fvalue, 8 293(p_yy) add cnt = -8, cnt // subtract 294;; } 295 296.move_bytes_from_alignment: 297{ .mib 298 cmp.eq p_scr, p0 = cnt, r0 299 tbit.nz.unc p_y, p0 = cnt, 2 // should we terminate with a st4 ? 300(p_scr) br.cond.dpnt.few .restore_and_exit 301;; } 302{ .mib 303(p_y) st4 [ptr1] = value,4 304 tbit.nz.unc p_yy, p0 = cnt, 1 // should we terminate with a st2 ? 305;; } 306{ .mib 307(p_yy) st2 [ptr1] = value,2 308 tbit.nz.unc p_y, p0 = cnt, 0 // should we terminate with a st1 ? 309;; } 310 311{ .mib 312(p_y) st1 [ptr1] = value 313;; } 314.restore_and_exit: 315{ .mib 316 nop.m 0 317 mov.i ar.lc = save_lc 318 br.ret.sptk.many rp 319;; } 320 321.move_bytes_unaligned: 322{ .mmi 323 .pred.rel "mutex",p_y, p_n 324 .pred.rel "mutex",p_yy, p_nn 325(p_n) cmp.le p_yy, p_nn = 4, cnt 326(p_y) cmp.le p_yy, p_nn = 5, cnt 327(p_n) add ptr2 = 2, ptr1 328} { .mmi 329(p_y) add ptr2 = 3, ptr1 330(p_y) st1 [ptr1] = value, 1 // fill 1 (odd-aligned) byte [15, 14 (or less) left] 331(p_y) add cnt = -1, cnt 332;; } 333{ .mmi 334(p_yy) cmp.le.unc p_y, p0 = 8, cnt 335 add ptr3 = ptr1, cnt // prepare last store 336 mov.i ar.lc = save_lc 337} { .mmi 338(p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes 339(p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes [11, 10 (o less) left] 340(p_yy) add cnt = -4, cnt 341;; } 342{ .mmi 343(p_y) cmp.le.unc p_yy, p0 = 8, cnt 344 add ptr3 = -1, ptr3 // last store 345 tbit.nz p_scr, p0 = cnt, 1 // will there be a st2 at the end ? 346} { .mmi 347(p_y) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes 348(p_y) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes [7, 6 (or less) left] 349(p_y) add cnt = -4, cnt 350;; } 351{ .mmi 352(p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes 353(p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes [3, 2 (or less) left] 354 tbit.nz p_y, p0 = cnt, 0 // will there be a st1 at the end ? 355} { .mmi 356(p_yy) add cnt = -4, cnt 357;; } 358{ .mmb 359(p_scr) st2 [ptr1] = value // fill 2 (aligned) bytes 360(p_y) st1 [ptr3] = value // fill last byte (using ptr3) 361 br.ret.sptk.many rp 362} 363END(memset) 364EXPORT_SYMBOL(memset) 365