11da177e4SLinus Torvalds/* 21da177e4SLinus Torvalds * 31da177e4SLinus Torvalds * Optimized version of the copy_user() routine. 41da177e4SLinus Torvalds * It is used to copy date across the kernel/user boundary. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * The source and destination are always on opposite side of 71da177e4SLinus Torvalds * the boundary. When reading from user space we must catch 81da177e4SLinus Torvalds * faults on loads. When writing to user space we must catch 91da177e4SLinus Torvalds * errors on stores. Note that because of the nature of the copy 101da177e4SLinus Torvalds * we don't need to worry about overlapping regions. 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * Inputs: 141da177e4SLinus Torvalds * in0 address of source buffer 151da177e4SLinus Torvalds * in1 address of destination buffer 161da177e4SLinus Torvalds * in2 number of bytes to copy 171da177e4SLinus Torvalds * 181da177e4SLinus Torvalds * Outputs: 191da177e4SLinus Torvalds * ret0 0 in case of success. The number of bytes NOT copied in 201da177e4SLinus Torvalds * case of error. 211da177e4SLinus Torvalds * 221da177e4SLinus Torvalds * Copyright (C) 2000-2001 Hewlett-Packard Co 231da177e4SLinus Torvalds * Stephane Eranian <eranian@hpl.hp.com> 241da177e4SLinus Torvalds * 251da177e4SLinus Torvalds * Fixme: 261da177e4SLinus Torvalds * - handle the case where we have more than 16 bytes and the alignment 271da177e4SLinus Torvalds * are different. 281da177e4SLinus Torvalds * - more benchmarking 291da177e4SLinus Torvalds * - fix extraneous stop bit introduced by the EX() macro. 301da177e4SLinus Torvalds */ 311da177e4SLinus Torvalds 321da177e4SLinus Torvalds#include <asm/asmmacro.h> 33e007c533SAl Viro#include <asm/export.h> 341da177e4SLinus Torvalds 351da177e4SLinus Torvalds// 361da177e4SLinus Torvalds// Tuneable parameters 371da177e4SLinus Torvalds// 381da177e4SLinus Torvalds#define COPY_BREAK 16 // we do byte copy below (must be >=16) 391da177e4SLinus Torvalds#define PIPE_DEPTH 21 // pipe depth 401da177e4SLinus Torvalds 411da177e4SLinus Torvalds#define EPI p[PIPE_DEPTH-1] 421da177e4SLinus Torvalds 431da177e4SLinus Torvalds// 441da177e4SLinus Torvalds// arguments 451da177e4SLinus Torvalds// 461da177e4SLinus Torvalds#define dst in0 471da177e4SLinus Torvalds#define src in1 481da177e4SLinus Torvalds#define len in2 491da177e4SLinus Torvalds 501da177e4SLinus Torvalds// 511da177e4SLinus Torvalds// local registers 521da177e4SLinus Torvalds// 531da177e4SLinus Torvalds#define t1 r2 // rshift in bytes 541da177e4SLinus Torvalds#define t2 r3 // lshift in bytes 551da177e4SLinus Torvalds#define rshift r14 // right shift in bits 561da177e4SLinus Torvalds#define lshift r15 // left shift in bits 571da177e4SLinus Torvalds#define word1 r16 581da177e4SLinus Torvalds#define word2 r17 591da177e4SLinus Torvalds#define cnt r18 601da177e4SLinus Torvalds#define len2 r19 611da177e4SLinus Torvalds#define saved_lc r20 621da177e4SLinus Torvalds#define saved_pr r21 631da177e4SLinus Torvalds#define tmp r22 641da177e4SLinus Torvalds#define val r23 651da177e4SLinus Torvalds#define src1 r24 661da177e4SLinus Torvalds#define dst1 r25 671da177e4SLinus Torvalds#define src2 r26 681da177e4SLinus Torvalds#define dst2 r27 691da177e4SLinus Torvalds#define len1 r28 701da177e4SLinus Torvalds#define enddst r29 711da177e4SLinus Torvalds#define endsrc r30 721da177e4SLinus Torvalds#define saved_pfs r31 731da177e4SLinus Torvalds 741da177e4SLinus TorvaldsGLOBAL_ENTRY(__copy_user) 751da177e4SLinus Torvalds .prologue 761da177e4SLinus Torvalds .save ar.pfs, saved_pfs 771da177e4SLinus Torvalds alloc saved_pfs=ar.pfs,3,((2*PIPE_DEPTH+7)&~7),0,((2*PIPE_DEPTH+7)&~7) 781da177e4SLinus Torvalds 791da177e4SLinus Torvalds .rotr val1[PIPE_DEPTH],val2[PIPE_DEPTH] 801da177e4SLinus Torvalds .rotp p[PIPE_DEPTH] 811da177e4SLinus Torvalds 821da177e4SLinus Torvalds adds len2=-1,len // br.ctop is repeat/until 831da177e4SLinus Torvalds mov ret0=r0 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds ;; // RAW of cfm when len=0 861da177e4SLinus Torvalds cmp.eq p8,p0=r0,len // check for zero length 871da177e4SLinus Torvalds .save ar.lc, saved_lc 881da177e4SLinus Torvalds mov saved_lc=ar.lc // preserve ar.lc (slow) 891da177e4SLinus Torvalds(p8) br.ret.spnt.many rp // empty mempcy() 901da177e4SLinus Torvalds ;; 911da177e4SLinus Torvalds add enddst=dst,len // first byte after end of source 921da177e4SLinus Torvalds add endsrc=src,len // first byte after end of destination 931da177e4SLinus Torvalds .save pr, saved_pr 941da177e4SLinus Torvalds mov saved_pr=pr // preserve predicates 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds .body 971da177e4SLinus Torvalds 981da177e4SLinus Torvalds mov dst1=dst // copy because of rotation 991da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 1001da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds mov src1=src // copy because of rotation 1031da177e4SLinus Torvalds mov ar.lc=len2 // initialize lc for small count 1041da177e4SLinus Torvalds cmp.lt p10,p7=COPY_BREAK,len // if len > COPY_BREAK then long copy 1051da177e4SLinus Torvalds 1061da177e4SLinus Torvalds xor tmp=src,dst // same alignment test prepare 1071da177e4SLinus Torvalds(p10) br.cond.dptk .long_copy_user 1081da177e4SLinus Torvalds ;; // RAW pr.rot/p16 ? 1091da177e4SLinus Torvalds // 1101da177e4SLinus Torvalds // Now we do the byte by byte loop with software pipeline 1111da177e4SLinus Torvalds // 1121da177e4SLinus Torvalds // p7 is necessarily false by now 1131da177e4SLinus Torvalds1: 1141da177e4SLinus Torvalds EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1) 1151da177e4SLinus Torvalds EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1) 1161da177e4SLinus Torvalds br.ctop.dptk.few 1b 1171da177e4SLinus Torvalds ;; 1181da177e4SLinus Torvalds mov ar.lc=saved_lc 1191da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 1201da177e4SLinus Torvalds mov ar.pfs=saved_pfs // restore ar.ec 1211da177e4SLinus Torvalds br.ret.sptk.many rp // end of short memcpy 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds // 1241da177e4SLinus Torvalds // Not 8-byte aligned 1251da177e4SLinus Torvalds // 1261da177e4SLinus Torvalds.diff_align_copy_user: 1271da177e4SLinus Torvalds // At this point we know we have more than 16 bytes to copy 1281da177e4SLinus Torvalds // and also that src and dest do _not_ have the same alignment. 1291da177e4SLinus Torvalds and src2=0x7,src1 // src offset 1301da177e4SLinus Torvalds and dst2=0x7,dst1 // dst offset 1311da177e4SLinus Torvalds ;; 1321da177e4SLinus Torvalds // The basic idea is that we copy byte-by-byte at the head so 1331da177e4SLinus Torvalds // that we can reach 8-byte alignment for both src1 and dst1. 1341da177e4SLinus Torvalds // Then copy the body using software pipelined 8-byte copy, 1351da177e4SLinus Torvalds // shifting the two back-to-back words right and left, then copy 1361da177e4SLinus Torvalds // the tail by copying byte-by-byte. 1371da177e4SLinus Torvalds // 1381da177e4SLinus Torvalds // Fault handling. If the byte-by-byte at the head fails on the 1391da177e4SLinus Torvalds // load, then restart and finish the pipleline by copying zeros 1401da177e4SLinus Torvalds // to the dst1. Then copy zeros for the rest of dst1. 1411da177e4SLinus Torvalds // If 8-byte software pipeline fails on the load, do the same as 1421da177e4SLinus Torvalds // failure_in3 does. If the byte-by-byte at the tail fails, it is 1431da177e4SLinus Torvalds // handled simply by failure_in_pipe1. 1441da177e4SLinus Torvalds // 1451da177e4SLinus Torvalds // The case p14 represents the source has more bytes in the 1461da177e4SLinus Torvalds // the first word (by the shifted part), whereas the p15 needs to 1471da177e4SLinus Torvalds // copy some bytes from the 2nd word of the source that has the 1481da177e4SLinus Torvalds // tail of the 1st of the destination. 1491da177e4SLinus Torvalds // 1501da177e4SLinus Torvalds 1511da177e4SLinus Torvalds // 1521da177e4SLinus Torvalds // Optimization. If dst1 is 8-byte aligned (quite common), we don't need 1531da177e4SLinus Torvalds // to copy the head to dst1, to start 8-byte copy software pipeline. 1541da177e4SLinus Torvalds // We know src1 is not 8-byte aligned in this case. 1551da177e4SLinus Torvalds // 1561da177e4SLinus Torvalds cmp.eq p14,p15=r0,dst2 1571da177e4SLinus Torvalds(p15) br.cond.spnt 1f 1581da177e4SLinus Torvalds ;; 1591da177e4SLinus Torvalds sub t1=8,src2 1601da177e4SLinus Torvalds mov t2=src2 1611da177e4SLinus Torvalds ;; 1621da177e4SLinus Torvalds shl rshift=t2,3 1631da177e4SLinus Torvalds sub len1=len,t1 // set len1 1641da177e4SLinus Torvalds ;; 1651da177e4SLinus Torvalds sub lshift=64,rshift 1661da177e4SLinus Torvalds ;; 1671da177e4SLinus Torvalds br.cond.spnt .word_copy_user 1681da177e4SLinus Torvalds ;; 1691da177e4SLinus Torvalds1: 1701da177e4SLinus Torvalds cmp.leu p14,p15=src2,dst2 1711da177e4SLinus Torvalds sub t1=dst2,src2 1721da177e4SLinus Torvalds ;; 1731da177e4SLinus Torvalds .pred.rel "mutex", p14, p15 1741da177e4SLinus Torvalds(p14) sub word1=8,src2 // (8 - src offset) 1751da177e4SLinus Torvalds(p15) sub t1=r0,t1 // absolute value 1761da177e4SLinus Torvalds(p15) sub word1=8,dst2 // (8 - dst offset) 1771da177e4SLinus Torvalds ;; 1781da177e4SLinus Torvalds // For the case p14, we don't need to copy the shifted part to 1791da177e4SLinus Torvalds // the 1st word of destination. 1801da177e4SLinus Torvalds sub t2=8,t1 1811da177e4SLinus Torvalds(p14) sub word1=word1,t1 1821da177e4SLinus Torvalds ;; 1831da177e4SLinus Torvalds sub len1=len,word1 // resulting len 1841da177e4SLinus Torvalds(p15) shl rshift=t1,3 // in bits 1851da177e4SLinus Torvalds(p14) shl rshift=t2,3 1861da177e4SLinus Torvalds ;; 1871da177e4SLinus Torvalds(p14) sub len1=len1,t1 1881da177e4SLinus Torvalds adds cnt=-1,word1 1891da177e4SLinus Torvalds ;; 1901da177e4SLinus Torvalds sub lshift=64,rshift 1911da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 1921da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 1931da177e4SLinus Torvalds mov ar.lc=cnt 1941da177e4SLinus Torvalds ;; 1951da177e4SLinus Torvalds2: 1961da177e4SLinus Torvalds EX(.failure_in_pipe2,(p16) ld1 val1[0]=[src1],1) 1971da177e4SLinus Torvalds EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1) 1981da177e4SLinus Torvalds br.ctop.dptk.few 2b 1991da177e4SLinus Torvalds ;; 2001da177e4SLinus Torvalds clrrrb 2011da177e4SLinus Torvalds ;; 2021da177e4SLinus Torvalds.word_copy_user: 2031da177e4SLinus Torvalds cmp.gtu p9,p0=16,len1 2041da177e4SLinus Torvalds(p9) br.cond.spnt 4f // if (16 > len1) skip 8-byte copy 2051da177e4SLinus Torvalds ;; 2061da177e4SLinus Torvalds shr.u cnt=len1,3 // number of 64-bit words 2071da177e4SLinus Torvalds ;; 2081da177e4SLinus Torvalds adds cnt=-1,cnt 2091da177e4SLinus Torvalds ;; 2101da177e4SLinus Torvalds .pred.rel "mutex", p14, p15 2111da177e4SLinus Torvalds(p14) sub src1=src1,t2 2121da177e4SLinus Torvalds(p15) sub src1=src1,t1 2131da177e4SLinus Torvalds // 2141da177e4SLinus Torvalds // Now both src1 and dst1 point to an 8-byte aligned address. And 2151da177e4SLinus Torvalds // we have more than 8 bytes to copy. 2161da177e4SLinus Torvalds // 2171da177e4SLinus Torvalds mov ar.lc=cnt 2181da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 2191da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 2201da177e4SLinus Torvalds ;; 2211da177e4SLinus Torvalds3: 2221da177e4SLinus Torvalds // 2231da177e4SLinus Torvalds // The pipleline consists of 3 stages: 2241da177e4SLinus Torvalds // 1 (p16): Load a word from src1 2251da177e4SLinus Torvalds // 2 (EPI_1): Shift right pair, saving to tmp 2261da177e4SLinus Torvalds // 3 (EPI): Store tmp to dst1 2271da177e4SLinus Torvalds // 2281da177e4SLinus Torvalds // To make it simple, use at least 2 (p16) loops to set up val1[n] 2291da177e4SLinus Torvalds // because we need 2 back-to-back val1[] to get tmp. 2301da177e4SLinus Torvalds // Note that this implies EPI_2 must be p18 or greater. 2311da177e4SLinus Torvalds // 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds#define EPI_1 p[PIPE_DEPTH-2] 2341da177e4SLinus Torvalds#define SWITCH(pred, shift) cmp.eq pred,p0=shift,rshift 2351da177e4SLinus Torvalds#define CASE(pred, shift) \ 2361da177e4SLinus Torvalds (pred) br.cond.spnt .copy_user_bit##shift 2371da177e4SLinus Torvalds#define BODY(rshift) \ 2381da177e4SLinus Torvalds.copy_user_bit##rshift: \ 2391da177e4SLinus Torvalds1: \ 2401da177e4SLinus Torvalds EX(.failure_out,(EPI) st8 [dst1]=tmp,8); \ 2411da177e4SLinus Torvalds(EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \ 2421da177e4SLinus Torvalds EX(3f,(p16) ld8 val1[1]=[src1],8); \ 2431da177e4SLinus Torvalds(p16) mov val1[0]=r0; \ 2441da177e4SLinus Torvalds br.ctop.dptk 1b; \ 2451da177e4SLinus Torvalds ;; \ 2461da177e4SLinus Torvalds br.cond.sptk.many .diff_align_do_tail; \ 2471da177e4SLinus Torvalds2: \ 2481da177e4SLinus Torvalds(EPI) st8 [dst1]=tmp,8; \ 2491da177e4SLinus Torvalds(EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \ 2501da177e4SLinus Torvalds3: \ 2511da177e4SLinus Torvalds(p16) mov val1[1]=r0; \ 2521da177e4SLinus Torvalds(p16) mov val1[0]=r0; \ 2531da177e4SLinus Torvalds br.ctop.dptk 2b; \ 2541da177e4SLinus Torvalds ;; \ 2551da177e4SLinus Torvalds br.cond.sptk.many .failure_in2 2561da177e4SLinus Torvalds 2571da177e4SLinus Torvalds // 2581da177e4SLinus Torvalds // Since the instruction 'shrp' requires a fixed 128-bit value 2591da177e4SLinus Torvalds // specifying the bits to shift, we need to provide 7 cases 2601da177e4SLinus Torvalds // below. 2611da177e4SLinus Torvalds // 2621da177e4SLinus Torvalds SWITCH(p6, 8) 2631da177e4SLinus Torvalds SWITCH(p7, 16) 2641da177e4SLinus Torvalds SWITCH(p8, 24) 2651da177e4SLinus Torvalds SWITCH(p9, 32) 2661da177e4SLinus Torvalds SWITCH(p10, 40) 2671da177e4SLinus Torvalds SWITCH(p11, 48) 2681da177e4SLinus Torvalds SWITCH(p12, 56) 2691da177e4SLinus Torvalds ;; 2701da177e4SLinus Torvalds CASE(p6, 8) 2711da177e4SLinus Torvalds CASE(p7, 16) 2721da177e4SLinus Torvalds CASE(p8, 24) 2731da177e4SLinus Torvalds CASE(p9, 32) 2741da177e4SLinus Torvalds CASE(p10, 40) 2751da177e4SLinus Torvalds CASE(p11, 48) 2761da177e4SLinus Torvalds CASE(p12, 56) 2771da177e4SLinus Torvalds ;; 2781da177e4SLinus Torvalds BODY(8) 2791da177e4SLinus Torvalds BODY(16) 2801da177e4SLinus Torvalds BODY(24) 2811da177e4SLinus Torvalds BODY(32) 2821da177e4SLinus Torvalds BODY(40) 2831da177e4SLinus Torvalds BODY(48) 2841da177e4SLinus Torvalds BODY(56) 2851da177e4SLinus Torvalds ;; 2861da177e4SLinus Torvalds.diff_align_do_tail: 2871da177e4SLinus Torvalds .pred.rel "mutex", p14, p15 2881da177e4SLinus Torvalds(p14) sub src1=src1,t1 2891da177e4SLinus Torvalds(p14) adds dst1=-8,dst1 2901da177e4SLinus Torvalds(p15) sub dst1=dst1,t1 2911da177e4SLinus Torvalds ;; 2921da177e4SLinus Torvalds4: 2931da177e4SLinus Torvalds // Tail correction. 2941da177e4SLinus Torvalds // 2951da177e4SLinus Torvalds // The problem with this piplelined loop is that the last word is not 2961da177e4SLinus Torvalds // loaded and thus parf of the last word written is not correct. 2971da177e4SLinus Torvalds // To fix that, we simply copy the tail byte by byte. 2981da177e4SLinus Torvalds 2991da177e4SLinus Torvalds sub len1=endsrc,src1,1 3001da177e4SLinus Torvalds clrrrb 3011da177e4SLinus Torvalds ;; 3021da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 3031da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 3041da177e4SLinus Torvalds mov ar.lc=len1 3051da177e4SLinus Torvalds ;; 3061da177e4SLinus Torvalds5: 3071da177e4SLinus Torvalds EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1) 3081da177e4SLinus Torvalds EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1) 3091da177e4SLinus Torvalds br.ctop.dptk.few 5b 3101da177e4SLinus Torvalds ;; 3111da177e4SLinus Torvalds mov ar.lc=saved_lc 3121da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 3131da177e4SLinus Torvalds mov ar.pfs=saved_pfs 3141da177e4SLinus Torvalds br.ret.sptk.many rp 3151da177e4SLinus Torvalds 3161da177e4SLinus Torvalds // 3171da177e4SLinus Torvalds // Beginning of long mempcy (i.e. > 16 bytes) 3181da177e4SLinus Torvalds // 3191da177e4SLinus Torvalds.long_copy_user: 3201da177e4SLinus Torvalds tbit.nz p6,p7=src1,0 // odd alignment 3211da177e4SLinus Torvalds and tmp=7,tmp 3221da177e4SLinus Torvalds ;; 3231da177e4SLinus Torvalds cmp.eq p10,p8=r0,tmp 3241da177e4SLinus Torvalds mov len1=len // copy because of rotation 3251da177e4SLinus Torvalds(p8) br.cond.dpnt .diff_align_copy_user 3261da177e4SLinus Torvalds ;; 3271da177e4SLinus Torvalds // At this point we know we have more than 16 bytes to copy 3281da177e4SLinus Torvalds // and also that both src and dest have the same alignment 3291da177e4SLinus Torvalds // which may not be the one we want. So for now we must move 3301da177e4SLinus Torvalds // forward slowly until we reach 16byte alignment: no need to 3311da177e4SLinus Torvalds // worry about reaching the end of buffer. 3321da177e4SLinus Torvalds // 3331da177e4SLinus Torvalds EX(.failure_in1,(p6) ld1 val1[0]=[src1],1) // 1-byte aligned 3341da177e4SLinus Torvalds(p6) adds len1=-1,len1;; 3351da177e4SLinus Torvalds tbit.nz p7,p0=src1,1 3361da177e4SLinus Torvalds ;; 3371da177e4SLinus Torvalds EX(.failure_in1,(p7) ld2 val1[1]=[src1],2) // 2-byte aligned 3381da177e4SLinus Torvalds(p7) adds len1=-2,len1;; 3391da177e4SLinus Torvalds tbit.nz p8,p0=src1,2 3401da177e4SLinus Torvalds ;; 3411da177e4SLinus Torvalds // 3421da177e4SLinus Torvalds // Stop bit not required after ld4 because if we fail on ld4 3431da177e4SLinus Torvalds // we have never executed the ld1, therefore st1 is not executed. 3441da177e4SLinus Torvalds // 3451da177e4SLinus Torvalds EX(.failure_in1,(p8) ld4 val2[0]=[src1],4) // 4-byte aligned 3461da177e4SLinus Torvalds ;; 3471da177e4SLinus Torvalds EX(.failure_out,(p6) st1 [dst1]=val1[0],1) 3481da177e4SLinus Torvalds tbit.nz p9,p0=src1,3 3491da177e4SLinus Torvalds ;; 3501da177e4SLinus Torvalds // 3511da177e4SLinus Torvalds // Stop bit not required after ld8 because if we fail on ld8 3521da177e4SLinus Torvalds // we have never executed the ld2, therefore st2 is not executed. 3531da177e4SLinus Torvalds // 3541da177e4SLinus Torvalds EX(.failure_in1,(p9) ld8 val2[1]=[src1],8) // 8-byte aligned 3551da177e4SLinus Torvalds EX(.failure_out,(p7) st2 [dst1]=val1[1],2) 3561da177e4SLinus Torvalds(p8) adds len1=-4,len1 3571da177e4SLinus Torvalds ;; 3581da177e4SLinus Torvalds EX(.failure_out, (p8) st4 [dst1]=val2[0],4) 3591da177e4SLinus Torvalds(p9) adds len1=-8,len1;; 3601da177e4SLinus Torvalds shr.u cnt=len1,4 // number of 128-bit (2x64bit) words 3611da177e4SLinus Torvalds ;; 3621da177e4SLinus Torvalds EX(.failure_out, (p9) st8 [dst1]=val2[1],8) 3631da177e4SLinus Torvalds tbit.nz p6,p0=len1,3 3641da177e4SLinus Torvalds cmp.eq p7,p0=r0,cnt 3651da177e4SLinus Torvalds adds tmp=-1,cnt // br.ctop is repeat/until 3661da177e4SLinus Torvalds(p7) br.cond.dpnt .dotail // we have less than 16 bytes left 3671da177e4SLinus Torvalds ;; 3681da177e4SLinus Torvalds adds src2=8,src1 3691da177e4SLinus Torvalds adds dst2=8,dst1 3701da177e4SLinus Torvalds mov ar.lc=tmp 3711da177e4SLinus Torvalds ;; 3721da177e4SLinus Torvalds // 3731da177e4SLinus Torvalds // 16bytes/iteration 3741da177e4SLinus Torvalds // 3751da177e4SLinus Torvalds2: 3761da177e4SLinus Torvalds EX(.failure_in3,(p16) ld8 val1[0]=[src1],16) 3771da177e4SLinus Torvalds(p16) ld8 val2[0]=[src2],16 3781da177e4SLinus Torvalds 3791da177e4SLinus Torvalds EX(.failure_out, (EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16) 3801da177e4SLinus Torvalds(EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16 3811da177e4SLinus Torvalds br.ctop.dptk 2b 3821da177e4SLinus Torvalds ;; // RAW on src1 when fall through from loop 3831da177e4SLinus Torvalds // 3841da177e4SLinus Torvalds // Tail correction based on len only 3851da177e4SLinus Torvalds // 3861da177e4SLinus Torvalds // No matter where we come from (loop or test) the src1 pointer 3871da177e4SLinus Torvalds // is 16 byte aligned AND we have less than 16 bytes to copy. 3881da177e4SLinus Torvalds // 3891da177e4SLinus Torvalds.dotail: 3901da177e4SLinus Torvalds EX(.failure_in1,(p6) ld8 val1[0]=[src1],8) // at least 8 bytes 3911da177e4SLinus Torvalds tbit.nz p7,p0=len1,2 3921da177e4SLinus Torvalds ;; 3931da177e4SLinus Torvalds EX(.failure_in1,(p7) ld4 val1[1]=[src1],4) // at least 4 bytes 3941da177e4SLinus Torvalds tbit.nz p8,p0=len1,1 3951da177e4SLinus Torvalds ;; 3961da177e4SLinus Torvalds EX(.failure_in1,(p8) ld2 val2[0]=[src1],2) // at least 2 bytes 3971da177e4SLinus Torvalds tbit.nz p9,p0=len1,0 3981da177e4SLinus Torvalds ;; 3991da177e4SLinus Torvalds EX(.failure_out, (p6) st8 [dst1]=val1[0],8) 4001da177e4SLinus Torvalds ;; 4011da177e4SLinus Torvalds EX(.failure_in1,(p9) ld1 val2[1]=[src1]) // only 1 byte left 4021da177e4SLinus Torvalds mov ar.lc=saved_lc 4031da177e4SLinus Torvalds ;; 4041da177e4SLinus Torvalds EX(.failure_out,(p7) st4 [dst1]=val1[1],4) 4051da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 4061da177e4SLinus Torvalds ;; 4071da177e4SLinus Torvalds EX(.failure_out, (p8) st2 [dst1]=val2[0],2) 4081da177e4SLinus Torvalds mov ar.pfs=saved_pfs 4091da177e4SLinus Torvalds ;; 4101da177e4SLinus Torvalds EX(.failure_out, (p9) st1 [dst1]=val2[1]) 4111da177e4SLinus Torvalds br.ret.sptk.many rp 4121da177e4SLinus Torvalds 4131da177e4SLinus Torvalds 4141da177e4SLinus Torvalds // 4151da177e4SLinus Torvalds // Here we handle the case where the byte by byte copy fails 4161da177e4SLinus Torvalds // on the load. 4171da177e4SLinus Torvalds // Several factors make the zeroing of the rest of the buffer kind of 4181da177e4SLinus Torvalds // tricky: 4191da177e4SLinus Torvalds // - the pipeline: loads/stores are not in sync (pipeline) 4201da177e4SLinus Torvalds // 4211da177e4SLinus Torvalds // In the same loop iteration, the dst1 pointer does not directly 4221da177e4SLinus Torvalds // reflect where the faulty load was. 4231da177e4SLinus Torvalds // 4241da177e4SLinus Torvalds // - pipeline effect 4251da177e4SLinus Torvalds // When you get a fault on load, you may have valid data from 4261da177e4SLinus Torvalds // previous loads not yet store in transit. Such data must be 4271da177e4SLinus Torvalds // store normally before moving onto zeroing the rest. 4281da177e4SLinus Torvalds // 4291da177e4SLinus Torvalds // - single/multi dispersal independence. 4301da177e4SLinus Torvalds // 4311da177e4SLinus Torvalds // solution: 4321da177e4SLinus Torvalds // - we don't disrupt the pipeline, i.e. data in transit in 4331da177e4SLinus Torvalds // the software pipeline will be eventually move to memory. 4341da177e4SLinus Torvalds // We simply replace the load with a simple mov and keep the 4351da177e4SLinus Torvalds // pipeline going. We can't really do this inline because 4361da177e4SLinus Torvalds // p16 is always reset to 1 when lc > 0. 4371da177e4SLinus Torvalds // 4381da177e4SLinus Torvalds.failure_in_pipe1: 4391da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 4401da177e4SLinus Torvalds1: 4411da177e4SLinus Torvalds(p16) mov val1[0]=r0 4421da177e4SLinus Torvalds(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1 4431da177e4SLinus Torvalds br.ctop.dptk 1b 4441da177e4SLinus Torvalds ;; 4451da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 4461da177e4SLinus Torvalds mov ar.lc=saved_lc 4471da177e4SLinus Torvalds mov ar.pfs=saved_pfs 4481da177e4SLinus Torvalds br.ret.sptk.many rp 4491da177e4SLinus Torvalds 4501da177e4SLinus Torvalds // 4511da177e4SLinus Torvalds // This is the case where the byte by byte copy fails on the load 4521da177e4SLinus Torvalds // when we copy the head. We need to finish the pipeline and copy 4531da177e4SLinus Torvalds // zeros for the rest of the destination. Since this happens 4541da177e4SLinus Torvalds // at the top we still need to fill the body and tail. 4551da177e4SLinus Torvalds.failure_in_pipe2: 4561da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 4571da177e4SLinus Torvalds2: 4581da177e4SLinus Torvalds(p16) mov val1[0]=r0 4591da177e4SLinus Torvalds(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1 4601da177e4SLinus Torvalds br.ctop.dptk 2b 4611da177e4SLinus Torvalds ;; 4621da177e4SLinus Torvalds sub len=enddst,dst1,1 // precompute len 4631da177e4SLinus Torvalds br.cond.dptk.many .failure_in1bis 4641da177e4SLinus Torvalds ;; 4651da177e4SLinus Torvalds 4661da177e4SLinus Torvalds // 4671da177e4SLinus Torvalds // Here we handle the head & tail part when we check for alignment. 4681da177e4SLinus Torvalds // The following code handles only the load failures. The 4691da177e4SLinus Torvalds // main diffculty comes from the fact that loads/stores are 4701da177e4SLinus Torvalds // scheduled. So when you fail on a load, the stores corresponding 4711da177e4SLinus Torvalds // to previous successful loads must be executed. 4721da177e4SLinus Torvalds // 4731da177e4SLinus Torvalds // However some simplifications are possible given the way 4741da177e4SLinus Torvalds // things work. 4751da177e4SLinus Torvalds // 4761da177e4SLinus Torvalds // 1) HEAD 4771da177e4SLinus Torvalds // Theory of operation: 4781da177e4SLinus Torvalds // 4791da177e4SLinus Torvalds // Page A | Page B 4801da177e4SLinus Torvalds // ---------|----- 4811da177e4SLinus Torvalds // 1|8 x 4821da177e4SLinus Torvalds // 1 2|8 x 4831da177e4SLinus Torvalds // 4|8 x 4841da177e4SLinus Torvalds // 1 4|8 x 4851da177e4SLinus Torvalds // 2 4|8 x 4861da177e4SLinus Torvalds // 1 2 4|8 x 4871da177e4SLinus Torvalds // |1 4881da177e4SLinus Torvalds // |2 x 4891da177e4SLinus Torvalds // |4 x 4901da177e4SLinus Torvalds // 4911da177e4SLinus Torvalds // page_size >= 4k (2^12). (x means 4, 2, 1) 4921da177e4SLinus Torvalds // Here we suppose Page A exists and Page B does not. 4931da177e4SLinus Torvalds // 4941da177e4SLinus Torvalds // As we move towards eight byte alignment we may encounter faults. 4951da177e4SLinus Torvalds // The numbers on each page show the size of the load (current alignment). 4961da177e4SLinus Torvalds // 4971da177e4SLinus Torvalds // Key point: 4981da177e4SLinus Torvalds // - if you fail on 1, 2, 4 then you have never executed any smaller 4991da177e4SLinus Torvalds // size loads, e.g. failing ld4 means no ld1 nor ld2 executed 5001da177e4SLinus Torvalds // before. 5011da177e4SLinus Torvalds // 5021da177e4SLinus Torvalds // This allows us to simplify the cleanup code, because basically you 5031da177e4SLinus Torvalds // only have to worry about "pending" stores in the case of a failing 5041da177e4SLinus Torvalds // ld8(). Given the way the code is written today, this means only 5051da177e4SLinus Torvalds // worry about st2, st4. There we can use the information encapsulated 5061da177e4SLinus Torvalds // into the predicates. 5071da177e4SLinus Torvalds // 5081da177e4SLinus Torvalds // Other key point: 5091da177e4SLinus Torvalds // - if you fail on the ld8 in the head, it means you went straight 5101da177e4SLinus Torvalds // to it, i.e. 8byte alignment within an unexisting page. 5111da177e4SLinus Torvalds // Again this comes from the fact that if you crossed just for the ld8 then 5121da177e4SLinus Torvalds // you are 8byte aligned but also 16byte align, therefore you would 5131da177e4SLinus Torvalds // either go for the 16byte copy loop OR the ld8 in the tail part. 5141da177e4SLinus Torvalds // The combination ld1, ld2, ld4, ld8 where you fail on ld8 is impossible 5151da177e4SLinus Torvalds // because it would mean you had 15bytes to copy in which case you 5161da177e4SLinus Torvalds // would have defaulted to the byte by byte copy. 5171da177e4SLinus Torvalds // 5181da177e4SLinus Torvalds // 5191da177e4SLinus Torvalds // 2) TAIL 5201da177e4SLinus Torvalds // Here we now we have less than 16 bytes AND we are either 8 or 16 byte 5211da177e4SLinus Torvalds // aligned. 5221da177e4SLinus Torvalds // 5231da177e4SLinus Torvalds // Key point: 5241da177e4SLinus Torvalds // This means that we either: 5251da177e4SLinus Torvalds // - are right on a page boundary 5261da177e4SLinus Torvalds // OR 5271da177e4SLinus Torvalds // - are at more than 16 bytes from a page boundary with 5281da177e4SLinus Torvalds // at most 15 bytes to copy: no chance of crossing. 5291da177e4SLinus Torvalds // 5301da177e4SLinus Torvalds // This allows us to assume that if we fail on a load we haven't possibly 5311da177e4SLinus Torvalds // executed any of the previous (tail) ones, so we don't need to do 5321da177e4SLinus Torvalds // any stores. For instance, if we fail on ld2, this means we had 5331da177e4SLinus Torvalds // 2 or 3 bytes left to copy and we did not execute the ld8 nor ld4. 5341da177e4SLinus Torvalds // 5351da177e4SLinus Torvalds // This means that we are in a situation similar the a fault in the 5361da177e4SLinus Torvalds // head part. That's nice! 5371da177e4SLinus Torvalds // 5381da177e4SLinus Torvalds.failure_in1: 5391da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 5401da177e4SLinus Torvalds sub len=endsrc,src1,1 5411da177e4SLinus Torvalds // 5421da177e4SLinus Torvalds // we know that ret0 can never be zero at this point 5431da177e4SLinus Torvalds // because we failed why trying to do a load, i.e. there is still 5441da177e4SLinus Torvalds // some work to do. 5451da177e4SLinus Torvalds // The failure_in1bis and length problem is taken care of at the 5461da177e4SLinus Torvalds // calling side. 5471da177e4SLinus Torvalds // 5481da177e4SLinus Torvalds ;; 5491da177e4SLinus Torvalds.failure_in1bis: // from (.failure_in3) 5501da177e4SLinus Torvalds mov ar.lc=len // Continue with a stupid byte store. 5511da177e4SLinus Torvalds ;; 5521da177e4SLinus Torvalds5: 5531da177e4SLinus Torvalds st1 [dst1]=r0,1 5541da177e4SLinus Torvalds br.cloop.dptk 5b 5551da177e4SLinus Torvalds ;; 5561da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 5571da177e4SLinus Torvalds mov ar.lc=saved_lc 5581da177e4SLinus Torvalds mov ar.pfs=saved_pfs 5591da177e4SLinus Torvalds br.ret.sptk.many rp 5601da177e4SLinus Torvalds 5611da177e4SLinus Torvalds // 5621da177e4SLinus Torvalds // Here we simply restart the loop but instead 5631da177e4SLinus Torvalds // of doing loads we fill the pipeline with zeroes 5641da177e4SLinus Torvalds // We can't simply store r0 because we may have valid 5651da177e4SLinus Torvalds // data in transit in the pipeline. 5661da177e4SLinus Torvalds // ar.lc and ar.ec are setup correctly at this point 5671da177e4SLinus Torvalds // 5681da177e4SLinus Torvalds // we MUST use src1/endsrc here and not dst1/enddst because 5691da177e4SLinus Torvalds // of the pipeline effect. 5701da177e4SLinus Torvalds // 5711da177e4SLinus Torvalds.failure_in3: 5721da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 5731da177e4SLinus Torvalds ;; 5741da177e4SLinus Torvalds2: 5751da177e4SLinus Torvalds(p16) mov val1[0]=r0 5761da177e4SLinus Torvalds(p16) mov val2[0]=r0 5771da177e4SLinus Torvalds(EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16 5781da177e4SLinus Torvalds(EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16 5791da177e4SLinus Torvalds br.ctop.dptk 2b 5801da177e4SLinus Torvalds ;; 5811da177e4SLinus Torvalds cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ? 5821da177e4SLinus Torvalds sub len=enddst,dst1,1 // precompute len 5831da177e4SLinus Torvalds(p6) br.cond.dptk .failure_in1bis 5841da177e4SLinus Torvalds ;; 5851da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 5861da177e4SLinus Torvalds mov ar.lc=saved_lc 5871da177e4SLinus Torvalds mov ar.pfs=saved_pfs 5881da177e4SLinus Torvalds br.ret.sptk.many rp 5891da177e4SLinus Torvalds 5901da177e4SLinus Torvalds.failure_in2: 5911da177e4SLinus Torvalds sub ret0=endsrc,src1 5921da177e4SLinus Torvalds cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ? 5931da177e4SLinus Torvalds sub len=enddst,dst1,1 // precompute len 5941da177e4SLinus Torvalds(p6) br.cond.dptk .failure_in1bis 5951da177e4SLinus Torvalds ;; 5961da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 5971da177e4SLinus Torvalds mov ar.lc=saved_lc 5981da177e4SLinus Torvalds mov ar.pfs=saved_pfs 5991da177e4SLinus Torvalds br.ret.sptk.many rp 6001da177e4SLinus Torvalds 6011da177e4SLinus Torvalds // 6021da177e4SLinus Torvalds // handling of failures on stores: that's the easy part 6031da177e4SLinus Torvalds // 6041da177e4SLinus Torvalds.failure_out: 6051da177e4SLinus Torvalds sub ret0=enddst,dst1 6061da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 6071da177e4SLinus Torvalds mov ar.lc=saved_lc 6081da177e4SLinus Torvalds 6091da177e4SLinus Torvalds mov ar.pfs=saved_pfs 6101da177e4SLinus Torvalds br.ret.sptk.many rp 6111da177e4SLinus TorvaldsEND(__copy_user) 612e007c533SAl ViroEXPORT_SYMBOL(__copy_user) 613