1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 21da177e4SLinus Torvalds/* 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Optimized version of the copy_user() routine. 51da177e4SLinus Torvalds * It is used to copy date across the kernel/user boundary. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * The source and destination are always on opposite side of 81da177e4SLinus Torvalds * the boundary. When reading from user space we must catch 91da177e4SLinus Torvalds * faults on loads. When writing to user space we must catch 101da177e4SLinus Torvalds * errors on stores. Note that because of the nature of the copy 111da177e4SLinus Torvalds * we don't need to worry about overlapping regions. 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * 141da177e4SLinus Torvalds * Inputs: 151da177e4SLinus Torvalds * in0 address of source buffer 161da177e4SLinus Torvalds * in1 address of destination buffer 171da177e4SLinus Torvalds * in2 number of bytes to copy 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * Outputs: 201da177e4SLinus Torvalds * ret0 0 in case of success. The number of bytes NOT copied in 211da177e4SLinus Torvalds * case of error. 221da177e4SLinus Torvalds * 231da177e4SLinus Torvalds * Copyright (C) 2000-2001 Hewlett-Packard Co 241da177e4SLinus Torvalds * Stephane Eranian <eranian@hpl.hp.com> 251da177e4SLinus Torvalds * 261da177e4SLinus Torvalds * Fixme: 271da177e4SLinus Torvalds * - handle the case where we have more than 16 bytes and the alignment 281da177e4SLinus Torvalds * are different. 291da177e4SLinus Torvalds * - more benchmarking 301da177e4SLinus Torvalds * - fix extraneous stop bit introduced by the EX() macro. 311da177e4SLinus Torvalds */ 321da177e4SLinus Torvalds 33*ab03e604SMasahiro Yamada#include <linux/export.h> 341da177e4SLinus Torvalds#include <asm/asmmacro.h> 351da177e4SLinus Torvalds 361da177e4SLinus Torvalds// 371da177e4SLinus Torvalds// Tuneable parameters 381da177e4SLinus Torvalds// 391da177e4SLinus Torvalds#define COPY_BREAK 16 // we do byte copy below (must be >=16) 401da177e4SLinus Torvalds#define PIPE_DEPTH 21 // pipe depth 411da177e4SLinus Torvalds 421da177e4SLinus Torvalds#define EPI p[PIPE_DEPTH-1] 431da177e4SLinus Torvalds 441da177e4SLinus Torvalds// 451da177e4SLinus Torvalds// arguments 461da177e4SLinus Torvalds// 471da177e4SLinus Torvalds#define dst in0 481da177e4SLinus Torvalds#define src in1 491da177e4SLinus Torvalds#define len in2 501da177e4SLinus Torvalds 511da177e4SLinus Torvalds// 521da177e4SLinus Torvalds// local registers 531da177e4SLinus Torvalds// 541da177e4SLinus Torvalds#define t1 r2 // rshift in bytes 551da177e4SLinus Torvalds#define t2 r3 // lshift in bytes 561da177e4SLinus Torvalds#define rshift r14 // right shift in bits 571da177e4SLinus Torvalds#define lshift r15 // left shift in bits 581da177e4SLinus Torvalds#define word1 r16 591da177e4SLinus Torvalds#define word2 r17 601da177e4SLinus Torvalds#define cnt r18 611da177e4SLinus Torvalds#define len2 r19 621da177e4SLinus Torvalds#define saved_lc r20 631da177e4SLinus Torvalds#define saved_pr r21 641da177e4SLinus Torvalds#define tmp r22 651da177e4SLinus Torvalds#define val r23 661da177e4SLinus Torvalds#define src1 r24 671da177e4SLinus Torvalds#define dst1 r25 681da177e4SLinus Torvalds#define src2 r26 691da177e4SLinus Torvalds#define dst2 r27 701da177e4SLinus Torvalds#define len1 r28 711da177e4SLinus Torvalds#define enddst r29 721da177e4SLinus Torvalds#define endsrc r30 731da177e4SLinus Torvalds#define saved_pfs r31 741da177e4SLinus Torvalds 751da177e4SLinus TorvaldsGLOBAL_ENTRY(__copy_user) 761da177e4SLinus Torvalds .prologue 771da177e4SLinus Torvalds .save ar.pfs, saved_pfs 781da177e4SLinus Torvalds alloc saved_pfs=ar.pfs,3,((2*PIPE_DEPTH+7)&~7),0,((2*PIPE_DEPTH+7)&~7) 791da177e4SLinus Torvalds 801da177e4SLinus Torvalds .rotr val1[PIPE_DEPTH],val2[PIPE_DEPTH] 811da177e4SLinus Torvalds .rotp p[PIPE_DEPTH] 821da177e4SLinus Torvalds 831da177e4SLinus Torvalds adds len2=-1,len // br.ctop is repeat/until 841da177e4SLinus Torvalds mov ret0=r0 851da177e4SLinus Torvalds 861da177e4SLinus Torvalds ;; // RAW of cfm when len=0 871da177e4SLinus Torvalds cmp.eq p8,p0=r0,len // check for zero length 881da177e4SLinus Torvalds .save ar.lc, saved_lc 891da177e4SLinus Torvalds mov saved_lc=ar.lc // preserve ar.lc (slow) 901da177e4SLinus Torvalds(p8) br.ret.spnt.many rp // empty mempcy() 911da177e4SLinus Torvalds ;; 921da177e4SLinus Torvalds add enddst=dst,len // first byte after end of source 931da177e4SLinus Torvalds add endsrc=src,len // first byte after end of destination 941da177e4SLinus Torvalds .save pr, saved_pr 951da177e4SLinus Torvalds mov saved_pr=pr // preserve predicates 961da177e4SLinus Torvalds 971da177e4SLinus Torvalds .body 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds mov dst1=dst // copy because of rotation 1001da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 1011da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 1021da177e4SLinus Torvalds 1031da177e4SLinus Torvalds mov src1=src // copy because of rotation 1041da177e4SLinus Torvalds mov ar.lc=len2 // initialize lc for small count 1051da177e4SLinus Torvalds cmp.lt p10,p7=COPY_BREAK,len // if len > COPY_BREAK then long copy 1061da177e4SLinus Torvalds 1071da177e4SLinus Torvalds xor tmp=src,dst // same alignment test prepare 1081da177e4SLinus Torvalds(p10) br.cond.dptk .long_copy_user 1091da177e4SLinus Torvalds ;; // RAW pr.rot/p16 ? 1101da177e4SLinus Torvalds // 1111da177e4SLinus Torvalds // Now we do the byte by byte loop with software pipeline 1121da177e4SLinus Torvalds // 1131da177e4SLinus Torvalds // p7 is necessarily false by now 1141da177e4SLinus Torvalds1: 1151da177e4SLinus Torvalds EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1) 1161da177e4SLinus Torvalds EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1) 1171da177e4SLinus Torvalds br.ctop.dptk.few 1b 1181da177e4SLinus Torvalds ;; 1191da177e4SLinus Torvalds mov ar.lc=saved_lc 1201da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 1211da177e4SLinus Torvalds mov ar.pfs=saved_pfs // restore ar.ec 1221da177e4SLinus Torvalds br.ret.sptk.many rp // end of short memcpy 1231da177e4SLinus Torvalds 1241da177e4SLinus Torvalds // 1251da177e4SLinus Torvalds // Not 8-byte aligned 1261da177e4SLinus Torvalds // 1271da177e4SLinus Torvalds.diff_align_copy_user: 1281da177e4SLinus Torvalds // At this point we know we have more than 16 bytes to copy 1291da177e4SLinus Torvalds // and also that src and dest do _not_ have the same alignment. 1301da177e4SLinus Torvalds and src2=0x7,src1 // src offset 1311da177e4SLinus Torvalds and dst2=0x7,dst1 // dst offset 1321da177e4SLinus Torvalds ;; 1331da177e4SLinus Torvalds // The basic idea is that we copy byte-by-byte at the head so 1341da177e4SLinus Torvalds // that we can reach 8-byte alignment for both src1 and dst1. 1351da177e4SLinus Torvalds // Then copy the body using software pipelined 8-byte copy, 1361da177e4SLinus Torvalds // shifting the two back-to-back words right and left, then copy 1371da177e4SLinus Torvalds // the tail by copying byte-by-byte. 1381da177e4SLinus Torvalds // 1391da177e4SLinus Torvalds // Fault handling. If the byte-by-byte at the head fails on the 1401da177e4SLinus Torvalds // load, then restart and finish the pipleline by copying zeros 1411da177e4SLinus Torvalds // to the dst1. Then copy zeros for the rest of dst1. 1421da177e4SLinus Torvalds // If 8-byte software pipeline fails on the load, do the same as 1431da177e4SLinus Torvalds // failure_in3 does. If the byte-by-byte at the tail fails, it is 1441da177e4SLinus Torvalds // handled simply by failure_in_pipe1. 1451da177e4SLinus Torvalds // 1461da177e4SLinus Torvalds // The case p14 represents the source has more bytes in the 1471da177e4SLinus Torvalds // the first word (by the shifted part), whereas the p15 needs to 1481da177e4SLinus Torvalds // copy some bytes from the 2nd word of the source that has the 1491da177e4SLinus Torvalds // tail of the 1st of the destination. 1501da177e4SLinus Torvalds // 1511da177e4SLinus Torvalds 1521da177e4SLinus Torvalds // 1531da177e4SLinus Torvalds // Optimization. If dst1 is 8-byte aligned (quite common), we don't need 1541da177e4SLinus Torvalds // to copy the head to dst1, to start 8-byte copy software pipeline. 1551da177e4SLinus Torvalds // We know src1 is not 8-byte aligned in this case. 1561da177e4SLinus Torvalds // 1571da177e4SLinus Torvalds cmp.eq p14,p15=r0,dst2 1581da177e4SLinus Torvalds(p15) br.cond.spnt 1f 1591da177e4SLinus Torvalds ;; 1601da177e4SLinus Torvalds sub t1=8,src2 1611da177e4SLinus Torvalds mov t2=src2 1621da177e4SLinus Torvalds ;; 1631da177e4SLinus Torvalds shl rshift=t2,3 1641da177e4SLinus Torvalds sub len1=len,t1 // set len1 1651da177e4SLinus Torvalds ;; 1661da177e4SLinus Torvalds sub lshift=64,rshift 1671da177e4SLinus Torvalds ;; 1681da177e4SLinus Torvalds br.cond.spnt .word_copy_user 1691da177e4SLinus Torvalds ;; 1701da177e4SLinus Torvalds1: 1711da177e4SLinus Torvalds cmp.leu p14,p15=src2,dst2 1721da177e4SLinus Torvalds sub t1=dst2,src2 1731da177e4SLinus Torvalds ;; 1741da177e4SLinus Torvalds .pred.rel "mutex", p14, p15 1751da177e4SLinus Torvalds(p14) sub word1=8,src2 // (8 - src offset) 1761da177e4SLinus Torvalds(p15) sub t1=r0,t1 // absolute value 1771da177e4SLinus Torvalds(p15) sub word1=8,dst2 // (8 - dst offset) 1781da177e4SLinus Torvalds ;; 1791da177e4SLinus Torvalds // For the case p14, we don't need to copy the shifted part to 1801da177e4SLinus Torvalds // the 1st word of destination. 1811da177e4SLinus Torvalds sub t2=8,t1 1821da177e4SLinus Torvalds(p14) sub word1=word1,t1 1831da177e4SLinus Torvalds ;; 1841da177e4SLinus Torvalds sub len1=len,word1 // resulting len 1851da177e4SLinus Torvalds(p15) shl rshift=t1,3 // in bits 1861da177e4SLinus Torvalds(p14) shl rshift=t2,3 1871da177e4SLinus Torvalds ;; 1881da177e4SLinus Torvalds(p14) sub len1=len1,t1 1891da177e4SLinus Torvalds adds cnt=-1,word1 1901da177e4SLinus Torvalds ;; 1911da177e4SLinus Torvalds sub lshift=64,rshift 1921da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 1931da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 1941da177e4SLinus Torvalds mov ar.lc=cnt 1951da177e4SLinus Torvalds ;; 1961da177e4SLinus Torvalds2: 1971da177e4SLinus Torvalds EX(.failure_in_pipe2,(p16) ld1 val1[0]=[src1],1) 1981da177e4SLinus Torvalds EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1) 1991da177e4SLinus Torvalds br.ctop.dptk.few 2b 2001da177e4SLinus Torvalds ;; 2011da177e4SLinus Torvalds clrrrb 2021da177e4SLinus Torvalds ;; 2031da177e4SLinus Torvalds.word_copy_user: 2041da177e4SLinus Torvalds cmp.gtu p9,p0=16,len1 2051da177e4SLinus Torvalds(p9) br.cond.spnt 4f // if (16 > len1) skip 8-byte copy 2061da177e4SLinus Torvalds ;; 2071da177e4SLinus Torvalds shr.u cnt=len1,3 // number of 64-bit words 2081da177e4SLinus Torvalds ;; 2091da177e4SLinus Torvalds adds cnt=-1,cnt 2101da177e4SLinus Torvalds ;; 2111da177e4SLinus Torvalds .pred.rel "mutex", p14, p15 2121da177e4SLinus Torvalds(p14) sub src1=src1,t2 2131da177e4SLinus Torvalds(p15) sub src1=src1,t1 2141da177e4SLinus Torvalds // 2151da177e4SLinus Torvalds // Now both src1 and dst1 point to an 8-byte aligned address. And 2161da177e4SLinus Torvalds // we have more than 8 bytes to copy. 2171da177e4SLinus Torvalds // 2181da177e4SLinus Torvalds mov ar.lc=cnt 2191da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 2201da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 2211da177e4SLinus Torvalds ;; 2221da177e4SLinus Torvalds3: 2231da177e4SLinus Torvalds // 2241da177e4SLinus Torvalds // The pipleline consists of 3 stages: 2251da177e4SLinus Torvalds // 1 (p16): Load a word from src1 2261da177e4SLinus Torvalds // 2 (EPI_1): Shift right pair, saving to tmp 2271da177e4SLinus Torvalds // 3 (EPI): Store tmp to dst1 2281da177e4SLinus Torvalds // 2291da177e4SLinus Torvalds // To make it simple, use at least 2 (p16) loops to set up val1[n] 2301da177e4SLinus Torvalds // because we need 2 back-to-back val1[] to get tmp. 2311da177e4SLinus Torvalds // Note that this implies EPI_2 must be p18 or greater. 2321da177e4SLinus Torvalds // 2331da177e4SLinus Torvalds 2341da177e4SLinus Torvalds#define EPI_1 p[PIPE_DEPTH-2] 2351da177e4SLinus Torvalds#define SWITCH(pred, shift) cmp.eq pred,p0=shift,rshift 2361da177e4SLinus Torvalds#define CASE(pred, shift) \ 2371da177e4SLinus Torvalds (pred) br.cond.spnt .copy_user_bit##shift 2381da177e4SLinus Torvalds#define BODY(rshift) \ 2391da177e4SLinus Torvalds.copy_user_bit##rshift: \ 2401da177e4SLinus Torvalds1: \ 2411da177e4SLinus Torvalds EX(.failure_out,(EPI) st8 [dst1]=tmp,8); \ 2421da177e4SLinus Torvalds(EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \ 2431da177e4SLinus Torvalds EX(3f,(p16) ld8 val1[1]=[src1],8); \ 2441da177e4SLinus Torvalds(p16) mov val1[0]=r0; \ 2451da177e4SLinus Torvalds br.ctop.dptk 1b; \ 2461da177e4SLinus Torvalds ;; \ 2471da177e4SLinus Torvalds br.cond.sptk.many .diff_align_do_tail; \ 2481da177e4SLinus Torvalds2: \ 2491da177e4SLinus Torvalds(EPI) st8 [dst1]=tmp,8; \ 2501da177e4SLinus Torvalds(EPI_1) shrp tmp=val1[PIPE_DEPTH-2],val1[PIPE_DEPTH-1],rshift; \ 2511da177e4SLinus Torvalds3: \ 2521da177e4SLinus Torvalds(p16) mov val1[1]=r0; \ 2531da177e4SLinus Torvalds(p16) mov val1[0]=r0; \ 2541da177e4SLinus Torvalds br.ctop.dptk 2b; \ 2551da177e4SLinus Torvalds ;; \ 2561da177e4SLinus Torvalds br.cond.sptk.many .failure_in2 2571da177e4SLinus Torvalds 2581da177e4SLinus Torvalds // 2591da177e4SLinus Torvalds // Since the instruction 'shrp' requires a fixed 128-bit value 2601da177e4SLinus Torvalds // specifying the bits to shift, we need to provide 7 cases 2611da177e4SLinus Torvalds // below. 2621da177e4SLinus Torvalds // 2631da177e4SLinus Torvalds SWITCH(p6, 8) 2641da177e4SLinus Torvalds SWITCH(p7, 16) 2651da177e4SLinus Torvalds SWITCH(p8, 24) 2661da177e4SLinus Torvalds SWITCH(p9, 32) 2671da177e4SLinus Torvalds SWITCH(p10, 40) 2681da177e4SLinus Torvalds SWITCH(p11, 48) 2691da177e4SLinus Torvalds SWITCH(p12, 56) 2701da177e4SLinus Torvalds ;; 2711da177e4SLinus Torvalds CASE(p6, 8) 2721da177e4SLinus Torvalds CASE(p7, 16) 2731da177e4SLinus Torvalds CASE(p8, 24) 2741da177e4SLinus Torvalds CASE(p9, 32) 2751da177e4SLinus Torvalds CASE(p10, 40) 2761da177e4SLinus Torvalds CASE(p11, 48) 2771da177e4SLinus Torvalds CASE(p12, 56) 2781da177e4SLinus Torvalds ;; 2791da177e4SLinus Torvalds BODY(8) 2801da177e4SLinus Torvalds BODY(16) 2811da177e4SLinus Torvalds BODY(24) 2821da177e4SLinus Torvalds BODY(32) 2831da177e4SLinus Torvalds BODY(40) 2841da177e4SLinus Torvalds BODY(48) 2851da177e4SLinus Torvalds BODY(56) 2861da177e4SLinus Torvalds ;; 2871da177e4SLinus Torvalds.diff_align_do_tail: 2881da177e4SLinus Torvalds .pred.rel "mutex", p14, p15 2891da177e4SLinus Torvalds(p14) sub src1=src1,t1 2901da177e4SLinus Torvalds(p14) adds dst1=-8,dst1 2911da177e4SLinus Torvalds(p15) sub dst1=dst1,t1 2921da177e4SLinus Torvalds ;; 2931da177e4SLinus Torvalds4: 2941da177e4SLinus Torvalds // Tail correction. 2951da177e4SLinus Torvalds // 2961da177e4SLinus Torvalds // The problem with this piplelined loop is that the last word is not 2971da177e4SLinus Torvalds // loaded and thus parf of the last word written is not correct. 2981da177e4SLinus Torvalds // To fix that, we simply copy the tail byte by byte. 2991da177e4SLinus Torvalds 3001da177e4SLinus Torvalds sub len1=endsrc,src1,1 3011da177e4SLinus Torvalds clrrrb 3021da177e4SLinus Torvalds ;; 3031da177e4SLinus Torvalds mov ar.ec=PIPE_DEPTH 3041da177e4SLinus Torvalds mov pr.rot=1<<16 // p16=true all others are false 3051da177e4SLinus Torvalds mov ar.lc=len1 3061da177e4SLinus Torvalds ;; 3071da177e4SLinus Torvalds5: 3081da177e4SLinus Torvalds EX(.failure_in_pipe1,(p16) ld1 val1[0]=[src1],1) 3091da177e4SLinus Torvalds EX(.failure_out,(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1) 3101da177e4SLinus Torvalds br.ctop.dptk.few 5b 3111da177e4SLinus Torvalds ;; 3121da177e4SLinus Torvalds mov ar.lc=saved_lc 3131da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 3141da177e4SLinus Torvalds mov ar.pfs=saved_pfs 3151da177e4SLinus Torvalds br.ret.sptk.many rp 3161da177e4SLinus Torvalds 3171da177e4SLinus Torvalds // 3181da177e4SLinus Torvalds // Beginning of long mempcy (i.e. > 16 bytes) 3191da177e4SLinus Torvalds // 3201da177e4SLinus Torvalds.long_copy_user: 3211da177e4SLinus Torvalds tbit.nz p6,p7=src1,0 // odd alignment 3221da177e4SLinus Torvalds and tmp=7,tmp 3231da177e4SLinus Torvalds ;; 3241da177e4SLinus Torvalds cmp.eq p10,p8=r0,tmp 3251da177e4SLinus Torvalds mov len1=len // copy because of rotation 3261da177e4SLinus Torvalds(p8) br.cond.dpnt .diff_align_copy_user 3271da177e4SLinus Torvalds ;; 3281da177e4SLinus Torvalds // At this point we know we have more than 16 bytes to copy 3291da177e4SLinus Torvalds // and also that both src and dest have the same alignment 3301da177e4SLinus Torvalds // which may not be the one we want. So for now we must move 3311da177e4SLinus Torvalds // forward slowly until we reach 16byte alignment: no need to 3321da177e4SLinus Torvalds // worry about reaching the end of buffer. 3331da177e4SLinus Torvalds // 3341da177e4SLinus Torvalds EX(.failure_in1,(p6) ld1 val1[0]=[src1],1) // 1-byte aligned 3351da177e4SLinus Torvalds(p6) adds len1=-1,len1;; 3361da177e4SLinus Torvalds tbit.nz p7,p0=src1,1 3371da177e4SLinus Torvalds ;; 3381da177e4SLinus Torvalds EX(.failure_in1,(p7) ld2 val1[1]=[src1],2) // 2-byte aligned 3391da177e4SLinus Torvalds(p7) adds len1=-2,len1;; 3401da177e4SLinus Torvalds tbit.nz p8,p0=src1,2 3411da177e4SLinus Torvalds ;; 3421da177e4SLinus Torvalds // 3431da177e4SLinus Torvalds // Stop bit not required after ld4 because if we fail on ld4 3441da177e4SLinus Torvalds // we have never executed the ld1, therefore st1 is not executed. 3451da177e4SLinus Torvalds // 3461da177e4SLinus Torvalds EX(.failure_in1,(p8) ld4 val2[0]=[src1],4) // 4-byte aligned 3471da177e4SLinus Torvalds ;; 3481da177e4SLinus Torvalds EX(.failure_out,(p6) st1 [dst1]=val1[0],1) 3491da177e4SLinus Torvalds tbit.nz p9,p0=src1,3 3501da177e4SLinus Torvalds ;; 3511da177e4SLinus Torvalds // 3521da177e4SLinus Torvalds // Stop bit not required after ld8 because if we fail on ld8 3531da177e4SLinus Torvalds // we have never executed the ld2, therefore st2 is not executed. 3541da177e4SLinus Torvalds // 3551da177e4SLinus Torvalds EX(.failure_in1,(p9) ld8 val2[1]=[src1],8) // 8-byte aligned 3561da177e4SLinus Torvalds EX(.failure_out,(p7) st2 [dst1]=val1[1],2) 3571da177e4SLinus Torvalds(p8) adds len1=-4,len1 3581da177e4SLinus Torvalds ;; 3591da177e4SLinus Torvalds EX(.failure_out, (p8) st4 [dst1]=val2[0],4) 3601da177e4SLinus Torvalds(p9) adds len1=-8,len1;; 3611da177e4SLinus Torvalds shr.u cnt=len1,4 // number of 128-bit (2x64bit) words 3621da177e4SLinus Torvalds ;; 3631da177e4SLinus Torvalds EX(.failure_out, (p9) st8 [dst1]=val2[1],8) 3641da177e4SLinus Torvalds tbit.nz p6,p0=len1,3 3651da177e4SLinus Torvalds cmp.eq p7,p0=r0,cnt 3661da177e4SLinus Torvalds adds tmp=-1,cnt // br.ctop is repeat/until 3671da177e4SLinus Torvalds(p7) br.cond.dpnt .dotail // we have less than 16 bytes left 3681da177e4SLinus Torvalds ;; 3691da177e4SLinus Torvalds adds src2=8,src1 3701da177e4SLinus Torvalds adds dst2=8,dst1 3711da177e4SLinus Torvalds mov ar.lc=tmp 3721da177e4SLinus Torvalds ;; 3731da177e4SLinus Torvalds // 3741da177e4SLinus Torvalds // 16bytes/iteration 3751da177e4SLinus Torvalds // 3761da177e4SLinus Torvalds2: 3771da177e4SLinus Torvalds EX(.failure_in3,(p16) ld8 val1[0]=[src1],16) 3781da177e4SLinus Torvalds(p16) ld8 val2[0]=[src2],16 3791da177e4SLinus Torvalds 3801da177e4SLinus Torvalds EX(.failure_out, (EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16) 3811da177e4SLinus Torvalds(EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16 3821da177e4SLinus Torvalds br.ctop.dptk 2b 3831da177e4SLinus Torvalds ;; // RAW on src1 when fall through from loop 3841da177e4SLinus Torvalds // 3851da177e4SLinus Torvalds // Tail correction based on len only 3861da177e4SLinus Torvalds // 3871da177e4SLinus Torvalds // No matter where we come from (loop or test) the src1 pointer 3881da177e4SLinus Torvalds // is 16 byte aligned AND we have less than 16 bytes to copy. 3891da177e4SLinus Torvalds // 3901da177e4SLinus Torvalds.dotail: 3911da177e4SLinus Torvalds EX(.failure_in1,(p6) ld8 val1[0]=[src1],8) // at least 8 bytes 3921da177e4SLinus Torvalds tbit.nz p7,p0=len1,2 3931da177e4SLinus Torvalds ;; 3941da177e4SLinus Torvalds EX(.failure_in1,(p7) ld4 val1[1]=[src1],4) // at least 4 bytes 3951da177e4SLinus Torvalds tbit.nz p8,p0=len1,1 3961da177e4SLinus Torvalds ;; 3971da177e4SLinus Torvalds EX(.failure_in1,(p8) ld2 val2[0]=[src1],2) // at least 2 bytes 3981da177e4SLinus Torvalds tbit.nz p9,p0=len1,0 3991da177e4SLinus Torvalds ;; 4001da177e4SLinus Torvalds EX(.failure_out, (p6) st8 [dst1]=val1[0],8) 4011da177e4SLinus Torvalds ;; 4021da177e4SLinus Torvalds EX(.failure_in1,(p9) ld1 val2[1]=[src1]) // only 1 byte left 4031da177e4SLinus Torvalds mov ar.lc=saved_lc 4041da177e4SLinus Torvalds ;; 4051da177e4SLinus Torvalds EX(.failure_out,(p7) st4 [dst1]=val1[1],4) 4061da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 4071da177e4SLinus Torvalds ;; 4081da177e4SLinus Torvalds EX(.failure_out, (p8) st2 [dst1]=val2[0],2) 4091da177e4SLinus Torvalds mov ar.pfs=saved_pfs 4101da177e4SLinus Torvalds ;; 4111da177e4SLinus Torvalds EX(.failure_out, (p9) st1 [dst1]=val2[1]) 4121da177e4SLinus Torvalds br.ret.sptk.many rp 4131da177e4SLinus Torvalds 4141da177e4SLinus Torvalds 4151da177e4SLinus Torvalds // 4161da177e4SLinus Torvalds // Here we handle the case where the byte by byte copy fails 4171da177e4SLinus Torvalds // on the load. 4181da177e4SLinus Torvalds // Several factors make the zeroing of the rest of the buffer kind of 4191da177e4SLinus Torvalds // tricky: 4201da177e4SLinus Torvalds // - the pipeline: loads/stores are not in sync (pipeline) 4211da177e4SLinus Torvalds // 4221da177e4SLinus Torvalds // In the same loop iteration, the dst1 pointer does not directly 4231da177e4SLinus Torvalds // reflect where the faulty load was. 4241da177e4SLinus Torvalds // 4251da177e4SLinus Torvalds // - pipeline effect 4261da177e4SLinus Torvalds // When you get a fault on load, you may have valid data from 4271da177e4SLinus Torvalds // previous loads not yet store in transit. Such data must be 4281da177e4SLinus Torvalds // store normally before moving onto zeroing the rest. 4291da177e4SLinus Torvalds // 4301da177e4SLinus Torvalds // - single/multi dispersal independence. 4311da177e4SLinus Torvalds // 4321da177e4SLinus Torvalds // solution: 4331da177e4SLinus Torvalds // - we don't disrupt the pipeline, i.e. data in transit in 4341da177e4SLinus Torvalds // the software pipeline will be eventually move to memory. 4351da177e4SLinus Torvalds // We simply replace the load with a simple mov and keep the 4361da177e4SLinus Torvalds // pipeline going. We can't really do this inline because 4371da177e4SLinus Torvalds // p16 is always reset to 1 when lc > 0. 4381da177e4SLinus Torvalds // 4391da177e4SLinus Torvalds.failure_in_pipe1: 4401da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 4411da177e4SLinus Torvalds1: 4421da177e4SLinus Torvalds(p16) mov val1[0]=r0 4431da177e4SLinus Torvalds(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1 4441da177e4SLinus Torvalds br.ctop.dptk 1b 4451da177e4SLinus Torvalds ;; 4461da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 4471da177e4SLinus Torvalds mov ar.lc=saved_lc 4481da177e4SLinus Torvalds mov ar.pfs=saved_pfs 4491da177e4SLinus Torvalds br.ret.sptk.many rp 4501da177e4SLinus Torvalds 4511da177e4SLinus Torvalds // 4521da177e4SLinus Torvalds // This is the case where the byte by byte copy fails on the load 4531da177e4SLinus Torvalds // when we copy the head. We need to finish the pipeline and copy 4541da177e4SLinus Torvalds // zeros for the rest of the destination. Since this happens 4551da177e4SLinus Torvalds // at the top we still need to fill the body and tail. 4561da177e4SLinus Torvalds.failure_in_pipe2: 4571da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 4581da177e4SLinus Torvalds2: 4591da177e4SLinus Torvalds(p16) mov val1[0]=r0 4601da177e4SLinus Torvalds(EPI) st1 [dst1]=val1[PIPE_DEPTH-1],1 4611da177e4SLinus Torvalds br.ctop.dptk 2b 4621da177e4SLinus Torvalds ;; 4631da177e4SLinus Torvalds sub len=enddst,dst1,1 // precompute len 4641da177e4SLinus Torvalds br.cond.dptk.many .failure_in1bis 4651da177e4SLinus Torvalds ;; 4661da177e4SLinus Torvalds 4671da177e4SLinus Torvalds // 4681da177e4SLinus Torvalds // Here we handle the head & tail part when we check for alignment. 4691da177e4SLinus Torvalds // The following code handles only the load failures. The 4701da177e4SLinus Torvalds // main diffculty comes from the fact that loads/stores are 4711da177e4SLinus Torvalds // scheduled. So when you fail on a load, the stores corresponding 4721da177e4SLinus Torvalds // to previous successful loads must be executed. 4731da177e4SLinus Torvalds // 4741da177e4SLinus Torvalds // However some simplifications are possible given the way 4751da177e4SLinus Torvalds // things work. 4761da177e4SLinus Torvalds // 4771da177e4SLinus Torvalds // 1) HEAD 4781da177e4SLinus Torvalds // Theory of operation: 4791da177e4SLinus Torvalds // 4801da177e4SLinus Torvalds // Page A | Page B 4811da177e4SLinus Torvalds // ---------|----- 4821da177e4SLinus Torvalds // 1|8 x 4831da177e4SLinus Torvalds // 1 2|8 x 4841da177e4SLinus Torvalds // 4|8 x 4851da177e4SLinus Torvalds // 1 4|8 x 4861da177e4SLinus Torvalds // 2 4|8 x 4871da177e4SLinus Torvalds // 1 2 4|8 x 4881da177e4SLinus Torvalds // |1 4891da177e4SLinus Torvalds // |2 x 4901da177e4SLinus Torvalds // |4 x 4911da177e4SLinus Torvalds // 4921da177e4SLinus Torvalds // page_size >= 4k (2^12). (x means 4, 2, 1) 4931da177e4SLinus Torvalds // Here we suppose Page A exists and Page B does not. 4941da177e4SLinus Torvalds // 4951da177e4SLinus Torvalds // As we move towards eight byte alignment we may encounter faults. 4961da177e4SLinus Torvalds // The numbers on each page show the size of the load (current alignment). 4971da177e4SLinus Torvalds // 4981da177e4SLinus Torvalds // Key point: 4991da177e4SLinus Torvalds // - if you fail on 1, 2, 4 then you have never executed any smaller 5001da177e4SLinus Torvalds // size loads, e.g. failing ld4 means no ld1 nor ld2 executed 5011da177e4SLinus Torvalds // before. 5021da177e4SLinus Torvalds // 5031da177e4SLinus Torvalds // This allows us to simplify the cleanup code, because basically you 5041da177e4SLinus Torvalds // only have to worry about "pending" stores in the case of a failing 5051da177e4SLinus Torvalds // ld8(). Given the way the code is written today, this means only 5061da177e4SLinus Torvalds // worry about st2, st4. There we can use the information encapsulated 5071da177e4SLinus Torvalds // into the predicates. 5081da177e4SLinus Torvalds // 5091da177e4SLinus Torvalds // Other key point: 5101da177e4SLinus Torvalds // - if you fail on the ld8 in the head, it means you went straight 5111da177e4SLinus Torvalds // to it, i.e. 8byte alignment within an unexisting page. 5121da177e4SLinus Torvalds // Again this comes from the fact that if you crossed just for the ld8 then 5131da177e4SLinus Torvalds // you are 8byte aligned but also 16byte align, therefore you would 5141da177e4SLinus Torvalds // either go for the 16byte copy loop OR the ld8 in the tail part. 5151da177e4SLinus Torvalds // The combination ld1, ld2, ld4, ld8 where you fail on ld8 is impossible 5161da177e4SLinus Torvalds // because it would mean you had 15bytes to copy in which case you 5171da177e4SLinus Torvalds // would have defaulted to the byte by byte copy. 5181da177e4SLinus Torvalds // 5191da177e4SLinus Torvalds // 5201da177e4SLinus Torvalds // 2) TAIL 5211da177e4SLinus Torvalds // Here we now we have less than 16 bytes AND we are either 8 or 16 byte 5221da177e4SLinus Torvalds // aligned. 5231da177e4SLinus Torvalds // 5241da177e4SLinus Torvalds // Key point: 5251da177e4SLinus Torvalds // This means that we either: 5261da177e4SLinus Torvalds // - are right on a page boundary 5271da177e4SLinus Torvalds // OR 5281da177e4SLinus Torvalds // - are at more than 16 bytes from a page boundary with 5291da177e4SLinus Torvalds // at most 15 bytes to copy: no chance of crossing. 5301da177e4SLinus Torvalds // 5311da177e4SLinus Torvalds // This allows us to assume that if we fail on a load we haven't possibly 5321da177e4SLinus Torvalds // executed any of the previous (tail) ones, so we don't need to do 5331da177e4SLinus Torvalds // any stores. For instance, if we fail on ld2, this means we had 5341da177e4SLinus Torvalds // 2 or 3 bytes left to copy and we did not execute the ld8 nor ld4. 5351da177e4SLinus Torvalds // 5361da177e4SLinus Torvalds // This means that we are in a situation similar the a fault in the 5371da177e4SLinus Torvalds // head part. That's nice! 5381da177e4SLinus Torvalds // 5391da177e4SLinus Torvalds.failure_in1: 5401da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 5411da177e4SLinus Torvalds sub len=endsrc,src1,1 5421da177e4SLinus Torvalds // 5431da177e4SLinus Torvalds // we know that ret0 can never be zero at this point 5441da177e4SLinus Torvalds // because we failed why trying to do a load, i.e. there is still 5451da177e4SLinus Torvalds // some work to do. 5461da177e4SLinus Torvalds // The failure_in1bis and length problem is taken care of at the 5471da177e4SLinus Torvalds // calling side. 5481da177e4SLinus Torvalds // 5491da177e4SLinus Torvalds ;; 5501da177e4SLinus Torvalds.failure_in1bis: // from (.failure_in3) 5511da177e4SLinus Torvalds mov ar.lc=len // Continue with a stupid byte store. 5521da177e4SLinus Torvalds ;; 5531da177e4SLinus Torvalds5: 5541da177e4SLinus Torvalds st1 [dst1]=r0,1 5551da177e4SLinus Torvalds br.cloop.dptk 5b 5561da177e4SLinus Torvalds ;; 5571da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 5581da177e4SLinus Torvalds mov ar.lc=saved_lc 5591da177e4SLinus Torvalds mov ar.pfs=saved_pfs 5601da177e4SLinus Torvalds br.ret.sptk.many rp 5611da177e4SLinus Torvalds 5621da177e4SLinus Torvalds // 5631da177e4SLinus Torvalds // Here we simply restart the loop but instead 5641da177e4SLinus Torvalds // of doing loads we fill the pipeline with zeroes 5651da177e4SLinus Torvalds // We can't simply store r0 because we may have valid 5661da177e4SLinus Torvalds // data in transit in the pipeline. 5671da177e4SLinus Torvalds // ar.lc and ar.ec are setup correctly at this point 5681da177e4SLinus Torvalds // 5691da177e4SLinus Torvalds // we MUST use src1/endsrc here and not dst1/enddst because 5701da177e4SLinus Torvalds // of the pipeline effect. 5711da177e4SLinus Torvalds // 5721da177e4SLinus Torvalds.failure_in3: 5731da177e4SLinus Torvalds sub ret0=endsrc,src1 // number of bytes to zero, i.e. not copied 5741da177e4SLinus Torvalds ;; 5751da177e4SLinus Torvalds2: 5761da177e4SLinus Torvalds(p16) mov val1[0]=r0 5771da177e4SLinus Torvalds(p16) mov val2[0]=r0 5781da177e4SLinus Torvalds(EPI) st8 [dst1]=val1[PIPE_DEPTH-1],16 5791da177e4SLinus Torvalds(EPI) st8 [dst2]=val2[PIPE_DEPTH-1],16 5801da177e4SLinus Torvalds br.ctop.dptk 2b 5811da177e4SLinus Torvalds ;; 5821da177e4SLinus Torvalds cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ? 5831da177e4SLinus Torvalds sub len=enddst,dst1,1 // precompute len 5841da177e4SLinus Torvalds(p6) br.cond.dptk .failure_in1bis 5851da177e4SLinus Torvalds ;; 5861da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 5871da177e4SLinus Torvalds mov ar.lc=saved_lc 5881da177e4SLinus Torvalds mov ar.pfs=saved_pfs 5891da177e4SLinus Torvalds br.ret.sptk.many rp 5901da177e4SLinus Torvalds 5911da177e4SLinus Torvalds.failure_in2: 5921da177e4SLinus Torvalds sub ret0=endsrc,src1 5931da177e4SLinus Torvalds cmp.ne p6,p0=dst1,enddst // Do we need to finish the tail ? 5941da177e4SLinus Torvalds sub len=enddst,dst1,1 // precompute len 5951da177e4SLinus Torvalds(p6) br.cond.dptk .failure_in1bis 5961da177e4SLinus Torvalds ;; 5971da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 5981da177e4SLinus Torvalds mov ar.lc=saved_lc 5991da177e4SLinus Torvalds mov ar.pfs=saved_pfs 6001da177e4SLinus Torvalds br.ret.sptk.many rp 6011da177e4SLinus Torvalds 6021da177e4SLinus Torvalds // 6031da177e4SLinus Torvalds // handling of failures on stores: that's the easy part 6041da177e4SLinus Torvalds // 6051da177e4SLinus Torvalds.failure_out: 6061da177e4SLinus Torvalds sub ret0=enddst,dst1 6071da177e4SLinus Torvalds mov pr=saved_pr,0xffffffffffff0000 6081da177e4SLinus Torvalds mov ar.lc=saved_lc 6091da177e4SLinus Torvalds 6101da177e4SLinus Torvalds mov ar.pfs=saved_pfs 6111da177e4SLinus Torvalds br.ret.sptk.many rp 6121da177e4SLinus TorvaldsEND(__copy_user) 613e007c533SAl ViroEXPORT_SYMBOL(__copy_user) 614