Lines Matching +full:3 +full:- +full:d

1 /* SPDX-License-Identifier: GPL-2.0 */
3 * McKinley-optimized version of copy_page().
5 * Copyright (C) 2002 Hewlett-Packard Co
15 * - use regular loads and stores to prefetch data to avoid consuming M-slot just for
16 * lfetches => good for in-cache performance
17 * - avoid l2 bank-conflicts by not storing into the same 16-byte bank within a single
21 * First, note that L1 has a line-size of 64 bytes and L2 a line-size of 128 bytes.
22 * To avoid secondary misses in L2, we prefetch both source and destination with a line-size
28 * We use a software-pipelined loop to control the overall operation. The pipeline
30 * source cache-lines. The second PREFETCH_DIST stages are used for prefetching destination
31 * cache-lines, the last K stages are used to copy the cache-line words not copied by
32 * the prefetches. The four relevant points in the pipelined are called A, B, C, D:
33 * p[A] is TRUE if a source-line should be prefetched, p[B] is TRUE if a destination-line
35 * into L1D and p[D] is TRUE if a cacheline needs to be copied.
37 * This all sounds very complicated, but thanks to the modulo-scheduled loop support,
42 * main-loop (.line_copy), but has all known-to-be-predicated-off instructions removed,
48 * +------+------+---
57 * +------+------+---
60 * to fetch the second-half of the L2 cache line into L1, and the tX words are copied in
98 #define D (C + 3) macro
99 #define N (D + 1)
104 alloc r8 = ar.pfs, 2, Nrot-2, 0, Nrot
106 .rotr v[2*PREFETCH_DIST], n[D-C+1]
120 mov ar.lc = 2*PREFETCH_DIST - 1
125 add src1 = 3*8, in1 // first t3 src
127 add dst1 = 3*8, in0 // first t3 dst
128 mov t1 = (PAGE_SIZE/128) - (2*PREFETCH_DIST) - 1
132 // same as .line_copy loop, but with all predicated-off instructions removed:
143 (p[D]) ld8 t2 = [src0], 3*8 // M0
144 (p[D]) ld8 t4 = [src1], 3*8 // M1
146 (p[D]) st8 [dst_pre_l2] = n[D-C], 128 // M3 prefetch dst from L2
150 (p[D]) st8 [dst0] = t1, 8 // M2
151 (p[D]) st8 [dst1] = t3, 8 // M3
153 (p[D]) ld8 t5 = [src0], 8
154 (p[D]) ld8 t7 = [src1], 3*8
155 (p[D]) st8 [dst0] = t2, 3*8
156 (p[D]) st8 [dst1] = t4, 3*8
158 (p[D]) ld8 t6 = [src0], 3*8
159 (p[D]) ld8 t10 = [src1], 8
160 (p[D]) st8 [dst0] = t5, 8
161 (p[D]) st8 [dst1] = t7, 3*8
163 (p[D]) ld8 t9 = [src0], 3*8
164 (p[D]) ld8 t11 = [src1], 3*8
165 (p[D]) st8 [dst0] = t6, 3*8
166 (p[D]) st8 [dst1] = t10, 8
168 (p[D]) ld8 t12 = [src0], 8
169 (p[D]) ld8 t14 = [src1], 8
170 (p[D]) st8 [dst0] = t9, 3*8
171 (p[D]) st8 [dst1] = t11, 3*8
173 (p[D]) ld8 t13 = [src0], 4*8
174 (p[D]) ld8 t15 = [src1], 4*8
175 (p[D]) st8 [dst0] = t12, 8
176 (p[D]) st8 [dst1] = t14, 8
178 (p[D-1])ld8 t1 = [src0], 8
179 (p[D-1])ld8 t3 = [src1], 8
180 (p[D]) st8 [dst0] = t13, 4*8
181 (p[D]) st8 [dst1] = t15, 4*8
185 mov pr = saved_pr, -1