ia64/lib/copy_page_mck.S

1 /* SPDX-License-Identifier: GPL-2.0 */
3  * McKinley-optimized version of copy_page().
5  * Copyright (C) 2002 Hewlett-Packard Co
15  *	- use regular loads and stores to prefetch data to avoid consuming M-slot just for
16  *	  lfetches => good for in-cache performance
17  *	- avoid l2 bank-conflicts by not storing into the same 16-byte bank within a single
21  *	First, note that L1 has a line-size of 64 bytes and L2 a line-size of 128 bytes.
22  *	To avoid secondary misses in L2, we prefetch both source and destination with a line-size
28  *	We use a software-pipelined loop to control the overall operation.  The pipeline
30  *	source cache-lines.  The second PREFETCH_DIST stages are used for prefetching destination
31  *	cache-lines, the last K stages are used to copy the cache-line words not copied by
32  *	the prefetches.  The four relevant points in the pipelined are called A, B, C, D:
33  *	p[A] is TRUE if a source-line should be prefetched, p[B] is TRUE if a destination-line
35  *	into L1D and p[D] is TRUE if a cacheline needs to be copied.
37  *	This all sounds very complicated, but thanks to the modulo-scheduled loop support,
40  *	As a secondary optimization, the first 2*PREFETCH_DIST iterations are implemented
42  *	main-loop (.line_copy), but has all known-to-be-predicated-off instructions removed,
48  *            +------+------+---
57  *	      +------+------+---
60  *	to fetch the second-half of the L2 cache line into L1, and the tX words are copied in
67 #define PREFETCH_DIST	8		// McKinley sustains 16 outstanding L2 misses (8 ld, 8 st)
95 #define	A	0  macro
104 	alloc r8 = ar.pfs, 2, Nrot-2, 0, Nrot
106 	.rotr v[2*PREFETCH_DIST], n[D-C+1]
120 	mov ar.lc = 2*PREFETCH_DIST - 1
122 	add src_pre_l2 = 8*8, in1
123 	add dst_pre_l2 = 8*8, in0
124 	add src0 = 8, in1			// first t1 src
125 	add src1 = 3*8, in1			// first t3 src
126 	add dst0 = 8, in0			// first t1 dst
127 	add dst1 = 3*8, in0			// first t3 dst
128 	mov t1 = (PAGE_SIZE/128) - (2*PREFETCH_DIST) - 1
132 	// same as .line_copy loop, but with all predicated-off instructions removed:
134 (p[A])	ld8 v[A] = [src_pre_mem], 128		// M0
139 	mov ar.lc = t1				// with 64KB pages, t1 is too big to fit in 8 bits!
143 (p[D])	ld8 t2 = [src0], 3*8			// M0
144 (p[D])	ld8 t4 = [src1], 3*8			// M1
146 (p[D])	st8 [dst_pre_l2] = n[D-C], 128		// M3 prefetch dst from L2
148 (p[A])	ld8 v[A] = [src_pre_mem], 128		// M0 prefetch src from memory
150 (p[D])	st8 [dst0] =  t1, 8			// M2
151 (p[D])	st8 [dst1] =  t3, 8			// M3
153 (p[D])	ld8  t5 = [src0], 8
154 (p[D])	ld8  t7 = [src1], 3*8
155 (p[D])	st8 [dst0] =  t2, 3*8
156 (p[D])	st8 [dst1] =  t4, 3*8
158 (p[D])	ld8  t6 = [src0], 3*8
159 (p[D])	ld8 t10 = [src1], 8
160 (p[D])	st8 [dst0] =  t5, 8
161 (p[D])	st8 [dst1] =  t7, 3*8
163 (p[D])	ld8  t9 = [src0], 3*8
164 (p[D])	ld8 t11 = [src1], 3*8
165 (p[D])	st8 [dst0] =  t6, 3*8
166 (p[D])	st8 [dst1] = t10, 8
168 (p[D])	ld8 t12 = [src0], 8
169 (p[D])	ld8 t14 = [src1], 8
170 (p[D])	st8 [dst0] =  t9, 3*8
171 (p[D])	st8 [dst1] = t11, 3*8
173 (p[D])	ld8 t13 = [src0], 4*8
174 (p[D])	ld8 t15 = [src1], 4*8
175 (p[D])	st8 [dst0] = t12, 8
176 (p[D])	st8 [dst1] = t14, 8
178 (p[D-1])ld8  t1 = [src0], 8
179 (p[D-1])ld8  t3 = [src1], 8
180 (p[D])	st8 [dst0] = t13, 4*8
181 (p[D])	st8 [dst1] = t15, 4*8
185 	mov pr = saved_pr, -1