xref: /openbmc/linux/lib/raid6/avx512.c (revision 0edff03d) 
1 /* -*- linux-c -*- --------------------------------------------------------
2  *
3  *   Copyright (C) 2016 Intel Corporation
4  *
5  *   Author: Gayatri Kammela <gayatri.kammela@intel.com>
6  *   Author: Megha Dey <megha.dey@linux.intel.com>
7  *
8  *   Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
9  *   Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
10  *
11  *   This program is free software; you can redistribute it and/or modify
12  *   it under the terms of the GNU General Public License as published by
13  *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
14  *   Boston MA 02111-1307, USA; either version 2 of the License, or
15  *   (at your option) any later version; incorporated herein by reference.
16  *
17  * -----------------------------------------------------------------------
18  */
19 
20 /*
21  * AVX512 implementation of RAID-6 syndrome functions
22  *
23  */
24 
25 #ifdef CONFIG_AS_AVX512
26 
27 #include <linux/raid/pq.h>
28 #include "x86.h"
29 
30 static const struct raid6_avx512_constants {
31 	u64 x1d[8];
32 } raid6_avx512_constants __aligned(512/8) = {
33 	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
34 	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
35 	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
36 	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
37 };
38 
39 static int raid6_have_avx512(void)
40 {
41 	return boot_cpu_has(X86_FEATURE_AVX2) &&
42 		boot_cpu_has(X86_FEATURE_AVX) &&
43 		boot_cpu_has(X86_FEATURE_AVX512F) &&
44 		boot_cpu_has(X86_FEATURE_AVX512BW) &&
45 		boot_cpu_has(X86_FEATURE_AVX512VL) &&
46 		boot_cpu_has(X86_FEATURE_AVX512DQ);
47 }
48 
49 static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
50 {
51 	u8 **dptr = (u8 **)ptrs;
52 	u8 *p, *q;
53 	int d, z, z0;
54 
55 	z0 = disks - 3;         /* Highest data disk */
56 	p = dptr[z0+1];         /* XOR parity */
57 	q = dptr[z0+2];         /* RS syndrome */
58 
59 	kernel_fpu_begin();
60 
61 	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
62 		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
63 		     :
64 		     : "m" (raid6_avx512_constants.x1d[0]));
65 
66 	for (d = 0; d < bytes; d += 64) {
67 		asm volatile("prefetchnta %0\n\t"
68 			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
69 			     "prefetchnta %1\n\t"
70 			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
71 			     "vmovdqa64 %1,%%zmm6"
72 			     :
73 			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
74 		for (z = z0-2; z >= 0; z--) {
75 			asm volatile("prefetchnta %0\n\t"
76 				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
77 				     "vpmovm2b %%k1,%%zmm5\n\t"
78 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
79 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
80 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
81 				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
82 				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
83 				     "vmovdqa64 %0,%%zmm6"
84 				     :
85 				     : "m" (dptr[z][d]));
86 		}
87 		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
88 			     "vpmovm2b %%k1,%%zmm5\n\t"
89 			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
90 			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
91 			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
92 			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
93 			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
94 			     "vmovntdq %%zmm2,%0\n\t"
95 			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
96 			     "vmovntdq %%zmm4,%1\n\t"
97 			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
98 			     :
99 			     : "m" (p[d]), "m" (q[d]));
100 	}
101 
102 	asm volatile("sfence" : : : "memory");
103 	kernel_fpu_end();
104 }
105 
106 static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
107 				       size_t bytes, void **ptrs)
108 {
109 	u8 **dptr = (u8 **)ptrs;
110 	u8 *p, *q;
111 	int d, z, z0;
112 
113 	z0 = stop;		/* P/Q right side optimization */
114 	p = dptr[disks-2];	/* XOR parity */
115 	q = dptr[disks-1];	/* RS syndrome */
116 
117 	kernel_fpu_begin();
118 
119 	asm volatile("vmovdqa64 %0,%%zmm0"
120 		     : : "m" (raid6_avx512_constants.x1d[0]));
121 
122 	for (d = 0 ; d < bytes ; d += 64) {
123 		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
124 			     "vmovdqa64 %1,%%zmm2\n\t"
125 			     "vpxorq %%zmm4,%%zmm2,%%zmm2"
126 			     :
127 			     : "m" (dptr[z0][d]),  "m" (p[d]));
128 		/* P/Q data pages */
129 		for (z = z0-1 ; z >= start ; z--) {
130 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
131 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
132 				     "vpmovm2b %%k1,%%zmm5\n\t"
133 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
134 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
135 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
136 				     "vmovdqa64 %0,%%zmm5\n\t"
137 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
138 				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
139 				     :
140 				     : "m" (dptr[z][d]));
141 		}
142 		/* P/Q left side optimization */
143 		for (z = start-1 ; z >= 0 ; z--) {
144 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
145 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
146 				     "vpmovm2b %%k1,%%zmm5\n\t"
147 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
148 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
149 				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
150 				     :
151 				     : );
152 		}
153 		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
154 		/* Don't use movntdq for r/w memory area < cache line */
155 			     "vmovdqa64 %%zmm4,%0\n\t"
156 			     "vmovdqa64 %%zmm2,%1"
157 			     :
158 			     : "m" (q[d]), "m" (p[d]));
159 	}
160 
161 	asm volatile("sfence" : : : "memory");
162 	kernel_fpu_end();
163 }
164 
165 const struct raid6_calls raid6_avx512x1 = {
166 	raid6_avx5121_gen_syndrome,
167 	raid6_avx5121_xor_syndrome,
168 	raid6_have_avx512,
169 	"avx512x1",
170 	1                       /* Has cache hints */
171 };
172 
173 /*
174  * Unrolled-by-2 AVX512 implementation
175  */
176 static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
177 {
178 	u8 **dptr = (u8 **)ptrs;
179 	u8 *p, *q;
180 	int d, z, z0;
181 
182 	z0 = disks - 3;         /* Highest data disk */
183 	p = dptr[z0+1];         /* XOR parity */
184 	q = dptr[z0+2];         /* RS syndrome */
185 
186 	kernel_fpu_begin();
187 
188 	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
189 		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
190 		     :
191 		     : "m" (raid6_avx512_constants.x1d[0]));
192 
193 	/* We uniformly assume a single prefetch covers at least 64 bytes */
194 	for (d = 0; d < bytes; d += 128) {
195 		asm volatile("prefetchnta %0\n\t"
196 			     "prefetchnta %1\n\t"
197 			     "vmovdqa64 %0,%%zmm2\n\t"      /* P[0] */
198 			     "vmovdqa64 %1,%%zmm3\n\t"      /* P[1] */
199 			     "vmovdqa64 %%zmm2,%%zmm4\n\t"  /* Q[0] */
200 			     "vmovdqa64 %%zmm3,%%zmm6"      /* Q[1] */
201 			     :
202 			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
203 		for (z = z0-1; z >= 0; z--) {
204 			asm volatile("prefetchnta %0\n\t"
205 				     "prefetchnta %1\n\t"
206 				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
207 				     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
208 				     "vpmovm2b %%k1,%%zmm5\n\t"
209 				     "vpmovm2b %%k2,%%zmm7\n\t"
210 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
211 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
212 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
213 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
214 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
215 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
216 				     "vmovdqa64 %0,%%zmm5\n\t"
217 				     "vmovdqa64 %1,%%zmm7\n\t"
218 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
219 				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
220 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
221 				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
222 				     :
223 				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
224 		}
225 		asm volatile("vmovntdq %%zmm2,%0\n\t"
226 			     "vmovntdq %%zmm3,%1\n\t"
227 			     "vmovntdq %%zmm4,%2\n\t"
228 			     "vmovntdq %%zmm6,%3"
229 			     :
230 			     : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
231 			       "m" (q[d+64]));
232 	}
233 
234 	asm volatile("sfence" : : : "memory");
235 	kernel_fpu_end();
236 }
237 
238 static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
239 				       size_t bytes, void **ptrs)
240 {
241 	u8 **dptr = (u8 **)ptrs;
242 	u8 *p, *q;
243 	int d, z, z0;
244 
245 	z0 = stop;		/* P/Q right side optimization */
246 	p = dptr[disks-2];	/* XOR parity */
247 	q = dptr[disks-1];	/* RS syndrome */
248 
249 	kernel_fpu_begin();
250 
251 	asm volatile("vmovdqa64 %0,%%zmm0"
252 		     : : "m" (raid6_avx512_constants.x1d[0]));
253 
254 	for (d = 0 ; d < bytes ; d += 128) {
255 		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
256 			     "vmovdqa64 %1,%%zmm6\n\t"
257 			     "vmovdqa64 %2,%%zmm2\n\t"
258 			     "vmovdqa64 %3,%%zmm3\n\t"
259 			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
260 			     "vpxorq %%zmm6,%%zmm3,%%zmm3"
261 			     :
262 			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
263 			       "m" (p[d]), "m" (p[d+64]));
264 		/* P/Q data pages */
265 		for (z = z0-1 ; z >= start ; z--) {
266 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
267 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
268 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
269 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
270 				     "vpmovm2b %%k1,%%zmm5\n\t"
271 				     "vpmovm2b %%k2,%%zmm7\n\t"
272 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
273 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
274 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
275 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
276 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
277 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
278 				     "vmovdqa64 %0,%%zmm5\n\t"
279 				     "vmovdqa64 %1,%%zmm7\n\t"
280 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
281 				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
282 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
283 				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
284 				     :
285 				     : "m" (dptr[z][d]),  "m" (dptr[z][d+64]));
286 		}
287 		/* P/Q left side optimization */
288 		for (z = start-1 ; z >= 0 ; z--) {
289 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
290 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
291 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
292 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
293 				     "vpmovm2b %%k1,%%zmm5\n\t"
294 				     "vpmovm2b %%k2,%%zmm7\n\t"
295 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
296 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
297 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
298 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
299 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
300 				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
301 				     :
302 				     : );
303 		}
304 		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
305 			     "vpxorq %1,%%zmm6,%%zmm6\n\t"
306 			     /* Don't use movntdq for r/w
307 			      * memory area < cache line
308 			      */
309 			     "vmovdqa64 %%zmm4,%0\n\t"
310 			     "vmovdqa64 %%zmm6,%1\n\t"
311 			     "vmovdqa64 %%zmm2,%2\n\t"
312 			     "vmovdqa64 %%zmm3,%3"
313 			     :
314 			     : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
315 			       "m" (p[d+64]));
316 	}
317 
318 	asm volatile("sfence" : : : "memory");
319 	kernel_fpu_end();
320 }
321 
322 const struct raid6_calls raid6_avx512x2 = {
323 	raid6_avx5122_gen_syndrome,
324 	raid6_avx5122_xor_syndrome,
325 	raid6_have_avx512,
326 	"avx512x2",
327 	1                       /* Has cache hints */
328 };
329 
330 #ifdef CONFIG_X86_64
331 
332 /*
333  * Unrolled-by-4 AVX2 implementation
334  */
335 static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
336 {
337 	u8 **dptr = (u8 **)ptrs;
338 	u8 *p, *q;
339 	int d, z, z0;
340 
341 	z0 = disks - 3;         /* Highest data disk */
342 	p = dptr[z0+1];         /* XOR parity */
343 	q = dptr[z0+2];         /* RS syndrome */
344 
345 	kernel_fpu_begin();
346 
347 	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
348 		     "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t"       /* Zero temp */
349 		     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"       /* P[0] */
350 		     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"       /* P[1] */
351 		     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"       /* Q[0] */
352 		     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"       /* Q[1] */
353 		     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"    /* P[2] */
354 		     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"    /* P[3] */
355 		     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"    /* Q[2] */
356 		     "vpxorq %%zmm14,%%zmm14,%%zmm14"        /* Q[3] */
357 		     :
358 		     : "m" (raid6_avx512_constants.x1d[0]));
359 
360 	for (d = 0; d < bytes; d += 256) {
361 		for (z = z0; z >= 0; z--) {
362 		asm volatile("prefetchnta %0\n\t"
363 			     "prefetchnta %1\n\t"
364 			     "prefetchnta %2\n\t"
365 			     "prefetchnta %3\n\t"
366 			     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
367 			     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
368 			     "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
369 			     "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
370 			     "vpmovm2b %%k1,%%zmm5\n\t"
371 			     "vpmovm2b %%k2,%%zmm7\n\t"
372 			     "vpmovm2b %%k3,%%zmm13\n\t"
373 			     "vpmovm2b %%k4,%%zmm15\n\t"
374 			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
375 			     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
376 			     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
377 			     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
378 			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
379 			     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
380 			     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
381 			     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
382 			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
383 			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
384 			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
385 			     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
386 			     "vmovdqa64 %0,%%zmm5\n\t"
387 			     "vmovdqa64 %1,%%zmm7\n\t"
388 			     "vmovdqa64 %2,%%zmm13\n\t"
389 			     "vmovdqa64 %3,%%zmm15\n\t"
390 			     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
391 			     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
392 			     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
393 			     "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
394 			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
395 			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
396 			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
397 			     "vpxorq %%zmm15,%%zmm14,%%zmm14"
398 			     :
399 			     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
400 			       "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
401 		}
402 		asm volatile("vmovntdq %%zmm2,%0\n\t"
403 			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
404 			     "vmovntdq %%zmm3,%1\n\t"
405 			     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
406 			     "vmovntdq %%zmm10,%2\n\t"
407 			     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
408 			     "vmovntdq %%zmm11,%3\n\t"
409 			     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
410 			     "vmovntdq %%zmm4,%4\n\t"
411 			     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
412 			     "vmovntdq %%zmm6,%5\n\t"
413 			     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
414 			     "vmovntdq %%zmm12,%6\n\t"
415 			     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
416 			     "vmovntdq %%zmm14,%7\n\t"
417 			     "vpxorq %%zmm14,%%zmm14,%%zmm14"
418 			     :
419 			     : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
420 			       "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
421 			       "m" (q[d+128]), "m" (q[d+192]));
422 	}
423 
424 	asm volatile("sfence" : : : "memory");
425 	kernel_fpu_end();
426 }
427 
428 static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
429 				       size_t bytes, void **ptrs)
430 {
431 	u8 **dptr = (u8 **)ptrs;
432 	u8 *p, *q;
433 	int d, z, z0;
434 
435 	z0 = stop;		/* P/Q right side optimization */
436 	p = dptr[disks-2];	/* XOR parity */
437 	q = dptr[disks-1];	/* RS syndrome */
438 
439 	kernel_fpu_begin();
440 
441 	asm volatile("vmovdqa64 %0,%%zmm0"
442 		     :: "m" (raid6_avx512_constants.x1d[0]));
443 
444 	for (d = 0 ; d < bytes ; d += 256) {
445 		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
446 			     "vmovdqa64 %1,%%zmm6\n\t"
447 			     "vmovdqa64 %2,%%zmm12\n\t"
448 			     "vmovdqa64 %3,%%zmm14\n\t"
449 			     "vmovdqa64 %4,%%zmm2\n\t"
450 			     "vmovdqa64 %5,%%zmm3\n\t"
451 			     "vmovdqa64 %6,%%zmm10\n\t"
452 			     "vmovdqa64 %7,%%zmm11\n\t"
453 			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
454 			     "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
455 			     "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
456 			     "vpxorq %%zmm14,%%zmm11,%%zmm11"
457 			     :
458 			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
459 			       "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
460 			       "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
461 			       "m" (p[d+192]));
462 		/* P/Q data pages */
463 		for (z = z0-1 ; z >= start ; z--) {
464 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
465 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
466 				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
467 				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
468 				     "prefetchnta %0\n\t"
469 				     "prefetchnta %2\n\t"
470 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
471 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
472 				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
473 				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
474 				     "vpmovm2b %%k1,%%zmm5\n\t"
475 				     "vpmovm2b %%k2,%%zmm7\n\t"
476 				     "vpmovm2b %%k3,%%zmm13\n\t"
477 				     "vpmovm2b %%k4,%%zmm15\n\t"
478 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
479 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
480 				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
481 				     "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
482 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
483 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
484 				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
485 				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
486 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
487 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
488 				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
489 				     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
490 				     "vmovdqa64 %0,%%zmm5\n\t"
491 				     "vmovdqa64 %1,%%zmm7\n\t"
492 				     "vmovdqa64 %2,%%zmm13\n\t"
493 				     "vmovdqa64 %3,%%zmm15\n\t"
494 				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
495 				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
496 				     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
497 				     "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
498 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
499 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
500 				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
501 				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
502 				     :
503 				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
504 				       "m" (dptr[z][d+128]),
505 				       "m" (dptr[z][d+192]));
506 		}
507 		asm volatile("prefetchnta %0\n\t"
508 			     "prefetchnta %1\n\t"
509 			     :
510 			     : "m" (q[d]), "m" (q[d+128]));
511 		/* P/Q left side optimization */
512 		for (z = start-1 ; z >= 0 ; z--) {
513 			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
514 				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
515 				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
516 				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
517 				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
518 				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
519 				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
520 				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
521 				     "vpmovm2b %%k1,%%zmm5\n\t"
522 				     "vpmovm2b %%k2,%%zmm7\n\t"
523 				     "vpmovm2b %%k3,%%zmm13\n\t"
524 				     "vpmovm2b %%k4,%%zmm15\n\t"
525 				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
526 				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
527 				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
528 				     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
529 				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
530 				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
531 				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
532 				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
533 				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
534 				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
535 				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
536 				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
537 				     :
538 				     : );
539 		}
540 		asm volatile("vmovntdq %%zmm2,%0\n\t"
541 			     "vmovntdq %%zmm3,%1\n\t"
542 			     "vmovntdq %%zmm10,%2\n\t"
543 			     "vmovntdq %%zmm11,%3\n\t"
544 			     "vpxorq %4,%%zmm4,%%zmm4\n\t"
545 			     "vpxorq %5,%%zmm6,%%zmm6\n\t"
546 			     "vpxorq %6,%%zmm12,%%zmm12\n\t"
547 			     "vpxorq %7,%%zmm14,%%zmm14\n\t"
548 			     "vmovntdq %%zmm4,%4\n\t"
549 			     "vmovntdq %%zmm6,%5\n\t"
550 			     "vmovntdq %%zmm12,%6\n\t"
551 			     "vmovntdq %%zmm14,%7"
552 			     :
553 			     : "m" (p[d]),  "m" (p[d+64]), "m" (p[d+128]),
554 			       "m" (p[d+192]), "m" (q[d]),  "m" (q[d+64]),
555 			       "m" (q[d+128]), "m" (q[d+192]));
556 	}
557 	asm volatile("sfence" : : : "memory");
558 	kernel_fpu_end();
559 }
560 const struct raid6_calls raid6_avx512x4 = {
561 	raid6_avx5124_gen_syndrome,
562 	raid6_avx5124_xor_syndrome,
563 	raid6_have_avx512,
564 	"avx512x4",
565 	1                       /* Has cache hints */
566 };
567 #endif
568 
569 #endif /* CONFIG_AS_AVX512 */
570