xref: /openbmc/linux/lib/raid6/recov_neon.c (revision 3de13550)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Intel Corporation
4  * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5  */
6 
7 #include <linux/raid/pq.h>
8 
9 #ifdef __KERNEL__
10 #include <asm/neon.h>
11 #include "neon.h"
12 #else
13 #define kernel_neon_begin()
14 #define kernel_neon_end()
15 #define cpu_has_neon()		(1)
16 #endif
17 
raid6_has_neon(void)18 static int raid6_has_neon(void)
19 {
20 	return cpu_has_neon();
21 }
22 
raid6_2data_recov_neon(int disks,size_t bytes,int faila,int failb,void ** ptrs)23 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
24 		int failb, void **ptrs)
25 {
26 	u8 *p, *q, *dp, *dq;
27 	const u8 *pbmul;	/* P multiplier table for B data */
28 	const u8 *qmul;		/* Q multiplier table (for both) */
29 
30 	p = (u8 *)ptrs[disks - 2];
31 	q = (u8 *)ptrs[disks - 1];
32 
33 	/*
34 	 * Compute syndrome with zero for the missing data pages
35 	 * Use the dead data pages as temporary storage for
36 	 * delta p and delta q
37 	 */
38 	dp = (u8 *)ptrs[faila];
39 	ptrs[faila] = (void *)raid6_empty_zero_page;
40 	ptrs[disks - 2] = dp;
41 	dq = (u8 *)ptrs[failb];
42 	ptrs[failb] = (void *)raid6_empty_zero_page;
43 	ptrs[disks - 1] = dq;
44 
45 	raid6_call.gen_syndrome(disks, bytes, ptrs);
46 
47 	/* Restore pointer table */
48 	ptrs[faila]     = dp;
49 	ptrs[failb]     = dq;
50 	ptrs[disks - 2] = p;
51 	ptrs[disks - 1] = q;
52 
53 	/* Now, pick the proper data tables */
54 	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
55 	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
56 					 raid6_gfexp[failb]]];
57 
58 	kernel_neon_begin();
59 	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
60 	kernel_neon_end();
61 }
62 
raid6_datap_recov_neon(int disks,size_t bytes,int faila,void ** ptrs)63 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
64 		void **ptrs)
65 {
66 	u8 *p, *q, *dq;
67 	const u8 *qmul;		/* Q multiplier table */
68 
69 	p = (u8 *)ptrs[disks - 2];
70 	q = (u8 *)ptrs[disks - 1];
71 
72 	/*
73 	 * Compute syndrome with zero for the missing data page
74 	 * Use the dead data page as temporary storage for delta q
75 	 */
76 	dq = (u8 *)ptrs[faila];
77 	ptrs[faila] = (void *)raid6_empty_zero_page;
78 	ptrs[disks - 1] = dq;
79 
80 	raid6_call.gen_syndrome(disks, bytes, ptrs);
81 
82 	/* Restore pointer table */
83 	ptrs[faila]     = dq;
84 	ptrs[disks - 1] = q;
85 
86 	/* Now, pick the proper data tables */
87 	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
88 
89 	kernel_neon_begin();
90 	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
91 	kernel_neon_end();
92 }
93 
94 const struct raid6_recov_calls raid6_recov_neon = {
95 	.data2		= raid6_2data_recov_neon,
96 	.datap		= raid6_datap_recov_neon,
97 	.valid		= raid6_has_neon,
98 	.name		= "neon",
99 	.priority	= 10,
100 };
101