xref: /openbmc/linux/lib/raid6/recov_neon.c (revision 2cf1c348)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Intel Corporation
4  * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5  */
6 
7 #include <linux/raid/pq.h>
8 
9 #ifdef __KERNEL__
10 #include <asm/neon.h>
11 #else
12 #define kernel_neon_begin()
13 #define kernel_neon_end()
14 #define cpu_has_neon()		(1)
15 #endif
16 
17 static int raid6_has_neon(void)
18 {
19 	return cpu_has_neon();
20 }
21 
22 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
23 			      uint8_t *dq, const uint8_t *pbmul,
24 			      const uint8_t *qmul);
25 
26 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
27 			      const uint8_t *qmul);
28 
29 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
30 		int failb, void **ptrs)
31 {
32 	u8 *p, *q, *dp, *dq;
33 	const u8 *pbmul;	/* P multiplier table for B data */
34 	const u8 *qmul;		/* Q multiplier table (for both) */
35 
36 	p = (u8 *)ptrs[disks - 2];
37 	q = (u8 *)ptrs[disks - 1];
38 
39 	/*
40 	 * Compute syndrome with zero for the missing data pages
41 	 * Use the dead data pages as temporary storage for
42 	 * delta p and delta q
43 	 */
44 	dp = (u8 *)ptrs[faila];
45 	ptrs[faila] = (void *)raid6_empty_zero_page;
46 	ptrs[disks - 2] = dp;
47 	dq = (u8 *)ptrs[failb];
48 	ptrs[failb] = (void *)raid6_empty_zero_page;
49 	ptrs[disks - 1] = dq;
50 
51 	raid6_call.gen_syndrome(disks, bytes, ptrs);
52 
53 	/* Restore pointer table */
54 	ptrs[faila]     = dp;
55 	ptrs[failb]     = dq;
56 	ptrs[disks - 2] = p;
57 	ptrs[disks - 1] = q;
58 
59 	/* Now, pick the proper data tables */
60 	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
61 	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
62 					 raid6_gfexp[failb]]];
63 
64 	kernel_neon_begin();
65 	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
66 	kernel_neon_end();
67 }
68 
69 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
70 		void **ptrs)
71 {
72 	u8 *p, *q, *dq;
73 	const u8 *qmul;		/* Q multiplier table */
74 
75 	p = (u8 *)ptrs[disks - 2];
76 	q = (u8 *)ptrs[disks - 1];
77 
78 	/*
79 	 * Compute syndrome with zero for the missing data page
80 	 * Use the dead data page as temporary storage for delta q
81 	 */
82 	dq = (u8 *)ptrs[faila];
83 	ptrs[faila] = (void *)raid6_empty_zero_page;
84 	ptrs[disks - 1] = dq;
85 
86 	raid6_call.gen_syndrome(disks, bytes, ptrs);
87 
88 	/* Restore pointer table */
89 	ptrs[faila]     = dq;
90 	ptrs[disks - 1] = q;
91 
92 	/* Now, pick the proper data tables */
93 	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
94 
95 	kernel_neon_begin();
96 	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
97 	kernel_neon_end();
98 }
99 
100 const struct raid6_recov_calls raid6_recov_neon = {
101 	.data2		= raid6_2data_recov_neon,
102 	.datap		= raid6_datap_recov_neon,
103 	.valid		= raid6_has_neon,
104 	.name		= "neon",
105 	.priority	= 10,
106 };
107