1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Intel Corporation 4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 */ 6 7 #include <linux/raid/pq.h> 8 9 #ifdef __KERNEL__ 10 #include <asm/neon.h> 11 #else 12 #define kernel_neon_begin() 13 #define kernel_neon_end() 14 #define cpu_has_neon() (1) 15 #endif 16 17 static int raid6_has_neon(void) 18 { 19 return cpu_has_neon(); 20 } 21 22 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp, 23 uint8_t *dq, const uint8_t *pbmul, 24 const uint8_t *qmul); 25 26 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq, 27 const uint8_t *qmul); 28 29 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, 30 int failb, void **ptrs) 31 { 32 u8 *p, *q, *dp, *dq; 33 const u8 *pbmul; /* P multiplier table for B data */ 34 const u8 *qmul; /* Q multiplier table (for both) */ 35 36 p = (u8 *)ptrs[disks - 2]; 37 q = (u8 *)ptrs[disks - 1]; 38 39 /* 40 * Compute syndrome with zero for the missing data pages 41 * Use the dead data pages as temporary storage for 42 * delta p and delta q 43 */ 44 dp = (u8 *)ptrs[faila]; 45 ptrs[faila] = (void *)raid6_empty_zero_page; 46 ptrs[disks - 2] = dp; 47 dq = (u8 *)ptrs[failb]; 48 ptrs[failb] = (void *)raid6_empty_zero_page; 49 ptrs[disks - 1] = dq; 50 51 raid6_call.gen_syndrome(disks, bytes, ptrs); 52 53 /* Restore pointer table */ 54 ptrs[faila] = dp; 55 ptrs[failb] = dq; 56 ptrs[disks - 2] = p; 57 ptrs[disks - 1] = q; 58 59 /* Now, pick the proper data tables */ 60 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; 61 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 62 raid6_gfexp[failb]]]; 63 64 kernel_neon_begin(); 65 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); 66 kernel_neon_end(); 67 } 68 69 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, 70 void **ptrs) 71 { 72 u8 *p, *q, *dq; 73 const u8 *qmul; /* Q multiplier table */ 74 75 p = (u8 *)ptrs[disks - 2]; 76 q = (u8 *)ptrs[disks - 1]; 77 78 /* 79 * Compute syndrome with zero for the missing data page 80 * Use the dead data page as temporary storage for delta q 81 */ 82 dq = (u8 *)ptrs[faila]; 83 ptrs[faila] = (void *)raid6_empty_zero_page; 84 ptrs[disks - 1] = dq; 85 86 raid6_call.gen_syndrome(disks, bytes, ptrs); 87 88 /* Restore pointer table */ 89 ptrs[faila] = dq; 90 ptrs[disks - 1] = q; 91 92 /* Now, pick the proper data tables */ 93 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 94 95 kernel_neon_begin(); 96 __raid6_datap_recov_neon(bytes, p, q, dq, qmul); 97 kernel_neon_end(); 98 } 99 100 const struct raid6_recov_calls raid6_recov_neon = { 101 .data2 = raid6_2data_recov_neon, 102 .datap = raid6_datap_recov_neon, 103 .valid = raid6_has_neon, 104 .name = "neon", 105 .priority = 10, 106 }; 107