1 /* 2 * Copyright (C) 2012 Intel Corporation 3 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; version 2 8 * of the License. 9 */ 10 11 #include <linux/raid/pq.h> 12 13 #ifdef __KERNEL__ 14 #include <asm/neon.h> 15 #else 16 #define kernel_neon_begin() 17 #define kernel_neon_end() 18 #define cpu_has_neon() (1) 19 #endif 20 21 static int raid6_has_neon(void) 22 { 23 return cpu_has_neon(); 24 } 25 26 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp, 27 uint8_t *dq, const uint8_t *pbmul, 28 const uint8_t *qmul); 29 30 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq, 31 const uint8_t *qmul); 32 33 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, 34 int failb, void **ptrs) 35 { 36 u8 *p, *q, *dp, *dq; 37 const u8 *pbmul; /* P multiplier table for B data */ 38 const u8 *qmul; /* Q multiplier table (for both) */ 39 40 p = (u8 *)ptrs[disks - 2]; 41 q = (u8 *)ptrs[disks - 1]; 42 43 /* 44 * Compute syndrome with zero for the missing data pages 45 * Use the dead data pages as temporary storage for 46 * delta p and delta q 47 */ 48 dp = (u8 *)ptrs[faila]; 49 ptrs[faila] = (void *)raid6_empty_zero_page; 50 ptrs[disks - 2] = dp; 51 dq = (u8 *)ptrs[failb]; 52 ptrs[failb] = (void *)raid6_empty_zero_page; 53 ptrs[disks - 1] = dq; 54 55 raid6_call.gen_syndrome(disks, bytes, ptrs); 56 57 /* Restore pointer table */ 58 ptrs[faila] = dp; 59 ptrs[failb] = dq; 60 ptrs[disks - 2] = p; 61 ptrs[disks - 1] = q; 62 63 /* Now, pick the proper data tables */ 64 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; 65 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 66 raid6_gfexp[failb]]]; 67 68 kernel_neon_begin(); 69 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); 70 kernel_neon_end(); 71 } 72 73 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, 74 void **ptrs) 75 { 76 u8 *p, *q, *dq; 77 const u8 *qmul; /* Q multiplier table */ 78 79 p = (u8 *)ptrs[disks - 2]; 80 q = (u8 *)ptrs[disks - 1]; 81 82 /* 83 * Compute syndrome with zero for the missing data page 84 * Use the dead data page as temporary storage for delta q 85 */ 86 dq = (u8 *)ptrs[faila]; 87 ptrs[faila] = (void *)raid6_empty_zero_page; 88 ptrs[disks - 1] = dq; 89 90 raid6_call.gen_syndrome(disks, bytes, ptrs); 91 92 /* Restore pointer table */ 93 ptrs[faila] = dq; 94 ptrs[disks - 1] = q; 95 96 /* Now, pick the proper data tables */ 97 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 98 99 kernel_neon_begin(); 100 __raid6_datap_recov_neon(bytes, p, q, dq, qmul); 101 kernel_neon_end(); 102 } 103 104 const struct raid6_recov_calls raid6_recov_neon = { 105 .data2 = raid6_2data_recov_neon, 106 .datap = raid6_datap_recov_neon, 107 .valid = raid6_has_neon, 108 .name = "neon", 109 .priority = 10, 110 }; 111