1// SPDX-License-Identifier: GPL-2.0 2/* 3 * raid6_vx$#.c 4 * 5 * $#-way unrolled RAID6 gen/xor functions for s390 6 * based on the vector facility 7 * 8 * Copyright IBM Corp. 2016 9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 10 * 11 * This file is postprocessed using unroll.awk. 12 */ 13 14#include <linux/raid/pq.h> 15#include <asm/fpu/api.h> 16 17asm(".include \"asm/vx-insn.h\"\n"); 18 19#define NSIZE 16 20 21static inline void LOAD_CONST(void) 22{ 23 asm volatile("VREPIB %v24,7"); 24 asm volatile("VREPIB %v25,0x1d"); 25} 26 27/* 28 * The SHLBYTE() operation shifts each of the 16 bytes in 29 * vector register y left by 1 bit and stores the result in 30 * vector register x. 31 */ 32static inline void SHLBYTE(int x, int y) 33{ 34 asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y)); 35} 36 37/* 38 * For each of the 16 bytes in the vector register y the MASK() 39 * operation returns 0xFF if the high bit of the byte is 1, 40 * or 0x00 if the high bit is 0. The result is stored in vector 41 * register x. 42 */ 43static inline void MASK(int x, int y) 44{ 45 asm volatile ("VESRAVB %0,%1,24" : : "i" (x), "i" (y)); 46} 47 48static inline void AND(int x, int y, int z) 49{ 50 asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z)); 51} 52 53static inline void XOR(int x, int y, int z) 54{ 55 asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z)); 56} 57 58static inline void LOAD_DATA(int x, int n, u8 *ptr) 59{ 60 typedef struct { u8 _[16*n]; } addrtype; 61 register addrtype *__ptr asm("1") = (addrtype *) ptr; 62 63 asm volatile ("VLM %2,%3,0,%r1" 64 : : "m" (*__ptr), "a" (__ptr), "i" (x), "i" (x + n - 1)); 65} 66 67static inline void STORE_DATA(int x, int n, u8 *ptr) 68{ 69 typedef struct { u8 _[16*n]; } addrtype; 70 register addrtype *__ptr asm("1") = (addrtype *) ptr; 71 72 asm volatile ("VSTM %2,%3,0,1" 73 : "=m" (*__ptr) : "a" (__ptr), "i" (x), "i" (x + n - 1)); 74} 75 76static inline void COPY_VEC(int x, int y) 77{ 78 asm volatile ("VLR %0,%1" : : "i" (x), "i" (y)); 79} 80 81static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) 82{ 83 struct kernel_fpu vxstate; 84 u8 **dptr, *p, *q; 85 int d, z, z0; 86 87 kernel_fpu_begin(&vxstate, KERNEL_VXR); 88 LOAD_CONST(); 89 90 dptr = (u8 **) ptrs; 91 z0 = disks - 3; /* Highest data disk */ 92 p = dptr[z0 + 1]; /* XOR parity */ 93 q = dptr[z0 + 2]; /* RS syndrome */ 94 95 for (d = 0; d < bytes; d += $#*NSIZE) { 96 LOAD_DATA(0,$#,&dptr[z0][d]); 97 COPY_VEC(8+$$,0+$$); 98 for (z = z0 - 1; z >= 0; z--) { 99 MASK(16+$$,8+$$); 100 AND(16+$$,16+$$,25); 101 SHLBYTE(8+$$,8+$$); 102 XOR(8+$$,8+$$,16+$$); 103 LOAD_DATA(16,$#,&dptr[z][d]); 104 XOR(0+$$,0+$$,16+$$); 105 XOR(8+$$,8+$$,16+$$); 106 } 107 STORE_DATA(0,$#,&p[d]); 108 STORE_DATA(8,$#,&q[d]); 109 } 110 kernel_fpu_end(&vxstate, KERNEL_VXR); 111} 112 113static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, 114 size_t bytes, void **ptrs) 115{ 116 struct kernel_fpu vxstate; 117 u8 **dptr, *p, *q; 118 int d, z, z0; 119 120 dptr = (u8 **) ptrs; 121 z0 = stop; /* P/Q right side optimization */ 122 p = dptr[disks - 2]; /* XOR parity */ 123 q = dptr[disks - 1]; /* RS syndrome */ 124 125 kernel_fpu_begin(&vxstate, KERNEL_VXR); 126 LOAD_CONST(); 127 128 for (d = 0; d < bytes; d += $#*NSIZE) { 129 /* P/Q data pages */ 130 LOAD_DATA(0,$#,&dptr[z0][d]); 131 COPY_VEC(8+$$,0+$$); 132 for (z = z0 - 1; z >= start; z--) { 133 MASK(16+$$,8+$$); 134 AND(16+$$,16+$$,25); 135 SHLBYTE(8+$$,8+$$); 136 XOR(8+$$,8+$$,16+$$); 137 LOAD_DATA(16,$#,&dptr[z][d]); 138 XOR(0+$$,0+$$,16+$$); 139 XOR(8+$$,8+$$,16+$$); 140 } 141 /* P/Q left side optimization */ 142 for (z = start - 1; z >= 0; z--) { 143 MASK(16+$$,8+$$); 144 AND(16+$$,16+$$,25); 145 SHLBYTE(8+$$,8+$$); 146 XOR(8+$$,8+$$,16+$$); 147 } 148 LOAD_DATA(16,$#,&p[d]); 149 XOR(16+$$,16+$$,0+$$); 150 STORE_DATA(16,$#,&p[d]); 151 LOAD_DATA(16,$#,&q[d]); 152 XOR(16+$$,16+$$,8+$$); 153 STORE_DATA(16,$#,&q[d]); 154 } 155 kernel_fpu_end(&vxstate, KERNEL_VXR); 156} 157 158static int raid6_s390vx$#_valid(void) 159{ 160 return MACHINE_HAS_VX; 161} 162 163const struct raid6_calls raid6_s390vx$# = { 164 raid6_s390vx$#_gen_syndrome, 165 raid6_s390vx$#_xor_syndrome, 166 raid6_s390vx$#_valid, 167 "vx128x$#", 168 1 169}; 170