1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Optimized xor_block operation for RAID4/5 4 * 5 * Copyright IBM Corp. 2016 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/export.h> 11 #include <linux/raid/xor.h> 12 #include <asm/xor.h> 13 14 static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1, 15 const unsigned long * __restrict p2) 16 { 17 asm volatile( 18 " larl 1,2f\n" 19 " aghi %0,-1\n" 20 " jm 3f\n" 21 " srlg 0,%0,8\n" 22 " ltgr 0,0\n" 23 " jz 1f\n" 24 "0: xc 0(256,%1),0(%2)\n" 25 " la %1,256(%1)\n" 26 " la %2,256(%2)\n" 27 " brctg 0,0b\n" 28 "1: ex %0,0(1)\n" 29 " j 3f\n" 30 "2: xc 0(1,%1),0(%2)\n" 31 "3:\n" 32 : : "d" (bytes), "a" (p1), "a" (p2) 33 : "0", "1", "cc", "memory"); 34 } 35 36 static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1, 37 const unsigned long * __restrict p2, 38 const unsigned long * __restrict p3) 39 { 40 asm volatile( 41 " larl 1,2f\n" 42 " aghi %0,-1\n" 43 " jm 3f\n" 44 " srlg 0,%0,8\n" 45 " ltgr 0,0\n" 46 " jz 1f\n" 47 "0: xc 0(256,%1),0(%2)\n" 48 " xc 0(256,%1),0(%3)\n" 49 " la %1,256(%1)\n" 50 " la %2,256(%2)\n" 51 " la %3,256(%3)\n" 52 " brctg 0,0b\n" 53 "1: ex %0,0(1)\n" 54 " ex %0,6(1)\n" 55 " j 3f\n" 56 "2: xc 0(1,%1),0(%2)\n" 57 " xc 0(1,%1),0(%3)\n" 58 "3:\n" 59 : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3) 60 : : "0", "1", "cc", "memory"); 61 } 62 63 static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1, 64 const unsigned long * __restrict p2, 65 const unsigned long * __restrict p3, 66 const unsigned long * __restrict p4) 67 { 68 asm volatile( 69 " larl 1,2f\n" 70 " aghi %0,-1\n" 71 " jm 3f\n" 72 " srlg 0,%0,8\n" 73 " ltgr 0,0\n" 74 " jz 1f\n" 75 "0: xc 0(256,%1),0(%2)\n" 76 " xc 0(256,%1),0(%3)\n" 77 " xc 0(256,%1),0(%4)\n" 78 " la %1,256(%1)\n" 79 " la %2,256(%2)\n" 80 " la %3,256(%3)\n" 81 " la %4,256(%4)\n" 82 " brctg 0,0b\n" 83 "1: ex %0,0(1)\n" 84 " ex %0,6(1)\n" 85 " ex %0,12(1)\n" 86 " j 3f\n" 87 "2: xc 0(1,%1),0(%2)\n" 88 " xc 0(1,%1),0(%3)\n" 89 " xc 0(1,%1),0(%4)\n" 90 "3:\n" 91 : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4) 92 : : "0", "1", "cc", "memory"); 93 } 94 95 static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1, 96 const unsigned long * __restrict p2, 97 const unsigned long * __restrict p3, 98 const unsigned long * __restrict p4, 99 const unsigned long * __restrict p5) 100 { 101 asm volatile( 102 " larl 1,2f\n" 103 " aghi %0,-1\n" 104 " jm 3f\n" 105 " srlg 0,%0,8\n" 106 " ltgr 0,0\n" 107 " jz 1f\n" 108 "0: xc 0(256,%1),0(%2)\n" 109 " xc 0(256,%1),0(%3)\n" 110 " xc 0(256,%1),0(%4)\n" 111 " xc 0(256,%1),0(%5)\n" 112 " la %1,256(%1)\n" 113 " la %2,256(%2)\n" 114 " la %3,256(%3)\n" 115 " la %4,256(%4)\n" 116 " la %5,256(%5)\n" 117 " brctg 0,0b\n" 118 "1: ex %0,0(1)\n" 119 " ex %0,6(1)\n" 120 " ex %0,12(1)\n" 121 " ex %0,18(1)\n" 122 " j 3f\n" 123 "2: xc 0(1,%1),0(%2)\n" 124 " xc 0(1,%1),0(%3)\n" 125 " xc 0(1,%1),0(%4)\n" 126 " xc 0(1,%1),0(%5)\n" 127 "3:\n" 128 : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4), 129 "+a" (p5) 130 : : "0", "1", "cc", "memory"); 131 } 132 133 struct xor_block_template xor_block_xc = { 134 .name = "xc", 135 .do_2 = xor_xc_2, 136 .do_3 = xor_xc_3, 137 .do_4 = xor_xc_4, 138 .do_5 = xor_xc_5, 139 }; 140 EXPORT_SYMBOL(xor_block_xc); 141