1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/string.h> 3 #include <linux/export.h> 4 5 #undef memcpy 6 #undef memset 7 8 __visible void *memcpy(void *to, const void *from, size_t n) 9 { 10 return __memcpy(to, from, n); 11 } 12 EXPORT_SYMBOL(memcpy); 13 14 __visible void *memset(void *s, int c, size_t count) 15 { 16 return __memset(s, c, count); 17 } 18 EXPORT_SYMBOL(memset); 19 20 __visible void *memmove(void *dest, const void *src, size_t n) 21 { 22 int d0,d1,d2,d3,d4,d5; 23 char *ret = dest; 24 25 __asm__ __volatile__( 26 /* Handle more 16 bytes in loop */ 27 "cmp $0x10, %0\n\t" 28 "jb 1f\n\t" 29 30 /* Decide forward/backward copy mode */ 31 "cmp %2, %1\n\t" 32 "jb 2f\n\t" 33 34 /* 35 * movs instruction have many startup latency 36 * so we handle small size by general register. 37 */ 38 "cmp $680, %0\n\t" 39 "jb 3f\n\t" 40 /* 41 * movs instruction is only good for aligned case. 42 */ 43 "mov %1, %3\n\t" 44 "xor %2, %3\n\t" 45 "and $0xff, %3\n\t" 46 "jz 4f\n\t" 47 "3:\n\t" 48 "sub $0x10, %0\n\t" 49 50 /* 51 * We gobble 16 bytes forward in each loop. 52 */ 53 "3:\n\t" 54 "sub $0x10, %0\n\t" 55 "mov 0*4(%1), %3\n\t" 56 "mov 1*4(%1), %4\n\t" 57 "mov %3, 0*4(%2)\n\t" 58 "mov %4, 1*4(%2)\n\t" 59 "mov 2*4(%1), %3\n\t" 60 "mov 3*4(%1), %4\n\t" 61 "mov %3, 2*4(%2)\n\t" 62 "mov %4, 3*4(%2)\n\t" 63 "lea 0x10(%1), %1\n\t" 64 "lea 0x10(%2), %2\n\t" 65 "jae 3b\n\t" 66 "add $0x10, %0\n\t" 67 "jmp 1f\n\t" 68 69 /* 70 * Handle data forward by movs. 71 */ 72 ".p2align 4\n\t" 73 "4:\n\t" 74 "mov -4(%1, %0), %3\n\t" 75 "lea -4(%2, %0), %4\n\t" 76 "shr $2, %0\n\t" 77 "rep movsl\n\t" 78 "mov %3, (%4)\n\t" 79 "jmp 11f\n\t" 80 /* 81 * Handle data backward by movs. 82 */ 83 ".p2align 4\n\t" 84 "6:\n\t" 85 "mov (%1), %3\n\t" 86 "mov %2, %4\n\t" 87 "lea -4(%1, %0), %1\n\t" 88 "lea -4(%2, %0), %2\n\t" 89 "shr $2, %0\n\t" 90 "std\n\t" 91 "rep movsl\n\t" 92 "mov %3,(%4)\n\t" 93 "cld\n\t" 94 "jmp 11f\n\t" 95 96 /* 97 * Start to prepare for backward copy. 98 */ 99 ".p2align 4\n\t" 100 "2:\n\t" 101 "cmp $680, %0\n\t" 102 "jb 5f\n\t" 103 "mov %1, %3\n\t" 104 "xor %2, %3\n\t" 105 "and $0xff, %3\n\t" 106 "jz 6b\n\t" 107 108 /* 109 * Calculate copy position to tail. 110 */ 111 "5:\n\t" 112 "add %0, %1\n\t" 113 "add %0, %2\n\t" 114 "sub $0x10, %0\n\t" 115 116 /* 117 * We gobble 16 bytes backward in each loop. 118 */ 119 "7:\n\t" 120 "sub $0x10, %0\n\t" 121 122 "mov -1*4(%1), %3\n\t" 123 "mov -2*4(%1), %4\n\t" 124 "mov %3, -1*4(%2)\n\t" 125 "mov %4, -2*4(%2)\n\t" 126 "mov -3*4(%1), %3\n\t" 127 "mov -4*4(%1), %4\n\t" 128 "mov %3, -3*4(%2)\n\t" 129 "mov %4, -4*4(%2)\n\t" 130 "lea -0x10(%1), %1\n\t" 131 "lea -0x10(%2), %2\n\t" 132 "jae 7b\n\t" 133 /* 134 * Calculate copy position to head. 135 */ 136 "add $0x10, %0\n\t" 137 "sub %0, %1\n\t" 138 "sub %0, %2\n\t" 139 140 /* 141 * Move data from 8 bytes to 15 bytes. 142 */ 143 ".p2align 4\n\t" 144 "1:\n\t" 145 "cmp $8, %0\n\t" 146 "jb 8f\n\t" 147 "mov 0*4(%1), %3\n\t" 148 "mov 1*4(%1), %4\n\t" 149 "mov -2*4(%1, %0), %5\n\t" 150 "mov -1*4(%1, %0), %1\n\t" 151 152 "mov %3, 0*4(%2)\n\t" 153 "mov %4, 1*4(%2)\n\t" 154 "mov %5, -2*4(%2, %0)\n\t" 155 "mov %1, -1*4(%2, %0)\n\t" 156 "jmp 11f\n\t" 157 158 /* 159 * Move data from 4 bytes to 7 bytes. 160 */ 161 ".p2align 4\n\t" 162 "8:\n\t" 163 "cmp $4, %0\n\t" 164 "jb 9f\n\t" 165 "mov 0*4(%1), %3\n\t" 166 "mov -1*4(%1, %0), %4\n\t" 167 "mov %3, 0*4(%2)\n\t" 168 "mov %4, -1*4(%2, %0)\n\t" 169 "jmp 11f\n\t" 170 171 /* 172 * Move data from 2 bytes to 3 bytes. 173 */ 174 ".p2align 4\n\t" 175 "9:\n\t" 176 "cmp $2, %0\n\t" 177 "jb 10f\n\t" 178 "movw 0*2(%1), %%dx\n\t" 179 "movw -1*2(%1, %0), %%bx\n\t" 180 "movw %%dx, 0*2(%2)\n\t" 181 "movw %%bx, -1*2(%2, %0)\n\t" 182 "jmp 11f\n\t" 183 184 /* 185 * Move data for 1 byte. 186 */ 187 ".p2align 4\n\t" 188 "10:\n\t" 189 "cmp $1, %0\n\t" 190 "jb 11f\n\t" 191 "movb (%1), %%cl\n\t" 192 "movb %%cl, (%2)\n\t" 193 ".p2align 4\n\t" 194 "11:" 195 : "=&c" (d0), "=&S" (d1), "=&D" (d2), 196 "=r" (d3),"=r" (d4), "=r"(d5) 197 :"0" (n), 198 "1" (src), 199 "2" (dest) 200 :"memory"); 201 202 return ret; 203 204 } 205 EXPORT_SYMBOL(memmove); 206