1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * csum_partial_copy - do IP checksumming and copy 4 * 5 * (C) Copyright 1996 Linus Torvalds 6 * accelerated versions (and 21264 assembly versions ) contributed by 7 * Rick Gorton <rick.gorton@alpha-processor.com> 8 * 9 * Don't look at this too closely - you'll go mad. The things 10 * we do for performance.. 11 */ 12 13 #include <linux/types.h> 14 #include <linux/string.h> 15 #include <linux/uaccess.h> 16 17 18 #define ldq_u(x,y) \ 19 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y))) 20 21 #define stq_u(x,y) \ 22 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x)) 23 24 #define extql(x,y,z) \ 25 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 26 27 #define extqh(x,y,z) \ 28 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 29 30 #define mskql(x,y,z) \ 31 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 32 33 #define mskqh(x,y,z) \ 34 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 35 36 #define insql(x,y,z) \ 37 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 38 39 #define insqh(x,y,z) \ 40 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 41 42 #define __get_word(insn,x,ptr) \ 43 ({ \ 44 long __guu_err; \ 45 __asm__ __volatile__( \ 46 "1: "#insn" %0,%2\n" \ 47 "2:\n" \ 48 EXC(1b,2b,%0,%1) \ 49 : "=r"(x), "=r"(__guu_err) \ 50 : "m"(__m(ptr)), "1"(0)); \ 51 __guu_err; \ 52 }) 53 54 static inline unsigned short from64to16(unsigned long x) 55 { 56 /* Using extract instructions is a bit more efficient 57 than the original shift/bitmask version. */ 58 59 union { 60 unsigned long ul; 61 unsigned int ui[2]; 62 unsigned short us[4]; 63 } in_v, tmp_v, out_v; 64 65 in_v.ul = x; 66 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1]; 67 68 /* Since the bits of tmp_v.sh[3] are going to always be zero, 69 we don't have to bother to add that in. */ 70 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1] 71 + (unsigned long) tmp_v.us[2]; 72 73 /* Similarly, out_v.us[2] is always zero for the final add. */ 74 return out_v.us[0] + out_v.us[1]; 75 } 76 77 78 79 /* 80 * Ok. This isn't fun, but this is the EASY case. 81 */ 82 static inline unsigned long 83 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst, 84 long len) 85 { 86 unsigned long checksum = ~0U; 87 unsigned long carry = 0; 88 89 while (len >= 0) { 90 unsigned long word; 91 if (__get_word(ldq, word, src)) 92 return 0; 93 checksum += carry; 94 src++; 95 checksum += word; 96 len -= 8; 97 carry = checksum < word; 98 *dst = word; 99 dst++; 100 } 101 len += 8; 102 checksum += carry; 103 if (len) { 104 unsigned long word, tmp; 105 if (__get_word(ldq, word, src)) 106 return 0; 107 tmp = *dst; 108 mskql(word, len, word); 109 checksum += word; 110 mskqh(tmp, len, tmp); 111 carry = checksum < word; 112 *dst = word | tmp; 113 checksum += carry; 114 } 115 return checksum; 116 } 117 118 /* 119 * This is even less fun, but this is still reasonably 120 * easy. 121 */ 122 static inline unsigned long 123 csum_partial_cfu_dest_aligned(const unsigned long __user *src, 124 unsigned long *dst, 125 unsigned long soff, 126 long len) 127 { 128 unsigned long first; 129 unsigned long word, carry; 130 unsigned long lastsrc = 7+len+(unsigned long)src; 131 unsigned long checksum = ~0U; 132 133 if (__get_word(ldq_u, first,src)) 134 return 0; 135 carry = 0; 136 while (len >= 0) { 137 unsigned long second; 138 139 if (__get_word(ldq_u, second, src+1)) 140 return 0; 141 extql(first, soff, word); 142 len -= 8; 143 src++; 144 extqh(second, soff, first); 145 checksum += carry; 146 word |= first; 147 first = second; 148 checksum += word; 149 *dst = word; 150 dst++; 151 carry = checksum < word; 152 } 153 len += 8; 154 checksum += carry; 155 if (len) { 156 unsigned long tmp; 157 unsigned long second; 158 if (__get_word(ldq_u, second, lastsrc)) 159 return 0; 160 tmp = *dst; 161 extql(first, soff, word); 162 extqh(second, soff, first); 163 word |= first; 164 mskql(word, len, word); 165 checksum += word; 166 mskqh(tmp, len, tmp); 167 carry = checksum < word; 168 *dst = word | tmp; 169 checksum += carry; 170 } 171 return checksum; 172 } 173 174 /* 175 * This is slightly less fun than the above.. 176 */ 177 static inline unsigned long 178 csum_partial_cfu_src_aligned(const unsigned long __user *src, 179 unsigned long *dst, 180 unsigned long doff, 181 long len, 182 unsigned long partial_dest) 183 { 184 unsigned long carry = 0; 185 unsigned long word; 186 unsigned long second_dest; 187 unsigned long checksum = ~0U; 188 189 mskql(partial_dest, doff, partial_dest); 190 while (len >= 0) { 191 if (__get_word(ldq, word, src)) 192 return 0; 193 len -= 8; 194 insql(word, doff, second_dest); 195 checksum += carry; 196 stq_u(partial_dest | second_dest, dst); 197 src++; 198 checksum += word; 199 insqh(word, doff, partial_dest); 200 carry = checksum < word; 201 dst++; 202 } 203 len += 8; 204 if (len) { 205 checksum += carry; 206 if (__get_word(ldq, word, src)) 207 return 0; 208 mskql(word, len, word); 209 len -= 8; 210 checksum += word; 211 insql(word, doff, second_dest); 212 len += doff; 213 carry = checksum < word; 214 partial_dest |= second_dest; 215 if (len >= 0) { 216 stq_u(partial_dest, dst); 217 if (!len) goto out; 218 dst++; 219 insqh(word, doff, partial_dest); 220 } 221 doff = len; 222 } 223 ldq_u(second_dest, dst); 224 mskqh(second_dest, doff, second_dest); 225 stq_u(partial_dest | second_dest, dst); 226 out: 227 checksum += carry; 228 return checksum; 229 } 230 231 /* 232 * This is so totally un-fun that it's frightening. Don't 233 * look at this too closely, you'll go blind. 234 */ 235 static inline unsigned long 236 csum_partial_cfu_unaligned(const unsigned long __user * src, 237 unsigned long * dst, 238 unsigned long soff, unsigned long doff, 239 long len, unsigned long partial_dest) 240 { 241 unsigned long carry = 0; 242 unsigned long first; 243 unsigned long lastsrc; 244 unsigned long checksum = ~0U; 245 246 if (__get_word(ldq_u, first, src)) 247 return 0; 248 lastsrc = 7+len+(unsigned long)src; 249 mskql(partial_dest, doff, partial_dest); 250 while (len >= 0) { 251 unsigned long second, word; 252 unsigned long second_dest; 253 254 if (__get_word(ldq_u, second, src+1)) 255 return 0; 256 extql(first, soff, word); 257 checksum += carry; 258 len -= 8; 259 extqh(second, soff, first); 260 src++; 261 word |= first; 262 first = second; 263 insql(word, doff, second_dest); 264 checksum += word; 265 stq_u(partial_dest | second_dest, dst); 266 carry = checksum < word; 267 insqh(word, doff, partial_dest); 268 dst++; 269 } 270 len += doff; 271 checksum += carry; 272 if (len >= 0) { 273 unsigned long second, word; 274 unsigned long second_dest; 275 276 if (__get_word(ldq_u, second, lastsrc)) 277 return 0; 278 extql(first, soff, word); 279 extqh(second, soff, first); 280 word |= first; 281 first = second; 282 mskql(word, len-doff, word); 283 checksum += word; 284 insql(word, doff, second_dest); 285 carry = checksum < word; 286 stq_u(partial_dest | second_dest, dst); 287 if (len) { 288 ldq_u(second_dest, dst+1); 289 insqh(word, doff, partial_dest); 290 mskqh(second_dest, len, second_dest); 291 stq_u(partial_dest | second_dest, dst+1); 292 } 293 checksum += carry; 294 } else { 295 unsigned long second, word; 296 unsigned long second_dest; 297 298 if (__get_word(ldq_u, second, lastsrc)) 299 return 0; 300 extql(first, soff, word); 301 extqh(second, soff, first); 302 word |= first; 303 ldq_u(second_dest, dst); 304 mskql(word, len-doff, word); 305 checksum += word; 306 mskqh(second_dest, len, second_dest); 307 carry = checksum < word; 308 insql(word, doff, word); 309 stq_u(partial_dest | word | second_dest, dst); 310 checksum += carry; 311 } 312 return checksum; 313 } 314 315 static __wsum __csum_and_copy(const void __user *src, void *dst, int len) 316 { 317 unsigned long soff = 7 & (unsigned long) src; 318 unsigned long doff = 7 & (unsigned long) dst; 319 unsigned long checksum; 320 321 if (!doff) { 322 if (!soff) 323 checksum = csum_partial_cfu_aligned( 324 (const unsigned long __user *) src, 325 (unsigned long *) dst, len-8); 326 else 327 checksum = csum_partial_cfu_dest_aligned( 328 (const unsigned long __user *) src, 329 (unsigned long *) dst, 330 soff, len-8); 331 } else { 332 unsigned long partial_dest; 333 ldq_u(partial_dest, dst); 334 if (!soff) 335 checksum = csum_partial_cfu_src_aligned( 336 (const unsigned long __user *) src, 337 (unsigned long *) dst, 338 doff, len-8, partial_dest); 339 else 340 checksum = csum_partial_cfu_unaligned( 341 (const unsigned long __user *) src, 342 (unsigned long *) dst, 343 soff, doff, len-8, partial_dest); 344 } 345 return (__force __wsum)from64to16 (checksum); 346 } 347 348 __wsum 349 csum_and_copy_from_user(const void __user *src, void *dst, int len) 350 { 351 if (!access_ok(src, len)) 352 return 0; 353 return __csum_and_copy(src, dst, len); 354 } 355 EXPORT_SYMBOL(csum_and_copy_from_user); 356 357 __wsum 358 csum_partial_copy_nocheck(const void *src, void *dst, int len) 359 { 360 return __csum_and_copy((__force const void __user *)src, 361 dst, len); 362 } 363 EXPORT_SYMBOL(csum_partial_copy_nocheck); 364