1 /* 2 * csum_partial_copy - do IP checksumming and copy 3 * 4 * (C) Copyright 1996 Linus Torvalds 5 * accelerated versions (and 21264 assembly versions ) contributed by 6 * Rick Gorton <rick.gorton@alpha-processor.com> 7 * 8 * Don't look at this too closely - you'll go mad. The things 9 * we do for performance.. 10 */ 11 12 #include <linux/types.h> 13 #include <linux/string.h> 14 #include <linux/uaccess.h> 15 16 17 #define ldq_u(x,y) \ 18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y))) 19 20 #define stq_u(x,y) \ 21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x)) 22 23 #define extql(x,y,z) \ 24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 25 26 #define extqh(x,y,z) \ 27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 28 29 #define mskql(x,y,z) \ 30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 31 32 #define mskqh(x,y,z) \ 33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 34 35 #define insql(x,y,z) \ 36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 37 38 #define insqh(x,y,z) \ 39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 40 41 42 #define __get_user_u(x,ptr) \ 43 ({ \ 44 long __guu_err; \ 45 __asm__ __volatile__( \ 46 "1: ldq_u %0,%2\n" \ 47 "2:\n" \ 48 EXC(1b,2b,%0,%1) \ 49 : "=r"(x), "=r"(__guu_err) \ 50 : "m"(__m(ptr)), "1"(0)); \ 51 __guu_err; \ 52 }) 53 54 #define __put_user_u(x,ptr) \ 55 ({ \ 56 long __puu_err; \ 57 __asm__ __volatile__( \ 58 "1: stq_u %2,%1\n" \ 59 "2:\n" \ 60 EXC(1b,2b,$31,%0) \ 61 : "=r"(__puu_err) \ 62 : "m"(__m(addr)), "rJ"(x), "0"(0)); \ 63 __puu_err; \ 64 }) 65 66 67 static inline unsigned short from64to16(unsigned long x) 68 { 69 /* Using extract instructions is a bit more efficient 70 than the original shift/bitmask version. */ 71 72 union { 73 unsigned long ul; 74 unsigned int ui[2]; 75 unsigned short us[4]; 76 } in_v, tmp_v, out_v; 77 78 in_v.ul = x; 79 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1]; 80 81 /* Since the bits of tmp_v.sh[3] are going to always be zero, 82 we don't have to bother to add that in. */ 83 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1] 84 + (unsigned long) tmp_v.us[2]; 85 86 /* Similarly, out_v.us[2] is always zero for the final add. */ 87 return out_v.us[0] + out_v.us[1]; 88 } 89 90 91 92 /* 93 * Ok. This isn't fun, but this is the EASY case. 94 */ 95 static inline unsigned long 96 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst, 97 long len, unsigned long checksum, 98 int *errp) 99 { 100 unsigned long carry = 0; 101 int err = 0; 102 103 while (len >= 0) { 104 unsigned long word; 105 err |= __get_user(word, src); 106 checksum += carry; 107 src++; 108 checksum += word; 109 len -= 8; 110 carry = checksum < word; 111 *dst = word; 112 dst++; 113 } 114 len += 8; 115 checksum += carry; 116 if (len) { 117 unsigned long word, tmp; 118 err |= __get_user(word, src); 119 tmp = *dst; 120 mskql(word, len, word); 121 checksum += word; 122 mskqh(tmp, len, tmp); 123 carry = checksum < word; 124 *dst = word | tmp; 125 checksum += carry; 126 } 127 if (err && errp) *errp = err; 128 return checksum; 129 } 130 131 /* 132 * This is even less fun, but this is still reasonably 133 * easy. 134 */ 135 static inline unsigned long 136 csum_partial_cfu_dest_aligned(const unsigned long __user *src, 137 unsigned long *dst, 138 unsigned long soff, 139 long len, unsigned long checksum, 140 int *errp) 141 { 142 unsigned long first; 143 unsigned long word, carry; 144 unsigned long lastsrc = 7+len+(unsigned long)src; 145 int err = 0; 146 147 err |= __get_user_u(first,src); 148 carry = 0; 149 while (len >= 0) { 150 unsigned long second; 151 152 err |= __get_user_u(second, src+1); 153 extql(first, soff, word); 154 len -= 8; 155 src++; 156 extqh(second, soff, first); 157 checksum += carry; 158 word |= first; 159 first = second; 160 checksum += word; 161 *dst = word; 162 dst++; 163 carry = checksum < word; 164 } 165 len += 8; 166 checksum += carry; 167 if (len) { 168 unsigned long tmp; 169 unsigned long second; 170 err |= __get_user_u(second, lastsrc); 171 tmp = *dst; 172 extql(first, soff, word); 173 extqh(second, soff, first); 174 word |= first; 175 mskql(word, len, word); 176 checksum += word; 177 mskqh(tmp, len, tmp); 178 carry = checksum < word; 179 *dst = word | tmp; 180 checksum += carry; 181 } 182 if (err && errp) *errp = err; 183 return checksum; 184 } 185 186 /* 187 * This is slightly less fun than the above.. 188 */ 189 static inline unsigned long 190 csum_partial_cfu_src_aligned(const unsigned long __user *src, 191 unsigned long *dst, 192 unsigned long doff, 193 long len, unsigned long checksum, 194 unsigned long partial_dest, 195 int *errp) 196 { 197 unsigned long carry = 0; 198 unsigned long word; 199 unsigned long second_dest; 200 int err = 0; 201 202 mskql(partial_dest, doff, partial_dest); 203 while (len >= 0) { 204 err |= __get_user(word, src); 205 len -= 8; 206 insql(word, doff, second_dest); 207 checksum += carry; 208 stq_u(partial_dest | second_dest, dst); 209 src++; 210 checksum += word; 211 insqh(word, doff, partial_dest); 212 carry = checksum < word; 213 dst++; 214 } 215 len += 8; 216 if (len) { 217 checksum += carry; 218 err |= __get_user(word, src); 219 mskql(word, len, word); 220 len -= 8; 221 checksum += word; 222 insql(word, doff, second_dest); 223 len += doff; 224 carry = checksum < word; 225 partial_dest |= second_dest; 226 if (len >= 0) { 227 stq_u(partial_dest, dst); 228 if (!len) goto out; 229 dst++; 230 insqh(word, doff, partial_dest); 231 } 232 doff = len; 233 } 234 ldq_u(second_dest, dst); 235 mskqh(second_dest, doff, second_dest); 236 stq_u(partial_dest | second_dest, dst); 237 out: 238 checksum += carry; 239 if (err && errp) *errp = err; 240 return checksum; 241 } 242 243 /* 244 * This is so totally un-fun that it's frightening. Don't 245 * look at this too closely, you'll go blind. 246 */ 247 static inline unsigned long 248 csum_partial_cfu_unaligned(const unsigned long __user * src, 249 unsigned long * dst, 250 unsigned long soff, unsigned long doff, 251 long len, unsigned long checksum, 252 unsigned long partial_dest, 253 int *errp) 254 { 255 unsigned long carry = 0; 256 unsigned long first; 257 unsigned long lastsrc; 258 int err = 0; 259 260 err |= __get_user_u(first, src); 261 lastsrc = 7+len+(unsigned long)src; 262 mskql(partial_dest, doff, partial_dest); 263 while (len >= 0) { 264 unsigned long second, word; 265 unsigned long second_dest; 266 267 err |= __get_user_u(second, src+1); 268 extql(first, soff, word); 269 checksum += carry; 270 len -= 8; 271 extqh(second, soff, first); 272 src++; 273 word |= first; 274 first = second; 275 insql(word, doff, second_dest); 276 checksum += word; 277 stq_u(partial_dest | second_dest, dst); 278 carry = checksum < word; 279 insqh(word, doff, partial_dest); 280 dst++; 281 } 282 len += doff; 283 checksum += carry; 284 if (len >= 0) { 285 unsigned long second, word; 286 unsigned long second_dest; 287 288 err |= __get_user_u(second, lastsrc); 289 extql(first, soff, word); 290 extqh(second, soff, first); 291 word |= first; 292 first = second; 293 mskql(word, len-doff, word); 294 checksum += word; 295 insql(word, doff, second_dest); 296 carry = checksum < word; 297 stq_u(partial_dest | second_dest, dst); 298 if (len) { 299 ldq_u(second_dest, dst+1); 300 insqh(word, doff, partial_dest); 301 mskqh(second_dest, len, second_dest); 302 stq_u(partial_dest | second_dest, dst+1); 303 } 304 checksum += carry; 305 } else { 306 unsigned long second, word; 307 unsigned long second_dest; 308 309 err |= __get_user_u(second, lastsrc); 310 extql(first, soff, word); 311 extqh(second, soff, first); 312 word |= first; 313 ldq_u(second_dest, dst); 314 mskql(word, len-doff, word); 315 checksum += word; 316 mskqh(second_dest, len, second_dest); 317 carry = checksum < word; 318 insql(word, doff, word); 319 stq_u(partial_dest | word | second_dest, dst); 320 checksum += carry; 321 } 322 if (err && errp) *errp = err; 323 return checksum; 324 } 325 326 __wsum 327 csum_partial_copy_from_user(const void __user *src, void *dst, int len, 328 __wsum sum, int *errp) 329 { 330 unsigned long checksum = (__force u32) sum; 331 unsigned long soff = 7 & (unsigned long) src; 332 unsigned long doff = 7 & (unsigned long) dst; 333 334 if (len) { 335 if (!access_ok(VERIFY_READ, src, len)) { 336 if (errp) *errp = -EFAULT; 337 memset(dst, 0, len); 338 return sum; 339 } 340 if (!doff) { 341 if (!soff) 342 checksum = csum_partial_cfu_aligned( 343 (const unsigned long __user *) src, 344 (unsigned long *) dst, 345 len-8, checksum, errp); 346 else 347 checksum = csum_partial_cfu_dest_aligned( 348 (const unsigned long __user *) src, 349 (unsigned long *) dst, 350 soff, len-8, checksum, errp); 351 } else { 352 unsigned long partial_dest; 353 ldq_u(partial_dest, dst); 354 if (!soff) 355 checksum = csum_partial_cfu_src_aligned( 356 (const unsigned long __user *) src, 357 (unsigned long *) dst, 358 doff, len-8, checksum, 359 partial_dest, errp); 360 else 361 checksum = csum_partial_cfu_unaligned( 362 (const unsigned long __user *) src, 363 (unsigned long *) dst, 364 soff, doff, len-8, checksum, 365 partial_dest, errp); 366 } 367 checksum = from64to16 (checksum); 368 } 369 return (__force __wsum)checksum; 370 } 371 EXPORT_SYMBOL(csum_partial_copy_from_user); 372 373 __wsum 374 csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) 375 { 376 __wsum checksum; 377 mm_segment_t oldfs = get_fs(); 378 set_fs(KERNEL_DS); 379 checksum = csum_partial_copy_from_user((__force const void __user *)src, 380 dst, len, sum, NULL); 381 set_fs(oldfs); 382 return checksum; 383 } 384 EXPORT_SYMBOL(csum_partial_copy_nocheck); 385