1 /* 2 * csum_partial_copy - do IP checksumming and copy 3 * 4 * (C) Copyright 1996 Linus Torvalds 5 * accelerated versions (and 21264 assembly versions ) contributed by 6 * Rick Gorton <rick.gorton@alpha-processor.com> 7 * 8 * Don't look at this too closely - you'll go mad. The things 9 * we do for performance.. 10 */ 11 12 #include <linux/types.h> 13 #include <linux/string.h> 14 #include <asm/uaccess.h> 15 16 17 #define ldq_u(x,y) \ 18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y))) 19 20 #define stq_u(x,y) \ 21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x)) 22 23 #define extql(x,y,z) \ 24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 25 26 #define extqh(x,y,z) \ 27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 28 29 #define mskql(x,y,z) \ 30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 31 32 #define mskqh(x,y,z) \ 33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 34 35 #define insql(x,y,z) \ 36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 37 38 #define insqh(x,y,z) \ 39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) 40 41 42 #define __get_user_u(x,ptr) \ 43 ({ \ 44 long __guu_err; \ 45 __asm__ __volatile__( \ 46 "1: ldq_u %0,%2\n" \ 47 "2:\n" \ 48 ".section __ex_table,\"a\"\n" \ 49 " .long 1b - .\n" \ 50 " lda %0,2b-1b(%1)\n" \ 51 ".previous" \ 52 : "=r"(x), "=r"(__guu_err) \ 53 : "m"(__m(ptr)), "1"(0)); \ 54 __guu_err; \ 55 }) 56 57 #define __put_user_u(x,ptr) \ 58 ({ \ 59 long __puu_err; \ 60 __asm__ __volatile__( \ 61 "1: stq_u %2,%1\n" \ 62 "2:\n" \ 63 ".section __ex_table,\"a\"\n" \ 64 " .long 1b - ." \ 65 " lda $31,2b-1b(%0)\n" \ 66 ".previous" \ 67 : "=r"(__puu_err) \ 68 : "m"(__m(addr)), "rJ"(x), "0"(0)); \ 69 __puu_err; \ 70 }) 71 72 73 static inline unsigned short from64to16(unsigned long x) 74 { 75 /* Using extract instructions is a bit more efficient 76 than the original shift/bitmask version. */ 77 78 union { 79 unsigned long ul; 80 unsigned int ui[2]; 81 unsigned short us[4]; 82 } in_v, tmp_v, out_v; 83 84 in_v.ul = x; 85 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1]; 86 87 /* Since the bits of tmp_v.sh[3] are going to always be zero, 88 we don't have to bother to add that in. */ 89 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1] 90 + (unsigned long) tmp_v.us[2]; 91 92 /* Similarly, out_v.us[2] is always zero for the final add. */ 93 return out_v.us[0] + out_v.us[1]; 94 } 95 96 97 98 /* 99 * Ok. This isn't fun, but this is the EASY case. 100 */ 101 static inline unsigned long 102 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst, 103 long len, unsigned long checksum, 104 int *errp) 105 { 106 unsigned long carry = 0; 107 int err = 0; 108 109 while (len >= 0) { 110 unsigned long word; 111 err |= __get_user(word, src); 112 checksum += carry; 113 src++; 114 checksum += word; 115 len -= 8; 116 carry = checksum < word; 117 *dst = word; 118 dst++; 119 } 120 len += 8; 121 checksum += carry; 122 if (len) { 123 unsigned long word, tmp; 124 err |= __get_user(word, src); 125 tmp = *dst; 126 mskql(word, len, word); 127 checksum += word; 128 mskqh(tmp, len, tmp); 129 carry = checksum < word; 130 *dst = word | tmp; 131 checksum += carry; 132 } 133 if (err) *errp = err; 134 return checksum; 135 } 136 137 /* 138 * This is even less fun, but this is still reasonably 139 * easy. 140 */ 141 static inline unsigned long 142 csum_partial_cfu_dest_aligned(const unsigned long __user *src, 143 unsigned long *dst, 144 unsigned long soff, 145 long len, unsigned long checksum, 146 int *errp) 147 { 148 unsigned long first; 149 unsigned long word, carry; 150 unsigned long lastsrc = 7+len+(unsigned long)src; 151 int err = 0; 152 153 err |= __get_user_u(first,src); 154 carry = 0; 155 while (len >= 0) { 156 unsigned long second; 157 158 err |= __get_user_u(second, src+1); 159 extql(first, soff, word); 160 len -= 8; 161 src++; 162 extqh(second, soff, first); 163 checksum += carry; 164 word |= first; 165 first = second; 166 checksum += word; 167 *dst = word; 168 dst++; 169 carry = checksum < word; 170 } 171 len += 8; 172 checksum += carry; 173 if (len) { 174 unsigned long tmp; 175 unsigned long second; 176 err |= __get_user_u(second, lastsrc); 177 tmp = *dst; 178 extql(first, soff, word); 179 extqh(second, soff, first); 180 word |= first; 181 mskql(word, len, word); 182 checksum += word; 183 mskqh(tmp, len, tmp); 184 carry = checksum < word; 185 *dst = word | tmp; 186 checksum += carry; 187 } 188 if (err) *errp = err; 189 return checksum; 190 } 191 192 /* 193 * This is slightly less fun than the above.. 194 */ 195 static inline unsigned long 196 csum_partial_cfu_src_aligned(const unsigned long __user *src, 197 unsigned long *dst, 198 unsigned long doff, 199 long len, unsigned long checksum, 200 unsigned long partial_dest, 201 int *errp) 202 { 203 unsigned long carry = 0; 204 unsigned long word; 205 unsigned long second_dest; 206 int err = 0; 207 208 mskql(partial_dest, doff, partial_dest); 209 while (len >= 0) { 210 err |= __get_user(word, src); 211 len -= 8; 212 insql(word, doff, second_dest); 213 checksum += carry; 214 stq_u(partial_dest | second_dest, dst); 215 src++; 216 checksum += word; 217 insqh(word, doff, partial_dest); 218 carry = checksum < word; 219 dst++; 220 } 221 len += 8; 222 if (len) { 223 checksum += carry; 224 err |= __get_user(word, src); 225 mskql(word, len, word); 226 len -= 8; 227 checksum += word; 228 insql(word, doff, second_dest); 229 len += doff; 230 carry = checksum < word; 231 partial_dest |= second_dest; 232 if (len >= 0) { 233 stq_u(partial_dest, dst); 234 if (!len) goto out; 235 dst++; 236 insqh(word, doff, partial_dest); 237 } 238 doff = len; 239 } 240 ldq_u(second_dest, dst); 241 mskqh(second_dest, doff, second_dest); 242 stq_u(partial_dest | second_dest, dst); 243 out: 244 checksum += carry; 245 if (err) *errp = err; 246 return checksum; 247 } 248 249 /* 250 * This is so totally un-fun that it's frightening. Don't 251 * look at this too closely, you'll go blind. 252 */ 253 static inline unsigned long 254 csum_partial_cfu_unaligned(const unsigned long __user * src, 255 unsigned long * dst, 256 unsigned long soff, unsigned long doff, 257 long len, unsigned long checksum, 258 unsigned long partial_dest, 259 int *errp) 260 { 261 unsigned long carry = 0; 262 unsigned long first; 263 unsigned long lastsrc; 264 int err = 0; 265 266 err |= __get_user_u(first, src); 267 lastsrc = 7+len+(unsigned long)src; 268 mskql(partial_dest, doff, partial_dest); 269 while (len >= 0) { 270 unsigned long second, word; 271 unsigned long second_dest; 272 273 err |= __get_user_u(second, src+1); 274 extql(first, soff, word); 275 checksum += carry; 276 len -= 8; 277 extqh(second, soff, first); 278 src++; 279 word |= first; 280 first = second; 281 insql(word, doff, second_dest); 282 checksum += word; 283 stq_u(partial_dest | second_dest, dst); 284 carry = checksum < word; 285 insqh(word, doff, partial_dest); 286 dst++; 287 } 288 len += doff; 289 checksum += carry; 290 if (len >= 0) { 291 unsigned long second, word; 292 unsigned long second_dest; 293 294 err |= __get_user_u(second, lastsrc); 295 extql(first, soff, word); 296 extqh(second, soff, first); 297 word |= first; 298 first = second; 299 mskql(word, len-doff, word); 300 checksum += word; 301 insql(word, doff, second_dest); 302 carry = checksum < word; 303 stq_u(partial_dest | second_dest, dst); 304 if (len) { 305 ldq_u(second_dest, dst+1); 306 insqh(word, doff, partial_dest); 307 mskqh(second_dest, len, second_dest); 308 stq_u(partial_dest | second_dest, dst+1); 309 } 310 checksum += carry; 311 } else { 312 unsigned long second, word; 313 unsigned long second_dest; 314 315 err |= __get_user_u(second, lastsrc); 316 extql(first, soff, word); 317 extqh(second, soff, first); 318 word |= first; 319 ldq_u(second_dest, dst); 320 mskql(word, len-doff, word); 321 checksum += word; 322 mskqh(second_dest, len, second_dest); 323 carry = checksum < word; 324 insql(word, doff, word); 325 stq_u(partial_dest | word | second_dest, dst); 326 checksum += carry; 327 } 328 if (err) *errp = err; 329 return checksum; 330 } 331 332 __wsum 333 csum_partial_copy_from_user(const void __user *src, void *dst, int len, 334 __wsum sum, int *errp) 335 { 336 unsigned long checksum = (__force u32) sum; 337 unsigned long soff = 7 & (unsigned long) src; 338 unsigned long doff = 7 & (unsigned long) dst; 339 340 if (len) { 341 if (!doff) { 342 if (!soff) 343 checksum = csum_partial_cfu_aligned( 344 (const unsigned long __user *) src, 345 (unsigned long *) dst, 346 len-8, checksum, errp); 347 else 348 checksum = csum_partial_cfu_dest_aligned( 349 (const unsigned long __user *) src, 350 (unsigned long *) dst, 351 soff, len-8, checksum, errp); 352 } else { 353 unsigned long partial_dest; 354 ldq_u(partial_dest, dst); 355 if (!soff) 356 checksum = csum_partial_cfu_src_aligned( 357 (const unsigned long __user *) src, 358 (unsigned long *) dst, 359 doff, len-8, checksum, 360 partial_dest, errp); 361 else 362 checksum = csum_partial_cfu_unaligned( 363 (const unsigned long __user *) src, 364 (unsigned long *) dst, 365 soff, doff, len-8, checksum, 366 partial_dest, errp); 367 } 368 checksum = from64to16 (checksum); 369 } 370 return (__force __wsum)checksum; 371 } 372 373 __wsum 374 csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) 375 { 376 return csum_partial_copy_from_user((__force const void __user *)src, 377 dst, len, sum, NULL); 378 } 379