1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 8 * Copyright (C) 2007 Maciej W. Rozycki 9 * Copyright (C) 2014, Imagination Technologies Ltd. 10 */ 11 #ifndef _ASM_UACCESS_H 12 #define _ASM_UACCESS_H 13 14 #include <linux/kernel.h> 15 #include <linux/errno.h> 16 #include <linux/thread_info.h> 17 #include <asm/asm-eva.h> 18 19 /* 20 * The fs value determines whether argument validity checking should be 21 * performed or not. If get_fs() == USER_DS, checking is performed, with 22 * get_fs() == KERNEL_DS, checking is bypassed. 23 * 24 * For historical reasons, these macros are grossly misnamed. 25 */ 26 #ifdef CONFIG_32BIT 27 28 #ifdef CONFIG_KVM_GUEST 29 #define __UA_LIMIT 0x40000000UL 30 #else 31 #define __UA_LIMIT 0x80000000UL 32 #endif 33 34 #define __UA_ADDR ".word" 35 #define __UA_LA "la" 36 #define __UA_ADDU "addu" 37 #define __UA_t0 "$8" 38 #define __UA_t1 "$9" 39 40 #endif /* CONFIG_32BIT */ 41 42 #ifdef CONFIG_64BIT 43 44 extern u64 __ua_limit; 45 46 #define __UA_LIMIT __ua_limit 47 48 #define __UA_ADDR ".dword" 49 #define __UA_LA "dla" 50 #define __UA_ADDU "daddu" 51 #define __UA_t0 "$12" 52 #define __UA_t1 "$13" 53 54 #endif /* CONFIG_64BIT */ 55 56 /* 57 * USER_DS is a bitmask that has the bits set that may not be set in a valid 58 * userspace address. Note that we limit 32-bit userspace to 0x7fff8000 but 59 * the arithmetic we're doing only works if the limit is a power of two, so 60 * we use 0x80000000 here on 32-bit kernels. If a process passes an invalid 61 * address in this range it's the process's problem, not ours :-) 62 */ 63 64 #ifdef CONFIG_KVM_GUEST 65 #define KERNEL_DS ((mm_segment_t) { 0x80000000UL }) 66 #define USER_DS ((mm_segment_t) { 0xC0000000UL }) 67 #else 68 #define KERNEL_DS ((mm_segment_t) { 0UL }) 69 #define USER_DS ((mm_segment_t) { __UA_LIMIT }) 70 #endif 71 72 #define VERIFY_READ 0 73 #define VERIFY_WRITE 1 74 75 #define get_ds() (KERNEL_DS) 76 #define get_fs() (current_thread_info()->addr_limit) 77 #define set_fs(x) (current_thread_info()->addr_limit = (x)) 78 79 #define segment_eq(a, b) ((a).seg == (b).seg) 80 81 82 /* 83 * Is a address valid? This does a straighforward calculation rather 84 * than tests. 85 * 86 * Address valid if: 87 * - "addr" doesn't have any high-bits set 88 * - AND "size" doesn't have any high-bits set 89 * - AND "addr+size" doesn't have any high-bits set 90 * - OR we are in kernel mode. 91 * 92 * __ua_size() is a trick to avoid runtime checking of positive constant 93 * sizes; for those we already know at compile time that the size is ok. 94 */ 95 #define __ua_size(size) \ 96 ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size)) 97 98 /* 99 * access_ok: - Checks if a user space pointer is valid 100 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that 101 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe 102 * to write to a block, it is always safe to read from it. 103 * @addr: User space pointer to start of block to check 104 * @size: Size of block to check 105 * 106 * Context: User context only. This function may sleep. 107 * 108 * Checks if a pointer to a block of memory in user space is valid. 109 * 110 * Returns true (nonzero) if the memory block may be valid, false (zero) 111 * if it is definitely invalid. 112 * 113 * Note that, depending on architecture, this function probably just 114 * checks that the pointer is in the user space range - after calling 115 * this function, memory access functions may still return -EFAULT. 116 */ 117 118 #define __access_mask get_fs().seg 119 120 #define __access_ok(addr, size, mask) \ 121 ({ \ 122 unsigned long __addr = (unsigned long) (addr); \ 123 unsigned long __size = size; \ 124 unsigned long __mask = mask; \ 125 unsigned long __ok; \ 126 \ 127 __chk_user_ptr(addr); \ 128 __ok = (signed long)(__mask & (__addr | (__addr + __size) | \ 129 __ua_size(__size))); \ 130 __ok == 0; \ 131 }) 132 133 #define access_ok(type, addr, size) \ 134 likely(__access_ok((addr), (size), __access_mask)) 135 136 /* 137 * put_user: - Write a simple value into user space. 138 * @x: Value to copy to user space. 139 * @ptr: Destination address, in user space. 140 * 141 * Context: User context only. This function may sleep. 142 * 143 * This macro copies a single simple value from kernel space to user 144 * space. It supports simple types like char and int, but not larger 145 * data types like structures or arrays. 146 * 147 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 148 * to the result of dereferencing @ptr. 149 * 150 * Returns zero on success, or -EFAULT on error. 151 */ 152 #define put_user(x,ptr) \ 153 __put_user_check((x), (ptr), sizeof(*(ptr))) 154 155 /* 156 * get_user: - Get a simple variable from user space. 157 * @x: Variable to store result. 158 * @ptr: Source address, in user space. 159 * 160 * Context: User context only. This function may sleep. 161 * 162 * This macro copies a single simple variable from user space to kernel 163 * space. It supports simple types like char and int, but not larger 164 * data types like structures or arrays. 165 * 166 * @ptr must have pointer-to-simple-variable type, and the result of 167 * dereferencing @ptr must be assignable to @x without a cast. 168 * 169 * Returns zero on success, or -EFAULT on error. 170 * On error, the variable @x is set to zero. 171 */ 172 #define get_user(x,ptr) \ 173 __get_user_check((x), (ptr), sizeof(*(ptr))) 174 175 /* 176 * __put_user: - Write a simple value into user space, with less checking. 177 * @x: Value to copy to user space. 178 * @ptr: Destination address, in user space. 179 * 180 * Context: User context only. This function may sleep. 181 * 182 * This macro copies a single simple value from kernel space to user 183 * space. It supports simple types like char and int, but not larger 184 * data types like structures or arrays. 185 * 186 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 187 * to the result of dereferencing @ptr. 188 * 189 * Caller must check the pointer with access_ok() before calling this 190 * function. 191 * 192 * Returns zero on success, or -EFAULT on error. 193 */ 194 #define __put_user(x,ptr) \ 195 __put_user_nocheck((x), (ptr), sizeof(*(ptr))) 196 197 /* 198 * __get_user: - Get a simple variable from user space, with less checking. 199 * @x: Variable to store result. 200 * @ptr: Source address, in user space. 201 * 202 * Context: User context only. This function may sleep. 203 * 204 * This macro copies a single simple variable from user space to kernel 205 * space. It supports simple types like char and int, but not larger 206 * data types like structures or arrays. 207 * 208 * @ptr must have pointer-to-simple-variable type, and the result of 209 * dereferencing @ptr must be assignable to @x without a cast. 210 * 211 * Caller must check the pointer with access_ok() before calling this 212 * function. 213 * 214 * Returns zero on success, or -EFAULT on error. 215 * On error, the variable @x is set to zero. 216 */ 217 #define __get_user(x,ptr) \ 218 __get_user_nocheck((x), (ptr), sizeof(*(ptr))) 219 220 struct __large_struct { unsigned long buf[100]; }; 221 #define __m(x) (*(struct __large_struct __user *)(x)) 222 223 /* 224 * Yuck. We need two variants, one for 64bit operation and one 225 * for 32 bit mode and old iron. 226 */ 227 #ifndef CONFIG_EVA 228 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr) 229 #else 230 /* 231 * Kernel specific functions for EVA. We need to use normal load instructions 232 * to read data from kernel when operating in EVA mode. We use these macros to 233 * avoid redefining __get_user_asm for EVA. 234 */ 235 #undef _loadd 236 #undef _loadw 237 #undef _loadh 238 #undef _loadb 239 #ifdef CONFIG_32BIT 240 #define _loadd _loadw 241 #else 242 #define _loadd(reg, addr) "ld " reg ", " addr 243 #endif 244 #define _loadw(reg, addr) "lw " reg ", " addr 245 #define _loadh(reg, addr) "lh " reg ", " addr 246 #define _loadb(reg, addr) "lb " reg ", " addr 247 248 #define __get_kernel_common(val, size, ptr) \ 249 do { \ 250 switch (size) { \ 251 case 1: __get_data_asm(val, _loadb, ptr); break; \ 252 case 2: __get_data_asm(val, _loadh, ptr); break; \ 253 case 4: __get_data_asm(val, _loadw, ptr); break; \ 254 case 8: __GET_DW(val, _loadd, ptr); break; \ 255 default: __get_user_unknown(); break; \ 256 } \ 257 } while (0) 258 #endif 259 260 #ifdef CONFIG_32BIT 261 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr) 262 #endif 263 #ifdef CONFIG_64BIT 264 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr) 265 #endif 266 267 extern void __get_user_unknown(void); 268 269 #define __get_user_common(val, size, ptr) \ 270 do { \ 271 switch (size) { \ 272 case 1: __get_data_asm(val, user_lb, ptr); break; \ 273 case 2: __get_data_asm(val, user_lh, ptr); break; \ 274 case 4: __get_data_asm(val, user_lw, ptr); break; \ 275 case 8: __GET_DW(val, user_ld, ptr); break; \ 276 default: __get_user_unknown(); break; \ 277 } \ 278 } while (0) 279 280 #define __get_user_nocheck(x, ptr, size) \ 281 ({ \ 282 int __gu_err; \ 283 \ 284 if (segment_eq(get_fs(), get_ds())) { \ 285 __get_kernel_common((x), size, ptr); \ 286 } else { \ 287 __chk_user_ptr(ptr); \ 288 __get_user_common((x), size, ptr); \ 289 } \ 290 __gu_err; \ 291 }) 292 293 #define __get_user_check(x, ptr, size) \ 294 ({ \ 295 int __gu_err = -EFAULT; \ 296 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ 297 \ 298 might_fault(); \ 299 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) { \ 300 if (segment_eq(get_fs(), get_ds())) \ 301 __get_kernel_common((x), size, __gu_ptr); \ 302 else \ 303 __get_user_common((x), size, __gu_ptr); \ 304 } else \ 305 (x) = 0; \ 306 \ 307 __gu_err; \ 308 }) 309 310 #define __get_data_asm(val, insn, addr) \ 311 { \ 312 long __gu_tmp; \ 313 \ 314 __asm__ __volatile__( \ 315 "1: "insn("%1", "%3")" \n" \ 316 "2: \n" \ 317 " .insn \n" \ 318 " .section .fixup,\"ax\" \n" \ 319 "3: li %0, %4 \n" \ 320 " move %1, $0 \n" \ 321 " j 2b \n" \ 322 " .previous \n" \ 323 " .section __ex_table,\"a\" \n" \ 324 " "__UA_ADDR "\t1b, 3b \n" \ 325 " .previous \n" \ 326 : "=r" (__gu_err), "=r" (__gu_tmp) \ 327 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \ 328 \ 329 (val) = (__typeof__(*(addr))) __gu_tmp; \ 330 } 331 332 /* 333 * Get a long long 64 using 32 bit registers. 334 */ 335 #define __get_data_asm_ll32(val, insn, addr) \ 336 { \ 337 union { \ 338 unsigned long long l; \ 339 __typeof__(*(addr)) t; \ 340 } __gu_tmp; \ 341 \ 342 __asm__ __volatile__( \ 343 "1: " insn("%1", "(%3)")" \n" \ 344 "2: " insn("%D1", "4(%3)")" \n" \ 345 "3: \n" \ 346 " .insn \n" \ 347 " .section .fixup,\"ax\" \n" \ 348 "4: li %0, %4 \n" \ 349 " move %1, $0 \n" \ 350 " move %D1, $0 \n" \ 351 " j 3b \n" \ 352 " .previous \n" \ 353 " .section __ex_table,\"a\" \n" \ 354 " " __UA_ADDR " 1b, 4b \n" \ 355 " " __UA_ADDR " 2b, 4b \n" \ 356 " .previous \n" \ 357 : "=r" (__gu_err), "=&r" (__gu_tmp.l) \ 358 : "0" (0), "r" (addr), "i" (-EFAULT)); \ 359 \ 360 (val) = __gu_tmp.t; \ 361 } 362 363 #ifndef CONFIG_EVA 364 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size) 365 #else 366 /* 367 * Kernel specific functions for EVA. We need to use normal load instructions 368 * to read data from kernel when operating in EVA mode. We use these macros to 369 * avoid redefining __get_data_asm for EVA. 370 */ 371 #undef _stored 372 #undef _storew 373 #undef _storeh 374 #undef _storeb 375 #ifdef CONFIG_32BIT 376 #define _stored _storew 377 #else 378 #define _stored(reg, addr) "ld " reg ", " addr 379 #endif 380 381 #define _storew(reg, addr) "sw " reg ", " addr 382 #define _storeh(reg, addr) "sh " reg ", " addr 383 #define _storeb(reg, addr) "sb " reg ", " addr 384 385 #define __put_kernel_common(ptr, size) \ 386 do { \ 387 switch (size) { \ 388 case 1: __put_data_asm(_storeb, ptr); break; \ 389 case 2: __put_data_asm(_storeh, ptr); break; \ 390 case 4: __put_data_asm(_storew, ptr); break; \ 391 case 8: __PUT_DW(_stored, ptr); break; \ 392 default: __put_user_unknown(); break; \ 393 } \ 394 } while(0) 395 #endif 396 397 /* 398 * Yuck. We need two variants, one for 64bit operation and one 399 * for 32 bit mode and old iron. 400 */ 401 #ifdef CONFIG_32BIT 402 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr) 403 #endif 404 #ifdef CONFIG_64BIT 405 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr) 406 #endif 407 408 #define __put_user_common(ptr, size) \ 409 do { \ 410 switch (size) { \ 411 case 1: __put_data_asm(user_sb, ptr); break; \ 412 case 2: __put_data_asm(user_sh, ptr); break; \ 413 case 4: __put_data_asm(user_sw, ptr); break; \ 414 case 8: __PUT_DW(user_sd, ptr); break; \ 415 default: __put_user_unknown(); break; \ 416 } \ 417 } while (0) 418 419 #define __put_user_nocheck(x, ptr, size) \ 420 ({ \ 421 __typeof__(*(ptr)) __pu_val; \ 422 int __pu_err = 0; \ 423 \ 424 __pu_val = (x); \ 425 if (segment_eq(get_fs(), get_ds())) { \ 426 __put_kernel_common(ptr, size); \ 427 } else { \ 428 __chk_user_ptr(ptr); \ 429 __put_user_common(ptr, size); \ 430 } \ 431 __pu_err; \ 432 }) 433 434 #define __put_user_check(x, ptr, size) \ 435 ({ \ 436 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 437 __typeof__(*(ptr)) __pu_val = (x); \ 438 int __pu_err = -EFAULT; \ 439 \ 440 might_fault(); \ 441 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) { \ 442 if (segment_eq(get_fs(), get_ds())) \ 443 __put_kernel_common(__pu_addr, size); \ 444 else \ 445 __put_user_common(__pu_addr, size); \ 446 } \ 447 \ 448 __pu_err; \ 449 }) 450 451 #define __put_data_asm(insn, ptr) \ 452 { \ 453 __asm__ __volatile__( \ 454 "1: "insn("%z2", "%3")" # __put_data_asm \n" \ 455 "2: \n" \ 456 " .insn \n" \ 457 " .section .fixup,\"ax\" \n" \ 458 "3: li %0, %4 \n" \ 459 " j 2b \n" \ 460 " .previous \n" \ 461 " .section __ex_table,\"a\" \n" \ 462 " " __UA_ADDR " 1b, 3b \n" \ 463 " .previous \n" \ 464 : "=r" (__pu_err) \ 465 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \ 466 "i" (-EFAULT)); \ 467 } 468 469 #define __put_data_asm_ll32(insn, ptr) \ 470 { \ 471 __asm__ __volatile__( \ 472 "1: "insn("%2", "(%3)")" # __put_data_asm_ll32 \n" \ 473 "2: "insn("%D2", "4(%3)")" \n" \ 474 "3: \n" \ 475 " .insn \n" \ 476 " .section .fixup,\"ax\" \n" \ 477 "4: li %0, %4 \n" \ 478 " j 3b \n" \ 479 " .previous \n" \ 480 " .section __ex_table,\"a\" \n" \ 481 " " __UA_ADDR " 1b, 4b \n" \ 482 " " __UA_ADDR " 2b, 4b \n" \ 483 " .previous" \ 484 : "=r" (__pu_err) \ 485 : "0" (0), "r" (__pu_val), "r" (ptr), \ 486 "i" (-EFAULT)); \ 487 } 488 489 extern void __put_user_unknown(void); 490 491 /* 492 * ul{b,h,w} are macros and there are no equivalent macros for EVA. 493 * EVA unaligned access is handled in the ADE exception handler. 494 */ 495 #ifndef CONFIG_EVA 496 /* 497 * put_user_unaligned: - Write a simple value into user space. 498 * @x: Value to copy to user space. 499 * @ptr: Destination address, in user space. 500 * 501 * Context: User context only. This function may sleep. 502 * 503 * This macro copies a single simple value from kernel space to user 504 * space. It supports simple types like char and int, but not larger 505 * data types like structures or arrays. 506 * 507 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 508 * to the result of dereferencing @ptr. 509 * 510 * Returns zero on success, or -EFAULT on error. 511 */ 512 #define put_user_unaligned(x,ptr) \ 513 __put_user_unaligned_check((x),(ptr),sizeof(*(ptr))) 514 515 /* 516 * get_user_unaligned: - Get a simple variable from user space. 517 * @x: Variable to store result. 518 * @ptr: Source address, in user space. 519 * 520 * Context: User context only. This function may sleep. 521 * 522 * This macro copies a single simple variable from user space to kernel 523 * space. It supports simple types like char and int, but not larger 524 * data types like structures or arrays. 525 * 526 * @ptr must have pointer-to-simple-variable type, and the result of 527 * dereferencing @ptr must be assignable to @x without a cast. 528 * 529 * Returns zero on success, or -EFAULT on error. 530 * On error, the variable @x is set to zero. 531 */ 532 #define get_user_unaligned(x,ptr) \ 533 __get_user_unaligned_check((x),(ptr),sizeof(*(ptr))) 534 535 /* 536 * __put_user_unaligned: - Write a simple value into user space, with less checking. 537 * @x: Value to copy to user space. 538 * @ptr: Destination address, in user space. 539 * 540 * Context: User context only. This function may sleep. 541 * 542 * This macro copies a single simple value from kernel space to user 543 * space. It supports simple types like char and int, but not larger 544 * data types like structures or arrays. 545 * 546 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 547 * to the result of dereferencing @ptr. 548 * 549 * Caller must check the pointer with access_ok() before calling this 550 * function. 551 * 552 * Returns zero on success, or -EFAULT on error. 553 */ 554 #define __put_user_unaligned(x,ptr) \ 555 __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr))) 556 557 /* 558 * __get_user_unaligned: - Get a simple variable from user space, with less checking. 559 * @x: Variable to store result. 560 * @ptr: Source address, in user space. 561 * 562 * Context: User context only. This function may sleep. 563 * 564 * This macro copies a single simple variable from user space to kernel 565 * space. It supports simple types like char and int, but not larger 566 * data types like structures or arrays. 567 * 568 * @ptr must have pointer-to-simple-variable type, and the result of 569 * dereferencing @ptr must be assignable to @x without a cast. 570 * 571 * Caller must check the pointer with access_ok() before calling this 572 * function. 573 * 574 * Returns zero on success, or -EFAULT on error. 575 * On error, the variable @x is set to zero. 576 */ 577 #define __get_user_unaligned(x,ptr) \ 578 __get_user__unalignednocheck((x),(ptr),sizeof(*(ptr))) 579 580 /* 581 * Yuck. We need two variants, one for 64bit operation and one 582 * for 32 bit mode and old iron. 583 */ 584 #ifdef CONFIG_32BIT 585 #define __GET_USER_UNALIGNED_DW(val, ptr) \ 586 __get_user_unaligned_asm_ll32(val, ptr) 587 #endif 588 #ifdef CONFIG_64BIT 589 #define __GET_USER_UNALIGNED_DW(val, ptr) \ 590 __get_user_unaligned_asm(val, "uld", ptr) 591 #endif 592 593 extern void __get_user_unaligned_unknown(void); 594 595 #define __get_user_unaligned_common(val, size, ptr) \ 596 do { \ 597 switch (size) { \ 598 case 1: __get_data_asm(val, "lb", ptr); break; \ 599 case 2: __get_user_unaligned_asm(val, "ulh", ptr); break; \ 600 case 4: __get_user_unaligned_asm(val, "ulw", ptr); break; \ 601 case 8: __GET_USER_UNALIGNED_DW(val, ptr); break; \ 602 default: __get_user_unaligned_unknown(); break; \ 603 } \ 604 } while (0) 605 606 #define __get_user_unaligned_nocheck(x,ptr,size) \ 607 ({ \ 608 int __gu_err; \ 609 \ 610 __get_user_unaligned_common((x), size, ptr); \ 611 __gu_err; \ 612 }) 613 614 #define __get_user_unaligned_check(x,ptr,size) \ 615 ({ \ 616 int __gu_err = -EFAULT; \ 617 const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ 618 \ 619 if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) \ 620 __get_user_unaligned_common((x), size, __gu_ptr); \ 621 \ 622 __gu_err; \ 623 }) 624 625 #define __get_data_unaligned_asm(val, insn, addr) \ 626 { \ 627 long __gu_tmp; \ 628 \ 629 __asm__ __volatile__( \ 630 "1: " insn " %1, %3 \n" \ 631 "2: \n" \ 632 " .insn \n" \ 633 " .section .fixup,\"ax\" \n" \ 634 "3: li %0, %4 \n" \ 635 " move %1, $0 \n" \ 636 " j 2b \n" \ 637 " .previous \n" \ 638 " .section __ex_table,\"a\" \n" \ 639 " "__UA_ADDR "\t1b, 3b \n" \ 640 " "__UA_ADDR "\t1b + 4, 3b \n" \ 641 " .previous \n" \ 642 : "=r" (__gu_err), "=r" (__gu_tmp) \ 643 : "0" (0), "o" (__m(addr)), "i" (-EFAULT)); \ 644 \ 645 (val) = (__typeof__(*(addr))) __gu_tmp; \ 646 } 647 648 /* 649 * Get a long long 64 using 32 bit registers. 650 */ 651 #define __get_user_unaligned_asm_ll32(val, addr) \ 652 { \ 653 unsigned long long __gu_tmp; \ 654 \ 655 __asm__ __volatile__( \ 656 "1: ulw %1, (%3) \n" \ 657 "2: ulw %D1, 4(%3) \n" \ 658 " move %0, $0 \n" \ 659 "3: \n" \ 660 " .insn \n" \ 661 " .section .fixup,\"ax\" \n" \ 662 "4: li %0, %4 \n" \ 663 " move %1, $0 \n" \ 664 " move %D1, $0 \n" \ 665 " j 3b \n" \ 666 " .previous \n" \ 667 " .section __ex_table,\"a\" \n" \ 668 " " __UA_ADDR " 1b, 4b \n" \ 669 " " __UA_ADDR " 1b + 4, 4b \n" \ 670 " " __UA_ADDR " 2b, 4b \n" \ 671 " " __UA_ADDR " 2b + 4, 4b \n" \ 672 " .previous \n" \ 673 : "=r" (__gu_err), "=&r" (__gu_tmp) \ 674 : "0" (0), "r" (addr), "i" (-EFAULT)); \ 675 (val) = (__typeof__(*(addr))) __gu_tmp; \ 676 } 677 678 /* 679 * Yuck. We need two variants, one for 64bit operation and one 680 * for 32 bit mode and old iron. 681 */ 682 #ifdef CONFIG_32BIT 683 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr) 684 #endif 685 #ifdef CONFIG_64BIT 686 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr) 687 #endif 688 689 #define __put_user_unaligned_common(ptr, size) \ 690 do { \ 691 switch (size) { \ 692 case 1: __put_data_asm("sb", ptr); break; \ 693 case 2: __put_user_unaligned_asm("ush", ptr); break; \ 694 case 4: __put_user_unaligned_asm("usw", ptr); break; \ 695 case 8: __PUT_USER_UNALIGNED_DW(ptr); break; \ 696 default: __put_user_unaligned_unknown(); break; \ 697 } while (0) 698 699 #define __put_user_unaligned_nocheck(x,ptr,size) \ 700 ({ \ 701 __typeof__(*(ptr)) __pu_val; \ 702 int __pu_err = 0; \ 703 \ 704 __pu_val = (x); \ 705 __put_user_unaligned_common(ptr, size); \ 706 __pu_err; \ 707 }) 708 709 #define __put_user_unaligned_check(x,ptr,size) \ 710 ({ \ 711 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 712 __typeof__(*(ptr)) __pu_val = (x); \ 713 int __pu_err = -EFAULT; \ 714 \ 715 if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) \ 716 __put_user_unaligned_common(__pu_addr, size); \ 717 \ 718 __pu_err; \ 719 }) 720 721 #define __put_user_unaligned_asm(insn, ptr) \ 722 { \ 723 __asm__ __volatile__( \ 724 "1: " insn " %z2, %3 # __put_user_unaligned_asm\n" \ 725 "2: \n" \ 726 " .insn \n" \ 727 " .section .fixup,\"ax\" \n" \ 728 "3: li %0, %4 \n" \ 729 " j 2b \n" \ 730 " .previous \n" \ 731 " .section __ex_table,\"a\" \n" \ 732 " " __UA_ADDR " 1b, 3b \n" \ 733 " .previous \n" \ 734 : "=r" (__pu_err) \ 735 : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)), \ 736 "i" (-EFAULT)); \ 737 } 738 739 #define __put_user_unaligned_asm_ll32(ptr) \ 740 { \ 741 __asm__ __volatile__( \ 742 "1: sw %2, (%3) # __put_user_unaligned_asm_ll32 \n" \ 743 "2: sw %D2, 4(%3) \n" \ 744 "3: \n" \ 745 " .insn \n" \ 746 " .section .fixup,\"ax\" \n" \ 747 "4: li %0, %4 \n" \ 748 " j 3b \n" \ 749 " .previous \n" \ 750 " .section __ex_table,\"a\" \n" \ 751 " " __UA_ADDR " 1b, 4b \n" \ 752 " " __UA_ADDR " 1b + 4, 4b \n" \ 753 " " __UA_ADDR " 2b, 4b \n" \ 754 " " __UA_ADDR " 2b + 4, 4b \n" \ 755 " .previous" \ 756 : "=r" (__pu_err) \ 757 : "0" (0), "r" (__pu_val), "r" (ptr), \ 758 "i" (-EFAULT)); \ 759 } 760 761 extern void __put_user_unaligned_unknown(void); 762 #endif 763 764 /* 765 * We're generating jump to subroutines which will be outside the range of 766 * jump instructions 767 */ 768 #ifdef MODULE 769 #define __MODULE_JAL(destination) \ 770 ".set\tnoat\n\t" \ 771 __UA_LA "\t$1, " #destination "\n\t" \ 772 "jalr\t$1\n\t" \ 773 ".set\tat\n\t" 774 #else 775 #define __MODULE_JAL(destination) \ 776 "jal\t" #destination "\n\t" 777 #endif 778 779 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) && \ 780 defined(CONFIG_CPU_HAS_PREFETCH)) 781 #define DADDI_SCRATCH "$3" 782 #else 783 #define DADDI_SCRATCH "$0" 784 #endif 785 786 extern size_t __copy_user(void *__to, const void *__from, size_t __n); 787 788 #ifndef CONFIG_EVA 789 #define __invoke_copy_to_user(to, from, n) \ 790 ({ \ 791 register void __user *__cu_to_r __asm__("$4"); \ 792 register const void *__cu_from_r __asm__("$5"); \ 793 register long __cu_len_r __asm__("$6"); \ 794 \ 795 __cu_to_r = (to); \ 796 __cu_from_r = (from); \ 797 __cu_len_r = (n); \ 798 __asm__ __volatile__( \ 799 __MODULE_JAL(__copy_user) \ 800 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 801 : \ 802 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 803 DADDI_SCRATCH, "memory"); \ 804 __cu_len_r; \ 805 }) 806 807 #define __invoke_copy_to_kernel(to, from, n) \ 808 __invoke_copy_to_user(to, from, n) 809 810 #endif 811 812 /* 813 * __copy_to_user: - Copy a block of data into user space, with less checking. 814 * @to: Destination address, in user space. 815 * @from: Source address, in kernel space. 816 * @n: Number of bytes to copy. 817 * 818 * Context: User context only. This function may sleep. 819 * 820 * Copy data from kernel space to user space. Caller must check 821 * the specified block with access_ok() before calling this function. 822 * 823 * Returns number of bytes that could not be copied. 824 * On success, this will be zero. 825 */ 826 #define __copy_to_user(to, from, n) \ 827 ({ \ 828 void __user *__cu_to; \ 829 const void *__cu_from; \ 830 long __cu_len; \ 831 \ 832 __cu_to = (to); \ 833 __cu_from = (from); \ 834 __cu_len = (n); \ 835 might_fault(); \ 836 if (segment_eq(get_fs(), get_ds())) \ 837 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \ 838 __cu_len); \ 839 else \ 840 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ 841 __cu_len); \ 842 __cu_len; \ 843 }) 844 845 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); 846 847 #define __copy_to_user_inatomic(to, from, n) \ 848 ({ \ 849 void __user *__cu_to; \ 850 const void *__cu_from; \ 851 long __cu_len; \ 852 \ 853 __cu_to = (to); \ 854 __cu_from = (from); \ 855 __cu_len = (n); \ 856 if (segment_eq(get_fs(), get_ds())) \ 857 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \ 858 __cu_len); \ 859 else \ 860 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ 861 __cu_len); \ 862 __cu_len; \ 863 }) 864 865 #define __copy_from_user_inatomic(to, from, n) \ 866 ({ \ 867 void *__cu_to; \ 868 const void __user *__cu_from; \ 869 long __cu_len; \ 870 \ 871 __cu_to = (to); \ 872 __cu_from = (from); \ 873 __cu_len = (n); \ 874 if (segment_eq(get_fs(), get_ds())) \ 875 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to, \ 876 __cu_from,\ 877 __cu_len);\ 878 else \ 879 __cu_len = __invoke_copy_from_user_inatomic(__cu_to, \ 880 __cu_from, \ 881 __cu_len); \ 882 __cu_len; \ 883 }) 884 885 /* 886 * copy_to_user: - Copy a block of data into user space. 887 * @to: Destination address, in user space. 888 * @from: Source address, in kernel space. 889 * @n: Number of bytes to copy. 890 * 891 * Context: User context only. This function may sleep. 892 * 893 * Copy data from kernel space to user space. 894 * 895 * Returns number of bytes that could not be copied. 896 * On success, this will be zero. 897 */ 898 #define copy_to_user(to, from, n) \ 899 ({ \ 900 void __user *__cu_to; \ 901 const void *__cu_from; \ 902 long __cu_len; \ 903 \ 904 __cu_to = (to); \ 905 __cu_from = (from); \ 906 __cu_len = (n); \ 907 if (segment_eq(get_fs(), get_ds())) { \ 908 __cu_len = __invoke_copy_to_kernel(__cu_to, \ 909 __cu_from, \ 910 __cu_len); \ 911 } else { \ 912 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \ 913 might_fault(); \ 914 __cu_len = __invoke_copy_to_user(__cu_to, \ 915 __cu_from, \ 916 __cu_len); \ 917 } \ 918 } \ 919 __cu_len; \ 920 }) 921 922 #ifndef CONFIG_EVA 923 924 #define __invoke_copy_from_user(to, from, n) \ 925 ({ \ 926 register void *__cu_to_r __asm__("$4"); \ 927 register const void __user *__cu_from_r __asm__("$5"); \ 928 register long __cu_len_r __asm__("$6"); \ 929 \ 930 __cu_to_r = (to); \ 931 __cu_from_r = (from); \ 932 __cu_len_r = (n); \ 933 __asm__ __volatile__( \ 934 ".set\tnoreorder\n\t" \ 935 __MODULE_JAL(__copy_user) \ 936 ".set\tnoat\n\t" \ 937 __UA_ADDU "\t$1, %1, %2\n\t" \ 938 ".set\tat\n\t" \ 939 ".set\treorder" \ 940 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 941 : \ 942 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 943 DADDI_SCRATCH, "memory"); \ 944 __cu_len_r; \ 945 }) 946 947 #define __invoke_copy_from_kernel(to, from, n) \ 948 __invoke_copy_from_user(to, from, n) 949 950 /* For userland <-> userland operations */ 951 #define ___invoke_copy_in_user(to, from, n) \ 952 __invoke_copy_from_user(to, from, n) 953 954 /* For kernel <-> kernel operations */ 955 #define ___invoke_copy_in_kernel(to, from, n) \ 956 __invoke_copy_from_user(to, from, n) 957 958 #define __invoke_copy_from_user_inatomic(to, from, n) \ 959 ({ \ 960 register void *__cu_to_r __asm__("$4"); \ 961 register const void __user *__cu_from_r __asm__("$5"); \ 962 register long __cu_len_r __asm__("$6"); \ 963 \ 964 __cu_to_r = (to); \ 965 __cu_from_r = (from); \ 966 __cu_len_r = (n); \ 967 __asm__ __volatile__( \ 968 ".set\tnoreorder\n\t" \ 969 __MODULE_JAL(__copy_user_inatomic) \ 970 ".set\tnoat\n\t" \ 971 __UA_ADDU "\t$1, %1, %2\n\t" \ 972 ".set\tat\n\t" \ 973 ".set\treorder" \ 974 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 975 : \ 976 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 977 DADDI_SCRATCH, "memory"); \ 978 __cu_len_r; \ 979 }) 980 981 #define __invoke_copy_from_kernel_inatomic(to, from, n) \ 982 __invoke_copy_from_user_inatomic(to, from, n) \ 983 984 #else 985 986 /* EVA specific functions */ 987 988 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from, 989 size_t __n); 990 extern size_t __copy_from_user_eva(void *__to, const void *__from, 991 size_t __n); 992 extern size_t __copy_to_user_eva(void *__to, const void *__from, 993 size_t __n); 994 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); 995 996 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr) \ 997 ({ \ 998 register void *__cu_to_r __asm__("$4"); \ 999 register const void __user *__cu_from_r __asm__("$5"); \ 1000 register long __cu_len_r __asm__("$6"); \ 1001 \ 1002 __cu_to_r = (to); \ 1003 __cu_from_r = (from); \ 1004 __cu_len_r = (n); \ 1005 __asm__ __volatile__( \ 1006 ".set\tnoreorder\n\t" \ 1007 __MODULE_JAL(func_ptr) \ 1008 ".set\tnoat\n\t" \ 1009 __UA_ADDU "\t$1, %1, %2\n\t" \ 1010 ".set\tat\n\t" \ 1011 ".set\treorder" \ 1012 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 1013 : \ 1014 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 1015 DADDI_SCRATCH, "memory"); \ 1016 __cu_len_r; \ 1017 }) 1018 1019 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr) \ 1020 ({ \ 1021 register void *__cu_to_r __asm__("$4"); \ 1022 register const void __user *__cu_from_r __asm__("$5"); \ 1023 register long __cu_len_r __asm__("$6"); \ 1024 \ 1025 __cu_to_r = (to); \ 1026 __cu_from_r = (from); \ 1027 __cu_len_r = (n); \ 1028 __asm__ __volatile__( \ 1029 __MODULE_JAL(func_ptr) \ 1030 : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ 1031 : \ 1032 : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ 1033 DADDI_SCRATCH, "memory"); \ 1034 __cu_len_r; \ 1035 }) 1036 1037 /* 1038 * Source or destination address is in userland. We need to go through 1039 * the TLB 1040 */ 1041 #define __invoke_copy_from_user(to, from, n) \ 1042 __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva) 1043 1044 #define __invoke_copy_from_user_inatomic(to, from, n) \ 1045 __invoke_copy_from_user_eva_generic(to, from, n, \ 1046 __copy_user_inatomic_eva) 1047 1048 #define __invoke_copy_to_user(to, from, n) \ 1049 __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva) 1050 1051 #define ___invoke_copy_in_user(to, from, n) \ 1052 __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva) 1053 1054 /* 1055 * Source or destination address in the kernel. We are not going through 1056 * the TLB 1057 */ 1058 #define __invoke_copy_from_kernel(to, from, n) \ 1059 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user) 1060 1061 #define __invoke_copy_from_kernel_inatomic(to, from, n) \ 1062 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic) 1063 1064 #define __invoke_copy_to_kernel(to, from, n) \ 1065 __invoke_copy_to_user_eva_generic(to, from, n, __copy_user) 1066 1067 #define ___invoke_copy_in_kernel(to, from, n) \ 1068 __invoke_copy_from_user_eva_generic(to, from, n, __copy_user) 1069 1070 #endif /* CONFIG_EVA */ 1071 1072 /* 1073 * __copy_from_user: - Copy a block of data from user space, with less checking. 1074 * @to: Destination address, in kernel space. 1075 * @from: Source address, in user space. 1076 * @n: Number of bytes to copy. 1077 * 1078 * Context: User context only. This function may sleep. 1079 * 1080 * Copy data from user space to kernel space. Caller must check 1081 * the specified block with access_ok() before calling this function. 1082 * 1083 * Returns number of bytes that could not be copied. 1084 * On success, this will be zero. 1085 * 1086 * If some data could not be copied, this function will pad the copied 1087 * data to the requested size using zero bytes. 1088 */ 1089 #define __copy_from_user(to, from, n) \ 1090 ({ \ 1091 void *__cu_to; \ 1092 const void __user *__cu_from; \ 1093 long __cu_len; \ 1094 \ 1095 __cu_to = (to); \ 1096 __cu_from = (from); \ 1097 __cu_len = (n); \ 1098 might_fault(); \ 1099 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ 1100 __cu_len); \ 1101 __cu_len; \ 1102 }) 1103 1104 /* 1105 * copy_from_user: - Copy a block of data from user space. 1106 * @to: Destination address, in kernel space. 1107 * @from: Source address, in user space. 1108 * @n: Number of bytes to copy. 1109 * 1110 * Context: User context only. This function may sleep. 1111 * 1112 * Copy data from user space to kernel space. 1113 * 1114 * Returns number of bytes that could not be copied. 1115 * On success, this will be zero. 1116 * 1117 * If some data could not be copied, this function will pad the copied 1118 * data to the requested size using zero bytes. 1119 */ 1120 #define copy_from_user(to, from, n) \ 1121 ({ \ 1122 void *__cu_to; \ 1123 const void __user *__cu_from; \ 1124 long __cu_len; \ 1125 \ 1126 __cu_to = (to); \ 1127 __cu_from = (from); \ 1128 __cu_len = (n); \ 1129 if (segment_eq(get_fs(), get_ds())) { \ 1130 __cu_len = __invoke_copy_from_kernel(__cu_to, \ 1131 __cu_from, \ 1132 __cu_len); \ 1133 } else { \ 1134 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \ 1135 might_fault(); \ 1136 __cu_len = __invoke_copy_from_user(__cu_to, \ 1137 __cu_from, \ 1138 __cu_len); \ 1139 } \ 1140 } \ 1141 __cu_len; \ 1142 }) 1143 1144 #define __copy_in_user(to, from, n) \ 1145 ({ \ 1146 void __user *__cu_to; \ 1147 const void __user *__cu_from; \ 1148 long __cu_len; \ 1149 \ 1150 __cu_to = (to); \ 1151 __cu_from = (from); \ 1152 __cu_len = (n); \ 1153 if (segment_eq(get_fs(), get_ds())) { \ 1154 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \ 1155 __cu_len); \ 1156 } else { \ 1157 might_fault(); \ 1158 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from, \ 1159 __cu_len); \ 1160 } \ 1161 __cu_len; \ 1162 }) 1163 1164 #define copy_in_user(to, from, n) \ 1165 ({ \ 1166 void __user *__cu_to; \ 1167 const void __user *__cu_from; \ 1168 long __cu_len; \ 1169 \ 1170 __cu_to = (to); \ 1171 __cu_from = (from); \ 1172 __cu_len = (n); \ 1173 if (segment_eq(get_fs(), get_ds())) { \ 1174 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from, \ 1175 __cu_len); \ 1176 } else { \ 1177 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\ 1178 access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\ 1179 might_fault(); \ 1180 __cu_len = ___invoke_copy_in_user(__cu_to, \ 1181 __cu_from, \ 1182 __cu_len); \ 1183 } \ 1184 } \ 1185 __cu_len; \ 1186 }) 1187 1188 /* 1189 * __clear_user: - Zero a block of memory in user space, with less checking. 1190 * @to: Destination address, in user space. 1191 * @n: Number of bytes to zero. 1192 * 1193 * Zero a block of memory in user space. Caller must check 1194 * the specified block with access_ok() before calling this function. 1195 * 1196 * Returns number of bytes that could not be cleared. 1197 * On success, this will be zero. 1198 */ 1199 static inline __kernel_size_t 1200 __clear_user(void __user *addr, __kernel_size_t size) 1201 { 1202 __kernel_size_t res; 1203 1204 might_fault(); 1205 __asm__ __volatile__( 1206 "move\t$4, %1\n\t" 1207 "move\t$5, $0\n\t" 1208 "move\t$6, %2\n\t" 1209 __MODULE_JAL(__bzero) 1210 "move\t%0, $6" 1211 : "=r" (res) 1212 : "r" (addr), "r" (size) 1213 : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31"); 1214 1215 return res; 1216 } 1217 1218 #define clear_user(addr,n) \ 1219 ({ \ 1220 void __user * __cl_addr = (addr); \ 1221 unsigned long __cl_size = (n); \ 1222 if (__cl_size && access_ok(VERIFY_WRITE, \ 1223 __cl_addr, __cl_size)) \ 1224 __cl_size = __clear_user(__cl_addr, __cl_size); \ 1225 __cl_size; \ 1226 }) 1227 1228 /* 1229 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. 1230 * @dst: Destination address, in kernel space. This buffer must be at 1231 * least @count bytes long. 1232 * @src: Source address, in user space. 1233 * @count: Maximum number of bytes to copy, including the trailing NUL. 1234 * 1235 * Copies a NUL-terminated string from userspace to kernel space. 1236 * Caller must check the specified block with access_ok() before calling 1237 * this function. 1238 * 1239 * On success, returns the length of the string (not including the trailing 1240 * NUL). 1241 * 1242 * If access to userspace fails, returns -EFAULT (some data may have been 1243 * copied). 1244 * 1245 * If @count is smaller than the length of the string, copies @count bytes 1246 * and returns @count. 1247 */ 1248 static inline long 1249 __strncpy_from_user(char *__to, const char __user *__from, long __len) 1250 { 1251 long res; 1252 1253 if (segment_eq(get_fs(), get_ds())) { 1254 __asm__ __volatile__( 1255 "move\t$4, %1\n\t" 1256 "move\t$5, %2\n\t" 1257 "move\t$6, %3\n\t" 1258 __MODULE_JAL(__strncpy_from_kernel_nocheck_asm) 1259 "move\t%0, $2" 1260 : "=r" (res) 1261 : "r" (__to), "r" (__from), "r" (__len) 1262 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1263 } else { 1264 might_fault(); 1265 __asm__ __volatile__( 1266 "move\t$4, %1\n\t" 1267 "move\t$5, %2\n\t" 1268 "move\t$6, %3\n\t" 1269 __MODULE_JAL(__strncpy_from_user_nocheck_asm) 1270 "move\t%0, $2" 1271 : "=r" (res) 1272 : "r" (__to), "r" (__from), "r" (__len) 1273 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1274 } 1275 1276 return res; 1277 } 1278 1279 /* 1280 * strncpy_from_user: - Copy a NUL terminated string from userspace. 1281 * @dst: Destination address, in kernel space. This buffer must be at 1282 * least @count bytes long. 1283 * @src: Source address, in user space. 1284 * @count: Maximum number of bytes to copy, including the trailing NUL. 1285 * 1286 * Copies a NUL-terminated string from userspace to kernel space. 1287 * 1288 * On success, returns the length of the string (not including the trailing 1289 * NUL). 1290 * 1291 * If access to userspace fails, returns -EFAULT (some data may have been 1292 * copied). 1293 * 1294 * If @count is smaller than the length of the string, copies @count bytes 1295 * and returns @count. 1296 */ 1297 static inline long 1298 strncpy_from_user(char *__to, const char __user *__from, long __len) 1299 { 1300 long res; 1301 1302 if (segment_eq(get_fs(), get_ds())) { 1303 __asm__ __volatile__( 1304 "move\t$4, %1\n\t" 1305 "move\t$5, %2\n\t" 1306 "move\t$6, %3\n\t" 1307 __MODULE_JAL(__strncpy_from_kernel_asm) 1308 "move\t%0, $2" 1309 : "=r" (res) 1310 : "r" (__to), "r" (__from), "r" (__len) 1311 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1312 } else { 1313 might_fault(); 1314 __asm__ __volatile__( 1315 "move\t$4, %1\n\t" 1316 "move\t$5, %2\n\t" 1317 "move\t$6, %3\n\t" 1318 __MODULE_JAL(__strncpy_from_user_asm) 1319 "move\t%0, $2" 1320 : "=r" (res) 1321 : "r" (__to), "r" (__from), "r" (__len) 1322 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory"); 1323 } 1324 1325 return res; 1326 } 1327 1328 /* 1329 * strlen_user: - Get the size of a string in user space. 1330 * @str: The string to measure. 1331 * 1332 * Context: User context only. This function may sleep. 1333 * 1334 * Get the size of a NUL-terminated string in user space. 1335 * 1336 * Returns the size of the string INCLUDING the terminating NUL. 1337 * On exception, returns 0. 1338 * 1339 * If there is a limit on the length of a valid string, you may wish to 1340 * consider using strnlen_user() instead. 1341 */ 1342 static inline long strlen_user(const char __user *s) 1343 { 1344 long res; 1345 1346 if (segment_eq(get_fs(), get_ds())) { 1347 __asm__ __volatile__( 1348 "move\t$4, %1\n\t" 1349 __MODULE_JAL(__strlen_kernel_asm) 1350 "move\t%0, $2" 1351 : "=r" (res) 1352 : "r" (s) 1353 : "$2", "$4", __UA_t0, "$31"); 1354 } else { 1355 might_fault(); 1356 __asm__ __volatile__( 1357 "move\t$4, %1\n\t" 1358 __MODULE_JAL(__strlen_kernel_asm) 1359 "move\t%0, $2" 1360 : "=r" (res) 1361 : "r" (s) 1362 : "$2", "$4", __UA_t0, "$31"); 1363 } 1364 1365 return res; 1366 } 1367 1368 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */ 1369 static inline long __strnlen_user(const char __user *s, long n) 1370 { 1371 long res; 1372 1373 if (segment_eq(get_fs(), get_ds())) { 1374 __asm__ __volatile__( 1375 "move\t$4, %1\n\t" 1376 "move\t$5, %2\n\t" 1377 __MODULE_JAL(__strnlen_kernel_nocheck_asm) 1378 "move\t%0, $2" 1379 : "=r" (res) 1380 : "r" (s), "r" (n) 1381 : "$2", "$4", "$5", __UA_t0, "$31"); 1382 } else { 1383 might_fault(); 1384 __asm__ __volatile__( 1385 "move\t$4, %1\n\t" 1386 "move\t$5, %2\n\t" 1387 __MODULE_JAL(__strnlen_user_nocheck_asm) 1388 "move\t%0, $2" 1389 : "=r" (res) 1390 : "r" (s), "r" (n) 1391 : "$2", "$4", "$5", __UA_t0, "$31"); 1392 } 1393 1394 return res; 1395 } 1396 1397 /* 1398 * strnlen_user: - Get the size of a string in user space. 1399 * @str: The string to measure. 1400 * 1401 * Context: User context only. This function may sleep. 1402 * 1403 * Get the size of a NUL-terminated string in user space. 1404 * 1405 * Returns the size of the string INCLUDING the terminating NUL. 1406 * On exception, returns 0. 1407 * If the string is too long, returns a value greater than @n. 1408 */ 1409 static inline long strnlen_user(const char __user *s, long n) 1410 { 1411 long res; 1412 1413 might_fault(); 1414 if (segment_eq(get_fs(), get_ds())) { 1415 __asm__ __volatile__( 1416 "move\t$4, %1\n\t" 1417 "move\t$5, %2\n\t" 1418 __MODULE_JAL(__strnlen_kernel_asm) 1419 "move\t%0, $2" 1420 : "=r" (res) 1421 : "r" (s), "r" (n) 1422 : "$2", "$4", "$5", __UA_t0, "$31"); 1423 } else { 1424 __asm__ __volatile__( 1425 "move\t$4, %1\n\t" 1426 "move\t$5, %2\n\t" 1427 __MODULE_JAL(__strnlen_user_asm) 1428 "move\t%0, $2" 1429 : "=r" (res) 1430 : "r" (s), "r" (n) 1431 : "$2", "$4", "$5", __UA_t0, "$31"); 1432 } 1433 1434 return res; 1435 } 1436 1437 struct exception_table_entry 1438 { 1439 unsigned long insn; 1440 unsigned long nextinsn; 1441 }; 1442 1443 extern int fixup_exception(struct pt_regs *regs); 1444 1445 #endif /* _ASM_UACCESS_H */ 1446