1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * S390 version 4 * Copyright IBM Corp. 1999, 2000 5 * Author(s): Hartmut Penner (hp@de.ibm.com), 6 * Martin Schwidefsky (schwidefsky@de.ibm.com) 7 * 8 * Derived from "include/asm-i386/uaccess.h" 9 */ 10 #ifndef __S390_UACCESS_H 11 #define __S390_UACCESS_H 12 13 /* 14 * User space memory access functions 15 */ 16 #include <asm/asm-extable.h> 17 #include <asm/processor.h> 18 #include <asm/ctl_reg.h> 19 #include <asm/extable.h> 20 #include <asm/facility.h> 21 #include <asm-generic/access_ok.h> 22 23 void debug_user_asce(int exit); 24 25 unsigned long __must_check 26 raw_copy_from_user(void *to, const void __user *from, unsigned long n); 27 28 unsigned long __must_check 29 raw_copy_to_user(void __user *to, const void *from, unsigned long n); 30 31 #ifndef CONFIG_KASAN 32 #define INLINE_COPY_FROM_USER 33 #define INLINE_COPY_TO_USER 34 #endif 35 36 unsigned long __must_check 37 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key); 38 39 static __always_inline unsigned long __must_check 40 copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key) 41 { 42 if (check_copy_size(to, n, false)) 43 n = _copy_from_user_key(to, from, n, key); 44 return n; 45 } 46 47 unsigned long __must_check 48 _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key); 49 50 static __always_inline unsigned long __must_check 51 copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key) 52 { 53 if (check_copy_size(from, n, true)) 54 n = _copy_to_user_key(to, from, n, key); 55 return n; 56 } 57 58 union oac { 59 unsigned int val; 60 struct { 61 struct { 62 unsigned short key : 4; 63 unsigned short : 4; 64 unsigned short as : 2; 65 unsigned short : 4; 66 unsigned short k : 1; 67 unsigned short a : 1; 68 } oac1; 69 struct { 70 unsigned short key : 4; 71 unsigned short : 4; 72 unsigned short as : 2; 73 unsigned short : 4; 74 unsigned short k : 1; 75 unsigned short a : 1; 76 } oac2; 77 }; 78 }; 79 80 int __noreturn __put_user_bad(void); 81 82 #define __put_user_asm(to, from, size) \ 83 ({ \ 84 union oac __oac_spec = { \ 85 .oac1.as = PSW_BITS_AS_SECONDARY, \ 86 .oac1.a = 1, \ 87 }; \ 88 int __rc; \ 89 \ 90 asm volatile( \ 91 " lr 0,%[spec]\n" \ 92 "0: mvcos %[_to],%[_from],%[_size]\n" \ 93 "1: xr %[rc],%[rc]\n" \ 94 "2:\n" \ 95 EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ 96 EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ 97 : [rc] "=&d" (__rc), [_to] "+Q" (*(to)) \ 98 : [_size] "d" (size), [_from] "Q" (*(from)), \ 99 [spec] "d" (__oac_spec.val) \ 100 : "cc", "0"); \ 101 __rc; \ 102 }) 103 104 static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) 105 { 106 int rc; 107 108 switch (size) { 109 case 1: 110 rc = __put_user_asm((unsigned char __user *)ptr, 111 (unsigned char *)x, 112 size); 113 break; 114 case 2: 115 rc = __put_user_asm((unsigned short __user *)ptr, 116 (unsigned short *)x, 117 size); 118 break; 119 case 4: 120 rc = __put_user_asm((unsigned int __user *)ptr, 121 (unsigned int *)x, 122 size); 123 break; 124 case 8: 125 rc = __put_user_asm((unsigned long __user *)ptr, 126 (unsigned long *)x, 127 size); 128 break; 129 default: 130 __put_user_bad(); 131 break; 132 } 133 return rc; 134 } 135 136 int __noreturn __get_user_bad(void); 137 138 #define __get_user_asm(to, from, size) \ 139 ({ \ 140 union oac __oac_spec = { \ 141 .oac2.as = PSW_BITS_AS_SECONDARY, \ 142 .oac2.a = 1, \ 143 }; \ 144 int __rc; \ 145 \ 146 asm volatile( \ 147 " lr 0,%[spec]\n" \ 148 "0: mvcos 0(%[_to]),%[_from],%[_size]\n" \ 149 "1: xr %[rc],%[rc]\n" \ 150 "2:\n" \ 151 EX_TABLE_UA_LOAD_MEM(0b, 2b, %[rc], %[_to], %[_ksize]) \ 152 EX_TABLE_UA_LOAD_MEM(1b, 2b, %[rc], %[_to], %[_ksize]) \ 153 : [rc] "=&d" (__rc), "=Q" (*(to)) \ 154 : [_size] "d" (size), [_from] "Q" (*(from)), \ 155 [spec] "d" (__oac_spec.val), [_to] "a" (to), \ 156 [_ksize] "K" (size) \ 157 : "cc", "0"); \ 158 __rc; \ 159 }) 160 161 static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) 162 { 163 int rc; 164 165 switch (size) { 166 case 1: 167 rc = __get_user_asm((unsigned char *)x, 168 (unsigned char __user *)ptr, 169 size); 170 break; 171 case 2: 172 rc = __get_user_asm((unsigned short *)x, 173 (unsigned short __user *)ptr, 174 size); 175 break; 176 case 4: 177 rc = __get_user_asm((unsigned int *)x, 178 (unsigned int __user *)ptr, 179 size); 180 break; 181 case 8: 182 rc = __get_user_asm((unsigned long *)x, 183 (unsigned long __user *)ptr, 184 size); 185 break; 186 default: 187 __get_user_bad(); 188 break; 189 } 190 return rc; 191 } 192 193 /* 194 * These are the main single-value transfer routines. They automatically 195 * use the right size if we just have the right pointer type. 196 */ 197 #define __put_user(x, ptr) \ 198 ({ \ 199 __typeof__(*(ptr)) __x = (x); \ 200 int __pu_err = -EFAULT; \ 201 \ 202 __chk_user_ptr(ptr); \ 203 switch (sizeof(*(ptr))) { \ 204 case 1: \ 205 case 2: \ 206 case 4: \ 207 case 8: \ 208 __pu_err = __put_user_fn(&__x, ptr, sizeof(*(ptr))); \ 209 break; \ 210 default: \ 211 __put_user_bad(); \ 212 break; \ 213 } \ 214 __builtin_expect(__pu_err, 0); \ 215 }) 216 217 #define put_user(x, ptr) \ 218 ({ \ 219 might_fault(); \ 220 __put_user(x, ptr); \ 221 }) 222 223 #define __get_user(x, ptr) \ 224 ({ \ 225 int __gu_err = -EFAULT; \ 226 \ 227 __chk_user_ptr(ptr); \ 228 switch (sizeof(*(ptr))) { \ 229 case 1: { \ 230 unsigned char __x; \ 231 \ 232 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 233 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 234 break; \ 235 }; \ 236 case 2: { \ 237 unsigned short __x; \ 238 \ 239 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 240 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 241 break; \ 242 }; \ 243 case 4: { \ 244 unsigned int __x; \ 245 \ 246 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 247 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 248 break; \ 249 }; \ 250 case 8: { \ 251 unsigned long __x; \ 252 \ 253 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 254 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 255 break; \ 256 }; \ 257 default: \ 258 __get_user_bad(); \ 259 break; \ 260 } \ 261 __builtin_expect(__gu_err, 0); \ 262 }) 263 264 #define get_user(x, ptr) \ 265 ({ \ 266 might_fault(); \ 267 __get_user(x, ptr); \ 268 }) 269 270 /* 271 * Copy a null terminated string from userspace. 272 */ 273 long __must_check strncpy_from_user(char *dst, const char __user *src, long count); 274 275 long __must_check strnlen_user(const char __user *src, long count); 276 277 /* 278 * Zero Userspace 279 */ 280 unsigned long __must_check __clear_user(void __user *to, unsigned long size); 281 282 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) 283 { 284 might_fault(); 285 return __clear_user(to, n); 286 } 287 288 void *s390_kernel_write(void *dst, const void *src, size_t size); 289 290 int __noreturn __put_kernel_bad(void); 291 292 #define __put_kernel_asm(val, to, insn) \ 293 ({ \ 294 int __rc; \ 295 \ 296 asm volatile( \ 297 "0: " insn " %[_val],%[_to]\n" \ 298 "1: xr %[rc],%[rc]\n" \ 299 "2:\n" \ 300 EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ 301 EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ 302 : [rc] "=d" (__rc), [_to] "+Q" (*(to)) \ 303 : [_val] "d" (val) \ 304 : "cc"); \ 305 __rc; \ 306 }) 307 308 #define __put_kernel_nofault(dst, src, type, err_label) \ 309 do { \ 310 unsigned long __x = (unsigned long)(*((type *)(src))); \ 311 int __pk_err; \ 312 \ 313 switch (sizeof(type)) { \ 314 case 1: \ 315 __pk_err = __put_kernel_asm(__x, (type *)(dst), "stc"); \ 316 break; \ 317 case 2: \ 318 __pk_err = __put_kernel_asm(__x, (type *)(dst), "sth"); \ 319 break; \ 320 case 4: \ 321 __pk_err = __put_kernel_asm(__x, (type *)(dst), "st"); \ 322 break; \ 323 case 8: \ 324 __pk_err = __put_kernel_asm(__x, (type *)(dst), "stg"); \ 325 break; \ 326 default: \ 327 __pk_err = __put_kernel_bad(); \ 328 break; \ 329 } \ 330 if (unlikely(__pk_err)) \ 331 goto err_label; \ 332 } while (0) 333 334 int __noreturn __get_kernel_bad(void); 335 336 #define __get_kernel_asm(val, from, insn) \ 337 ({ \ 338 int __rc; \ 339 \ 340 asm volatile( \ 341 "0: " insn " %[_val],%[_from]\n" \ 342 "1: xr %[rc],%[rc]\n" \ 343 "2:\n" \ 344 EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val]) \ 345 EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val]) \ 346 : [rc] "=d" (__rc), [_val] "=d" (val) \ 347 : [_from] "Q" (*(from)) \ 348 : "cc"); \ 349 __rc; \ 350 }) 351 352 #define __get_kernel_nofault(dst, src, type, err_label) \ 353 do { \ 354 int __gk_err; \ 355 \ 356 switch (sizeof(type)) { \ 357 case 1: { \ 358 unsigned char __x; \ 359 \ 360 __gk_err = __get_kernel_asm(__x, (type *)(src), "ic"); \ 361 *((type *)(dst)) = (type)__x; \ 362 break; \ 363 }; \ 364 case 2: { \ 365 unsigned short __x; \ 366 \ 367 __gk_err = __get_kernel_asm(__x, (type *)(src), "lh"); \ 368 *((type *)(dst)) = (type)__x; \ 369 break; \ 370 }; \ 371 case 4: { \ 372 unsigned int __x; \ 373 \ 374 __gk_err = __get_kernel_asm(__x, (type *)(src), "l"); \ 375 *((type *)(dst)) = (type)__x; \ 376 break; \ 377 }; \ 378 case 8: { \ 379 unsigned long __x; \ 380 \ 381 __gk_err = __get_kernel_asm(__x, (type *)(src), "lg"); \ 382 *((type *)(dst)) = (type)__x; \ 383 break; \ 384 }; \ 385 default: \ 386 __gk_err = __get_kernel_bad(); \ 387 break; \ 388 } \ 389 if (unlikely(__gk_err)) \ 390 goto err_label; \ 391 } while (0) 392 393 void __cmpxchg_user_key_called_with_bad_pointer(void); 394 395 static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 396 __uint128_t old, __uint128_t new, 397 unsigned long key, int size) 398 { 399 int rc = 0; 400 401 switch (size) { 402 case 1: { 403 unsigned int prev, shift, mask, _old, _new; 404 405 shift = (3 ^ (address & 3)) << 3; 406 address ^= address & 3; 407 _old = (old & 0xff) << shift; 408 _new = (new & 0xff) << shift; 409 mask = ~(0xff << shift); 410 asm volatile( 411 " spka 0(%[key])\n" 412 " sacf 256\n" 413 "0: l %[prev],%[address]\n" 414 "1: nr %[prev],%[mask]\n" 415 " xilf %[mask],0xffffffff\n" 416 " or %[new],%[prev]\n" 417 " or %[prev],%[tmp]\n" 418 "2: lr %[tmp],%[prev]\n" 419 "3: cs %[prev],%[new],%[address]\n" 420 "4: jnl 5f\n" 421 " xr %[tmp],%[prev]\n" 422 " xr %[new],%[tmp]\n" 423 " nr %[tmp],%[mask]\n" 424 " jz 2b\n" 425 "5: sacf 768\n" 426 " spka %[default_key]\n" 427 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) 428 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) 429 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) 430 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) 431 : [rc] "+&d" (rc), 432 [prev] "=&d" (prev), 433 [address] "+Q" (*(int *)address), 434 [tmp] "+&d" (_old), 435 [new] "+&d" (_new), 436 [mask] "+&d" (mask) 437 : [key] "a" (key << 4), 438 [default_key] "J" (PAGE_DEFAULT_KEY) 439 : "memory", "cc"); 440 *(unsigned char *)uval = prev >> shift; 441 return rc; 442 } 443 case 2: { 444 unsigned int prev, shift, mask, _old, _new; 445 446 shift = (2 ^ (address & 2)) << 3; 447 address ^= address & 2; 448 _old = (old & 0xffff) << shift; 449 _new = (new & 0xffff) << shift; 450 mask = ~(0xffff << shift); 451 asm volatile( 452 " spka 0(%[key])\n" 453 " sacf 256\n" 454 "0: l %[prev],%[address]\n" 455 "1: nr %[prev],%[mask]\n" 456 " xilf %[mask],0xffffffff\n" 457 " or %[new],%[prev]\n" 458 " or %[prev],%[tmp]\n" 459 "2: lr %[tmp],%[prev]\n" 460 "3: cs %[prev],%[new],%[address]\n" 461 "4: jnl 5f\n" 462 " xr %[tmp],%[prev]\n" 463 " xr %[new],%[tmp]\n" 464 " nr %[tmp],%[mask]\n" 465 " jz 2b\n" 466 "5: sacf 768\n" 467 " spka %[default_key]\n" 468 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) 469 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) 470 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) 471 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) 472 : [rc] "+&d" (rc), 473 [prev] "=&d" (prev), 474 [address] "+Q" (*(int *)address), 475 [tmp] "+&d" (_old), 476 [new] "+&d" (_new), 477 [mask] "+&d" (mask) 478 : [key] "a" (key << 4), 479 [default_key] "J" (PAGE_DEFAULT_KEY) 480 : "memory", "cc"); 481 *(unsigned short *)uval = prev >> shift; 482 return rc; 483 } 484 case 4: { 485 unsigned int prev = old; 486 487 asm volatile( 488 " spka 0(%[key])\n" 489 " sacf 256\n" 490 "0: cs %[prev],%[new],%[address]\n" 491 "1: sacf 768\n" 492 " spka %[default_key]\n" 493 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 494 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 495 : [rc] "+&d" (rc), 496 [prev] "+&d" (prev), 497 [address] "+Q" (*(int *)address) 498 : [new] "d" ((unsigned int)new), 499 [key] "a" (key << 4), 500 [default_key] "J" (PAGE_DEFAULT_KEY) 501 : "memory", "cc"); 502 *(unsigned int *)uval = prev; 503 return rc; 504 } 505 case 8: { 506 unsigned long prev = old; 507 508 asm volatile( 509 " spka 0(%[key])\n" 510 " sacf 256\n" 511 "0: csg %[prev],%[new],%[address]\n" 512 "1: sacf 768\n" 513 " spka %[default_key]\n" 514 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 515 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 516 : [rc] "+&d" (rc), 517 [prev] "+&d" (prev), 518 [address] "+QS" (*(long *)address) 519 : [new] "d" ((unsigned long)new), 520 [key] "a" (key << 4), 521 [default_key] "J" (PAGE_DEFAULT_KEY) 522 : "memory", "cc"); 523 *(unsigned long *)uval = prev; 524 return rc; 525 } 526 case 16: { 527 __uint128_t prev = old; 528 529 asm volatile( 530 " spka 0(%[key])\n" 531 " sacf 256\n" 532 "0: cdsg %[prev],%[new],%[address]\n" 533 "1: sacf 768\n" 534 " spka %[default_key]\n" 535 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) 536 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) 537 : [rc] "+&d" (rc), 538 [prev] "+&d" (prev), 539 [address] "+QS" (*(__int128_t *)address) 540 : [new] "d" (new), 541 [key] "a" (key << 4), 542 [default_key] "J" (PAGE_DEFAULT_KEY) 543 : "memory", "cc"); 544 *(__uint128_t *)uval = prev; 545 return rc; 546 } 547 } 548 __cmpxchg_user_key_called_with_bad_pointer(); 549 return rc; 550 } 551 552 /** 553 * cmpxchg_user_key() - cmpxchg with user space target, honoring storage keys 554 * @ptr: User space address of value to compare to @old and exchange with 555 * @new. Must be aligned to sizeof(*@ptr). 556 * @uval: Address where the old value of *@ptr is written to. 557 * @old: Old value. Compared to the content pointed to by @ptr in order to 558 * determine if the exchange occurs. The old value read from *@ptr is 559 * written to *@uval. 560 * @new: New value to place at *@ptr. 561 * @key: Access key to use for checking storage key protection. 562 * 563 * Perform a cmpxchg on a user space target, honoring storage key protection. 564 * @key alone determines how key checking is performed, neither 565 * storage-protection-override nor fetch-protection-override apply. 566 * The caller must compare *@uval and @old to determine if values have been 567 * exchanged. In case of an exception *@uval is set to zero. 568 * 569 * Return: 0: cmpxchg executed 570 * -EFAULT: an exception happened when trying to access *@ptr 571 */ 572 #define cmpxchg_user_key(ptr, uval, old, new, key) \ 573 ({ \ 574 __typeof__(ptr) __ptr = (ptr); \ 575 __typeof__(uval) __uval = (uval); \ 576 \ 577 BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval))); \ 578 might_fault(); \ 579 __chk_user_ptr(__ptr); \ 580 __cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \ 581 (old), (new), (key), sizeof(*(__ptr))); \ 582 }) 583 584 #endif /* __S390_UACCESS_H */ 585