1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * S390 version 4 * Copyright IBM Corp. 1999, 2000 5 * Author(s): Hartmut Penner (hp@de.ibm.com), 6 * Martin Schwidefsky (schwidefsky@de.ibm.com) 7 * 8 * Derived from "include/asm-i386/uaccess.h" 9 */ 10 #ifndef __S390_UACCESS_H 11 #define __S390_UACCESS_H 12 13 /* 14 * User space memory access functions 15 */ 16 #include <asm/asm-extable.h> 17 #include <asm/processor.h> 18 #include <asm/ctl_reg.h> 19 #include <asm/extable.h> 20 #include <asm/facility.h> 21 #include <asm-generic/access_ok.h> 22 23 void debug_user_asce(int exit); 24 25 unsigned long __must_check 26 raw_copy_from_user(void *to, const void __user *from, unsigned long n); 27 28 unsigned long __must_check 29 raw_copy_to_user(void __user *to, const void *from, unsigned long n); 30 31 #ifndef CONFIG_KASAN 32 #define INLINE_COPY_FROM_USER 33 #define INLINE_COPY_TO_USER 34 #endif 35 36 unsigned long __must_check 37 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key); 38 39 static __always_inline unsigned long __must_check 40 copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key) 41 { 42 if (check_copy_size(to, n, false)) 43 n = _copy_from_user_key(to, from, n, key); 44 return n; 45 } 46 47 unsigned long __must_check 48 _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key); 49 50 static __always_inline unsigned long __must_check 51 copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key) 52 { 53 if (check_copy_size(from, n, true)) 54 n = _copy_to_user_key(to, from, n, key); 55 return n; 56 } 57 58 union oac { 59 unsigned int val; 60 struct { 61 struct { 62 unsigned short key : 4; 63 unsigned short : 4; 64 unsigned short as : 2; 65 unsigned short : 4; 66 unsigned short k : 1; 67 unsigned short a : 1; 68 } oac1; 69 struct { 70 unsigned short key : 4; 71 unsigned short : 4; 72 unsigned short as : 2; 73 unsigned short : 4; 74 unsigned short k : 1; 75 unsigned short a : 1; 76 } oac2; 77 }; 78 }; 79 80 int __noreturn __put_user_bad(void); 81 82 #define __put_user_asm(to, from, size) \ 83 ({ \ 84 union oac __oac_spec = { \ 85 .oac1.as = PSW_BITS_AS_SECONDARY, \ 86 .oac1.a = 1, \ 87 }; \ 88 int __rc; \ 89 \ 90 asm volatile( \ 91 " lr 0,%[spec]\n" \ 92 "0: mvcos %[_to],%[_from],%[_size]\n" \ 93 "1: xr %[rc],%[rc]\n" \ 94 "2:\n" \ 95 EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ 96 EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ 97 : [rc] "=&d" (__rc), [_to] "+Q" (*(to)) \ 98 : [_size] "d" (size), [_from] "Q" (*(from)), \ 99 [spec] "d" (__oac_spec.val) \ 100 : "cc", "0"); \ 101 __rc; \ 102 }) 103 104 static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) 105 { 106 int rc; 107 108 switch (size) { 109 case 1: 110 rc = __put_user_asm((unsigned char __user *)ptr, 111 (unsigned char *)x, 112 size); 113 break; 114 case 2: 115 rc = __put_user_asm((unsigned short __user *)ptr, 116 (unsigned short *)x, 117 size); 118 break; 119 case 4: 120 rc = __put_user_asm((unsigned int __user *)ptr, 121 (unsigned int *)x, 122 size); 123 break; 124 case 8: 125 rc = __put_user_asm((unsigned long __user *)ptr, 126 (unsigned long *)x, 127 size); 128 break; 129 default: 130 __put_user_bad(); 131 break; 132 } 133 return rc; 134 } 135 136 int __noreturn __get_user_bad(void); 137 138 #define __get_user_asm(to, from, size) \ 139 ({ \ 140 union oac __oac_spec = { \ 141 .oac2.as = PSW_BITS_AS_SECONDARY, \ 142 .oac2.a = 1, \ 143 }; \ 144 int __rc; \ 145 \ 146 asm volatile( \ 147 " lr 0,%[spec]\n" \ 148 "0: mvcos 0(%[_to]),%[_from],%[_size]\n" \ 149 "1: xr %[rc],%[rc]\n" \ 150 "2:\n" \ 151 EX_TABLE_UA_LOAD_MEM(0b, 2b, %[rc], %[_to], %[_ksize]) \ 152 EX_TABLE_UA_LOAD_MEM(1b, 2b, %[rc], %[_to], %[_ksize]) \ 153 : [rc] "=&d" (__rc), "=Q" (*(to)) \ 154 : [_size] "d" (size), [_from] "Q" (*(from)), \ 155 [spec] "d" (__oac_spec.val), [_to] "a" (to), \ 156 [_ksize] "K" (size) \ 157 : "cc", "0"); \ 158 __rc; \ 159 }) 160 161 static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) 162 { 163 int rc; 164 165 switch (size) { 166 case 1: 167 rc = __get_user_asm((unsigned char *)x, 168 (unsigned char __user *)ptr, 169 size); 170 break; 171 case 2: 172 rc = __get_user_asm((unsigned short *)x, 173 (unsigned short __user *)ptr, 174 size); 175 break; 176 case 4: 177 rc = __get_user_asm((unsigned int *)x, 178 (unsigned int __user *)ptr, 179 size); 180 break; 181 case 8: 182 rc = __get_user_asm((unsigned long *)x, 183 (unsigned long __user *)ptr, 184 size); 185 break; 186 default: 187 __get_user_bad(); 188 break; 189 } 190 return rc; 191 } 192 193 /* 194 * These are the main single-value transfer routines. They automatically 195 * use the right size if we just have the right pointer type. 196 */ 197 #define __put_user(x, ptr) \ 198 ({ \ 199 __typeof__(*(ptr)) __x = (x); \ 200 int __pu_err = -EFAULT; \ 201 \ 202 __chk_user_ptr(ptr); \ 203 switch (sizeof(*(ptr))) { \ 204 case 1: \ 205 case 2: \ 206 case 4: \ 207 case 8: \ 208 __pu_err = __put_user_fn(&__x, ptr, sizeof(*(ptr))); \ 209 break; \ 210 default: \ 211 __put_user_bad(); \ 212 break; \ 213 } \ 214 __builtin_expect(__pu_err, 0); \ 215 }) 216 217 #define put_user(x, ptr) \ 218 ({ \ 219 might_fault(); \ 220 __put_user(x, ptr); \ 221 }) 222 223 #define __get_user(x, ptr) \ 224 ({ \ 225 int __gu_err = -EFAULT; \ 226 \ 227 __chk_user_ptr(ptr); \ 228 switch (sizeof(*(ptr))) { \ 229 case 1: { \ 230 unsigned char __x; \ 231 \ 232 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 233 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 234 break; \ 235 }; \ 236 case 2: { \ 237 unsigned short __x; \ 238 \ 239 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 240 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 241 break; \ 242 }; \ 243 case 4: { \ 244 unsigned int __x; \ 245 \ 246 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 247 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 248 break; \ 249 }; \ 250 case 8: { \ 251 unsigned long __x; \ 252 \ 253 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ 254 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 255 break; \ 256 }; \ 257 default: \ 258 __get_user_bad(); \ 259 break; \ 260 } \ 261 __builtin_expect(__gu_err, 0); \ 262 }) 263 264 #define get_user(x, ptr) \ 265 ({ \ 266 might_fault(); \ 267 __get_user(x, ptr); \ 268 }) 269 270 /* 271 * Copy a null terminated string from userspace. 272 */ 273 long __must_check strncpy_from_user(char *dst, const char __user *src, long count); 274 275 long __must_check strnlen_user(const char __user *src, long count); 276 277 /* 278 * Zero Userspace 279 */ 280 unsigned long __must_check __clear_user(void __user *to, unsigned long size); 281 282 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) 283 { 284 might_fault(); 285 return __clear_user(to, n); 286 } 287 288 void *s390_kernel_write(void *dst, const void *src, size_t size); 289 290 int __noreturn __put_kernel_bad(void); 291 292 #define __put_kernel_asm(val, to, insn) \ 293 ({ \ 294 int __rc; \ 295 \ 296 asm volatile( \ 297 "0: " insn " %[_val],%[_to]\n" \ 298 "1: xr %[rc],%[rc]\n" \ 299 "2:\n" \ 300 EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ 301 EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ 302 : [rc] "=d" (__rc), [_to] "+Q" (*(to)) \ 303 : [_val] "d" (val) \ 304 : "cc"); \ 305 __rc; \ 306 }) 307 308 #define __put_kernel_nofault(dst, src, type, err_label) \ 309 do { \ 310 unsigned long __x = (unsigned long)(*((type *)(src))); \ 311 int __pk_err; \ 312 \ 313 switch (sizeof(type)) { \ 314 case 1: \ 315 __pk_err = __put_kernel_asm(__x, (type *)(dst), "stc"); \ 316 break; \ 317 case 2: \ 318 __pk_err = __put_kernel_asm(__x, (type *)(dst), "sth"); \ 319 break; \ 320 case 4: \ 321 __pk_err = __put_kernel_asm(__x, (type *)(dst), "st"); \ 322 break; \ 323 case 8: \ 324 __pk_err = __put_kernel_asm(__x, (type *)(dst), "stg"); \ 325 break; \ 326 default: \ 327 __pk_err = __put_kernel_bad(); \ 328 break; \ 329 } \ 330 if (unlikely(__pk_err)) \ 331 goto err_label; \ 332 } while (0) 333 334 int __noreturn __get_kernel_bad(void); 335 336 #define __get_kernel_asm(val, from, insn) \ 337 ({ \ 338 int __rc; \ 339 \ 340 asm volatile( \ 341 "0: " insn " %[_val],%[_from]\n" \ 342 "1: xr %[rc],%[rc]\n" \ 343 "2:\n" \ 344 EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val]) \ 345 EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val]) \ 346 : [rc] "=d" (__rc), [_val] "=d" (val) \ 347 : [_from] "Q" (*(from)) \ 348 : "cc"); \ 349 __rc; \ 350 }) 351 352 #define __get_kernel_nofault(dst, src, type, err_label) \ 353 do { \ 354 int __gk_err; \ 355 \ 356 switch (sizeof(type)) { \ 357 case 1: { \ 358 unsigned char __x; \ 359 \ 360 __gk_err = __get_kernel_asm(__x, (type *)(src), "ic"); \ 361 *((type *)(dst)) = (type)__x; \ 362 break; \ 363 }; \ 364 case 2: { \ 365 unsigned short __x; \ 366 \ 367 __gk_err = __get_kernel_asm(__x, (type *)(src), "lh"); \ 368 *((type *)(dst)) = (type)__x; \ 369 break; \ 370 }; \ 371 case 4: { \ 372 unsigned int __x; \ 373 \ 374 __gk_err = __get_kernel_asm(__x, (type *)(src), "l"); \ 375 *((type *)(dst)) = (type)__x; \ 376 break; \ 377 }; \ 378 case 8: { \ 379 unsigned long __x; \ 380 \ 381 __gk_err = __get_kernel_asm(__x, (type *)(src), "lg"); \ 382 *((type *)(dst)) = (type)__x; \ 383 break; \ 384 }; \ 385 default: \ 386 __gk_err = __get_kernel_bad(); \ 387 break; \ 388 } \ 389 if (unlikely(__gk_err)) \ 390 goto err_label; \ 391 } while (0) 392 393 void __cmpxchg_user_key_called_with_bad_pointer(void); 394 395 static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 396 __uint128_t old, __uint128_t new, 397 unsigned long key, int size) 398 { 399 int rc = 0; 400 401 switch (size) { 402 case 1: { 403 unsigned int prev, tmp, shift; 404 405 shift = (3 ^ (address & 3)) << 3; 406 address ^= address & 3; 407 asm volatile( 408 " spka 0(%[key])\n" 409 " sacf 256\n" 410 "0: l %[prev],%[address]\n" 411 "1: nr %[prev],%[mask]\n" 412 " lr %[tmp],%[prev]\n" 413 " or %[prev],%[old]\n" 414 " or %[tmp],%[new]\n" 415 "2: cs %[prev],%[tmp],%[address]\n" 416 "3: jnl 4f\n" 417 " xr %[tmp],%[prev]\n" 418 " nr %[tmp],%[mask]\n" 419 " jnz 1b\n" 420 "4: sacf 768\n" 421 " spka %[default_key]\n" 422 EX_TABLE_UA_LOAD_REG(0b, 4b, %[rc], %[prev]) 423 EX_TABLE_UA_LOAD_REG(1b, 4b, %[rc], %[prev]) 424 EX_TABLE_UA_LOAD_REG(2b, 4b, %[rc], %[prev]) 425 EX_TABLE_UA_LOAD_REG(3b, 4b, %[rc], %[prev]) 426 : [rc] "+&d" (rc), 427 [prev] "=&d" (prev), 428 [tmp] "=&d" (tmp), 429 [address] "+Q" (*(int *)address) 430 : [old] "d" (((unsigned int)old & 0xff) << shift), 431 [new] "d" (((unsigned int)new & 0xff) << shift), 432 [mask] "d" (~(0xff << shift)), 433 [key] "a" (key << 4), 434 [default_key] "J" (PAGE_DEFAULT_KEY) 435 : "memory", "cc"); 436 *(unsigned char *)uval = prev >> shift; 437 return rc; 438 } 439 case 2: { 440 unsigned int prev, tmp, shift; 441 442 shift = (2 ^ (address & 2)) << 3; 443 address ^= address & 2; 444 asm volatile( 445 " spka 0(%[key])\n" 446 " sacf 256\n" 447 "0: l %[prev],%[address]\n" 448 "1: nr %[prev],%[mask]\n" 449 " lr %[tmp],%[prev]\n" 450 " or %[prev],%[old]\n" 451 " or %[tmp],%[new]\n" 452 "2: cs %[prev],%[tmp],%[address]\n" 453 "3: jnl 4f\n" 454 " xr %[tmp],%[prev]\n" 455 " nr %[tmp],%[mask]\n" 456 " jnz 1b\n" 457 "4: sacf 768\n" 458 " spka %[default_key]\n" 459 EX_TABLE_UA_LOAD_REG(0b, 4b, %[rc], %[prev]) 460 EX_TABLE_UA_LOAD_REG(1b, 4b, %[rc], %[prev]) 461 EX_TABLE_UA_LOAD_REG(2b, 4b, %[rc], %[prev]) 462 EX_TABLE_UA_LOAD_REG(3b, 4b, %[rc], %[prev]) 463 : [rc] "+&d" (rc), 464 [prev] "=&d" (prev), 465 [tmp] "=&d" (tmp), 466 [address] "+Q" (*(int *)address) 467 : [old] "d" (((unsigned int)old & 0xffff) << shift), 468 [new] "d" (((unsigned int)new & 0xffff) << shift), 469 [mask] "d" (~(0xffff << shift)), 470 [key] "a" (key << 4), 471 [default_key] "J" (PAGE_DEFAULT_KEY) 472 : "memory", "cc"); 473 *(unsigned short *)uval = prev >> shift; 474 return rc; 475 } 476 case 4: { 477 unsigned int prev = old; 478 479 asm volatile( 480 " spka 0(%[key])\n" 481 " sacf 256\n" 482 "0: cs %[prev],%[new],%[address]\n" 483 "1: sacf 768\n" 484 " spka %[default_key]\n" 485 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 486 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 487 : [rc] "+&d" (rc), 488 [prev] "+&d" (prev), 489 [address] "+Q" (*(int *)address) 490 : [new] "d" ((unsigned int)new), 491 [key] "a" (key << 4), 492 [default_key] "J" (PAGE_DEFAULT_KEY) 493 : "memory", "cc"); 494 *(unsigned int *)uval = prev; 495 return rc; 496 } 497 case 8: { 498 unsigned long prev = old; 499 500 asm volatile( 501 " spka 0(%[key])\n" 502 " sacf 256\n" 503 "0: csg %[prev],%[new],%[address]\n" 504 "1: sacf 768\n" 505 " spka %[default_key]\n" 506 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 507 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 508 : [rc] "+&d" (rc), 509 [prev] "+&d" (prev), 510 [address] "+QS" (*(long *)address) 511 : [new] "d" ((unsigned long)new), 512 [key] "a" (key << 4), 513 [default_key] "J" (PAGE_DEFAULT_KEY) 514 : "memory", "cc"); 515 *(unsigned long *)uval = prev; 516 return rc; 517 } 518 case 16: { 519 __uint128_t prev = old; 520 521 asm volatile( 522 " spka 0(%[key])\n" 523 " sacf 256\n" 524 "0: cdsg %[prev],%[new],%[address]\n" 525 "1: sacf 768\n" 526 " spka %[default_key]\n" 527 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) 528 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) 529 : [rc] "+&d" (rc), 530 [prev] "+&d" (prev), 531 [address] "+QS" (*(__int128_t *)address) 532 : [new] "d" (new), 533 [key] "a" (key << 4), 534 [default_key] "J" (PAGE_DEFAULT_KEY) 535 : "memory", "cc"); 536 *(__uint128_t *)uval = prev; 537 return rc; 538 } 539 } 540 __cmpxchg_user_key_called_with_bad_pointer(); 541 return rc; 542 } 543 544 /** 545 * cmpxchg_user_key() - cmpxchg with user space target, honoring storage keys 546 * @ptr: User space address of value to compare to @old and exchange with 547 * @new. Must be aligned to sizeof(*@ptr). 548 * @uval: Address where the old value of *@ptr is written to. 549 * @old: Old value. Compared to the content pointed to by @ptr in order to 550 * determine if the exchange occurs. The old value read from *@ptr is 551 * written to *@uval. 552 * @new: New value to place at *@ptr. 553 * @key: Access key to use for checking storage key protection. 554 * 555 * Perform a cmpxchg on a user space target, honoring storage key protection. 556 * @key alone determines how key checking is performed, neither 557 * storage-protection-override nor fetch-protection-override apply. 558 * The caller must compare *@uval and @old to determine if values have been 559 * exchanged. In case of an exception *@uval is set to zero. 560 * 561 * Return: 0: cmpxchg executed 562 * -EFAULT: an exception happened when trying to access *@ptr 563 */ 564 #define cmpxchg_user_key(ptr, uval, old, new, key) \ 565 ({ \ 566 __typeof__(ptr) __ptr = (ptr); \ 567 __typeof__(uval) __uval = (uval); \ 568 \ 569 BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval))); \ 570 might_fault(); \ 571 __chk_user_ptr(__ptr); \ 572 __cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \ 573 (old), (new), (key), sizeof(*(__ptr))); \ 574 }) 575 576 #endif /* __S390_UACCESS_H */ 577