1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Standard user space access functions based on mvcp/mvcs and doing 4 * interesting things in the secondary space mode. 5 * 6 * Copyright IBM Corp. 2006,2014 7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 8 * Gerald Schaefer (gerald.schaefer@de.ibm.com) 9 */ 10 11 #include <linux/jump_label.h> 12 #include <linux/uaccess.h> 13 #include <linux/export.h> 14 #include <linux/errno.h> 15 #include <linux/mm.h> 16 #include <asm/mmu_context.h> 17 #include <asm/facility.h> 18 19 #ifndef CONFIG_HAVE_MARCH_Z10_FEATURES 20 static DEFINE_STATIC_KEY_FALSE(have_mvcos); 21 22 static int __init uaccess_init(void) 23 { 24 if (test_facility(27)) 25 static_branch_enable(&have_mvcos); 26 return 0; 27 } 28 early_initcall(uaccess_init); 29 30 static inline int copy_with_mvcos(void) 31 { 32 if (static_branch_likely(&have_mvcos)) 33 return 1; 34 return 0; 35 } 36 #else 37 static inline int copy_with_mvcos(void) 38 { 39 return 1; 40 } 41 #endif 42 43 static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, 44 unsigned long size) 45 { 46 register unsigned long reg0 asm("0") = 0x81UL; 47 unsigned long tmp1, tmp2; 48 49 tmp1 = -4096UL; 50 asm volatile( 51 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n" 52 "6: jz 4f\n" 53 "1: algr %0,%3\n" 54 " slgr %1,%3\n" 55 " slgr %2,%3\n" 56 " j 0b\n" 57 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 58 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 59 " slgr %4,%1\n" 60 " clgr %0,%4\n" /* copy crosses next page boundary? */ 61 " jnh 5f\n" 62 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n" 63 "7: slgr %0,%4\n" 64 " j 5f\n" 65 "4: slgr %0,%0\n" 66 "5:\n" 67 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) 68 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 69 : "d" (reg0) : "cc", "memory"); 70 return size; 71 } 72 73 static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, 74 unsigned long size) 75 { 76 unsigned long tmp1, tmp2; 77 78 load_kernel_asce(); 79 tmp1 = -256UL; 80 asm volatile( 81 " sacf 0\n" 82 "0: mvcp 0(%0,%2),0(%1),%3\n" 83 "7: jz 5f\n" 84 "1: algr %0,%3\n" 85 " la %1,256(%1)\n" 86 " la %2,256(%2)\n" 87 "2: mvcp 0(%0,%2),0(%1),%3\n" 88 "8: jnz 1b\n" 89 " j 5f\n" 90 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ 91 " lghi %3,-4096\n" 92 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ 93 " slgr %4,%1\n" 94 " clgr %0,%4\n" /* copy crosses next page boundary? */ 95 " jnh 6f\n" 96 "4: mvcp 0(%4,%2),0(%1),%3\n" 97 "9: slgr %0,%4\n" 98 " j 6f\n" 99 "5: slgr %0,%0\n" 100 "6: sacf 768\n" 101 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) 102 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) 103 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 104 : : "cc", "memory"); 105 return size; 106 } 107 108 unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) 109 { 110 if (copy_with_mvcos()) 111 return copy_from_user_mvcos(to, from, n); 112 return copy_from_user_mvcp(to, from, n); 113 } 114 EXPORT_SYMBOL(raw_copy_from_user); 115 116 static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x, 117 unsigned long size) 118 { 119 register unsigned long reg0 asm("0") = 0x810000UL; 120 unsigned long tmp1, tmp2; 121 122 tmp1 = -4096UL; 123 asm volatile( 124 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 125 "6: jz 4f\n" 126 "1: algr %0,%3\n" 127 " slgr %1,%3\n" 128 " slgr %2,%3\n" 129 " j 0b\n" 130 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 131 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 132 " slgr %4,%1\n" 133 " clgr %0,%4\n" /* copy crosses next page boundary? */ 134 " jnh 5f\n" 135 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n" 136 "7: slgr %0,%4\n" 137 " j 5f\n" 138 "4: slgr %0,%0\n" 139 "5:\n" 140 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) 141 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 142 : "d" (reg0) : "cc", "memory"); 143 return size; 144 } 145 146 static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, 147 unsigned long size) 148 { 149 unsigned long tmp1, tmp2; 150 151 load_kernel_asce(); 152 tmp1 = -256UL; 153 asm volatile( 154 " sacf 0\n" 155 "0: mvcs 0(%0,%1),0(%2),%3\n" 156 "7: jz 5f\n" 157 "1: algr %0,%3\n" 158 " la %1,256(%1)\n" 159 " la %2,256(%2)\n" 160 "2: mvcs 0(%0,%1),0(%2),%3\n" 161 "8: jnz 1b\n" 162 " j 5f\n" 163 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ 164 " lghi %3,-4096\n" 165 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ 166 " slgr %4,%1\n" 167 " clgr %0,%4\n" /* copy crosses next page boundary? */ 168 " jnh 6f\n" 169 "4: mvcs 0(%4,%1),0(%2),%3\n" 170 "9: slgr %0,%4\n" 171 " j 6f\n" 172 "5: slgr %0,%0\n" 173 "6: sacf 768\n" 174 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) 175 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) 176 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 177 : : "cc", "memory"); 178 return size; 179 } 180 181 unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) 182 { 183 if (copy_with_mvcos()) 184 return copy_to_user_mvcos(to, from, n); 185 return copy_to_user_mvcs(to, from, n); 186 } 187 EXPORT_SYMBOL(raw_copy_to_user); 188 189 static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from, 190 unsigned long size) 191 { 192 register unsigned long reg0 asm("0") = 0x810081UL; 193 unsigned long tmp1, tmp2; 194 195 tmp1 = -4096UL; 196 /* FIXME: copy with reduced length. */ 197 asm volatile( 198 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 199 " jz 2f\n" 200 "1: algr %0,%3\n" 201 " slgr %1,%3\n" 202 " slgr %2,%3\n" 203 " j 0b\n" 204 "2:slgr %0,%0\n" 205 "3: \n" 206 EX_TABLE(0b,3b) 207 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2) 208 : "d" (reg0) : "cc", "memory"); 209 return size; 210 } 211 212 static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from, 213 unsigned long size) 214 { 215 unsigned long tmp1; 216 217 load_kernel_asce(); 218 asm volatile( 219 " sacf 256\n" 220 " aghi %0,-1\n" 221 " jo 5f\n" 222 " bras %3,3f\n" 223 "0: aghi %0,257\n" 224 "1: mvc 0(1,%1),0(%2)\n" 225 " la %1,1(%1)\n" 226 " la %2,1(%2)\n" 227 " aghi %0,-1\n" 228 " jnz 1b\n" 229 " j 5f\n" 230 "2: mvc 0(256,%1),0(%2)\n" 231 " la %1,256(%1)\n" 232 " la %2,256(%2)\n" 233 "3: aghi %0,-256\n" 234 " jnm 2b\n" 235 "4: ex %0,1b-0b(%3)\n" 236 "5: slgr %0,%0\n" 237 "6: sacf 768\n" 238 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) 239 : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) 240 : : "cc", "memory"); 241 return size; 242 } 243 244 unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) 245 { 246 if (copy_with_mvcos()) 247 return copy_in_user_mvcos(to, from, n); 248 return copy_in_user_mvc(to, from, n); 249 } 250 EXPORT_SYMBOL(raw_copy_in_user); 251 252 static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size) 253 { 254 register unsigned long reg0 asm("0") = 0x810000UL; 255 unsigned long tmp1, tmp2; 256 257 tmp1 = -4096UL; 258 asm volatile( 259 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n" 260 " jz 4f\n" 261 "1: algr %0,%2\n" 262 " slgr %1,%2\n" 263 " j 0b\n" 264 "2: la %3,4095(%1)\n"/* %4 = to + 4095 */ 265 " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */ 266 " slgr %3,%1\n" 267 " clgr %0,%3\n" /* copy crosses next page boundary? */ 268 " jnh 5f\n" 269 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n" 270 " slgr %0,%3\n" 271 " j 5f\n" 272 "4: slgr %0,%0\n" 273 "5:\n" 274 EX_TABLE(0b,2b) EX_TABLE(3b,5b) 275 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) 276 : "a" (empty_zero_page), "d" (reg0) : "cc", "memory"); 277 return size; 278 } 279 280 static inline unsigned long clear_user_xc(void __user *to, unsigned long size) 281 { 282 unsigned long tmp1, tmp2; 283 284 load_kernel_asce(); 285 asm volatile( 286 " sacf 256\n" 287 " aghi %0,-1\n" 288 " jo 5f\n" 289 " bras %3,3f\n" 290 " xc 0(1,%1),0(%1)\n" 291 "0: aghi %0,257\n" 292 " la %2,255(%1)\n" /* %2 = ptr + 255 */ 293 " srl %2,12\n" 294 " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */ 295 " slgr %2,%1\n" 296 " clgr %0,%2\n" /* clear crosses next page boundary? */ 297 " jnh 5f\n" 298 " aghi %2,-1\n" 299 "1: ex %2,0(%3)\n" 300 " aghi %2,1\n" 301 " slgr %0,%2\n" 302 " j 5f\n" 303 "2: xc 0(256,%1),0(%1)\n" 304 " la %1,256(%1)\n" 305 "3: aghi %0,-256\n" 306 " jnm 2b\n" 307 "4: ex %0,0(%3)\n" 308 "5: slgr %0,%0\n" 309 "6: sacf 768\n" 310 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) 311 : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) 312 : : "cc", "memory"); 313 return size; 314 } 315 316 unsigned long __clear_user(void __user *to, unsigned long size) 317 { 318 if (copy_with_mvcos()) 319 return clear_user_mvcos(to, size); 320 return clear_user_xc(to, size); 321 } 322 EXPORT_SYMBOL(__clear_user); 323 324 static inline unsigned long strnlen_user_srst(const char __user *src, 325 unsigned long size) 326 { 327 register unsigned long reg0 asm("0") = 0; 328 unsigned long tmp1, tmp2; 329 330 asm volatile( 331 " la %2,0(%1)\n" 332 " la %3,0(%0,%1)\n" 333 " slgr %0,%0\n" 334 " sacf 256\n" 335 "0: srst %3,%2\n" 336 " jo 0b\n" 337 " la %0,1(%3)\n" /* strnlen_user results includes \0 */ 338 " slgr %0,%1\n" 339 "1: sacf 768\n" 340 EX_TABLE(0b,1b) 341 : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2) 342 : "d" (reg0) : "cc", "memory"); 343 return size; 344 } 345 346 unsigned long __strnlen_user(const char __user *src, unsigned long size) 347 { 348 if (unlikely(!size)) 349 return 0; 350 load_kernel_asce(); 351 return strnlen_user_srst(src, size); 352 } 353 EXPORT_SYMBOL(__strnlen_user); 354 355 long __strncpy_from_user(char *dst, const char __user *src, long size) 356 { 357 size_t done, len, offset, len_str; 358 359 if (unlikely(size <= 0)) 360 return 0; 361 done = 0; 362 do { 363 offset = (size_t)src & (L1_CACHE_BYTES - 1); 364 len = min(size - done, L1_CACHE_BYTES - offset); 365 if (copy_from_user(dst, src, len)) 366 return -EFAULT; 367 len_str = strnlen(dst, len); 368 done += len_str; 369 src += len_str; 370 dst += len_str; 371 } while ((len_str == len) && (done < size)); 372 return done; 373 } 374 EXPORT_SYMBOL(__strncpy_from_user); 375