1 /* 2 * Standard user space access functions based on mvcp/mvcs and doing 3 * interesting things in the secondary space mode. 4 * 5 * Copyright IBM Corp. 2006,2014 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 7 * Gerald Schaefer (gerald.schaefer@de.ibm.com) 8 */ 9 10 #include <linux/jump_label.h> 11 #include <linux/uaccess.h> 12 #include <linux/export.h> 13 #include <linux/errno.h> 14 #include <linux/mm.h> 15 #include <asm/mmu_context.h> 16 #include <asm/facility.h> 17 18 static DEFINE_STATIC_KEY_FALSE(have_mvcos); 19 20 static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, 21 unsigned long size) 22 { 23 register unsigned long reg0 asm("0") = 0x81UL; 24 unsigned long tmp1, tmp2; 25 26 tmp1 = -4096UL; 27 asm volatile( 28 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n" 29 "6: jz 4f\n" 30 "1: algr %0,%3\n" 31 " slgr %1,%3\n" 32 " slgr %2,%3\n" 33 " j 0b\n" 34 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 35 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 36 " slgr %4,%1\n" 37 " clgr %0,%4\n" /* copy crosses next page boundary? */ 38 " jnh 5f\n" 39 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n" 40 "7: slgr %0,%4\n" 41 " j 5f\n" 42 "4: slgr %0,%0\n" 43 "5:\n" 44 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) 45 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 46 : "d" (reg0) : "cc", "memory"); 47 return size; 48 } 49 50 static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, 51 unsigned long size) 52 { 53 unsigned long tmp1, tmp2; 54 55 load_kernel_asce(); 56 tmp1 = -256UL; 57 asm volatile( 58 " sacf 0\n" 59 "0: mvcp 0(%0,%2),0(%1),%3\n" 60 "7: jz 5f\n" 61 "1: algr %0,%3\n" 62 " la %1,256(%1)\n" 63 " la %2,256(%2)\n" 64 "2: mvcp 0(%0,%2),0(%1),%3\n" 65 "8: jnz 1b\n" 66 " j 5f\n" 67 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ 68 " lghi %3,-4096\n" 69 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ 70 " slgr %4,%1\n" 71 " clgr %0,%4\n" /* copy crosses next page boundary? */ 72 " jnh 6f\n" 73 "4: mvcp 0(%4,%2),0(%1),%3\n" 74 "9: slgr %0,%4\n" 75 " j 6f\n" 76 "5: slgr %0,%0\n" 77 "6: sacf 768\n" 78 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) 79 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) 80 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 81 : : "cc", "memory"); 82 return size; 83 } 84 85 unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) 86 { 87 if (static_branch_likely(&have_mvcos)) 88 return copy_from_user_mvcos(to, from, n); 89 return copy_from_user_mvcp(to, from, n); 90 } 91 EXPORT_SYMBOL(raw_copy_from_user); 92 93 static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x, 94 unsigned long size) 95 { 96 register unsigned long reg0 asm("0") = 0x810000UL; 97 unsigned long tmp1, tmp2; 98 99 tmp1 = -4096UL; 100 asm volatile( 101 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 102 "6: jz 4f\n" 103 "1: algr %0,%3\n" 104 " slgr %1,%3\n" 105 " slgr %2,%3\n" 106 " j 0b\n" 107 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 108 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 109 " slgr %4,%1\n" 110 " clgr %0,%4\n" /* copy crosses next page boundary? */ 111 " jnh 5f\n" 112 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n" 113 "7: slgr %0,%4\n" 114 " j 5f\n" 115 "4: slgr %0,%0\n" 116 "5:\n" 117 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) 118 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 119 : "d" (reg0) : "cc", "memory"); 120 return size; 121 } 122 123 static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, 124 unsigned long size) 125 { 126 unsigned long tmp1, tmp2; 127 128 load_kernel_asce(); 129 tmp1 = -256UL; 130 asm volatile( 131 " sacf 0\n" 132 "0: mvcs 0(%0,%1),0(%2),%3\n" 133 "7: jz 5f\n" 134 "1: algr %0,%3\n" 135 " la %1,256(%1)\n" 136 " la %2,256(%2)\n" 137 "2: mvcs 0(%0,%1),0(%2),%3\n" 138 "8: jnz 1b\n" 139 " j 5f\n" 140 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ 141 " lghi %3,-4096\n" 142 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ 143 " slgr %4,%1\n" 144 " clgr %0,%4\n" /* copy crosses next page boundary? */ 145 " jnh 6f\n" 146 "4: mvcs 0(%4,%1),0(%2),%3\n" 147 "9: slgr %0,%4\n" 148 " j 6f\n" 149 "5: slgr %0,%0\n" 150 "6: sacf 768\n" 151 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) 152 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) 153 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 154 : : "cc", "memory"); 155 return size; 156 } 157 158 unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) 159 { 160 if (static_branch_likely(&have_mvcos)) 161 return copy_to_user_mvcos(to, from, n); 162 return copy_to_user_mvcs(to, from, n); 163 } 164 EXPORT_SYMBOL(raw_copy_to_user); 165 166 static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from, 167 unsigned long size) 168 { 169 register unsigned long reg0 asm("0") = 0x810081UL; 170 unsigned long tmp1, tmp2; 171 172 tmp1 = -4096UL; 173 /* FIXME: copy with reduced length. */ 174 asm volatile( 175 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 176 " jz 2f\n" 177 "1: algr %0,%3\n" 178 " slgr %1,%3\n" 179 " slgr %2,%3\n" 180 " j 0b\n" 181 "2:slgr %0,%0\n" 182 "3: \n" 183 EX_TABLE(0b,3b) 184 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2) 185 : "d" (reg0) : "cc", "memory"); 186 return size; 187 } 188 189 static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from, 190 unsigned long size) 191 { 192 unsigned long tmp1; 193 194 load_kernel_asce(); 195 asm volatile( 196 " sacf 256\n" 197 " aghi %0,-1\n" 198 " jo 5f\n" 199 " bras %3,3f\n" 200 "0: aghi %0,257\n" 201 "1: mvc 0(1,%1),0(%2)\n" 202 " la %1,1(%1)\n" 203 " la %2,1(%2)\n" 204 " aghi %0,-1\n" 205 " jnz 1b\n" 206 " j 5f\n" 207 "2: mvc 0(256,%1),0(%2)\n" 208 " la %1,256(%1)\n" 209 " la %2,256(%2)\n" 210 "3: aghi %0,-256\n" 211 " jnm 2b\n" 212 "4: ex %0,1b-0b(%3)\n" 213 "5: slgr %0,%0\n" 214 "6: sacf 768\n" 215 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) 216 : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) 217 : : "cc", "memory"); 218 return size; 219 } 220 221 unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) 222 { 223 if (static_branch_likely(&have_mvcos)) 224 return copy_in_user_mvcos(to, from, n); 225 return copy_in_user_mvc(to, from, n); 226 } 227 EXPORT_SYMBOL(raw_copy_in_user); 228 229 static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size) 230 { 231 register unsigned long reg0 asm("0") = 0x810000UL; 232 unsigned long tmp1, tmp2; 233 234 tmp1 = -4096UL; 235 asm volatile( 236 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n" 237 " jz 4f\n" 238 "1: algr %0,%2\n" 239 " slgr %1,%2\n" 240 " j 0b\n" 241 "2: la %3,4095(%1)\n"/* %4 = to + 4095 */ 242 " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */ 243 " slgr %3,%1\n" 244 " clgr %0,%3\n" /* copy crosses next page boundary? */ 245 " jnh 5f\n" 246 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n" 247 " slgr %0,%3\n" 248 " j 5f\n" 249 "4: slgr %0,%0\n" 250 "5:\n" 251 EX_TABLE(0b,2b) EX_TABLE(3b,5b) 252 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) 253 : "a" (empty_zero_page), "d" (reg0) : "cc", "memory"); 254 return size; 255 } 256 257 static inline unsigned long clear_user_xc(void __user *to, unsigned long size) 258 { 259 unsigned long tmp1, tmp2; 260 261 load_kernel_asce(); 262 asm volatile( 263 " sacf 256\n" 264 " aghi %0,-1\n" 265 " jo 5f\n" 266 " bras %3,3f\n" 267 " xc 0(1,%1),0(%1)\n" 268 "0: aghi %0,257\n" 269 " la %2,255(%1)\n" /* %2 = ptr + 255 */ 270 " srl %2,12\n" 271 " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */ 272 " slgr %2,%1\n" 273 " clgr %0,%2\n" /* clear crosses next page boundary? */ 274 " jnh 5f\n" 275 " aghi %2,-1\n" 276 "1: ex %2,0(%3)\n" 277 " aghi %2,1\n" 278 " slgr %0,%2\n" 279 " j 5f\n" 280 "2: xc 0(256,%1),0(%1)\n" 281 " la %1,256(%1)\n" 282 "3: aghi %0,-256\n" 283 " jnm 2b\n" 284 "4: ex %0,0(%3)\n" 285 "5: slgr %0,%0\n" 286 "6: sacf 768\n" 287 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) 288 : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) 289 : : "cc", "memory"); 290 return size; 291 } 292 293 unsigned long __clear_user(void __user *to, unsigned long size) 294 { 295 if (static_branch_likely(&have_mvcos)) 296 return clear_user_mvcos(to, size); 297 return clear_user_xc(to, size); 298 } 299 EXPORT_SYMBOL(__clear_user); 300 301 static inline unsigned long strnlen_user_srst(const char __user *src, 302 unsigned long size) 303 { 304 register unsigned long reg0 asm("0") = 0; 305 unsigned long tmp1, tmp2; 306 307 asm volatile( 308 " la %2,0(%1)\n" 309 " la %3,0(%0,%1)\n" 310 " slgr %0,%0\n" 311 " sacf 256\n" 312 "0: srst %3,%2\n" 313 " jo 0b\n" 314 " la %0,1(%3)\n" /* strnlen_user results includes \0 */ 315 " slgr %0,%1\n" 316 "1: sacf 768\n" 317 EX_TABLE(0b,1b) 318 : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2) 319 : "d" (reg0) : "cc", "memory"); 320 return size; 321 } 322 323 unsigned long __strnlen_user(const char __user *src, unsigned long size) 324 { 325 if (unlikely(!size)) 326 return 0; 327 load_kernel_asce(); 328 return strnlen_user_srst(src, size); 329 } 330 EXPORT_SYMBOL(__strnlen_user); 331 332 long __strncpy_from_user(char *dst, const char __user *src, long size) 333 { 334 size_t done, len, offset, len_str; 335 336 if (unlikely(size <= 0)) 337 return 0; 338 done = 0; 339 do { 340 offset = (size_t)src & (L1_CACHE_BYTES - 1); 341 len = min(size - done, L1_CACHE_BYTES - offset); 342 if (copy_from_user(dst, src, len)) 343 return -EFAULT; 344 len_str = strnlen(dst, len); 345 done += len_str; 346 src += len_str; 347 dst += len_str; 348 } while ((len_str == len) && (done < size)); 349 return done; 350 } 351 EXPORT_SYMBOL(__strncpy_from_user); 352 353 static int __init uaccess_init(void) 354 { 355 if (test_facility(27)) 356 static_branch_enable(&have_mvcos); 357 return 0; 358 } 359 early_initcall(uaccess_init); 360