1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Access kernel or user memory without faulting. 4 */ 5 #include <linux/export.h> 6 #include <linux/mm.h> 7 #include <linux/uaccess.h> 8 9 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, 10 size_t size) 11 { 12 return true; 13 } 14 15 #ifdef HAVE_GET_KERNEL_NOFAULT 16 17 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ 18 while (len >= sizeof(type)) { \ 19 __get_kernel_nofault(dst, src, type, err_label); \ 20 dst += sizeof(type); \ 21 src += sizeof(type); \ 22 len -= sizeof(type); \ 23 } 24 25 long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 26 { 27 if (!copy_from_kernel_nofault_allowed(src, size)) 28 return -ERANGE; 29 30 pagefault_disable(); 31 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); 32 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); 33 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); 34 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); 35 pagefault_enable(); 36 return 0; 37 Efault: 38 pagefault_enable(); 39 return -EFAULT; 40 } 41 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 42 43 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ 44 while (len >= sizeof(type)) { \ 45 __put_kernel_nofault(dst, src, type, err_label); \ 46 dst += sizeof(type); \ 47 src += sizeof(type); \ 48 len -= sizeof(type); \ 49 } 50 51 long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 52 { 53 pagefault_disable(); 54 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); 55 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); 56 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); 57 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); 58 pagefault_enable(); 59 return 0; 60 Efault: 61 pagefault_enable(); 62 return -EFAULT; 63 } 64 65 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 66 { 67 const void *src = unsafe_addr; 68 69 if (unlikely(count <= 0)) 70 return 0; 71 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 72 return -ERANGE; 73 74 pagefault_disable(); 75 do { 76 __get_kernel_nofault(dst, src, u8, Efault); 77 dst++; 78 src++; 79 } while (dst[-1] && src - unsafe_addr < count); 80 pagefault_enable(); 81 82 dst[-1] = '\0'; 83 return src - unsafe_addr; 84 Efault: 85 pagefault_enable(); 86 dst[-1] = '\0'; 87 return -EFAULT; 88 } 89 #else /* HAVE_GET_KERNEL_NOFAULT */ 90 /** 91 * copy_from_kernel_nofault(): safely attempt to read from kernel-space 92 * @dst: pointer to the buffer that shall take the data 93 * @src: address to read from 94 * @size: size of the data chunk 95 * 96 * Safely read from kernel address @src to the buffer at @dst. If a kernel 97 * fault happens, handle that and return -EFAULT. If @src is not a valid kernel 98 * address, return -ERANGE. 99 * 100 * We ensure that the copy_from_user is executed in atomic context so that 101 * do_page_fault() doesn't attempt to take mmap_lock. This makes 102 * copy_from_kernel_nofault() suitable for use within regions where the caller 103 * already holds mmap_lock, or other locks which nest inside mmap_lock. 104 */ 105 long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 106 { 107 long ret; 108 mm_segment_t old_fs = get_fs(); 109 110 if (!copy_from_kernel_nofault_allowed(src, size)) 111 return -ERANGE; 112 113 set_fs(KERNEL_DS); 114 pagefault_disable(); 115 ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, 116 size); 117 pagefault_enable(); 118 set_fs(old_fs); 119 120 if (ret) 121 return -EFAULT; 122 return 0; 123 } 124 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 125 126 /** 127 * copy_to_kernel_nofault(): safely attempt to write to a location 128 * @dst: address to write to 129 * @src: pointer to the data that shall be written 130 * @size: size of the data chunk 131 * 132 * Safely write to address @dst from the buffer at @src. If a kernel fault 133 * happens, handle that and return -EFAULT. 134 */ 135 long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 136 { 137 long ret; 138 mm_segment_t old_fs = get_fs(); 139 140 set_fs(KERNEL_DS); 141 pagefault_disable(); 142 ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); 143 pagefault_enable(); 144 set_fs(old_fs); 145 146 if (ret) 147 return -EFAULT; 148 return 0; 149 } 150 151 /** 152 * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe 153 * address. 154 * @dst: Destination address, in kernel space. This buffer must be at 155 * least @count bytes long. 156 * @unsafe_addr: Unsafe address. 157 * @count: Maximum number of bytes to copy, including the trailing NUL. 158 * 159 * Copies a NUL-terminated string from unsafe address to kernel buffer. 160 * 161 * On success, returns the length of the string INCLUDING the trailing NUL. 162 * 163 * If access fails, returns -EFAULT (some data may have been copied and the 164 * trailing NUL added). If @unsafe_addr is not a valid kernel address, return 165 * -ERANGE. 166 * 167 * If @count is smaller than the length of the string, copies @count-1 bytes, 168 * sets the last byte of @dst buffer to NUL and returns @count. 169 */ 170 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 171 { 172 mm_segment_t old_fs = get_fs(); 173 const void *src = unsafe_addr; 174 long ret; 175 176 if (unlikely(count <= 0)) 177 return 0; 178 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 179 return -ERANGE; 180 181 set_fs(KERNEL_DS); 182 pagefault_disable(); 183 184 do { 185 ret = __get_user(*dst++, (const char __user __force *)src++); 186 } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 187 188 dst[-1] = '\0'; 189 pagefault_enable(); 190 set_fs(old_fs); 191 192 return ret ? -EFAULT : src - unsafe_addr; 193 } 194 #endif /* HAVE_GET_KERNEL_NOFAULT */ 195 196 /** 197 * copy_from_user_nofault(): safely attempt to read from a user-space location 198 * @dst: pointer to the buffer that shall take the data 199 * @src: address to read from. This must be a user address. 200 * @size: size of the data chunk 201 * 202 * Safely read from user address @src to the buffer at @dst. If a kernel fault 203 * happens, handle that and return -EFAULT. 204 */ 205 long copy_from_user_nofault(void *dst, const void __user *src, size_t size) 206 { 207 long ret = -EFAULT; 208 mm_segment_t old_fs = get_fs(); 209 210 set_fs(USER_DS); 211 if (access_ok(src, size)) { 212 pagefault_disable(); 213 ret = __copy_from_user_inatomic(dst, src, size); 214 pagefault_enable(); 215 } 216 set_fs(old_fs); 217 218 if (ret) 219 return -EFAULT; 220 return 0; 221 } 222 EXPORT_SYMBOL_GPL(copy_from_user_nofault); 223 224 /** 225 * copy_to_user_nofault(): safely attempt to write to a user-space location 226 * @dst: address to write to 227 * @src: pointer to the data that shall be written 228 * @size: size of the data chunk 229 * 230 * Safely write to address @dst from the buffer at @src. If a kernel fault 231 * happens, handle that and return -EFAULT. 232 */ 233 long copy_to_user_nofault(void __user *dst, const void *src, size_t size) 234 { 235 long ret = -EFAULT; 236 mm_segment_t old_fs = get_fs(); 237 238 set_fs(USER_DS); 239 if (access_ok(dst, size)) { 240 pagefault_disable(); 241 ret = __copy_to_user_inatomic(dst, src, size); 242 pagefault_enable(); 243 } 244 set_fs(old_fs); 245 246 if (ret) 247 return -EFAULT; 248 return 0; 249 } 250 EXPORT_SYMBOL_GPL(copy_to_user_nofault); 251 252 /** 253 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 254 * address. 255 * @dst: Destination address, in kernel space. This buffer must be at 256 * least @count bytes long. 257 * @unsafe_addr: Unsafe user address. 258 * @count: Maximum number of bytes to copy, including the trailing NUL. 259 * 260 * Copies a NUL-terminated string from unsafe user address to kernel buffer. 261 * 262 * On success, returns the length of the string INCLUDING the trailing NUL. 263 * 264 * If access fails, returns -EFAULT (some data may have been copied 265 * and the trailing NUL added). 266 * 267 * If @count is smaller than the length of the string, copies @count-1 bytes, 268 * sets the last byte of @dst buffer to NUL and returns @count. 269 */ 270 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 271 long count) 272 { 273 mm_segment_t old_fs = get_fs(); 274 long ret; 275 276 if (unlikely(count <= 0)) 277 return 0; 278 279 set_fs(USER_DS); 280 pagefault_disable(); 281 ret = strncpy_from_user(dst, unsafe_addr, count); 282 pagefault_enable(); 283 set_fs(old_fs); 284 285 if (ret >= count) { 286 ret = count; 287 dst[ret - 1] = '\0'; 288 } else if (ret > 0) { 289 ret++; 290 } 291 292 return ret; 293 } 294 295 /** 296 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. 297 * @unsafe_addr: The string to measure. 298 * @count: Maximum count (including NUL) 299 * 300 * Get the size of a NUL-terminated string in user space without pagefault. 301 * 302 * Returns the size of the string INCLUDING the terminating NUL. 303 * 304 * If the string is too long, returns a number larger than @count. User 305 * has to check the return value against "> count". 306 * On exception (or invalid count), returns 0. 307 * 308 * Unlike strnlen_user, this can be used from IRQ handler etc. because 309 * it disables pagefaults. 310 */ 311 long strnlen_user_nofault(const void __user *unsafe_addr, long count) 312 { 313 mm_segment_t old_fs = get_fs(); 314 int ret; 315 316 set_fs(USER_DS); 317 pagefault_disable(); 318 ret = strnlen_user(unsafe_addr, count); 319 pagefault_enable(); 320 set_fs(old_fs); 321 322 return ret; 323 } 324