1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c33fa9f5SIngo Molnar /* 33f0acb1eSChristoph Hellwig * Access kernel or user memory without faulting. 4c33fa9f5SIngo Molnar */ 5b95f1b31SPaul Gortmaker #include <linux/export.h> 6c33fa9f5SIngo Molnar #include <linux/mm.h> 77c7fcf76SDavid Howells #include <linux/uaccess.h> 8c33fa9f5SIngo Molnar 93d708182SMasami Hiramatsu static __always_inline long 103d708182SMasami Hiramatsu probe_read_common(void *dst, const void __user *src, size_t size) 113d708182SMasami Hiramatsu { 123d708182SMasami Hiramatsu long ret; 133d708182SMasami Hiramatsu 143d708182SMasami Hiramatsu pagefault_disable(); 153d708182SMasami Hiramatsu ret = __copy_from_user_inatomic(dst, src, size); 163d708182SMasami Hiramatsu pagefault_enable(); 173d708182SMasami Hiramatsu 183d708182SMasami Hiramatsu return ret ? -EFAULT : 0; 193d708182SMasami Hiramatsu } 203d708182SMasami Hiramatsu 211d1585caSDaniel Borkmann static __always_inline long 221d1585caSDaniel Borkmann probe_write_common(void __user *dst, const void *src, size_t size) 231d1585caSDaniel Borkmann { 241d1585caSDaniel Borkmann long ret; 251d1585caSDaniel Borkmann 261d1585caSDaniel Borkmann pagefault_disable(); 271d1585caSDaniel Borkmann ret = __copy_to_user_inatomic(dst, src, size); 281d1585caSDaniel Borkmann pagefault_enable(); 291d1585caSDaniel Borkmann 301d1585caSDaniel Borkmann return ret ? -EFAULT : 0; 311d1585caSDaniel Borkmann } 321d1585caSDaniel Borkmann 33c33fa9f5SIngo Molnar /** 344f6de12bSChristoph Hellwig * probe_kernel_read(): safely attempt to read from any location 35c33fa9f5SIngo Molnar * @dst: pointer to the buffer that shall take the data 36c33fa9f5SIngo Molnar * @src: address to read from 37c33fa9f5SIngo Molnar * @size: size of the data chunk 38c33fa9f5SIngo Molnar * 394f6de12bSChristoph Hellwig * Same as probe_kernel_read_strict() except that for architectures with 404f6de12bSChristoph Hellwig * not fully separated user and kernel address spaces this function also works 414f6de12bSChristoph Hellwig * for user address tanges. 424f6de12bSChristoph Hellwig * 434f6de12bSChristoph Hellwig * DO NOT USE THIS FUNCTION - it is broken on architectures with entirely 444f6de12bSChristoph Hellwig * separate kernel and user address spaces, and also a bad idea otherwise. 454f6de12bSChristoph Hellwig */ 464f6de12bSChristoph Hellwig long __weak probe_kernel_read(void *dst, const void *src, size_t size) 474f6de12bSChristoph Hellwig __attribute__((alias("__probe_kernel_read"))); 484f6de12bSChristoph Hellwig 494f6de12bSChristoph Hellwig /** 504f6de12bSChristoph Hellwig * probe_kernel_read_strict(): safely attempt to read from kernel-space 514f6de12bSChristoph Hellwig * @dst: pointer to the buffer that shall take the data 524f6de12bSChristoph Hellwig * @src: address to read from 534f6de12bSChristoph Hellwig * @size: size of the data chunk 544f6de12bSChristoph Hellwig * 554f6de12bSChristoph Hellwig * Safely read from kernel address @src to the buffer at @dst. If a kernel 564f6de12bSChristoph Hellwig * fault happens, handle that and return -EFAULT. 570ab32b6fSAndrew Morton * 580ab32b6fSAndrew Morton * We ensure that the copy_from_user is executed in atomic context so that 59c1e8d7c6SMichel Lespinasse * do_page_fault() doesn't attempt to take mmap_lock. This makes 600ab32b6fSAndrew Morton * probe_kernel_read() suitable for use within regions where the caller 61c1e8d7c6SMichel Lespinasse * already holds mmap_lock, or other locks which nest inside mmap_lock. 62c33fa9f5SIngo Molnar */ 636144a85aSJason Wessel 6475a1a607SDaniel Borkmann long __weak probe_kernel_read_strict(void *dst, const void *src, size_t size) 6575a1a607SDaniel Borkmann __attribute__((alias("__probe_kernel_read"))); 6675a1a607SDaniel Borkmann 67f29c5041SSteven Rostedt long __probe_kernel_read(void *dst, const void *src, size_t size) 68c33fa9f5SIngo Molnar { 69c33fa9f5SIngo Molnar long ret; 70b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 71c33fa9f5SIngo Molnar 72b4b8ac52SJason Wessel set_fs(KERNEL_DS); 733d708182SMasami Hiramatsu ret = probe_read_common(dst, (__force const void __user *)src, size); 74b4b8ac52SJason Wessel set_fs(old_fs); 75c33fa9f5SIngo Molnar 763d708182SMasami Hiramatsu return ret; 77c33fa9f5SIngo Molnar } 78c33fa9f5SIngo Molnar EXPORT_SYMBOL_GPL(probe_kernel_read); 79c33fa9f5SIngo Molnar 80c33fa9f5SIngo Molnar /** 813d708182SMasami Hiramatsu * probe_user_read(): safely attempt to read from a user-space location 823d708182SMasami Hiramatsu * @dst: pointer to the buffer that shall take the data 833d708182SMasami Hiramatsu * @src: address to read from. This must be a user address. 843d708182SMasami Hiramatsu * @size: size of the data chunk 853d708182SMasami Hiramatsu * 863d708182SMasami Hiramatsu * Safely read from user address @src to the buffer at @dst. If a kernel fault 873d708182SMasami Hiramatsu * happens, handle that and return -EFAULT. 883d708182SMasami Hiramatsu */ 8948c49c0eSChristoph Hellwig long probe_user_read(void *dst, const void __user *src, size_t size) 903d708182SMasami Hiramatsu { 913d708182SMasami Hiramatsu long ret = -EFAULT; 923d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 933d708182SMasami Hiramatsu 943d708182SMasami Hiramatsu set_fs(USER_DS); 953d708182SMasami Hiramatsu if (access_ok(src, size)) 963d708182SMasami Hiramatsu ret = probe_read_common(dst, src, size); 973d708182SMasami Hiramatsu set_fs(old_fs); 983d708182SMasami Hiramatsu 993d708182SMasami Hiramatsu return ret; 1003d708182SMasami Hiramatsu } 1013d708182SMasami Hiramatsu EXPORT_SYMBOL_GPL(probe_user_read); 1023d708182SMasami Hiramatsu 1033d708182SMasami Hiramatsu /** 104c33fa9f5SIngo Molnar * probe_kernel_write(): safely attempt to write to a location 105c33fa9f5SIngo Molnar * @dst: address to write to 106c33fa9f5SIngo Molnar * @src: pointer to the data that shall be written 107c33fa9f5SIngo Molnar * @size: size of the data chunk 108c33fa9f5SIngo Molnar * 109c33fa9f5SIngo Molnar * Safely write to address @dst from the buffer at @src. If a kernel fault 110c33fa9f5SIngo Molnar * happens, handle that and return -EFAULT. 111c33fa9f5SIngo Molnar */ 11248c49c0eSChristoph Hellwig long probe_kernel_write(void *dst, const void *src, size_t size) 113c33fa9f5SIngo Molnar { 114c33fa9f5SIngo Molnar long ret; 115b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 116c33fa9f5SIngo Molnar 117b4b8ac52SJason Wessel set_fs(KERNEL_DS); 1181d1585caSDaniel Borkmann ret = probe_write_common((__force void __user *)dst, src, size); 119b4b8ac52SJason Wessel set_fs(old_fs); 120c33fa9f5SIngo Molnar 1211d1585caSDaniel Borkmann return ret; 122c33fa9f5SIngo Molnar } 123dbb7ee0eSAlexei Starovoitov 1241d1585caSDaniel Borkmann /** 1251d1585caSDaniel Borkmann * probe_user_write(): safely attempt to write to a user-space location 1261d1585caSDaniel Borkmann * @dst: address to write to 1271d1585caSDaniel Borkmann * @src: pointer to the data that shall be written 1281d1585caSDaniel Borkmann * @size: size of the data chunk 1291d1585caSDaniel Borkmann * 1301d1585caSDaniel Borkmann * Safely write to address @dst from the buffer at @src. If a kernel fault 1311d1585caSDaniel Borkmann * happens, handle that and return -EFAULT. 1321d1585caSDaniel Borkmann */ 13348c49c0eSChristoph Hellwig long probe_user_write(void __user *dst, const void *src, size_t size) 1341d1585caSDaniel Borkmann { 1351d1585caSDaniel Borkmann long ret = -EFAULT; 1361d1585caSDaniel Borkmann mm_segment_t old_fs = get_fs(); 1371d1585caSDaniel Borkmann 1381d1585caSDaniel Borkmann set_fs(USER_DS); 1391d1585caSDaniel Borkmann if (access_ok(dst, size)) 1401d1585caSDaniel Borkmann ret = probe_write_common(dst, src, size); 1411d1585caSDaniel Borkmann set_fs(old_fs); 1421d1585caSDaniel Borkmann 1431d1585caSDaniel Borkmann return ret; 1441d1585caSDaniel Borkmann } 1451d1585caSDaniel Borkmann EXPORT_SYMBOL_GPL(probe_user_write); 1463d708182SMasami Hiramatsu 147dbb7ee0eSAlexei Starovoitov /** 148dbb7ee0eSAlexei Starovoitov * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address. 149dbb7ee0eSAlexei Starovoitov * @dst: Destination address, in kernel space. This buffer must be at 150dbb7ee0eSAlexei Starovoitov * least @count bytes long. 151f144c390SMike Rapoport * @unsafe_addr: Unsafe address. 152dbb7ee0eSAlexei Starovoitov * @count: Maximum number of bytes to copy, including the trailing NUL. 153dbb7ee0eSAlexei Starovoitov * 154dbb7ee0eSAlexei Starovoitov * Copies a NUL-terminated string from unsafe address to kernel buffer. 155dbb7ee0eSAlexei Starovoitov * 156dbb7ee0eSAlexei Starovoitov * On success, returns the length of the string INCLUDING the trailing NUL. 157dbb7ee0eSAlexei Starovoitov * 158dbb7ee0eSAlexei Starovoitov * If access fails, returns -EFAULT (some data may have been copied 159dbb7ee0eSAlexei Starovoitov * and the trailing NUL added). 160dbb7ee0eSAlexei Starovoitov * 161dbb7ee0eSAlexei Starovoitov * If @count is smaller than the length of the string, copies @count-1 bytes, 162dbb7ee0eSAlexei Starovoitov * sets the last byte of @dst buffer to NUL and returns @count. 16375a1a607SDaniel Borkmann * 164c4cb1644SChristoph Hellwig * Same as strncpy_from_kernel_nofault() except that for architectures with 1654f6de12bSChristoph Hellwig * not fully separated user and kernel address spaces this function also works 1664f6de12bSChristoph Hellwig * for user address tanges. 1674f6de12bSChristoph Hellwig * 1684f6de12bSChristoph Hellwig * DO NOT USE THIS FUNCTION - it is broken on architectures with entirely 1694f6de12bSChristoph Hellwig * separate kernel and user address spaces, and also a bad idea otherwise. 170dbb7ee0eSAlexei Starovoitov */ 17175a1a607SDaniel Borkmann long __weak strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) 17275a1a607SDaniel Borkmann __attribute__((alias("__strncpy_from_unsafe"))); 17375a1a607SDaniel Borkmann 1744f6de12bSChristoph Hellwig /** 175c4cb1644SChristoph Hellwig * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe 1764f6de12bSChristoph Hellwig * address. 1774f6de12bSChristoph Hellwig * @dst: Destination address, in kernel space. This buffer must be at 1784f6de12bSChristoph Hellwig * least @count bytes long. 1794f6de12bSChristoph Hellwig * @unsafe_addr: Unsafe address. 1804f6de12bSChristoph Hellwig * @count: Maximum number of bytes to copy, including the trailing NUL. 1814f6de12bSChristoph Hellwig * 1824f6de12bSChristoph Hellwig * Copies a NUL-terminated string from unsafe address to kernel buffer. 1834f6de12bSChristoph Hellwig * 1844f6de12bSChristoph Hellwig * On success, returns the length of the string INCLUDING the trailing NUL. 1854f6de12bSChristoph Hellwig * 1864f6de12bSChristoph Hellwig * If access fails, returns -EFAULT (some data may have been copied 1874f6de12bSChristoph Hellwig * and the trailing NUL added). 1884f6de12bSChristoph Hellwig * 1894f6de12bSChristoph Hellwig * If @count is smaller than the length of the string, copies @count-1 bytes, 1904f6de12bSChristoph Hellwig * sets the last byte of @dst buffer to NUL and returns @count. 1914f6de12bSChristoph Hellwig */ 192c4cb1644SChristoph Hellwig long __weak strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, 19375a1a607SDaniel Borkmann long count) 19475a1a607SDaniel Borkmann __attribute__((alias("__strncpy_from_unsafe"))); 19575a1a607SDaniel Borkmann 19675a1a607SDaniel Borkmann long __strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) 197dbb7ee0eSAlexei Starovoitov { 198dbb7ee0eSAlexei Starovoitov mm_segment_t old_fs = get_fs(); 199dbb7ee0eSAlexei Starovoitov const void *src = unsafe_addr; 200dbb7ee0eSAlexei Starovoitov long ret; 201dbb7ee0eSAlexei Starovoitov 202dbb7ee0eSAlexei Starovoitov if (unlikely(count <= 0)) 203dbb7ee0eSAlexei Starovoitov return 0; 204dbb7ee0eSAlexei Starovoitov 205dbb7ee0eSAlexei Starovoitov set_fs(KERNEL_DS); 206dbb7ee0eSAlexei Starovoitov pagefault_disable(); 207dbb7ee0eSAlexei Starovoitov 208dbb7ee0eSAlexei Starovoitov do { 209bd28b145SLinus Torvalds ret = __get_user(*dst++, (const char __user __force *)src++); 210dbb7ee0eSAlexei Starovoitov } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 211dbb7ee0eSAlexei Starovoitov 212dbb7ee0eSAlexei Starovoitov dst[-1] = '\0'; 213dbb7ee0eSAlexei Starovoitov pagefault_enable(); 214dbb7ee0eSAlexei Starovoitov set_fs(old_fs); 215dbb7ee0eSAlexei Starovoitov 2169dd861d5SRasmus Villemoes return ret ? -EFAULT : src - unsafe_addr; 217dbb7ee0eSAlexei Starovoitov } 2183d708182SMasami Hiramatsu 2193d708182SMasami Hiramatsu /** 220bd88bb5dSChristoph Hellwig * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 2213d708182SMasami Hiramatsu * address. 2223d708182SMasami Hiramatsu * @dst: Destination address, in kernel space. This buffer must be at 2233d708182SMasami Hiramatsu * least @count bytes long. 2243d708182SMasami Hiramatsu * @unsafe_addr: Unsafe user address. 2253d708182SMasami Hiramatsu * @count: Maximum number of bytes to copy, including the trailing NUL. 2263d708182SMasami Hiramatsu * 2273d708182SMasami Hiramatsu * Copies a NUL-terminated string from unsafe user address to kernel buffer. 2283d708182SMasami Hiramatsu * 2293d708182SMasami Hiramatsu * On success, returns the length of the string INCLUDING the trailing NUL. 2303d708182SMasami Hiramatsu * 2313d708182SMasami Hiramatsu * If access fails, returns -EFAULT (some data may have been copied 2323d708182SMasami Hiramatsu * and the trailing NUL added). 2333d708182SMasami Hiramatsu * 2343d708182SMasami Hiramatsu * If @count is smaller than the length of the string, copies @count-1 bytes, 2353d708182SMasami Hiramatsu * sets the last byte of @dst buffer to NUL and returns @count. 2363d708182SMasami Hiramatsu */ 237bd88bb5dSChristoph Hellwig long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 2383d708182SMasami Hiramatsu long count) 2393d708182SMasami Hiramatsu { 2403d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2413d708182SMasami Hiramatsu long ret; 2423d708182SMasami Hiramatsu 2433d708182SMasami Hiramatsu if (unlikely(count <= 0)) 2443d708182SMasami Hiramatsu return 0; 2453d708182SMasami Hiramatsu 2463d708182SMasami Hiramatsu set_fs(USER_DS); 2473d708182SMasami Hiramatsu pagefault_disable(); 2483d708182SMasami Hiramatsu ret = strncpy_from_user(dst, unsafe_addr, count); 2493d708182SMasami Hiramatsu pagefault_enable(); 2503d708182SMasami Hiramatsu set_fs(old_fs); 2513d708182SMasami Hiramatsu 2523d708182SMasami Hiramatsu if (ret >= count) { 2533d708182SMasami Hiramatsu ret = count; 2543d708182SMasami Hiramatsu dst[ret - 1] = '\0'; 2553d708182SMasami Hiramatsu } else if (ret > 0) { 2563d708182SMasami Hiramatsu ret++; 2573d708182SMasami Hiramatsu } 2583d708182SMasami Hiramatsu 2593d708182SMasami Hiramatsu return ret; 2603d708182SMasami Hiramatsu } 2613d708182SMasami Hiramatsu 2623d708182SMasami Hiramatsu /** 2633d708182SMasami Hiramatsu * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL. 2643d708182SMasami Hiramatsu * @unsafe_addr: The string to measure. 2653d708182SMasami Hiramatsu * @count: Maximum count (including NUL) 2663d708182SMasami Hiramatsu * 2673d708182SMasami Hiramatsu * Get the size of a NUL-terminated string in user space without pagefault. 2683d708182SMasami Hiramatsu * 2693d708182SMasami Hiramatsu * Returns the size of the string INCLUDING the terminating NUL. 2703d708182SMasami Hiramatsu * 2713d708182SMasami Hiramatsu * If the string is too long, returns a number larger than @count. User 2723d708182SMasami Hiramatsu * has to check the return value against "> count". 2733d708182SMasami Hiramatsu * On exception (or invalid count), returns 0. 2743d708182SMasami Hiramatsu * 2753d708182SMasami Hiramatsu * Unlike strnlen_user, this can be used from IRQ handler etc. because 2763d708182SMasami Hiramatsu * it disables pagefaults. 2773d708182SMasami Hiramatsu */ 2783d708182SMasami Hiramatsu long strnlen_unsafe_user(const void __user *unsafe_addr, long count) 2793d708182SMasami Hiramatsu { 2803d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2813d708182SMasami Hiramatsu int ret; 2823d708182SMasami Hiramatsu 2833d708182SMasami Hiramatsu set_fs(USER_DS); 2843d708182SMasami Hiramatsu pagefault_disable(); 2853d708182SMasami Hiramatsu ret = strnlen_user(unsafe_addr, count); 2863d708182SMasami Hiramatsu pagefault_enable(); 2873d708182SMasami Hiramatsu set_fs(old_fs); 2883d708182SMasami Hiramatsu 2893d708182SMasami Hiramatsu return ret; 2903d708182SMasami Hiramatsu } 291