1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c33fa9f5SIngo Molnar /* 3c33fa9f5SIngo Molnar * Access kernel memory without faulting. 4c33fa9f5SIngo Molnar */ 5b95f1b31SPaul Gortmaker #include <linux/export.h> 6c33fa9f5SIngo Molnar #include <linux/mm.h> 77c7fcf76SDavid Howells #include <linux/uaccess.h> 8c33fa9f5SIngo Molnar 93d708182SMasami Hiramatsu static __always_inline long 103d708182SMasami Hiramatsu probe_read_common(void *dst, const void __user *src, size_t size) 113d708182SMasami Hiramatsu { 123d708182SMasami Hiramatsu long ret; 133d708182SMasami Hiramatsu 143d708182SMasami Hiramatsu pagefault_disable(); 153d708182SMasami Hiramatsu ret = __copy_from_user_inatomic(dst, src, size); 163d708182SMasami Hiramatsu pagefault_enable(); 173d708182SMasami Hiramatsu 183d708182SMasami Hiramatsu return ret ? -EFAULT : 0; 193d708182SMasami Hiramatsu } 203d708182SMasami Hiramatsu 211d1585caSDaniel Borkmann static __always_inline long 221d1585caSDaniel Borkmann probe_write_common(void __user *dst, const void *src, size_t size) 231d1585caSDaniel Borkmann { 241d1585caSDaniel Borkmann long ret; 251d1585caSDaniel Borkmann 261d1585caSDaniel Borkmann pagefault_disable(); 271d1585caSDaniel Borkmann ret = __copy_to_user_inatomic(dst, src, size); 281d1585caSDaniel Borkmann pagefault_enable(); 291d1585caSDaniel Borkmann 301d1585caSDaniel Borkmann return ret ? -EFAULT : 0; 311d1585caSDaniel Borkmann } 321d1585caSDaniel Borkmann 33c33fa9f5SIngo Molnar /** 343d708182SMasami Hiramatsu * probe_kernel_read(): safely attempt to read from a kernel-space location 35c33fa9f5SIngo Molnar * @dst: pointer to the buffer that shall take the data 36c33fa9f5SIngo Molnar * @src: address to read from 37c33fa9f5SIngo Molnar * @size: size of the data chunk 38c33fa9f5SIngo Molnar * 39c33fa9f5SIngo Molnar * Safely read from address @src to the buffer at @dst. If a kernel fault 40c33fa9f5SIngo Molnar * happens, handle that and return -EFAULT. 410ab32b6fSAndrew Morton * 420ab32b6fSAndrew Morton * We ensure that the copy_from_user is executed in atomic context so that 430ab32b6fSAndrew Morton * do_page_fault() doesn't attempt to take mmap_sem. This makes 440ab32b6fSAndrew Morton * probe_kernel_read() suitable for use within regions where the caller 450ab32b6fSAndrew Morton * already holds mmap_sem, or other locks which nest inside mmap_sem. 46c33fa9f5SIngo Molnar */ 476144a85aSJason Wessel 48f29c5041SSteven Rostedt long __weak probe_kernel_read(void *dst, const void *src, size_t size) 496144a85aSJason Wessel __attribute__((alias("__probe_kernel_read"))); 506144a85aSJason Wessel 51f29c5041SSteven Rostedt long __probe_kernel_read(void *dst, const void *src, size_t size) 52c33fa9f5SIngo Molnar { 53c33fa9f5SIngo Molnar long ret; 54b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 55c33fa9f5SIngo Molnar 56b4b8ac52SJason Wessel set_fs(KERNEL_DS); 573d708182SMasami Hiramatsu ret = probe_read_common(dst, (__force const void __user *)src, size); 58b4b8ac52SJason Wessel set_fs(old_fs); 59c33fa9f5SIngo Molnar 603d708182SMasami Hiramatsu return ret; 61c33fa9f5SIngo Molnar } 62c33fa9f5SIngo Molnar EXPORT_SYMBOL_GPL(probe_kernel_read); 63c33fa9f5SIngo Molnar 64c33fa9f5SIngo Molnar /** 653d708182SMasami Hiramatsu * probe_user_read(): safely attempt to read from a user-space location 663d708182SMasami Hiramatsu * @dst: pointer to the buffer that shall take the data 673d708182SMasami Hiramatsu * @src: address to read from. This must be a user address. 683d708182SMasami Hiramatsu * @size: size of the data chunk 693d708182SMasami Hiramatsu * 703d708182SMasami Hiramatsu * Safely read from user address @src to the buffer at @dst. If a kernel fault 713d708182SMasami Hiramatsu * happens, handle that and return -EFAULT. 723d708182SMasami Hiramatsu */ 733d708182SMasami Hiramatsu 743d708182SMasami Hiramatsu long __weak probe_user_read(void *dst, const void __user *src, size_t size) 753d708182SMasami Hiramatsu __attribute__((alias("__probe_user_read"))); 763d708182SMasami Hiramatsu 773d708182SMasami Hiramatsu long __probe_user_read(void *dst, const void __user *src, size_t size) 783d708182SMasami Hiramatsu { 793d708182SMasami Hiramatsu long ret = -EFAULT; 803d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 813d708182SMasami Hiramatsu 823d708182SMasami Hiramatsu set_fs(USER_DS); 833d708182SMasami Hiramatsu if (access_ok(src, size)) 843d708182SMasami Hiramatsu ret = probe_read_common(dst, src, size); 853d708182SMasami Hiramatsu set_fs(old_fs); 863d708182SMasami Hiramatsu 873d708182SMasami Hiramatsu return ret; 883d708182SMasami Hiramatsu } 893d708182SMasami Hiramatsu EXPORT_SYMBOL_GPL(probe_user_read); 903d708182SMasami Hiramatsu 913d708182SMasami Hiramatsu /** 92c33fa9f5SIngo Molnar * probe_kernel_write(): safely attempt to write to a location 93c33fa9f5SIngo Molnar * @dst: address to write to 94c33fa9f5SIngo Molnar * @src: pointer to the data that shall be written 95c33fa9f5SIngo Molnar * @size: size of the data chunk 96c33fa9f5SIngo Molnar * 97c33fa9f5SIngo Molnar * Safely write to address @dst from the buffer at @src. If a kernel fault 98c33fa9f5SIngo Molnar * happens, handle that and return -EFAULT. 99c33fa9f5SIngo Molnar */ 1001d1585caSDaniel Borkmann 101f29c5041SSteven Rostedt long __weak probe_kernel_write(void *dst, const void *src, size_t size) 1026144a85aSJason Wessel __attribute__((alias("__probe_kernel_write"))); 1036144a85aSJason Wessel 104f29c5041SSteven Rostedt long __probe_kernel_write(void *dst, const void *src, size_t size) 105c33fa9f5SIngo Molnar { 106c33fa9f5SIngo Molnar long ret; 107b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 108c33fa9f5SIngo Molnar 109b4b8ac52SJason Wessel set_fs(KERNEL_DS); 1101d1585caSDaniel Borkmann ret = probe_write_common((__force void __user *)dst, src, size); 111b4b8ac52SJason Wessel set_fs(old_fs); 112c33fa9f5SIngo Molnar 1131d1585caSDaniel Borkmann return ret; 114c33fa9f5SIngo Molnar } 115c33fa9f5SIngo Molnar EXPORT_SYMBOL_GPL(probe_kernel_write); 116dbb7ee0eSAlexei Starovoitov 1171d1585caSDaniel Borkmann /** 1181d1585caSDaniel Borkmann * probe_user_write(): safely attempt to write to a user-space location 1191d1585caSDaniel Borkmann * @dst: address to write to 1201d1585caSDaniel Borkmann * @src: pointer to the data that shall be written 1211d1585caSDaniel Borkmann * @size: size of the data chunk 1221d1585caSDaniel Borkmann * 1231d1585caSDaniel Borkmann * Safely write to address @dst from the buffer at @src. If a kernel fault 1241d1585caSDaniel Borkmann * happens, handle that and return -EFAULT. 1251d1585caSDaniel Borkmann */ 1261d1585caSDaniel Borkmann 1271d1585caSDaniel Borkmann long __weak probe_user_write(void __user *dst, const void *src, size_t size) 1281d1585caSDaniel Borkmann __attribute__((alias("__probe_user_write"))); 1291d1585caSDaniel Borkmann 1301d1585caSDaniel Borkmann long __probe_user_write(void __user *dst, const void *src, size_t size) 1311d1585caSDaniel Borkmann { 1321d1585caSDaniel Borkmann long ret = -EFAULT; 1331d1585caSDaniel Borkmann mm_segment_t old_fs = get_fs(); 1341d1585caSDaniel Borkmann 1351d1585caSDaniel Borkmann set_fs(USER_DS); 1361d1585caSDaniel Borkmann if (access_ok(dst, size)) 1371d1585caSDaniel Borkmann ret = probe_write_common(dst, src, size); 1381d1585caSDaniel Borkmann set_fs(old_fs); 1391d1585caSDaniel Borkmann 1401d1585caSDaniel Borkmann return ret; 1411d1585caSDaniel Borkmann } 1421d1585caSDaniel Borkmann EXPORT_SYMBOL_GPL(probe_user_write); 1433d708182SMasami Hiramatsu 144dbb7ee0eSAlexei Starovoitov /** 145dbb7ee0eSAlexei Starovoitov * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address. 146dbb7ee0eSAlexei Starovoitov * @dst: Destination address, in kernel space. This buffer must be at 147dbb7ee0eSAlexei Starovoitov * least @count bytes long. 148f144c390SMike Rapoport * @unsafe_addr: Unsafe address. 149dbb7ee0eSAlexei Starovoitov * @count: Maximum number of bytes to copy, including the trailing NUL. 150dbb7ee0eSAlexei Starovoitov * 151dbb7ee0eSAlexei Starovoitov * Copies a NUL-terminated string from unsafe address to kernel buffer. 152dbb7ee0eSAlexei Starovoitov * 153dbb7ee0eSAlexei Starovoitov * On success, returns the length of the string INCLUDING the trailing NUL. 154dbb7ee0eSAlexei Starovoitov * 155dbb7ee0eSAlexei Starovoitov * If access fails, returns -EFAULT (some data may have been copied 156dbb7ee0eSAlexei Starovoitov * and the trailing NUL added). 157dbb7ee0eSAlexei Starovoitov * 158dbb7ee0eSAlexei Starovoitov * If @count is smaller than the length of the string, copies @count-1 bytes, 159dbb7ee0eSAlexei Starovoitov * sets the last byte of @dst buffer to NUL and returns @count. 160dbb7ee0eSAlexei Starovoitov */ 161dbb7ee0eSAlexei Starovoitov long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) 162dbb7ee0eSAlexei Starovoitov { 163dbb7ee0eSAlexei Starovoitov mm_segment_t old_fs = get_fs(); 164dbb7ee0eSAlexei Starovoitov const void *src = unsafe_addr; 165dbb7ee0eSAlexei Starovoitov long ret; 166dbb7ee0eSAlexei Starovoitov 167dbb7ee0eSAlexei Starovoitov if (unlikely(count <= 0)) 168dbb7ee0eSAlexei Starovoitov return 0; 169dbb7ee0eSAlexei Starovoitov 170dbb7ee0eSAlexei Starovoitov set_fs(KERNEL_DS); 171dbb7ee0eSAlexei Starovoitov pagefault_disable(); 172dbb7ee0eSAlexei Starovoitov 173dbb7ee0eSAlexei Starovoitov do { 174bd28b145SLinus Torvalds ret = __get_user(*dst++, (const char __user __force *)src++); 175dbb7ee0eSAlexei Starovoitov } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 176dbb7ee0eSAlexei Starovoitov 177dbb7ee0eSAlexei Starovoitov dst[-1] = '\0'; 178dbb7ee0eSAlexei Starovoitov pagefault_enable(); 179dbb7ee0eSAlexei Starovoitov set_fs(old_fs); 180dbb7ee0eSAlexei Starovoitov 1819dd861d5SRasmus Villemoes return ret ? -EFAULT : src - unsafe_addr; 182dbb7ee0eSAlexei Starovoitov } 1833d708182SMasami Hiramatsu 1843d708182SMasami Hiramatsu /** 1853d708182SMasami Hiramatsu * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user 1863d708182SMasami Hiramatsu * address. 1873d708182SMasami Hiramatsu * @dst: Destination address, in kernel space. This buffer must be at 1883d708182SMasami Hiramatsu * least @count bytes long. 1893d708182SMasami Hiramatsu * @unsafe_addr: Unsafe user address. 1903d708182SMasami Hiramatsu * @count: Maximum number of bytes to copy, including the trailing NUL. 1913d708182SMasami Hiramatsu * 1923d708182SMasami Hiramatsu * Copies a NUL-terminated string from unsafe user address to kernel buffer. 1933d708182SMasami Hiramatsu * 1943d708182SMasami Hiramatsu * On success, returns the length of the string INCLUDING the trailing NUL. 1953d708182SMasami Hiramatsu * 1963d708182SMasami Hiramatsu * If access fails, returns -EFAULT (some data may have been copied 1973d708182SMasami Hiramatsu * and the trailing NUL added). 1983d708182SMasami Hiramatsu * 1993d708182SMasami Hiramatsu * If @count is smaller than the length of the string, copies @count-1 bytes, 2003d708182SMasami Hiramatsu * sets the last byte of @dst buffer to NUL and returns @count. 2013d708182SMasami Hiramatsu */ 2023d708182SMasami Hiramatsu long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, 2033d708182SMasami Hiramatsu long count) 2043d708182SMasami Hiramatsu { 2053d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2063d708182SMasami Hiramatsu long ret; 2073d708182SMasami Hiramatsu 2083d708182SMasami Hiramatsu if (unlikely(count <= 0)) 2093d708182SMasami Hiramatsu return 0; 2103d708182SMasami Hiramatsu 2113d708182SMasami Hiramatsu set_fs(USER_DS); 2123d708182SMasami Hiramatsu pagefault_disable(); 2133d708182SMasami Hiramatsu ret = strncpy_from_user(dst, unsafe_addr, count); 2143d708182SMasami Hiramatsu pagefault_enable(); 2153d708182SMasami Hiramatsu set_fs(old_fs); 2163d708182SMasami Hiramatsu 2173d708182SMasami Hiramatsu if (ret >= count) { 2183d708182SMasami Hiramatsu ret = count; 2193d708182SMasami Hiramatsu dst[ret - 1] = '\0'; 2203d708182SMasami Hiramatsu } else if (ret > 0) { 2213d708182SMasami Hiramatsu ret++; 2223d708182SMasami Hiramatsu } 2233d708182SMasami Hiramatsu 2243d708182SMasami Hiramatsu return ret; 2253d708182SMasami Hiramatsu } 2263d708182SMasami Hiramatsu 2273d708182SMasami Hiramatsu /** 2283d708182SMasami Hiramatsu * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL. 2293d708182SMasami Hiramatsu * @unsafe_addr: The string to measure. 2303d708182SMasami Hiramatsu * @count: Maximum count (including NUL) 2313d708182SMasami Hiramatsu * 2323d708182SMasami Hiramatsu * Get the size of a NUL-terminated string in user space without pagefault. 2333d708182SMasami Hiramatsu * 2343d708182SMasami Hiramatsu * Returns the size of the string INCLUDING the terminating NUL. 2353d708182SMasami Hiramatsu * 2363d708182SMasami Hiramatsu * If the string is too long, returns a number larger than @count. User 2373d708182SMasami Hiramatsu * has to check the return value against "> count". 2383d708182SMasami Hiramatsu * On exception (or invalid count), returns 0. 2393d708182SMasami Hiramatsu * 2403d708182SMasami Hiramatsu * Unlike strnlen_user, this can be used from IRQ handler etc. because 2413d708182SMasami Hiramatsu * it disables pagefaults. 2423d708182SMasami Hiramatsu */ 2433d708182SMasami Hiramatsu long strnlen_unsafe_user(const void __user *unsafe_addr, long count) 2443d708182SMasami Hiramatsu { 2453d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2463d708182SMasami Hiramatsu int ret; 2473d708182SMasami Hiramatsu 2483d708182SMasami Hiramatsu set_fs(USER_DS); 2493d708182SMasami Hiramatsu pagefault_disable(); 2503d708182SMasami Hiramatsu ret = strnlen_user(unsafe_addr, count); 2513d708182SMasami Hiramatsu pagefault_enable(); 2523d708182SMasami Hiramatsu set_fs(old_fs); 2533d708182SMasami Hiramatsu 2543d708182SMasami Hiramatsu return ret; 2553d708182SMasami Hiramatsu } 256