1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c33fa9f5SIngo Molnar /* 3c33fa9f5SIngo Molnar * Access kernel memory without faulting. 4c33fa9f5SIngo Molnar */ 5b95f1b31SPaul Gortmaker #include <linux/export.h> 6c33fa9f5SIngo Molnar #include <linux/mm.h> 77c7fcf76SDavid Howells #include <linux/uaccess.h> 8c33fa9f5SIngo Molnar 93d708182SMasami Hiramatsu static __always_inline long 103d708182SMasami Hiramatsu probe_read_common(void *dst, const void __user *src, size_t size) 113d708182SMasami Hiramatsu { 123d708182SMasami Hiramatsu long ret; 133d708182SMasami Hiramatsu 143d708182SMasami Hiramatsu pagefault_disable(); 153d708182SMasami Hiramatsu ret = __copy_from_user_inatomic(dst, src, size); 163d708182SMasami Hiramatsu pagefault_enable(); 173d708182SMasami Hiramatsu 183d708182SMasami Hiramatsu return ret ? -EFAULT : 0; 193d708182SMasami Hiramatsu } 203d708182SMasami Hiramatsu 211d1585caSDaniel Borkmann static __always_inline long 221d1585caSDaniel Borkmann probe_write_common(void __user *dst, const void *src, size_t size) 231d1585caSDaniel Borkmann { 241d1585caSDaniel Borkmann long ret; 251d1585caSDaniel Borkmann 261d1585caSDaniel Borkmann pagefault_disable(); 271d1585caSDaniel Borkmann ret = __copy_to_user_inatomic(dst, src, size); 281d1585caSDaniel Borkmann pagefault_enable(); 291d1585caSDaniel Borkmann 301d1585caSDaniel Borkmann return ret ? -EFAULT : 0; 311d1585caSDaniel Borkmann } 321d1585caSDaniel Borkmann 33c33fa9f5SIngo Molnar /** 343d708182SMasami Hiramatsu * probe_kernel_read(): safely attempt to read from a kernel-space location 35c33fa9f5SIngo Molnar * @dst: pointer to the buffer that shall take the data 36c33fa9f5SIngo Molnar * @src: address to read from 37c33fa9f5SIngo Molnar * @size: size of the data chunk 38c33fa9f5SIngo Molnar * 39c33fa9f5SIngo Molnar * Safely read from address @src to the buffer at @dst. If a kernel fault 40c33fa9f5SIngo Molnar * happens, handle that and return -EFAULT. 410ab32b6fSAndrew Morton * 420ab32b6fSAndrew Morton * We ensure that the copy_from_user is executed in atomic context so that 430ab32b6fSAndrew Morton * do_page_fault() doesn't attempt to take mmap_sem. This makes 440ab32b6fSAndrew Morton * probe_kernel_read() suitable for use within regions where the caller 450ab32b6fSAndrew Morton * already holds mmap_sem, or other locks which nest inside mmap_sem. 4675a1a607SDaniel Borkmann * 4775a1a607SDaniel Borkmann * probe_kernel_read_strict() is the same as probe_kernel_read() except for 4875a1a607SDaniel Borkmann * the case where architectures have non-overlapping user and kernel address 4975a1a607SDaniel Borkmann * ranges: probe_kernel_read_strict() will additionally return -EFAULT for 5075a1a607SDaniel Borkmann * probing memory on a user address range where probe_user_read() is supposed 5175a1a607SDaniel Borkmann * to be used instead. 52c33fa9f5SIngo Molnar */ 536144a85aSJason Wessel 54f29c5041SSteven Rostedt long __weak probe_kernel_read(void *dst, const void *src, size_t size) 556144a85aSJason Wessel __attribute__((alias("__probe_kernel_read"))); 566144a85aSJason Wessel 5775a1a607SDaniel Borkmann long __weak probe_kernel_read_strict(void *dst, const void *src, size_t size) 5875a1a607SDaniel Borkmann __attribute__((alias("__probe_kernel_read"))); 5975a1a607SDaniel Borkmann 60f29c5041SSteven Rostedt long __probe_kernel_read(void *dst, const void *src, size_t size) 61c33fa9f5SIngo Molnar { 62c33fa9f5SIngo Molnar long ret; 63b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 64c33fa9f5SIngo Molnar 65b4b8ac52SJason Wessel set_fs(KERNEL_DS); 663d708182SMasami Hiramatsu ret = probe_read_common(dst, (__force const void __user *)src, size); 67b4b8ac52SJason Wessel set_fs(old_fs); 68c33fa9f5SIngo Molnar 693d708182SMasami Hiramatsu return ret; 70c33fa9f5SIngo Molnar } 71c33fa9f5SIngo Molnar EXPORT_SYMBOL_GPL(probe_kernel_read); 72c33fa9f5SIngo Molnar 73c33fa9f5SIngo Molnar /** 743d708182SMasami Hiramatsu * probe_user_read(): safely attempt to read from a user-space location 753d708182SMasami Hiramatsu * @dst: pointer to the buffer that shall take the data 763d708182SMasami Hiramatsu * @src: address to read from. This must be a user address. 773d708182SMasami Hiramatsu * @size: size of the data chunk 783d708182SMasami Hiramatsu * 793d708182SMasami Hiramatsu * Safely read from user address @src to the buffer at @dst. If a kernel fault 803d708182SMasami Hiramatsu * happens, handle that and return -EFAULT. 813d708182SMasami Hiramatsu */ 823d708182SMasami Hiramatsu 833d708182SMasami Hiramatsu long __weak probe_user_read(void *dst, const void __user *src, size_t size) 843d708182SMasami Hiramatsu __attribute__((alias("__probe_user_read"))); 853d708182SMasami Hiramatsu 863d708182SMasami Hiramatsu long __probe_user_read(void *dst, const void __user *src, size_t size) 873d708182SMasami Hiramatsu { 883d708182SMasami Hiramatsu long ret = -EFAULT; 893d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 903d708182SMasami Hiramatsu 913d708182SMasami Hiramatsu set_fs(USER_DS); 923d708182SMasami Hiramatsu if (access_ok(src, size)) 933d708182SMasami Hiramatsu ret = probe_read_common(dst, src, size); 943d708182SMasami Hiramatsu set_fs(old_fs); 953d708182SMasami Hiramatsu 963d708182SMasami Hiramatsu return ret; 973d708182SMasami Hiramatsu } 983d708182SMasami Hiramatsu EXPORT_SYMBOL_GPL(probe_user_read); 993d708182SMasami Hiramatsu 1003d708182SMasami Hiramatsu /** 101c33fa9f5SIngo Molnar * probe_kernel_write(): safely attempt to write to a location 102c33fa9f5SIngo Molnar * @dst: address to write to 103c33fa9f5SIngo Molnar * @src: pointer to the data that shall be written 104c33fa9f5SIngo Molnar * @size: size of the data chunk 105c33fa9f5SIngo Molnar * 106c33fa9f5SIngo Molnar * Safely write to address @dst from the buffer at @src. If a kernel fault 107c33fa9f5SIngo Molnar * happens, handle that and return -EFAULT. 108c33fa9f5SIngo Molnar */ 1091d1585caSDaniel Borkmann 110f29c5041SSteven Rostedt long __weak probe_kernel_write(void *dst, const void *src, size_t size) 1116144a85aSJason Wessel __attribute__((alias("__probe_kernel_write"))); 1126144a85aSJason Wessel 113f29c5041SSteven Rostedt long __probe_kernel_write(void *dst, const void *src, size_t size) 114c33fa9f5SIngo Molnar { 115c33fa9f5SIngo Molnar long ret; 116b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 117c33fa9f5SIngo Molnar 118b4b8ac52SJason Wessel set_fs(KERNEL_DS); 1191d1585caSDaniel Borkmann ret = probe_write_common((__force void __user *)dst, src, size); 120b4b8ac52SJason Wessel set_fs(old_fs); 121c33fa9f5SIngo Molnar 1221d1585caSDaniel Borkmann return ret; 123c33fa9f5SIngo Molnar } 124c33fa9f5SIngo Molnar EXPORT_SYMBOL_GPL(probe_kernel_write); 125dbb7ee0eSAlexei Starovoitov 1261d1585caSDaniel Borkmann /** 1271d1585caSDaniel Borkmann * probe_user_write(): safely attempt to write to a user-space location 1281d1585caSDaniel Borkmann * @dst: address to write to 1291d1585caSDaniel Borkmann * @src: pointer to the data that shall be written 1301d1585caSDaniel Borkmann * @size: size of the data chunk 1311d1585caSDaniel Borkmann * 1321d1585caSDaniel Borkmann * Safely write to address @dst from the buffer at @src. If a kernel fault 1331d1585caSDaniel Borkmann * happens, handle that and return -EFAULT. 1341d1585caSDaniel Borkmann */ 1351d1585caSDaniel Borkmann 1361d1585caSDaniel Borkmann long __weak probe_user_write(void __user *dst, const void *src, size_t size) 1371d1585caSDaniel Borkmann __attribute__((alias("__probe_user_write"))); 1381d1585caSDaniel Borkmann 1391d1585caSDaniel Borkmann long __probe_user_write(void __user *dst, const void *src, size_t size) 1401d1585caSDaniel Borkmann { 1411d1585caSDaniel Borkmann long ret = -EFAULT; 1421d1585caSDaniel Borkmann mm_segment_t old_fs = get_fs(); 1431d1585caSDaniel Borkmann 1441d1585caSDaniel Borkmann set_fs(USER_DS); 1451d1585caSDaniel Borkmann if (access_ok(dst, size)) 1461d1585caSDaniel Borkmann ret = probe_write_common(dst, src, size); 1471d1585caSDaniel Borkmann set_fs(old_fs); 1481d1585caSDaniel Borkmann 1491d1585caSDaniel Borkmann return ret; 1501d1585caSDaniel Borkmann } 1511d1585caSDaniel Borkmann EXPORT_SYMBOL_GPL(probe_user_write); 1523d708182SMasami Hiramatsu 153dbb7ee0eSAlexei Starovoitov /** 154dbb7ee0eSAlexei Starovoitov * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address. 155dbb7ee0eSAlexei Starovoitov * @dst: Destination address, in kernel space. This buffer must be at 156dbb7ee0eSAlexei Starovoitov * least @count bytes long. 157f144c390SMike Rapoport * @unsafe_addr: Unsafe address. 158dbb7ee0eSAlexei Starovoitov * @count: Maximum number of bytes to copy, including the trailing NUL. 159dbb7ee0eSAlexei Starovoitov * 160dbb7ee0eSAlexei Starovoitov * Copies a NUL-terminated string from unsafe address to kernel buffer. 161dbb7ee0eSAlexei Starovoitov * 162dbb7ee0eSAlexei Starovoitov * On success, returns the length of the string INCLUDING the trailing NUL. 163dbb7ee0eSAlexei Starovoitov * 164dbb7ee0eSAlexei Starovoitov * If access fails, returns -EFAULT (some data may have been copied 165dbb7ee0eSAlexei Starovoitov * and the trailing NUL added). 166dbb7ee0eSAlexei Starovoitov * 167dbb7ee0eSAlexei Starovoitov * If @count is smaller than the length of the string, copies @count-1 bytes, 168dbb7ee0eSAlexei Starovoitov * sets the last byte of @dst buffer to NUL and returns @count. 16975a1a607SDaniel Borkmann * 17075a1a607SDaniel Borkmann * strncpy_from_unsafe_strict() is the same as strncpy_from_unsafe() except 17175a1a607SDaniel Borkmann * for the case where architectures have non-overlapping user and kernel address 17275a1a607SDaniel Borkmann * ranges: strncpy_from_unsafe_strict() will additionally return -EFAULT for 17375a1a607SDaniel Borkmann * probing memory on a user address range where strncpy_from_unsafe_user() is 17475a1a607SDaniel Borkmann * supposed to be used instead. 175dbb7ee0eSAlexei Starovoitov */ 17675a1a607SDaniel Borkmann 17775a1a607SDaniel Borkmann long __weak strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) 17875a1a607SDaniel Borkmann __attribute__((alias("__strncpy_from_unsafe"))); 17975a1a607SDaniel Borkmann 18075a1a607SDaniel Borkmann long __weak strncpy_from_unsafe_strict(char *dst, const void *unsafe_addr, 18175a1a607SDaniel Borkmann long count) 18275a1a607SDaniel Borkmann __attribute__((alias("__strncpy_from_unsafe"))); 18375a1a607SDaniel Borkmann 18475a1a607SDaniel Borkmann long __strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) 185dbb7ee0eSAlexei Starovoitov { 186dbb7ee0eSAlexei Starovoitov mm_segment_t old_fs = get_fs(); 187dbb7ee0eSAlexei Starovoitov const void *src = unsafe_addr; 188dbb7ee0eSAlexei Starovoitov long ret; 189dbb7ee0eSAlexei Starovoitov 190dbb7ee0eSAlexei Starovoitov if (unlikely(count <= 0)) 191dbb7ee0eSAlexei Starovoitov return 0; 192dbb7ee0eSAlexei Starovoitov 193dbb7ee0eSAlexei Starovoitov set_fs(KERNEL_DS); 194dbb7ee0eSAlexei Starovoitov pagefault_disable(); 195dbb7ee0eSAlexei Starovoitov 196dbb7ee0eSAlexei Starovoitov do { 197bd28b145SLinus Torvalds ret = __get_user(*dst++, (const char __user __force *)src++); 198dbb7ee0eSAlexei Starovoitov } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 199dbb7ee0eSAlexei Starovoitov 200dbb7ee0eSAlexei Starovoitov dst[-1] = '\0'; 201dbb7ee0eSAlexei Starovoitov pagefault_enable(); 202dbb7ee0eSAlexei Starovoitov set_fs(old_fs); 203dbb7ee0eSAlexei Starovoitov 2049dd861d5SRasmus Villemoes return ret ? -EFAULT : src - unsafe_addr; 205dbb7ee0eSAlexei Starovoitov } 2063d708182SMasami Hiramatsu 2073d708182SMasami Hiramatsu /** 2083d708182SMasami Hiramatsu * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user 2093d708182SMasami Hiramatsu * address. 2103d708182SMasami Hiramatsu * @dst: Destination address, in kernel space. This buffer must be at 2113d708182SMasami Hiramatsu * least @count bytes long. 2123d708182SMasami Hiramatsu * @unsafe_addr: Unsafe user address. 2133d708182SMasami Hiramatsu * @count: Maximum number of bytes to copy, including the trailing NUL. 2143d708182SMasami Hiramatsu * 2153d708182SMasami Hiramatsu * Copies a NUL-terminated string from unsafe user address to kernel buffer. 2163d708182SMasami Hiramatsu * 2173d708182SMasami Hiramatsu * On success, returns the length of the string INCLUDING the trailing NUL. 2183d708182SMasami Hiramatsu * 2193d708182SMasami Hiramatsu * If access fails, returns -EFAULT (some data may have been copied 2203d708182SMasami Hiramatsu * and the trailing NUL added). 2213d708182SMasami Hiramatsu * 2223d708182SMasami Hiramatsu * If @count is smaller than the length of the string, copies @count-1 bytes, 2233d708182SMasami Hiramatsu * sets the last byte of @dst buffer to NUL and returns @count. 2243d708182SMasami Hiramatsu */ 2253d708182SMasami Hiramatsu long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, 2263d708182SMasami Hiramatsu long count) 2273d708182SMasami Hiramatsu { 2283d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2293d708182SMasami Hiramatsu long ret; 2303d708182SMasami Hiramatsu 2313d708182SMasami Hiramatsu if (unlikely(count <= 0)) 2323d708182SMasami Hiramatsu return 0; 2333d708182SMasami Hiramatsu 2343d708182SMasami Hiramatsu set_fs(USER_DS); 2353d708182SMasami Hiramatsu pagefault_disable(); 2363d708182SMasami Hiramatsu ret = strncpy_from_user(dst, unsafe_addr, count); 2373d708182SMasami Hiramatsu pagefault_enable(); 2383d708182SMasami Hiramatsu set_fs(old_fs); 2393d708182SMasami Hiramatsu 2403d708182SMasami Hiramatsu if (ret >= count) { 2413d708182SMasami Hiramatsu ret = count; 2423d708182SMasami Hiramatsu dst[ret - 1] = '\0'; 2433d708182SMasami Hiramatsu } else if (ret > 0) { 2443d708182SMasami Hiramatsu ret++; 2453d708182SMasami Hiramatsu } 2463d708182SMasami Hiramatsu 2473d708182SMasami Hiramatsu return ret; 2483d708182SMasami Hiramatsu } 2493d708182SMasami Hiramatsu 2503d708182SMasami Hiramatsu /** 2513d708182SMasami Hiramatsu * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL. 2523d708182SMasami Hiramatsu * @unsafe_addr: The string to measure. 2533d708182SMasami Hiramatsu * @count: Maximum count (including NUL) 2543d708182SMasami Hiramatsu * 2553d708182SMasami Hiramatsu * Get the size of a NUL-terminated string in user space without pagefault. 2563d708182SMasami Hiramatsu * 2573d708182SMasami Hiramatsu * Returns the size of the string INCLUDING the terminating NUL. 2583d708182SMasami Hiramatsu * 2593d708182SMasami Hiramatsu * If the string is too long, returns a number larger than @count. User 2603d708182SMasami Hiramatsu * has to check the return value against "> count". 2613d708182SMasami Hiramatsu * On exception (or invalid count), returns 0. 2623d708182SMasami Hiramatsu * 2633d708182SMasami Hiramatsu * Unlike strnlen_user, this can be used from IRQ handler etc. because 2643d708182SMasami Hiramatsu * it disables pagefaults. 2653d708182SMasami Hiramatsu */ 2663d708182SMasami Hiramatsu long strnlen_unsafe_user(const void __user *unsafe_addr, long count) 2673d708182SMasami Hiramatsu { 2683d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2693d708182SMasami Hiramatsu int ret; 2703d708182SMasami Hiramatsu 2713d708182SMasami Hiramatsu set_fs(USER_DS); 2723d708182SMasami Hiramatsu pagefault_disable(); 2733d708182SMasami Hiramatsu ret = strnlen_user(unsafe_addr, count); 2743d708182SMasami Hiramatsu pagefault_enable(); 2753d708182SMasami Hiramatsu set_fs(old_fs); 2763d708182SMasami Hiramatsu 2773d708182SMasami Hiramatsu return ret; 2783d708182SMasami Hiramatsu } 279