1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c33fa9f5SIngo Molnar /* 33f0acb1eSChristoph Hellwig * Access kernel or user memory without faulting. 4c33fa9f5SIngo Molnar */ 5b95f1b31SPaul Gortmaker #include <linux/export.h> 6c33fa9f5SIngo Molnar #include <linux/mm.h> 77c7fcf76SDavid Howells #include <linux/uaccess.h> 8c33fa9f5SIngo Molnar 9fe557319SChristoph Hellwig bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, 10fe557319SChristoph Hellwig size_t size) 11eab0c608SChristoph Hellwig { 12eab0c608SChristoph Hellwig return true; 13eab0c608SChristoph Hellwig } 14eab0c608SChristoph Hellwig 15b58294eaSChristoph Hellwig #ifdef HAVE_GET_KERNEL_NOFAULT 16b58294eaSChristoph Hellwig 17fe557319SChristoph Hellwig #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ 18b58294eaSChristoph Hellwig while (len >= sizeof(type)) { \ 19b58294eaSChristoph Hellwig __get_kernel_nofault(dst, src, type, err_label); \ 20b58294eaSChristoph Hellwig dst += sizeof(type); \ 21b58294eaSChristoph Hellwig src += sizeof(type); \ 22b58294eaSChristoph Hellwig len -= sizeof(type); \ 23b58294eaSChristoph Hellwig } 24b58294eaSChristoph Hellwig 25fe557319SChristoph Hellwig long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 26b58294eaSChristoph Hellwig { 27fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(src, size)) 282a71e81dSChristoph Hellwig return -ERANGE; 29b58294eaSChristoph Hellwig 30b58294eaSChristoph Hellwig pagefault_disable(); 31fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); 32fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); 33fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); 34fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); 35b58294eaSChristoph Hellwig pagefault_enable(); 36b58294eaSChristoph Hellwig return 0; 37b58294eaSChristoph Hellwig Efault: 38b58294eaSChristoph Hellwig pagefault_enable(); 39b58294eaSChristoph Hellwig return -EFAULT; 40b58294eaSChristoph Hellwig } 41fe557319SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 42b58294eaSChristoph Hellwig 43fe557319SChristoph Hellwig #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ 44b58294eaSChristoph Hellwig while (len >= sizeof(type)) { \ 45b58294eaSChristoph Hellwig __put_kernel_nofault(dst, src, type, err_label); \ 46b58294eaSChristoph Hellwig dst += sizeof(type); \ 47b58294eaSChristoph Hellwig src += sizeof(type); \ 48b58294eaSChristoph Hellwig len -= sizeof(type); \ 49b58294eaSChristoph Hellwig } 50b58294eaSChristoph Hellwig 51fe557319SChristoph Hellwig long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 52b58294eaSChristoph Hellwig { 53b58294eaSChristoph Hellwig pagefault_disable(); 54fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); 55fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); 56fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); 57fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); 58b58294eaSChristoph Hellwig pagefault_enable(); 59b58294eaSChristoph Hellwig return 0; 60b58294eaSChristoph Hellwig Efault: 61b58294eaSChristoph Hellwig pagefault_enable(); 62b58294eaSChristoph Hellwig return -EFAULT; 63b58294eaSChristoph Hellwig } 64b58294eaSChristoph Hellwig 65b58294eaSChristoph Hellwig long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 66b58294eaSChristoph Hellwig { 67b58294eaSChristoph Hellwig const void *src = unsafe_addr; 68b58294eaSChristoph Hellwig 69b58294eaSChristoph Hellwig if (unlikely(count <= 0)) 70b58294eaSChristoph Hellwig return 0; 71fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 722a71e81dSChristoph Hellwig return -ERANGE; 73b58294eaSChristoph Hellwig 74b58294eaSChristoph Hellwig pagefault_disable(); 75b58294eaSChristoph Hellwig do { 76b58294eaSChristoph Hellwig __get_kernel_nofault(dst, src, u8, Efault); 77b58294eaSChristoph Hellwig dst++; 78b58294eaSChristoph Hellwig src++; 79b58294eaSChristoph Hellwig } while (dst[-1] && src - unsafe_addr < count); 80b58294eaSChristoph Hellwig pagefault_enable(); 81b58294eaSChristoph Hellwig 82b58294eaSChristoph Hellwig dst[-1] = '\0'; 83b58294eaSChristoph Hellwig return src - unsafe_addr; 84b58294eaSChristoph Hellwig Efault: 85b58294eaSChristoph Hellwig pagefault_enable(); 86b58294eaSChristoph Hellwig dst[-1] = '\0'; 87b58294eaSChristoph Hellwig return -EFAULT; 88b58294eaSChristoph Hellwig } 89b58294eaSChristoph Hellwig #else /* HAVE_GET_KERNEL_NOFAULT */ 90c33fa9f5SIngo Molnar /** 91fe557319SChristoph Hellwig * copy_from_kernel_nofault(): safely attempt to read from kernel-space 924f6de12bSChristoph Hellwig * @dst: pointer to the buffer that shall take the data 934f6de12bSChristoph Hellwig * @src: address to read from 944f6de12bSChristoph Hellwig * @size: size of the data chunk 954f6de12bSChristoph Hellwig * 964f6de12bSChristoph Hellwig * Safely read from kernel address @src to the buffer at @dst. If a kernel 972a71e81dSChristoph Hellwig * fault happens, handle that and return -EFAULT. If @src is not a valid kernel 982a71e81dSChristoph Hellwig * address, return -ERANGE. 990ab32b6fSAndrew Morton * 1000ab32b6fSAndrew Morton * We ensure that the copy_from_user is executed in atomic context so that 101c1e8d7c6SMichel Lespinasse * do_page_fault() doesn't attempt to take mmap_lock. This makes 102fe557319SChristoph Hellwig * copy_from_kernel_nofault() suitable for use within regions where the caller 103c1e8d7c6SMichel Lespinasse * already holds mmap_lock, or other locks which nest inside mmap_lock. 104c33fa9f5SIngo Molnar */ 105fe557319SChristoph Hellwig long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 106c33fa9f5SIngo Molnar { 107c33fa9f5SIngo Molnar long ret; 108b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 109c33fa9f5SIngo Molnar 110fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(src, size)) 1112a71e81dSChristoph Hellwig return -ERANGE; 112eab0c608SChristoph Hellwig 113b4b8ac52SJason Wessel set_fs(KERNEL_DS); 114cd030905SChristoph Hellwig pagefault_disable(); 115cd030905SChristoph Hellwig ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, 116cd030905SChristoph Hellwig size); 117cd030905SChristoph Hellwig pagefault_enable(); 118b4b8ac52SJason Wessel set_fs(old_fs); 119c33fa9f5SIngo Molnar 120cd030905SChristoph Hellwig if (ret) 121cd030905SChristoph Hellwig return -EFAULT; 122cd030905SChristoph Hellwig return 0; 123c33fa9f5SIngo Molnar } 124fe557319SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 125c33fa9f5SIngo Molnar 126c33fa9f5SIngo Molnar /** 127fe557319SChristoph Hellwig * copy_to_kernel_nofault(): safely attempt to write to a location 128c33fa9f5SIngo Molnar * @dst: address to write to 129c33fa9f5SIngo Molnar * @src: pointer to the data that shall be written 130c33fa9f5SIngo Molnar * @size: size of the data chunk 131c33fa9f5SIngo Molnar * 132c33fa9f5SIngo Molnar * Safely write to address @dst from the buffer at @src. If a kernel fault 133c33fa9f5SIngo Molnar * happens, handle that and return -EFAULT. 134c33fa9f5SIngo Molnar */ 135fe557319SChristoph Hellwig long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 136c33fa9f5SIngo Molnar { 137c33fa9f5SIngo Molnar long ret; 138b4b8ac52SJason Wessel mm_segment_t old_fs = get_fs(); 139c33fa9f5SIngo Molnar 140b4b8ac52SJason Wessel set_fs(KERNEL_DS); 141cd030905SChristoph Hellwig pagefault_disable(); 142cd030905SChristoph Hellwig ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); 143cd030905SChristoph Hellwig pagefault_enable(); 144b4b8ac52SJason Wessel set_fs(old_fs); 145c33fa9f5SIngo Molnar 146cd030905SChristoph Hellwig if (ret) 147cd030905SChristoph Hellwig return -EFAULT; 148cd030905SChristoph Hellwig return 0; 149c33fa9f5SIngo Molnar } 150dbb7ee0eSAlexei Starovoitov 1511d1585caSDaniel Borkmann /** 152c4cb1644SChristoph Hellwig * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe 1534f6de12bSChristoph Hellwig * address. 1544f6de12bSChristoph Hellwig * @dst: Destination address, in kernel space. This buffer must be at 1554f6de12bSChristoph Hellwig * least @count bytes long. 1564f6de12bSChristoph Hellwig * @unsafe_addr: Unsafe address. 1574f6de12bSChristoph Hellwig * @count: Maximum number of bytes to copy, including the trailing NUL. 1584f6de12bSChristoph Hellwig * 1594f6de12bSChristoph Hellwig * Copies a NUL-terminated string from unsafe address to kernel buffer. 1604f6de12bSChristoph Hellwig * 1614f6de12bSChristoph Hellwig * On success, returns the length of the string INCLUDING the trailing NUL. 1624f6de12bSChristoph Hellwig * 1632a71e81dSChristoph Hellwig * If access fails, returns -EFAULT (some data may have been copied and the 1642a71e81dSChristoph Hellwig * trailing NUL added). If @unsafe_addr is not a valid kernel address, return 1652a71e81dSChristoph Hellwig * -ERANGE. 1664f6de12bSChristoph Hellwig * 1674f6de12bSChristoph Hellwig * If @count is smaller than the length of the string, copies @count-1 bytes, 1684f6de12bSChristoph Hellwig * sets the last byte of @dst buffer to NUL and returns @count. 1694f6de12bSChristoph Hellwig */ 170eab0c608SChristoph Hellwig long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 171eab0c608SChristoph Hellwig { 172dbb7ee0eSAlexei Starovoitov mm_segment_t old_fs = get_fs(); 173dbb7ee0eSAlexei Starovoitov const void *src = unsafe_addr; 174dbb7ee0eSAlexei Starovoitov long ret; 175dbb7ee0eSAlexei Starovoitov 176dbb7ee0eSAlexei Starovoitov if (unlikely(count <= 0)) 177dbb7ee0eSAlexei Starovoitov return 0; 178fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 1792a71e81dSChristoph Hellwig return -ERANGE; 180dbb7ee0eSAlexei Starovoitov 181dbb7ee0eSAlexei Starovoitov set_fs(KERNEL_DS); 182dbb7ee0eSAlexei Starovoitov pagefault_disable(); 183dbb7ee0eSAlexei Starovoitov 184dbb7ee0eSAlexei Starovoitov do { 185bd28b145SLinus Torvalds ret = __get_user(*dst++, (const char __user __force *)src++); 186dbb7ee0eSAlexei Starovoitov } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 187dbb7ee0eSAlexei Starovoitov 188dbb7ee0eSAlexei Starovoitov dst[-1] = '\0'; 189dbb7ee0eSAlexei Starovoitov pagefault_enable(); 190dbb7ee0eSAlexei Starovoitov set_fs(old_fs); 191dbb7ee0eSAlexei Starovoitov 1929dd861d5SRasmus Villemoes return ret ? -EFAULT : src - unsafe_addr; 193dbb7ee0eSAlexei Starovoitov } 194b58294eaSChristoph Hellwig #endif /* HAVE_GET_KERNEL_NOFAULT */ 1953d708182SMasami Hiramatsu 1963d708182SMasami Hiramatsu /** 197c0ee37e8SChristoph Hellwig * copy_from_user_nofault(): safely attempt to read from a user-space location 198fc3562d7SChristoph Hellwig * @dst: pointer to the buffer that shall take the data 199fc3562d7SChristoph Hellwig * @src: address to read from. This must be a user address. 200fc3562d7SChristoph Hellwig * @size: size of the data chunk 201fc3562d7SChristoph Hellwig * 202fc3562d7SChristoph Hellwig * Safely read from user address @src to the buffer at @dst. If a kernel fault 203fc3562d7SChristoph Hellwig * happens, handle that and return -EFAULT. 204fc3562d7SChristoph Hellwig */ 205c0ee37e8SChristoph Hellwig long copy_from_user_nofault(void *dst, const void __user *src, size_t size) 206fc3562d7SChristoph Hellwig { 207fc3562d7SChristoph Hellwig long ret = -EFAULT; 208fc3562d7SChristoph Hellwig mm_segment_t old_fs = get_fs(); 209fc3562d7SChristoph Hellwig 210fc3562d7SChristoph Hellwig set_fs(USER_DS); 211fc3562d7SChristoph Hellwig if (access_ok(src, size)) { 212fc3562d7SChristoph Hellwig pagefault_disable(); 213fc3562d7SChristoph Hellwig ret = __copy_from_user_inatomic(dst, src, size); 214fc3562d7SChristoph Hellwig pagefault_enable(); 215fc3562d7SChristoph Hellwig } 216fc3562d7SChristoph Hellwig set_fs(old_fs); 217fc3562d7SChristoph Hellwig 218fc3562d7SChristoph Hellwig if (ret) 219fc3562d7SChristoph Hellwig return -EFAULT; 220fc3562d7SChristoph Hellwig return 0; 221fc3562d7SChristoph Hellwig } 222c0ee37e8SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_user_nofault); 223fc3562d7SChristoph Hellwig 224fc3562d7SChristoph Hellwig /** 225c0ee37e8SChristoph Hellwig * copy_to_user_nofault(): safely attempt to write to a user-space location 226fc3562d7SChristoph Hellwig * @dst: address to write to 227fc3562d7SChristoph Hellwig * @src: pointer to the data that shall be written 228fc3562d7SChristoph Hellwig * @size: size of the data chunk 229fc3562d7SChristoph Hellwig * 230fc3562d7SChristoph Hellwig * Safely write to address @dst from the buffer at @src. If a kernel fault 231fc3562d7SChristoph Hellwig * happens, handle that and return -EFAULT. 232fc3562d7SChristoph Hellwig */ 233c0ee37e8SChristoph Hellwig long copy_to_user_nofault(void __user *dst, const void *src, size_t size) 234fc3562d7SChristoph Hellwig { 235fc3562d7SChristoph Hellwig long ret = -EFAULT; 236fc3562d7SChristoph Hellwig mm_segment_t old_fs = get_fs(); 237fc3562d7SChristoph Hellwig 238fc3562d7SChristoph Hellwig set_fs(USER_DS); 239fc3562d7SChristoph Hellwig if (access_ok(dst, size)) { 240fc3562d7SChristoph Hellwig pagefault_disable(); 241fc3562d7SChristoph Hellwig ret = __copy_to_user_inatomic(dst, src, size); 242fc3562d7SChristoph Hellwig pagefault_enable(); 243fc3562d7SChristoph Hellwig } 244fc3562d7SChristoph Hellwig set_fs(old_fs); 245fc3562d7SChristoph Hellwig 246fc3562d7SChristoph Hellwig if (ret) 247fc3562d7SChristoph Hellwig return -EFAULT; 248fc3562d7SChristoph Hellwig return 0; 249fc3562d7SChristoph Hellwig } 250c0ee37e8SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_to_user_nofault); 251fc3562d7SChristoph Hellwig 252fc3562d7SChristoph Hellwig /** 253bd88bb5dSChristoph Hellwig * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 2543d708182SMasami Hiramatsu * address. 2553d708182SMasami Hiramatsu * @dst: Destination address, in kernel space. This buffer must be at 2563d708182SMasami Hiramatsu * least @count bytes long. 2573d708182SMasami Hiramatsu * @unsafe_addr: Unsafe user address. 2583d708182SMasami Hiramatsu * @count: Maximum number of bytes to copy, including the trailing NUL. 2593d708182SMasami Hiramatsu * 2603d708182SMasami Hiramatsu * Copies a NUL-terminated string from unsafe user address to kernel buffer. 2613d708182SMasami Hiramatsu * 2623d708182SMasami Hiramatsu * On success, returns the length of the string INCLUDING the trailing NUL. 2633d708182SMasami Hiramatsu * 2643d708182SMasami Hiramatsu * If access fails, returns -EFAULT (some data may have been copied 2653d708182SMasami Hiramatsu * and the trailing NUL added). 2663d708182SMasami Hiramatsu * 2673d708182SMasami Hiramatsu * If @count is smaller than the length of the string, copies @count-1 bytes, 2683d708182SMasami Hiramatsu * sets the last byte of @dst buffer to NUL and returns @count. 2693d708182SMasami Hiramatsu */ 270bd88bb5dSChristoph Hellwig long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 2713d708182SMasami Hiramatsu long count) 2723d708182SMasami Hiramatsu { 2733d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 2743d708182SMasami Hiramatsu long ret; 2753d708182SMasami Hiramatsu 2763d708182SMasami Hiramatsu if (unlikely(count <= 0)) 2773d708182SMasami Hiramatsu return 0; 2783d708182SMasami Hiramatsu 2793d708182SMasami Hiramatsu set_fs(USER_DS); 2803d708182SMasami Hiramatsu pagefault_disable(); 2813d708182SMasami Hiramatsu ret = strncpy_from_user(dst, unsafe_addr, count); 2823d708182SMasami Hiramatsu pagefault_enable(); 2833d708182SMasami Hiramatsu set_fs(old_fs); 2843d708182SMasami Hiramatsu 2853d708182SMasami Hiramatsu if (ret >= count) { 2863d708182SMasami Hiramatsu ret = count; 2873d708182SMasami Hiramatsu dst[ret - 1] = '\0'; 2883d708182SMasami Hiramatsu } else if (ret > 0) { 2893d708182SMasami Hiramatsu ret++; 2903d708182SMasami Hiramatsu } 2913d708182SMasami Hiramatsu 2923d708182SMasami Hiramatsu return ret; 2933d708182SMasami Hiramatsu } 2943d708182SMasami Hiramatsu 2953d708182SMasami Hiramatsu /** 29602dddb16SChristoph Hellwig * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. 2973d708182SMasami Hiramatsu * @unsafe_addr: The string to measure. 2983d708182SMasami Hiramatsu * @count: Maximum count (including NUL) 2993d708182SMasami Hiramatsu * 3003d708182SMasami Hiramatsu * Get the size of a NUL-terminated string in user space without pagefault. 3013d708182SMasami Hiramatsu * 3023d708182SMasami Hiramatsu * Returns the size of the string INCLUDING the terminating NUL. 3033d708182SMasami Hiramatsu * 3043d708182SMasami Hiramatsu * If the string is too long, returns a number larger than @count. User 3053d708182SMasami Hiramatsu * has to check the return value against "> count". 3063d708182SMasami Hiramatsu * On exception (or invalid count), returns 0. 3073d708182SMasami Hiramatsu * 3083d708182SMasami Hiramatsu * Unlike strnlen_user, this can be used from IRQ handler etc. because 3093d708182SMasami Hiramatsu * it disables pagefaults. 3103d708182SMasami Hiramatsu */ 31102dddb16SChristoph Hellwig long strnlen_user_nofault(const void __user *unsafe_addr, long count) 3123d708182SMasami Hiramatsu { 3133d708182SMasami Hiramatsu mm_segment_t old_fs = get_fs(); 3143d708182SMasami Hiramatsu int ret; 3153d708182SMasami Hiramatsu 3163d708182SMasami Hiramatsu set_fs(USER_DS); 3173d708182SMasami Hiramatsu pagefault_disable(); 3183d708182SMasami Hiramatsu ret = strnlen_user(unsafe_addr, count); 3193d708182SMasami Hiramatsu pagefault_enable(); 3203d708182SMasami Hiramatsu set_fs(old_fs); 3213d708182SMasami Hiramatsu 3223d708182SMasami Hiramatsu return ret; 3233d708182SMasami Hiramatsu } 324