1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2c33fa9f5SIngo Molnar /* 33f0acb1eSChristoph Hellwig * Access kernel or user memory without faulting. 4c33fa9f5SIngo Molnar */ 5b95f1b31SPaul Gortmaker #include <linux/export.h> 6c33fa9f5SIngo Molnar #include <linux/mm.h> 77c7fcf76SDavid Howells #include <linux/uaccess.h> 8c33fa9f5SIngo Molnar 9fe557319SChristoph Hellwig bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, 10fe557319SChristoph Hellwig size_t size) 11eab0c608SChristoph Hellwig { 12eab0c608SChristoph Hellwig return true; 13eab0c608SChristoph Hellwig } 14eab0c608SChristoph Hellwig 15fe557319SChristoph Hellwig #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ 16b58294eaSChristoph Hellwig while (len >= sizeof(type)) { \ 17b58294eaSChristoph Hellwig __get_kernel_nofault(dst, src, type, err_label); \ 18b58294eaSChristoph Hellwig dst += sizeof(type); \ 19b58294eaSChristoph Hellwig src += sizeof(type); \ 20b58294eaSChristoph Hellwig len -= sizeof(type); \ 21b58294eaSChristoph Hellwig } 22b58294eaSChristoph Hellwig 23fe557319SChristoph Hellwig long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 24b58294eaSChristoph Hellwig { 252423de2eSArnd Bergmann unsigned long align = 0; 262423de2eSArnd Bergmann 272423de2eSArnd Bergmann if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 282423de2eSArnd Bergmann align = (unsigned long)dst | (unsigned long)src; 292423de2eSArnd Bergmann 30fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(src, size)) 312a71e81dSChristoph Hellwig return -ERANGE; 32b58294eaSChristoph Hellwig 33b58294eaSChristoph Hellwig pagefault_disable(); 342423de2eSArnd Bergmann if (!(align & 7)) 35fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); 362423de2eSArnd Bergmann if (!(align & 3)) 37fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); 382423de2eSArnd Bergmann if (!(align & 1)) 39fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); 40fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); 41b58294eaSChristoph Hellwig pagefault_enable(); 42b58294eaSChristoph Hellwig return 0; 43b58294eaSChristoph Hellwig Efault: 44b58294eaSChristoph Hellwig pagefault_enable(); 45b58294eaSChristoph Hellwig return -EFAULT; 46b58294eaSChristoph Hellwig } 47fe557319SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 48b58294eaSChristoph Hellwig 49fe557319SChristoph Hellwig #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ 50b58294eaSChristoph Hellwig while (len >= sizeof(type)) { \ 51b58294eaSChristoph Hellwig __put_kernel_nofault(dst, src, type, err_label); \ 52b58294eaSChristoph Hellwig dst += sizeof(type); \ 53b58294eaSChristoph Hellwig src += sizeof(type); \ 54b58294eaSChristoph Hellwig len -= sizeof(type); \ 55b58294eaSChristoph Hellwig } 56b58294eaSChristoph Hellwig 57fe557319SChristoph Hellwig long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 58b58294eaSChristoph Hellwig { 592423de2eSArnd Bergmann unsigned long align = 0; 602423de2eSArnd Bergmann 612423de2eSArnd Bergmann if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 622423de2eSArnd Bergmann align = (unsigned long)dst | (unsigned long)src; 632423de2eSArnd Bergmann 64b58294eaSChristoph Hellwig pagefault_disable(); 652423de2eSArnd Bergmann if (!(align & 7)) 66fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); 672423de2eSArnd Bergmann if (!(align & 3)) 68fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); 692423de2eSArnd Bergmann if (!(align & 1)) 70fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); 71fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); 72b58294eaSChristoph Hellwig pagefault_enable(); 73b58294eaSChristoph Hellwig return 0; 74b58294eaSChristoph Hellwig Efault: 75b58294eaSChristoph Hellwig pagefault_enable(); 76b58294eaSChristoph Hellwig return -EFAULT; 77b58294eaSChristoph Hellwig } 78b58294eaSChristoph Hellwig 79b58294eaSChristoph Hellwig long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 80b58294eaSChristoph Hellwig { 81b58294eaSChristoph Hellwig const void *src = unsafe_addr; 82b58294eaSChristoph Hellwig 83b58294eaSChristoph Hellwig if (unlikely(count <= 0)) 84b58294eaSChristoph Hellwig return 0; 85fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 862a71e81dSChristoph Hellwig return -ERANGE; 87b58294eaSChristoph Hellwig 88b58294eaSChristoph Hellwig pagefault_disable(); 89b58294eaSChristoph Hellwig do { 90b58294eaSChristoph Hellwig __get_kernel_nofault(dst, src, u8, Efault); 91b58294eaSChristoph Hellwig dst++; 92b58294eaSChristoph Hellwig src++; 93b58294eaSChristoph Hellwig } while (dst[-1] && src - unsafe_addr < count); 94b58294eaSChristoph Hellwig pagefault_enable(); 95b58294eaSChristoph Hellwig 96b58294eaSChristoph Hellwig dst[-1] = '\0'; 97b58294eaSChristoph Hellwig return src - unsafe_addr; 98b58294eaSChristoph Hellwig Efault: 99b58294eaSChristoph Hellwig pagefault_enable(); 100*8678ea06SAlban Crequy dst[0] = '\0'; 101b58294eaSChristoph Hellwig return -EFAULT; 102b58294eaSChristoph Hellwig } 1033d708182SMasami Hiramatsu 1043d708182SMasami Hiramatsu /** 105c0ee37e8SChristoph Hellwig * copy_from_user_nofault(): safely attempt to read from a user-space location 106fc3562d7SChristoph Hellwig * @dst: pointer to the buffer that shall take the data 107fc3562d7SChristoph Hellwig * @src: address to read from. This must be a user address. 108fc3562d7SChristoph Hellwig * @size: size of the data chunk 109fc3562d7SChristoph Hellwig * 110fc3562d7SChristoph Hellwig * Safely read from user address @src to the buffer at @dst. If a kernel fault 111fc3562d7SChristoph Hellwig * happens, handle that and return -EFAULT. 112fc3562d7SChristoph Hellwig */ 113c0ee37e8SChristoph Hellwig long copy_from_user_nofault(void *dst, const void __user *src, size_t size) 114fc3562d7SChristoph Hellwig { 115fc3562d7SChristoph Hellwig long ret = -EFAULT; 116fc3562d7SChristoph Hellwig if (access_ok(src, size)) { 117fc3562d7SChristoph Hellwig pagefault_disable(); 118fc3562d7SChristoph Hellwig ret = __copy_from_user_inatomic(dst, src, size); 119fc3562d7SChristoph Hellwig pagefault_enable(); 120fc3562d7SChristoph Hellwig } 121fc3562d7SChristoph Hellwig 122fc3562d7SChristoph Hellwig if (ret) 123fc3562d7SChristoph Hellwig return -EFAULT; 124fc3562d7SChristoph Hellwig return 0; 125fc3562d7SChristoph Hellwig } 126c0ee37e8SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_user_nofault); 127fc3562d7SChristoph Hellwig 128fc3562d7SChristoph Hellwig /** 129c0ee37e8SChristoph Hellwig * copy_to_user_nofault(): safely attempt to write to a user-space location 130fc3562d7SChristoph Hellwig * @dst: address to write to 131fc3562d7SChristoph Hellwig * @src: pointer to the data that shall be written 132fc3562d7SChristoph Hellwig * @size: size of the data chunk 133fc3562d7SChristoph Hellwig * 134fc3562d7SChristoph Hellwig * Safely write to address @dst from the buffer at @src. If a kernel fault 135fc3562d7SChristoph Hellwig * happens, handle that and return -EFAULT. 136fc3562d7SChristoph Hellwig */ 137c0ee37e8SChristoph Hellwig long copy_to_user_nofault(void __user *dst, const void *src, size_t size) 138fc3562d7SChristoph Hellwig { 139fc3562d7SChristoph Hellwig long ret = -EFAULT; 140fc3562d7SChristoph Hellwig 141fc3562d7SChristoph Hellwig if (access_ok(dst, size)) { 142fc3562d7SChristoph Hellwig pagefault_disable(); 143fc3562d7SChristoph Hellwig ret = __copy_to_user_inatomic(dst, src, size); 144fc3562d7SChristoph Hellwig pagefault_enable(); 145fc3562d7SChristoph Hellwig } 146fc3562d7SChristoph Hellwig 147fc3562d7SChristoph Hellwig if (ret) 148fc3562d7SChristoph Hellwig return -EFAULT; 149fc3562d7SChristoph Hellwig return 0; 150fc3562d7SChristoph Hellwig } 151c0ee37e8SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_to_user_nofault); 152fc3562d7SChristoph Hellwig 153fc3562d7SChristoph Hellwig /** 154bd88bb5dSChristoph Hellwig * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 1553d708182SMasami Hiramatsu * address. 1563d708182SMasami Hiramatsu * @dst: Destination address, in kernel space. This buffer must be at 1573d708182SMasami Hiramatsu * least @count bytes long. 1583d708182SMasami Hiramatsu * @unsafe_addr: Unsafe user address. 1593d708182SMasami Hiramatsu * @count: Maximum number of bytes to copy, including the trailing NUL. 1603d708182SMasami Hiramatsu * 1613d708182SMasami Hiramatsu * Copies a NUL-terminated string from unsafe user address to kernel buffer. 1623d708182SMasami Hiramatsu * 1633d708182SMasami Hiramatsu * On success, returns the length of the string INCLUDING the trailing NUL. 1643d708182SMasami Hiramatsu * 1653d708182SMasami Hiramatsu * If access fails, returns -EFAULT (some data may have been copied 1663d708182SMasami Hiramatsu * and the trailing NUL added). 1673d708182SMasami Hiramatsu * 1683d708182SMasami Hiramatsu * If @count is smaller than the length of the string, copies @count-1 bytes, 1693d708182SMasami Hiramatsu * sets the last byte of @dst buffer to NUL and returns @count. 1703d708182SMasami Hiramatsu */ 171bd88bb5dSChristoph Hellwig long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 1723d708182SMasami Hiramatsu long count) 1733d708182SMasami Hiramatsu { 1743d708182SMasami Hiramatsu long ret; 1753d708182SMasami Hiramatsu 1763d708182SMasami Hiramatsu if (unlikely(count <= 0)) 1773d708182SMasami Hiramatsu return 0; 1783d708182SMasami Hiramatsu 1793d708182SMasami Hiramatsu pagefault_disable(); 1803d708182SMasami Hiramatsu ret = strncpy_from_user(dst, unsafe_addr, count); 1813d708182SMasami Hiramatsu pagefault_enable(); 1823d708182SMasami Hiramatsu 1833d708182SMasami Hiramatsu if (ret >= count) { 1843d708182SMasami Hiramatsu ret = count; 1853d708182SMasami Hiramatsu dst[ret - 1] = '\0'; 1863d708182SMasami Hiramatsu } else if (ret > 0) { 1873d708182SMasami Hiramatsu ret++; 1883d708182SMasami Hiramatsu } 1893d708182SMasami Hiramatsu 1903d708182SMasami Hiramatsu return ret; 1913d708182SMasami Hiramatsu } 1923d708182SMasami Hiramatsu 1933d708182SMasami Hiramatsu /** 19402dddb16SChristoph Hellwig * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. 1953d708182SMasami Hiramatsu * @unsafe_addr: The string to measure. 1963d708182SMasami Hiramatsu * @count: Maximum count (including NUL) 1973d708182SMasami Hiramatsu * 1983d708182SMasami Hiramatsu * Get the size of a NUL-terminated string in user space without pagefault. 1993d708182SMasami Hiramatsu * 2003d708182SMasami Hiramatsu * Returns the size of the string INCLUDING the terminating NUL. 2013d708182SMasami Hiramatsu * 2023d708182SMasami Hiramatsu * If the string is too long, returns a number larger than @count. User 2033d708182SMasami Hiramatsu * has to check the return value against "> count". 2043d708182SMasami Hiramatsu * On exception (or invalid count), returns 0. 2053d708182SMasami Hiramatsu * 2063d708182SMasami Hiramatsu * Unlike strnlen_user, this can be used from IRQ handler etc. because 2073d708182SMasami Hiramatsu * it disables pagefaults. 2083d708182SMasami Hiramatsu */ 20902dddb16SChristoph Hellwig long strnlen_user_nofault(const void __user *unsafe_addr, long count) 2103d708182SMasami Hiramatsu { 2113d708182SMasami Hiramatsu int ret; 2123d708182SMasami Hiramatsu 2133d708182SMasami Hiramatsu pagefault_disable(); 2143d708182SMasami Hiramatsu ret = strnlen_user(unsafe_addr, count); 2153d708182SMasami Hiramatsu pagefault_enable(); 2163d708182SMasami Hiramatsu 2173d708182SMasami Hiramatsu return ret; 2183d708182SMasami Hiramatsu } 219ad7489d5SChristophe Leroy 220ad7489d5SChristophe Leroy void __copy_overflow(int size, unsigned long count) 221ad7489d5SChristophe Leroy { 222ad7489d5SChristophe Leroy WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); 223ad7489d5SChristophe Leroy } 224ad7489d5SChristophe Leroy EXPORT_SYMBOL(__copy_overflow); 225