1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Access kernel or user memory without faulting. 4 */ 5 #include <linux/export.h> 6 #include <linux/mm.h> 7 #include <linux/uaccess.h> 8 #include <asm/tlb.h> 9 10 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, 11 size_t size) 12 { 13 return true; 14 } 15 16 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ 17 while (len >= sizeof(type)) { \ 18 __get_kernel_nofault(dst, src, type, err_label); \ 19 dst += sizeof(type); \ 20 src += sizeof(type); \ 21 len -= sizeof(type); \ 22 } 23 24 long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 25 { 26 unsigned long align = 0; 27 28 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 29 align = (unsigned long)dst | (unsigned long)src; 30 31 if (!copy_from_kernel_nofault_allowed(src, size)) 32 return -ERANGE; 33 34 pagefault_disable(); 35 if (!(align & 7)) 36 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); 37 if (!(align & 3)) 38 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); 39 if (!(align & 1)) 40 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); 41 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); 42 pagefault_enable(); 43 return 0; 44 Efault: 45 pagefault_enable(); 46 return -EFAULT; 47 } 48 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 49 50 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ 51 while (len >= sizeof(type)) { \ 52 __put_kernel_nofault(dst, src, type, err_label); \ 53 dst += sizeof(type); \ 54 src += sizeof(type); \ 55 len -= sizeof(type); \ 56 } 57 58 long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 59 { 60 unsigned long align = 0; 61 62 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 63 align = (unsigned long)dst | (unsigned long)src; 64 65 pagefault_disable(); 66 if (!(align & 7)) 67 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); 68 if (!(align & 3)) 69 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); 70 if (!(align & 1)) 71 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); 72 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); 73 pagefault_enable(); 74 return 0; 75 Efault: 76 pagefault_enable(); 77 return -EFAULT; 78 } 79 80 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 81 { 82 const void *src = unsafe_addr; 83 84 if (unlikely(count <= 0)) 85 return 0; 86 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 87 return -ERANGE; 88 89 pagefault_disable(); 90 do { 91 __get_kernel_nofault(dst, src, u8, Efault); 92 dst++; 93 src++; 94 } while (dst[-1] && src - unsafe_addr < count); 95 pagefault_enable(); 96 97 dst[-1] = '\0'; 98 return src - unsafe_addr; 99 Efault: 100 pagefault_enable(); 101 dst[0] = '\0'; 102 return -EFAULT; 103 } 104 105 /** 106 * copy_from_user_nofault(): safely attempt to read from a user-space location 107 * @dst: pointer to the buffer that shall take the data 108 * @src: address to read from. This must be a user address. 109 * @size: size of the data chunk 110 * 111 * Safely read from user address @src to the buffer at @dst. If a kernel fault 112 * happens, handle that and return -EFAULT. 113 */ 114 long copy_from_user_nofault(void *dst, const void __user *src, size_t size) 115 { 116 long ret = -EFAULT; 117 118 if (!__access_ok(src, size)) 119 return ret; 120 121 if (!nmi_uaccess_okay()) 122 return ret; 123 124 pagefault_disable(); 125 ret = __copy_from_user_inatomic(dst, src, size); 126 pagefault_enable(); 127 128 if (ret) 129 return -EFAULT; 130 return 0; 131 } 132 EXPORT_SYMBOL_GPL(copy_from_user_nofault); 133 134 /** 135 * copy_to_user_nofault(): safely attempt to write to a user-space location 136 * @dst: address to write to 137 * @src: pointer to the data that shall be written 138 * @size: size of the data chunk 139 * 140 * Safely write to address @dst from the buffer at @src. If a kernel fault 141 * happens, handle that and return -EFAULT. 142 */ 143 long copy_to_user_nofault(void __user *dst, const void *src, size_t size) 144 { 145 long ret = -EFAULT; 146 147 if (access_ok(dst, size)) { 148 pagefault_disable(); 149 ret = __copy_to_user_inatomic(dst, src, size); 150 pagefault_enable(); 151 } 152 153 if (ret) 154 return -EFAULT; 155 return 0; 156 } 157 EXPORT_SYMBOL_GPL(copy_to_user_nofault); 158 159 /** 160 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 161 * address. 162 * @dst: Destination address, in kernel space. This buffer must be at 163 * least @count bytes long. 164 * @unsafe_addr: Unsafe user address. 165 * @count: Maximum number of bytes to copy, including the trailing NUL. 166 * 167 * Copies a NUL-terminated string from unsafe user address to kernel buffer. 168 * 169 * On success, returns the length of the string INCLUDING the trailing NUL. 170 * 171 * If access fails, returns -EFAULT (some data may have been copied 172 * and the trailing NUL added). 173 * 174 * If @count is smaller than the length of the string, copies @count-1 bytes, 175 * sets the last byte of @dst buffer to NUL and returns @count. 176 */ 177 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 178 long count) 179 { 180 long ret; 181 182 if (unlikely(count <= 0)) 183 return 0; 184 185 pagefault_disable(); 186 ret = strncpy_from_user(dst, unsafe_addr, count); 187 pagefault_enable(); 188 189 if (ret >= count) { 190 ret = count; 191 dst[ret - 1] = '\0'; 192 } else if (ret > 0) { 193 ret++; 194 } 195 196 return ret; 197 } 198 199 /** 200 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. 201 * @unsafe_addr: The string to measure. 202 * @count: Maximum count (including NUL) 203 * 204 * Get the size of a NUL-terminated string in user space without pagefault. 205 * 206 * Returns the size of the string INCLUDING the terminating NUL. 207 * 208 * If the string is too long, returns a number larger than @count. User 209 * has to check the return value against "> count". 210 * On exception (or invalid count), returns 0. 211 * 212 * Unlike strnlen_user, this can be used from IRQ handler etc. because 213 * it disables pagefaults. 214 */ 215 long strnlen_user_nofault(const void __user *unsafe_addr, long count) 216 { 217 int ret; 218 219 pagefault_disable(); 220 ret = strnlen_user(unsafe_addr, count); 221 pagefault_enable(); 222 223 return ret; 224 } 225 226 void __copy_overflow(int size, unsigned long count) 227 { 228 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); 229 } 230 EXPORT_SYMBOL(__copy_overflow); 231