1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/cpu.h> 8 #include <linux/kernel.h> 9 #include <linux/mm.h> 10 #include <linux/prctl.h> 11 #include <linux/sched.h> 12 #include <linux/sched/mm.h> 13 #include <linux/string.h> 14 #include <linux/swap.h> 15 #include <linux/swapops.h> 16 #include <linux/thread_info.h> 17 #include <linux/types.h> 18 #include <linux/uio.h> 19 20 #include <asm/barrier.h> 21 #include <asm/cpufeature.h> 22 #include <asm/mte.h> 23 #include <asm/ptrace.h> 24 #include <asm/sysreg.h> 25 26 static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred); 27 28 #ifdef CONFIG_KASAN_HW_TAGS 29 /* Whether the MTE asynchronous mode is enabled. */ 30 DEFINE_STATIC_KEY_FALSE(mte_async_mode); 31 EXPORT_SYMBOL_GPL(mte_async_mode); 32 #endif 33 34 static void mte_sync_page_tags(struct page *page, pte_t old_pte, 35 bool check_swap, bool pte_is_tagged) 36 { 37 if (check_swap && is_swap_pte(old_pte)) { 38 swp_entry_t entry = pte_to_swp_entry(old_pte); 39 40 if (!non_swap_entry(entry) && mte_restore_tags(entry, page)) 41 return; 42 } 43 44 if (!pte_is_tagged) 45 return; 46 47 page_kasan_tag_reset(page); 48 /* 49 * We need smp_wmb() in between setting the flags and clearing the 50 * tags because if another thread reads page->flags and builds a 51 * tagged address out of it, there is an actual dependency to the 52 * memory access, but on the current thread we do not guarantee that 53 * the new page->flags are visible before the tags were updated. 54 */ 55 smp_wmb(); 56 mte_clear_page_tags(page_address(page)); 57 } 58 59 void mte_sync_tags(pte_t old_pte, pte_t pte) 60 { 61 struct page *page = pte_page(pte); 62 long i, nr_pages = compound_nr(page); 63 bool check_swap = nr_pages == 1; 64 bool pte_is_tagged = pte_tagged(pte); 65 66 /* Early out if there's nothing to do */ 67 if (!check_swap && !pte_is_tagged) 68 return; 69 70 /* if PG_mte_tagged is set, tags have already been initialised */ 71 for (i = 0; i < nr_pages; i++, page++) { 72 if (!test_and_set_bit(PG_mte_tagged, &page->flags)) 73 mte_sync_page_tags(page, old_pte, check_swap, 74 pte_is_tagged); 75 } 76 } 77 78 int memcmp_pages(struct page *page1, struct page *page2) 79 { 80 char *addr1, *addr2; 81 int ret; 82 83 addr1 = page_address(page1); 84 addr2 = page_address(page2); 85 ret = memcmp(addr1, addr2, PAGE_SIZE); 86 87 if (!system_supports_mte() || ret) 88 return ret; 89 90 /* 91 * If the page content is identical but at least one of the pages is 92 * tagged, return non-zero to avoid KSM merging. If only one of the 93 * pages is tagged, set_pte_at() may zero or change the tags of the 94 * other page via mte_sync_tags(). 95 */ 96 if (test_bit(PG_mte_tagged, &page1->flags) || 97 test_bit(PG_mte_tagged, &page2->flags)) 98 return addr1 != addr2; 99 100 return ret; 101 } 102 103 static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) 104 { 105 /* Enable MTE Sync Mode for EL1. */ 106 sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, tcf); 107 isb(); 108 109 pr_info_once("MTE: enabled in %s mode at EL1\n", mode); 110 } 111 112 #ifdef CONFIG_KASAN_HW_TAGS 113 void mte_enable_kernel_sync(void) 114 { 115 /* 116 * Make sure we enter this function when no PE has set 117 * async mode previously. 118 */ 119 WARN_ONCE(system_uses_mte_async_mode(), 120 "MTE async mode enabled system wide!"); 121 122 __mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC); 123 } 124 125 void mte_enable_kernel_async(void) 126 { 127 __mte_enable_kernel("asynchronous", SCTLR_ELx_TCF_ASYNC); 128 129 /* 130 * MTE async mode is set system wide by the first PE that 131 * executes this function. 132 * 133 * Note: If in future KASAN acquires a runtime switching 134 * mode in between sync and async, this strategy needs 135 * to be reviewed. 136 */ 137 if (!system_uses_mte_async_mode()) 138 static_branch_enable(&mte_async_mode); 139 } 140 #endif 141 142 #ifdef CONFIG_KASAN_HW_TAGS 143 void mte_check_tfsr_el1(void) 144 { 145 u64 tfsr_el1; 146 147 if (!system_supports_mte()) 148 return; 149 150 tfsr_el1 = read_sysreg_s(SYS_TFSR_EL1); 151 152 if (unlikely(tfsr_el1 & SYS_TFSR_EL1_TF1)) { 153 /* 154 * Note: isb() is not required after this direct write 155 * because there is no indirect read subsequent to it 156 * (per ARM DDI 0487F.c table D13-1). 157 */ 158 write_sysreg_s(0, SYS_TFSR_EL1); 159 160 kasan_report_async(); 161 } 162 } 163 #endif 164 165 static void mte_update_sctlr_user(struct task_struct *task) 166 { 167 /* 168 * This must be called with preemption disabled and can only be called 169 * on the current or next task since the CPU must match where the thread 170 * is going to run. The caller is responsible for calling 171 * update_sctlr_el1() later in the same preemption disabled block. 172 */ 173 unsigned long sctlr = task->thread.sctlr_user; 174 unsigned long mte_ctrl = task->thread.mte_ctrl; 175 unsigned long pref, resolved_mte_tcf; 176 177 pref = __this_cpu_read(mte_tcf_preferred); 178 resolved_mte_tcf = (mte_ctrl & pref) ? pref : mte_ctrl; 179 sctlr &= ~SCTLR_EL1_TCF0_MASK; 180 if (resolved_mte_tcf & MTE_CTRL_TCF_ASYNC) 181 sctlr |= SCTLR_EL1_TCF0_ASYNC; 182 else if (resolved_mte_tcf & MTE_CTRL_TCF_SYNC) 183 sctlr |= SCTLR_EL1_TCF0_SYNC; 184 task->thread.sctlr_user = sctlr; 185 } 186 187 void mte_thread_init_user(void) 188 { 189 if (!system_supports_mte()) 190 return; 191 192 /* clear any pending asynchronous tag fault */ 193 dsb(ish); 194 write_sysreg_s(0, SYS_TFSRE0_EL1); 195 clear_thread_flag(TIF_MTE_ASYNC_FAULT); 196 /* disable tag checking and reset tag generation mask */ 197 set_mte_ctrl(current, 0); 198 } 199 200 void mte_thread_switch(struct task_struct *next) 201 { 202 mte_update_sctlr_user(next); 203 204 /* 205 * Check if an async tag exception occurred at EL1. 206 * 207 * Note: On the context switch path we rely on the dsb() present 208 * in __switch_to() to guarantee that the indirect writes to TFSR_EL1 209 * are synchronized before this point. 210 */ 211 isb(); 212 mte_check_tfsr_el1(); 213 } 214 215 void mte_suspend_enter(void) 216 { 217 if (!system_supports_mte()) 218 return; 219 220 /* 221 * The barriers are required to guarantee that the indirect writes 222 * to TFSR_EL1 are synchronized before we report the state. 223 */ 224 dsb(nsh); 225 isb(); 226 227 /* Report SYS_TFSR_EL1 before suspend entry */ 228 mte_check_tfsr_el1(); 229 } 230 231 long set_mte_ctrl(struct task_struct *task, unsigned long arg) 232 { 233 u64 mte_ctrl = (~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) & 234 SYS_GCR_EL1_EXCL_MASK) << MTE_CTRL_GCR_USER_EXCL_SHIFT; 235 236 if (!system_supports_mte()) 237 return 0; 238 239 if (arg & PR_MTE_TCF_ASYNC) 240 mte_ctrl |= MTE_CTRL_TCF_ASYNC; 241 if (arg & PR_MTE_TCF_SYNC) 242 mte_ctrl |= MTE_CTRL_TCF_SYNC; 243 244 task->thread.mte_ctrl = mte_ctrl; 245 if (task == current) { 246 preempt_disable(); 247 mte_update_sctlr_user(task); 248 update_sctlr_el1(task->thread.sctlr_user); 249 preempt_enable(); 250 } 251 252 return 0; 253 } 254 255 long get_mte_ctrl(struct task_struct *task) 256 { 257 unsigned long ret; 258 u64 mte_ctrl = task->thread.mte_ctrl; 259 u64 incl = (~mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) & 260 SYS_GCR_EL1_EXCL_MASK; 261 262 if (!system_supports_mte()) 263 return 0; 264 265 ret = incl << PR_MTE_TAG_SHIFT; 266 if (mte_ctrl & MTE_CTRL_TCF_ASYNC) 267 ret |= PR_MTE_TCF_ASYNC; 268 if (mte_ctrl & MTE_CTRL_TCF_SYNC) 269 ret |= PR_MTE_TCF_SYNC; 270 271 return ret; 272 } 273 274 /* 275 * Access MTE tags in another process' address space as given in mm. Update 276 * the number of tags copied. Return 0 if any tags copied, error otherwise. 277 * Inspired by __access_remote_vm(). 278 */ 279 static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, 280 struct iovec *kiov, unsigned int gup_flags) 281 { 282 struct vm_area_struct *vma; 283 void __user *buf = kiov->iov_base; 284 size_t len = kiov->iov_len; 285 int ret; 286 int write = gup_flags & FOLL_WRITE; 287 288 if (!access_ok(buf, len)) 289 return -EFAULT; 290 291 if (mmap_read_lock_killable(mm)) 292 return -EIO; 293 294 while (len) { 295 unsigned long tags, offset; 296 void *maddr; 297 struct page *page = NULL; 298 299 ret = get_user_pages_remote(mm, addr, 1, gup_flags, &page, 300 &vma, NULL); 301 if (ret <= 0) 302 break; 303 304 /* 305 * Only copy tags if the page has been mapped as PROT_MTE 306 * (PG_mte_tagged set). Otherwise the tags are not valid and 307 * not accessible to user. Moreover, an mprotect(PROT_MTE) 308 * would cause the existing tags to be cleared if the page 309 * was never mapped with PROT_MTE. 310 */ 311 if (!(vma->vm_flags & VM_MTE)) { 312 ret = -EOPNOTSUPP; 313 put_page(page); 314 break; 315 } 316 WARN_ON_ONCE(!test_bit(PG_mte_tagged, &page->flags)); 317 318 /* limit access to the end of the page */ 319 offset = offset_in_page(addr); 320 tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE); 321 322 maddr = page_address(page); 323 if (write) { 324 tags = mte_copy_tags_from_user(maddr + offset, buf, tags); 325 set_page_dirty_lock(page); 326 } else { 327 tags = mte_copy_tags_to_user(buf, maddr + offset, tags); 328 } 329 put_page(page); 330 331 /* error accessing the tracer's buffer */ 332 if (!tags) 333 break; 334 335 len -= tags; 336 buf += tags; 337 addr += tags * MTE_GRANULE_SIZE; 338 } 339 mmap_read_unlock(mm); 340 341 /* return an error if no tags copied */ 342 kiov->iov_len = buf - kiov->iov_base; 343 if (!kiov->iov_len) { 344 /* check for error accessing the tracee's address space */ 345 if (ret <= 0) 346 return -EIO; 347 else 348 return -EFAULT; 349 } 350 351 return 0; 352 } 353 354 /* 355 * Copy MTE tags in another process' address space at 'addr' to/from tracer's 356 * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm(). 357 */ 358 static int access_remote_tags(struct task_struct *tsk, unsigned long addr, 359 struct iovec *kiov, unsigned int gup_flags) 360 { 361 struct mm_struct *mm; 362 int ret; 363 364 mm = get_task_mm(tsk); 365 if (!mm) 366 return -EPERM; 367 368 if (!tsk->ptrace || (current != tsk->parent) || 369 ((get_dumpable(mm) != SUID_DUMP_USER) && 370 !ptracer_capable(tsk, mm->user_ns))) { 371 mmput(mm); 372 return -EPERM; 373 } 374 375 ret = __access_remote_tags(mm, addr, kiov, gup_flags); 376 mmput(mm); 377 378 return ret; 379 } 380 381 int mte_ptrace_copy_tags(struct task_struct *child, long request, 382 unsigned long addr, unsigned long data) 383 { 384 int ret; 385 struct iovec kiov; 386 struct iovec __user *uiov = (void __user *)data; 387 unsigned int gup_flags = FOLL_FORCE; 388 389 if (!system_supports_mte()) 390 return -EIO; 391 392 if (get_user(kiov.iov_base, &uiov->iov_base) || 393 get_user(kiov.iov_len, &uiov->iov_len)) 394 return -EFAULT; 395 396 if (request == PTRACE_POKEMTETAGS) 397 gup_flags |= FOLL_WRITE; 398 399 /* align addr to the MTE tag granule */ 400 addr &= MTE_GRANULE_MASK; 401 402 ret = access_remote_tags(child, addr, &kiov, gup_flags); 403 if (!ret) 404 ret = put_user(kiov.iov_len, &uiov->iov_len); 405 406 return ret; 407 } 408 409 static ssize_t mte_tcf_preferred_show(struct device *dev, 410 struct device_attribute *attr, char *buf) 411 { 412 switch (per_cpu(mte_tcf_preferred, dev->id)) { 413 case MTE_CTRL_TCF_ASYNC: 414 return sysfs_emit(buf, "async\n"); 415 case MTE_CTRL_TCF_SYNC: 416 return sysfs_emit(buf, "sync\n"); 417 default: 418 return sysfs_emit(buf, "???\n"); 419 } 420 } 421 422 static ssize_t mte_tcf_preferred_store(struct device *dev, 423 struct device_attribute *attr, 424 const char *buf, size_t count) 425 { 426 u64 tcf; 427 428 if (sysfs_streq(buf, "async")) 429 tcf = MTE_CTRL_TCF_ASYNC; 430 else if (sysfs_streq(buf, "sync")) 431 tcf = MTE_CTRL_TCF_SYNC; 432 else 433 return -EINVAL; 434 435 device_lock(dev); 436 per_cpu(mte_tcf_preferred, dev->id) = tcf; 437 device_unlock(dev); 438 439 return count; 440 } 441 static DEVICE_ATTR_RW(mte_tcf_preferred); 442 443 static int register_mte_tcf_preferred_sysctl(void) 444 { 445 unsigned int cpu; 446 447 if (!system_supports_mte()) 448 return 0; 449 450 for_each_possible_cpu(cpu) { 451 per_cpu(mte_tcf_preferred, cpu) = MTE_CTRL_TCF_ASYNC; 452 device_create_file(get_cpu_device(cpu), 453 &dev_attr_mte_tcf_preferred); 454 } 455 456 return 0; 457 } 458 subsys_initcall(register_mte_tcf_preferred_sysctl); 459