1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/kernel.h> 8 #include <linux/mm.h> 9 #include <linux/prctl.h> 10 #include <linux/sched.h> 11 #include <linux/sched/mm.h> 12 #include <linux/string.h> 13 #include <linux/swap.h> 14 #include <linux/swapops.h> 15 #include <linux/thread_info.h> 16 #include <linux/uio.h> 17 18 #include <asm/cpufeature.h> 19 #include <asm/mte.h> 20 #include <asm/ptrace.h> 21 #include <asm/sysreg.h> 22 23 static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) 24 { 25 pte_t old_pte = READ_ONCE(*ptep); 26 27 if (check_swap && is_swap_pte(old_pte)) { 28 swp_entry_t entry = pte_to_swp_entry(old_pte); 29 30 if (!non_swap_entry(entry) && mte_restore_tags(entry, page)) 31 return; 32 } 33 34 mte_clear_page_tags(page_address(page)); 35 } 36 37 void mte_sync_tags(pte_t *ptep, pte_t pte) 38 { 39 struct page *page = pte_page(pte); 40 long i, nr_pages = compound_nr(page); 41 bool check_swap = nr_pages == 1; 42 43 /* if PG_mte_tagged is set, tags have already been initialised */ 44 for (i = 0; i < nr_pages; i++, page++) { 45 if (!test_and_set_bit(PG_mte_tagged, &page->flags)) 46 mte_sync_page_tags(page, ptep, check_swap); 47 } 48 } 49 50 int memcmp_pages(struct page *page1, struct page *page2) 51 { 52 char *addr1, *addr2; 53 int ret; 54 55 addr1 = page_address(page1); 56 addr2 = page_address(page2); 57 ret = memcmp(addr1, addr2, PAGE_SIZE); 58 59 if (!system_supports_mte() || ret) 60 return ret; 61 62 /* 63 * If the page content is identical but at least one of the pages is 64 * tagged, return non-zero to avoid KSM merging. If only one of the 65 * pages is tagged, set_pte_at() may zero or change the tags of the 66 * other page via mte_sync_tags(). 67 */ 68 if (test_bit(PG_mte_tagged, &page1->flags) || 69 test_bit(PG_mte_tagged, &page2->flags)) 70 return addr1 != addr2; 71 72 return ret; 73 } 74 75 static void update_sctlr_el1_tcf0(u64 tcf0) 76 { 77 /* ISB required for the kernel uaccess routines */ 78 sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0); 79 isb(); 80 } 81 82 static void set_sctlr_el1_tcf0(u64 tcf0) 83 { 84 /* 85 * mte_thread_switch() checks current->thread.sctlr_tcf0 as an 86 * optimisation. Disable preemption so that it does not see 87 * the variable update before the SCTLR_EL1.TCF0 one. 88 */ 89 preempt_disable(); 90 current->thread.sctlr_tcf0 = tcf0; 91 update_sctlr_el1_tcf0(tcf0); 92 preempt_enable(); 93 } 94 95 static void update_gcr_el1_excl(u64 incl) 96 { 97 u64 excl = ~incl & SYS_GCR_EL1_EXCL_MASK; 98 99 /* 100 * Note that 'incl' is an include mask (controlled by the user via 101 * prctl()) while GCR_EL1 accepts an exclude mask. 102 * No need for ISB since this only affects EL0 currently, implicit 103 * with ERET. 104 */ 105 sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl); 106 } 107 108 static void set_gcr_el1_excl(u64 incl) 109 { 110 current->thread.gcr_user_incl = incl; 111 update_gcr_el1_excl(incl); 112 } 113 114 void flush_mte_state(void) 115 { 116 if (!system_supports_mte()) 117 return; 118 119 /* clear any pending asynchronous tag fault */ 120 dsb(ish); 121 write_sysreg_s(0, SYS_TFSRE0_EL1); 122 clear_thread_flag(TIF_MTE_ASYNC_FAULT); 123 /* disable tag checking */ 124 set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE); 125 /* reset tag generation mask */ 126 set_gcr_el1_excl(0); 127 } 128 129 void mte_thread_switch(struct task_struct *next) 130 { 131 if (!system_supports_mte()) 132 return; 133 134 /* avoid expensive SCTLR_EL1 accesses if no change */ 135 if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0) 136 update_sctlr_el1_tcf0(next->thread.sctlr_tcf0); 137 update_gcr_el1_excl(next->thread.gcr_user_incl); 138 } 139 140 void mte_suspend_exit(void) 141 { 142 if (!system_supports_mte()) 143 return; 144 145 update_gcr_el1_excl(current->thread.gcr_user_incl); 146 } 147 148 long set_mte_ctrl(struct task_struct *task, unsigned long arg) 149 { 150 u64 tcf0; 151 u64 gcr_incl = (arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT; 152 153 if (!system_supports_mte()) 154 return 0; 155 156 switch (arg & PR_MTE_TCF_MASK) { 157 case PR_MTE_TCF_NONE: 158 tcf0 = SCTLR_EL1_TCF0_NONE; 159 break; 160 case PR_MTE_TCF_SYNC: 161 tcf0 = SCTLR_EL1_TCF0_SYNC; 162 break; 163 case PR_MTE_TCF_ASYNC: 164 tcf0 = SCTLR_EL1_TCF0_ASYNC; 165 break; 166 default: 167 return -EINVAL; 168 } 169 170 if (task != current) { 171 task->thread.sctlr_tcf0 = tcf0; 172 task->thread.gcr_user_incl = gcr_incl; 173 } else { 174 set_sctlr_el1_tcf0(tcf0); 175 set_gcr_el1_excl(gcr_incl); 176 } 177 178 return 0; 179 } 180 181 long get_mte_ctrl(struct task_struct *task) 182 { 183 unsigned long ret; 184 185 if (!system_supports_mte()) 186 return 0; 187 188 ret = task->thread.gcr_user_incl << PR_MTE_TAG_SHIFT; 189 190 switch (task->thread.sctlr_tcf0) { 191 case SCTLR_EL1_TCF0_NONE: 192 ret |= PR_MTE_TCF_NONE; 193 break; 194 case SCTLR_EL1_TCF0_SYNC: 195 ret |= PR_MTE_TCF_SYNC; 196 break; 197 case SCTLR_EL1_TCF0_ASYNC: 198 ret |= PR_MTE_TCF_ASYNC; 199 break; 200 } 201 202 return ret; 203 } 204 205 /* 206 * Access MTE tags in another process' address space as given in mm. Update 207 * the number of tags copied. Return 0 if any tags copied, error otherwise. 208 * Inspired by __access_remote_vm(). 209 */ 210 static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, 211 struct iovec *kiov, unsigned int gup_flags) 212 { 213 struct vm_area_struct *vma; 214 void __user *buf = kiov->iov_base; 215 size_t len = kiov->iov_len; 216 int ret; 217 int write = gup_flags & FOLL_WRITE; 218 219 if (!access_ok(buf, len)) 220 return -EFAULT; 221 222 if (mmap_read_lock_killable(mm)) 223 return -EIO; 224 225 while (len) { 226 unsigned long tags, offset; 227 void *maddr; 228 struct page *page = NULL; 229 230 ret = get_user_pages_remote(mm, addr, 1, gup_flags, &page, 231 &vma, NULL); 232 if (ret <= 0) 233 break; 234 235 /* 236 * Only copy tags if the page has been mapped as PROT_MTE 237 * (PG_mte_tagged set). Otherwise the tags are not valid and 238 * not accessible to user. Moreover, an mprotect(PROT_MTE) 239 * would cause the existing tags to be cleared if the page 240 * was never mapped with PROT_MTE. 241 */ 242 if (!test_bit(PG_mte_tagged, &page->flags)) { 243 ret = -EOPNOTSUPP; 244 put_page(page); 245 break; 246 } 247 248 /* limit access to the end of the page */ 249 offset = offset_in_page(addr); 250 tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE); 251 252 maddr = page_address(page); 253 if (write) { 254 tags = mte_copy_tags_from_user(maddr + offset, buf, tags); 255 set_page_dirty_lock(page); 256 } else { 257 tags = mte_copy_tags_to_user(buf, maddr + offset, tags); 258 } 259 put_page(page); 260 261 /* error accessing the tracer's buffer */ 262 if (!tags) 263 break; 264 265 len -= tags; 266 buf += tags; 267 addr += tags * MTE_GRANULE_SIZE; 268 } 269 mmap_read_unlock(mm); 270 271 /* return an error if no tags copied */ 272 kiov->iov_len = buf - kiov->iov_base; 273 if (!kiov->iov_len) { 274 /* check for error accessing the tracee's address space */ 275 if (ret <= 0) 276 return -EIO; 277 else 278 return -EFAULT; 279 } 280 281 return 0; 282 } 283 284 /* 285 * Copy MTE tags in another process' address space at 'addr' to/from tracer's 286 * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm(). 287 */ 288 static int access_remote_tags(struct task_struct *tsk, unsigned long addr, 289 struct iovec *kiov, unsigned int gup_flags) 290 { 291 struct mm_struct *mm; 292 int ret; 293 294 mm = get_task_mm(tsk); 295 if (!mm) 296 return -EPERM; 297 298 if (!tsk->ptrace || (current != tsk->parent) || 299 ((get_dumpable(mm) != SUID_DUMP_USER) && 300 !ptracer_capable(tsk, mm->user_ns))) { 301 mmput(mm); 302 return -EPERM; 303 } 304 305 ret = __access_remote_tags(mm, addr, kiov, gup_flags); 306 mmput(mm); 307 308 return ret; 309 } 310 311 int mte_ptrace_copy_tags(struct task_struct *child, long request, 312 unsigned long addr, unsigned long data) 313 { 314 int ret; 315 struct iovec kiov; 316 struct iovec __user *uiov = (void __user *)data; 317 unsigned int gup_flags = FOLL_FORCE; 318 319 if (!system_supports_mte()) 320 return -EIO; 321 322 if (get_user(kiov.iov_base, &uiov->iov_base) || 323 get_user(kiov.iov_len, &uiov->iov_len)) 324 return -EFAULT; 325 326 if (request == PTRACE_POKEMTETAGS) 327 gup_flags |= FOLL_WRITE; 328 329 /* align addr to the MTE tag granule */ 330 addr &= MTE_GRANULE_MASK; 331 332 ret = access_remote_tags(child, addr, &kiov, gup_flags); 333 if (!ret) 334 ret = put_user(kiov.iov_len, &uiov->iov_len); 335 336 return ret; 337 } 338