1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5 #ifndef __ASM_MTE_H 6 #define __ASM_MTE_H 7 8 #include <asm/compiler.h> 9 #include <asm/mte-def.h> 10 11 #ifndef __ASSEMBLY__ 12 13 #include <linux/bitfield.h> 14 #include <linux/kasan-enabled.h> 15 #include <linux/page-flags.h> 16 #include <linux/sched.h> 17 #include <linux/types.h> 18 19 #include <asm/pgtable-types.h> 20 21 void mte_clear_page_tags(void *addr); 22 unsigned long mte_copy_tags_from_user(void *to, const void __user *from, 23 unsigned long n); 24 unsigned long mte_copy_tags_to_user(void __user *to, void *from, 25 unsigned long n); 26 int mte_save_tags(struct page *page); 27 void mte_save_page_tags(const void *page_addr, void *tag_storage); 28 void mte_restore_tags(swp_entry_t entry, struct page *page); 29 void mte_restore_page_tags(void *page_addr, const void *tag_storage); 30 void mte_invalidate_tags(int type, pgoff_t offset); 31 void mte_invalidate_tags_area(int type); 32 void *mte_allocate_tag_storage(void); 33 void mte_free_tag_storage(char *storage); 34 35 #ifdef CONFIG_ARM64_MTE 36 37 /* track which pages have valid allocation tags */ 38 #define PG_mte_tagged PG_arch_2 39 /* simple lock to avoid multiple threads tagging the same page */ 40 #define PG_mte_lock PG_arch_3 41 42 static inline void set_page_mte_tagged(struct page *page) 43 { 44 /* 45 * Ensure that the tags written prior to this function are visible 46 * before the page flags update. 47 */ 48 smp_wmb(); 49 set_bit(PG_mte_tagged, &page->flags); 50 } 51 52 static inline bool page_mte_tagged(struct page *page) 53 { 54 bool ret = test_bit(PG_mte_tagged, &page->flags); 55 56 /* 57 * If the page is tagged, ensure ordering with a likely subsequent 58 * read of the tags. 59 */ 60 if (ret) 61 smp_rmb(); 62 return ret; 63 } 64 65 /* 66 * Lock the page for tagging and return 'true' if the page can be tagged, 67 * 'false' if already tagged. PG_mte_tagged is never cleared and therefore the 68 * locking only happens once for page initialisation. 69 * 70 * The page MTE lock state: 71 * 72 * Locked: PG_mte_lock && !PG_mte_tagged 73 * Unlocked: !PG_mte_lock || PG_mte_tagged 74 * 75 * Acquire semantics only if the page is tagged (returning 'false'). 76 */ 77 static inline bool try_page_mte_tagging(struct page *page) 78 { 79 if (!test_and_set_bit(PG_mte_lock, &page->flags)) 80 return true; 81 82 /* 83 * The tags are either being initialised or may have been initialised 84 * already. Check if the PG_mte_tagged flag has been set or wait 85 * otherwise. 86 */ 87 smp_cond_load_acquire(&page->flags, VAL & (1UL << PG_mte_tagged)); 88 89 return false; 90 } 91 92 void mte_zero_clear_page_tags(void *addr); 93 void mte_sync_tags(pte_t old_pte, pte_t pte); 94 void mte_copy_page_tags(void *kto, const void *kfrom); 95 void mte_thread_init_user(void); 96 void mte_thread_switch(struct task_struct *next); 97 void mte_cpu_setup(void); 98 void mte_suspend_enter(void); 99 void mte_suspend_exit(void); 100 long set_mte_ctrl(struct task_struct *task, unsigned long arg); 101 long get_mte_ctrl(struct task_struct *task); 102 int mte_ptrace_copy_tags(struct task_struct *child, long request, 103 unsigned long addr, unsigned long data); 104 size_t mte_probe_user_range(const char __user *uaddr, size_t size); 105 106 #else /* CONFIG_ARM64_MTE */ 107 108 /* unused if !CONFIG_ARM64_MTE, silence the compiler */ 109 #define PG_mte_tagged 0 110 111 static inline void set_page_mte_tagged(struct page *page) 112 { 113 } 114 static inline bool page_mte_tagged(struct page *page) 115 { 116 return false; 117 } 118 static inline bool try_page_mte_tagging(struct page *page) 119 { 120 return false; 121 } 122 static inline void mte_zero_clear_page_tags(void *addr) 123 { 124 } 125 static inline void mte_sync_tags(pte_t old_pte, pte_t pte) 126 { 127 } 128 static inline void mte_copy_page_tags(void *kto, const void *kfrom) 129 { 130 } 131 static inline void mte_thread_init_user(void) 132 { 133 } 134 static inline void mte_thread_switch(struct task_struct *next) 135 { 136 } 137 static inline void mte_suspend_enter(void) 138 { 139 } 140 static inline void mte_suspend_exit(void) 141 { 142 } 143 static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg) 144 { 145 return 0; 146 } 147 static inline long get_mte_ctrl(struct task_struct *task) 148 { 149 return 0; 150 } 151 static inline int mte_ptrace_copy_tags(struct task_struct *child, 152 long request, unsigned long addr, 153 unsigned long data) 154 { 155 return -EIO; 156 } 157 158 #endif /* CONFIG_ARM64_MTE */ 159 160 static inline void mte_disable_tco_entry(struct task_struct *task) 161 { 162 if (!system_supports_mte()) 163 return; 164 165 /* 166 * Re-enable tag checking (TCO set on exception entry). This is only 167 * necessary if MTE is enabled in either the kernel or the userspace 168 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set 169 * for both). With MTE disabled in the kernel and disabled or 170 * asynchronous in userspace, tag check faults (including in uaccesses) 171 * are not reported, therefore there is no need to re-enable checking. 172 * This is beneficial on microarchitectures where re-enabling TCO is 173 * expensive. 174 */ 175 if (kasan_hw_tags_enabled() || 176 (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT))) 177 asm volatile(SET_PSTATE_TCO(0)); 178 } 179 180 #ifdef CONFIG_KASAN_HW_TAGS 181 /* Whether the MTE asynchronous mode is enabled. */ 182 DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); 183 184 static inline bool system_uses_mte_async_or_asymm_mode(void) 185 { 186 return static_branch_unlikely(&mte_async_or_asymm_mode); 187 } 188 189 void mte_check_tfsr_el1(void); 190 191 static inline void mte_check_tfsr_entry(void) 192 { 193 if (!system_supports_mte()) 194 return; 195 196 mte_check_tfsr_el1(); 197 } 198 199 static inline void mte_check_tfsr_exit(void) 200 { 201 if (!system_supports_mte()) 202 return; 203 204 /* 205 * The asynchronous faults are sync'ed automatically with 206 * TFSR_EL1 on kernel entry but for exit an explicit dsb() 207 * is required. 208 */ 209 dsb(nsh); 210 isb(); 211 212 mte_check_tfsr_el1(); 213 } 214 #else 215 static inline bool system_uses_mte_async_or_asymm_mode(void) 216 { 217 return false; 218 } 219 static inline void mte_check_tfsr_el1(void) 220 { 221 } 222 static inline void mte_check_tfsr_entry(void) 223 { 224 } 225 static inline void mte_check_tfsr_exit(void) 226 { 227 } 228 #endif /* CONFIG_KASAN_HW_TAGS */ 229 230 #endif /* __ASSEMBLY__ */ 231 #endif /* __ASM_MTE_H */ 232