1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5 #ifndef __ASM_MTE_H 6 #define __ASM_MTE_H 7 8 #include <asm/compiler.h> 9 #include <asm/mte-def.h> 10 11 #ifndef __ASSEMBLY__ 12 13 #include <linux/bitfield.h> 14 #include <linux/kasan-enabled.h> 15 #include <linux/page-flags.h> 16 #include <linux/sched.h> 17 #include <linux/types.h> 18 19 #include <asm/pgtable-types.h> 20 21 void mte_clear_page_tags(void *addr); 22 unsigned long mte_copy_tags_from_user(void *to, const void __user *from, 23 unsigned long n); 24 unsigned long mte_copy_tags_to_user(void __user *to, void *from, 25 unsigned long n); 26 int mte_save_tags(struct page *page); 27 void mte_save_page_tags(const void *page_addr, void *tag_storage); 28 bool mte_restore_tags(swp_entry_t entry, struct page *page); 29 void mte_restore_page_tags(void *page_addr, const void *tag_storage); 30 void mte_invalidate_tags(int type, pgoff_t offset); 31 void mte_invalidate_tags_area(int type); 32 void *mte_allocate_tag_storage(void); 33 void mte_free_tag_storage(char *storage); 34 35 #ifdef CONFIG_ARM64_MTE 36 37 /* track which pages have valid allocation tags */ 38 #define PG_mte_tagged PG_arch_2 39 40 void mte_zero_clear_page_tags(void *addr); 41 void mte_sync_tags(pte_t old_pte, pte_t pte); 42 void mte_copy_page_tags(void *kto, const void *kfrom); 43 void mte_thread_init_user(void); 44 void mte_thread_switch(struct task_struct *next); 45 void mte_cpu_setup(void); 46 void mte_suspend_enter(void); 47 void mte_suspend_exit(void); 48 long set_mte_ctrl(struct task_struct *task, unsigned long arg); 49 long get_mte_ctrl(struct task_struct *task); 50 int mte_ptrace_copy_tags(struct task_struct *child, long request, 51 unsigned long addr, unsigned long data); 52 size_t mte_probe_user_range(const char __user *uaddr, size_t size); 53 54 #else /* CONFIG_ARM64_MTE */ 55 56 /* unused if !CONFIG_ARM64_MTE, silence the compiler */ 57 #define PG_mte_tagged 0 58 59 static inline void mte_zero_clear_page_tags(void *addr) 60 { 61 } 62 static inline void mte_sync_tags(pte_t old_pte, pte_t pte) 63 { 64 } 65 static inline void mte_copy_page_tags(void *kto, const void *kfrom) 66 { 67 } 68 static inline void mte_thread_init_user(void) 69 { 70 } 71 static inline void mte_thread_switch(struct task_struct *next) 72 { 73 } 74 static inline void mte_suspend_enter(void) 75 { 76 } 77 static inline void mte_suspend_exit(void) 78 { 79 } 80 static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg) 81 { 82 return 0; 83 } 84 static inline long get_mte_ctrl(struct task_struct *task) 85 { 86 return 0; 87 } 88 static inline int mte_ptrace_copy_tags(struct task_struct *child, 89 long request, unsigned long addr, 90 unsigned long data) 91 { 92 return -EIO; 93 } 94 95 #endif /* CONFIG_ARM64_MTE */ 96 97 static inline void mte_disable_tco_entry(struct task_struct *task) 98 { 99 if (!system_supports_mte()) 100 return; 101 102 /* 103 * Re-enable tag checking (TCO set on exception entry). This is only 104 * necessary if MTE is enabled in either the kernel or the userspace 105 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set 106 * for both). With MTE disabled in the kernel and disabled or 107 * asynchronous in userspace, tag check faults (including in uaccesses) 108 * are not reported, therefore there is no need to re-enable checking. 109 * This is beneficial on microarchitectures where re-enabling TCO is 110 * expensive. 111 */ 112 if (kasan_hw_tags_enabled() || 113 (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT))) 114 asm volatile(SET_PSTATE_TCO(0)); 115 } 116 117 #ifdef CONFIG_KASAN_HW_TAGS 118 /* Whether the MTE asynchronous mode is enabled. */ 119 DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); 120 121 static inline bool system_uses_mte_async_or_asymm_mode(void) 122 { 123 return static_branch_unlikely(&mte_async_or_asymm_mode); 124 } 125 126 void mte_check_tfsr_el1(void); 127 128 static inline void mte_check_tfsr_entry(void) 129 { 130 if (!system_supports_mte()) 131 return; 132 133 mte_check_tfsr_el1(); 134 } 135 136 static inline void mte_check_tfsr_exit(void) 137 { 138 if (!system_supports_mte()) 139 return; 140 141 /* 142 * The asynchronous faults are sync'ed automatically with 143 * TFSR_EL1 on kernel entry but for exit an explicit dsb() 144 * is required. 145 */ 146 dsb(nsh); 147 isb(); 148 149 mte_check_tfsr_el1(); 150 } 151 #else 152 static inline bool system_uses_mte_async_or_asymm_mode(void) 153 { 154 return false; 155 } 156 static inline void mte_check_tfsr_el1(void) 157 { 158 } 159 static inline void mte_check_tfsr_entry(void) 160 { 161 } 162 static inline void mte_check_tfsr_exit(void) 163 { 164 } 165 #endif /* CONFIG_KASAN_HW_TAGS */ 166 167 #endif /* __ASSEMBLY__ */ 168 #endif /* __ASM_MTE_H */ 169