xref: /openbmc/linux/arch/arm64/include/asm/mte.h (revision 332c151c)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2020 ARM Ltd.
4  */
5 #ifndef __ASM_MTE_H
6 #define __ASM_MTE_H
7 
8 #include <asm/compiler.h>
9 #include <asm/mte-def.h>
10 
11 #ifndef __ASSEMBLY__
12 
13 #include <linux/bitfield.h>
14 #include <linux/kasan-enabled.h>
15 #include <linux/page-flags.h>
16 #include <linux/sched.h>
17 #include <linux/types.h>
18 
19 #include <asm/pgtable-types.h>
20 
21 void mte_clear_page_tags(void *addr);
22 unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
23 				      unsigned long n);
24 unsigned long mte_copy_tags_to_user(void __user *to, void *from,
25 				    unsigned long n);
26 int mte_save_tags(struct page *page);
27 void mte_save_page_tags(const void *page_addr, void *tag_storage);
28 void mte_restore_tags(swp_entry_t entry, struct page *page);
29 void mte_restore_page_tags(void *page_addr, const void *tag_storage);
30 void mte_invalidate_tags(int type, pgoff_t offset);
31 void mte_invalidate_tags_area(int type);
32 void *mte_allocate_tag_storage(void);
33 void mte_free_tag_storage(char *storage);
34 
35 #ifdef CONFIG_ARM64_MTE
36 
37 /* track which pages have valid allocation tags */
38 #define PG_mte_tagged	PG_arch_2
39 /* simple lock to avoid multiple threads tagging the same page */
40 #define PG_mte_lock	PG_arch_3
41 
set_page_mte_tagged(struct page * page)42 static inline void set_page_mte_tagged(struct page *page)
43 {
44 	/*
45 	 * Ensure that the tags written prior to this function are visible
46 	 * before the page flags update.
47 	 */
48 	smp_wmb();
49 	set_bit(PG_mte_tagged, &page->flags);
50 }
51 
page_mte_tagged(struct page * page)52 static inline bool page_mte_tagged(struct page *page)
53 {
54 	bool ret = test_bit(PG_mte_tagged, &page->flags);
55 
56 	/*
57 	 * If the page is tagged, ensure ordering with a likely subsequent
58 	 * read of the tags.
59 	 */
60 	if (ret)
61 		smp_rmb();
62 	return ret;
63 }
64 
65 /*
66  * Lock the page for tagging and return 'true' if the page can be tagged,
67  * 'false' if already tagged. PG_mte_tagged is never cleared and therefore the
68  * locking only happens once for page initialisation.
69  *
70  * The page MTE lock state:
71  *
72  *   Locked:	PG_mte_lock && !PG_mte_tagged
73  *   Unlocked:	!PG_mte_lock || PG_mte_tagged
74  *
75  * Acquire semantics only if the page is tagged (returning 'false').
76  */
try_page_mte_tagging(struct page * page)77 static inline bool try_page_mte_tagging(struct page *page)
78 {
79 	if (!test_and_set_bit(PG_mte_lock, &page->flags))
80 		return true;
81 
82 	/*
83 	 * The tags are either being initialised or may have been initialised
84 	 * already. Check if the PG_mte_tagged flag has been set or wait
85 	 * otherwise.
86 	 */
87 	smp_cond_load_acquire(&page->flags, VAL & (1UL << PG_mte_tagged));
88 
89 	return false;
90 }
91 
92 void mte_zero_clear_page_tags(void *addr);
93 void mte_sync_tags(pte_t pte);
94 void mte_copy_page_tags(void *kto, const void *kfrom);
95 void mte_thread_init_user(void);
96 void mte_thread_switch(struct task_struct *next);
97 void mte_cpu_setup(void);
98 void mte_suspend_enter(void);
99 void mte_suspend_exit(void);
100 long set_mte_ctrl(struct task_struct *task, unsigned long arg);
101 long get_mte_ctrl(struct task_struct *task);
102 int mte_ptrace_copy_tags(struct task_struct *child, long request,
103 			 unsigned long addr, unsigned long data);
104 size_t mte_probe_user_range(const char __user *uaddr, size_t size);
105 
106 #else /* CONFIG_ARM64_MTE */
107 
108 /* unused if !CONFIG_ARM64_MTE, silence the compiler */
109 #define PG_mte_tagged	0
110 
set_page_mte_tagged(struct page * page)111 static inline void set_page_mte_tagged(struct page *page)
112 {
113 }
page_mte_tagged(struct page * page)114 static inline bool page_mte_tagged(struct page *page)
115 {
116 	return false;
117 }
try_page_mte_tagging(struct page * page)118 static inline bool try_page_mte_tagging(struct page *page)
119 {
120 	return false;
121 }
mte_zero_clear_page_tags(void * addr)122 static inline void mte_zero_clear_page_tags(void *addr)
123 {
124 }
mte_sync_tags(pte_t pte)125 static inline void mte_sync_tags(pte_t pte)
126 {
127 }
mte_copy_page_tags(void * kto,const void * kfrom)128 static inline void mte_copy_page_tags(void *kto, const void *kfrom)
129 {
130 }
mte_thread_init_user(void)131 static inline void mte_thread_init_user(void)
132 {
133 }
mte_thread_switch(struct task_struct * next)134 static inline void mte_thread_switch(struct task_struct *next)
135 {
136 }
mte_suspend_enter(void)137 static inline void mte_suspend_enter(void)
138 {
139 }
mte_suspend_exit(void)140 static inline void mte_suspend_exit(void)
141 {
142 }
set_mte_ctrl(struct task_struct * task,unsigned long arg)143 static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
144 {
145 	return 0;
146 }
get_mte_ctrl(struct task_struct * task)147 static inline long get_mte_ctrl(struct task_struct *task)
148 {
149 	return 0;
150 }
mte_ptrace_copy_tags(struct task_struct * child,long request,unsigned long addr,unsigned long data)151 static inline int mte_ptrace_copy_tags(struct task_struct *child,
152 				       long request, unsigned long addr,
153 				       unsigned long data)
154 {
155 	return -EIO;
156 }
157 
158 #endif /* CONFIG_ARM64_MTE */
159 
mte_disable_tco_entry(struct task_struct * task)160 static inline void mte_disable_tco_entry(struct task_struct *task)
161 {
162 	if (!system_supports_mte())
163 		return;
164 
165 	/*
166 	 * Re-enable tag checking (TCO set on exception entry). This is only
167 	 * necessary if MTE is enabled in either the kernel or the userspace
168 	 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set
169 	 * for both). With MTE disabled in the kernel and disabled or
170 	 * asynchronous in userspace, tag check faults (including in uaccesses)
171 	 * are not reported, therefore there is no need to re-enable checking.
172 	 * This is beneficial on microarchitectures where re-enabling TCO is
173 	 * expensive.
174 	 */
175 	if (kasan_hw_tags_enabled() ||
176 	    (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT)))
177 		asm volatile(SET_PSTATE_TCO(0));
178 }
179 
180 #ifdef CONFIG_KASAN_HW_TAGS
181 void mte_check_tfsr_el1(void);
182 
mte_check_tfsr_entry(void)183 static inline void mte_check_tfsr_entry(void)
184 {
185 	if (!system_supports_mte())
186 		return;
187 
188 	mte_check_tfsr_el1();
189 }
190 
mte_check_tfsr_exit(void)191 static inline void mte_check_tfsr_exit(void)
192 {
193 	if (!system_supports_mte())
194 		return;
195 
196 	/*
197 	 * The asynchronous faults are sync'ed automatically with
198 	 * TFSR_EL1 on kernel entry but for exit an explicit dsb()
199 	 * is required.
200 	 */
201 	dsb(nsh);
202 	isb();
203 
204 	mte_check_tfsr_el1();
205 }
206 #else
mte_check_tfsr_el1(void)207 static inline void mte_check_tfsr_el1(void)
208 {
209 }
mte_check_tfsr_entry(void)210 static inline void mte_check_tfsr_entry(void)
211 {
212 }
mte_check_tfsr_exit(void)213 static inline void mte_check_tfsr_exit(void)
214 {
215 }
216 #endif /* CONFIG_KASAN_HW_TAGS */
217 
218 #endif /* __ASSEMBLY__ */
219 #endif /* __ASM_MTE_H  */
220