xref: /openbmc/qemu/target/arm/tcg/mte_helper.c (revision b7770d72)
1 /*
2  * ARM v8.5-MemTag Operations
3  *
4  * Copyright (c) 2020 Linaro, Ltd.
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "cpu.h"
23 #include "internals.h"
24 #include "exec/exec-all.h"
25 #include "exec/ram_addr.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/helper-proto.h"
28 #include "hw/core/tcg-cpu-ops.h"
29 #include "qapi/error.h"
30 #include "qemu/guest-random.h"
31 
32 
33 static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
34 {
35     if (exclude == 0xffff) {
36         return 0;
37     }
38     if (offset == 0) {
39         while (exclude & (1 << tag)) {
40             tag = (tag + 1) & 15;
41         }
42     } else {
43         do {
44             do {
45                 tag = (tag + 1) & 15;
46             } while (exclude & (1 << tag));
47         } while (--offset > 0);
48     }
49     return tag;
50 }
51 
52 /**
53  * allocation_tag_mem_probe:
54  * @env: the cpu environment
55  * @ptr_mmu_idx: the addressing regime to use for the virtual address
56  * @ptr: the virtual address for which to look up tag memory
57  * @ptr_access: the access to use for the virtual address
58  * @ptr_size: the number of bytes in the normal memory access
59  * @tag_access: the access to use for the tag memory
60  * @probe: true to merely probe, never taking an exception
61  * @ra: the return address for exception handling
62  *
63  * Our tag memory is formatted as a sequence of little-endian nibbles.
64  * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
65  * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
66  * for the higher addr.
67  *
68  * Here, resolve the physical address from the virtual address, and return
69  * a pointer to the corresponding tag byte.
70  *
71  * If there is no tag storage corresponding to @ptr, return NULL.
72  *
73  * If the page is inaccessible for @ptr_access, or has a watchpoint, there are
74  * three options:
75  * (1) probe = true, ra = 0 : pure probe -- we return NULL if the page is not
76  *     accessible, and do not take watchpoint traps. The calling code must
77  *     handle those cases in the right priority compared to MTE traps.
78  * (2) probe = false, ra = 0 : probe, no fault expected -- the caller guarantees
79  *     that the page is going to be accessible. We will take watchpoint traps.
80  * (3) probe = false, ra != 0 : non-probe -- we will take both memory access
81  *     traps and watchpoint traps.
82  * (probe = true, ra != 0 is invalid and will assert.)
83  */
84 static uint8_t *allocation_tag_mem_probe(CPUARMState *env, int ptr_mmu_idx,
85                                          uint64_t ptr, MMUAccessType ptr_access,
86                                          int ptr_size, MMUAccessType tag_access,
87                                          bool probe, uintptr_t ra)
88 {
89 #ifdef CONFIG_USER_ONLY
90     uint64_t clean_ptr = useronly_clean_ptr(ptr);
91     int flags = page_get_flags(clean_ptr);
92     uint8_t *tags;
93     uintptr_t index;
94 
95     assert(!(probe && ra));
96 
97     if (!(flags & (ptr_access == MMU_DATA_STORE ? PAGE_WRITE_ORG : PAGE_READ))) {
98         cpu_loop_exit_sigsegv(env_cpu(env), ptr, ptr_access,
99                               !(flags & PAGE_VALID), ra);
100     }
101 
102     /* Require both MAP_ANON and PROT_MTE for the page. */
103     if (!(flags & PAGE_ANON) || !(flags & PAGE_MTE)) {
104         return NULL;
105     }
106 
107     tags = page_get_target_data(clean_ptr);
108 
109     index = extract32(ptr, LOG2_TAG_GRANULE + 1,
110                       TARGET_PAGE_BITS - LOG2_TAG_GRANULE - 1);
111     return tags + index;
112 #else
113     CPUTLBEntryFull *full;
114     MemTxAttrs attrs;
115     int in_page, flags;
116     hwaddr ptr_paddr, tag_paddr, xlat;
117     MemoryRegion *mr;
118     ARMASIdx tag_asi;
119     AddressSpace *tag_as;
120     void *host;
121 
122     /*
123      * Probe the first byte of the virtual address.  This raises an
124      * exception for inaccessible pages, and resolves the virtual address
125      * into the softmmu tlb.
126      *
127      * When RA == 0, this is either a pure probe or a no-fault-expected probe.
128      * Indicate to probe_access_flags no-fault, then either return NULL
129      * for the pure probe, or assert that we received a valid page for the
130      * no-fault-expected probe.
131      */
132     flags = probe_access_full(env, ptr, 0, ptr_access, ptr_mmu_idx,
133                               ra == 0, &host, &full, ra);
134     if (probe && (flags & TLB_INVALID_MASK)) {
135         return NULL;
136     }
137     assert(!(flags & TLB_INVALID_MASK));
138 
139     /* If the virtual page MemAttr != Tagged, access unchecked. */
140     if (full->extra.arm.pte_attrs != 0xf0) {
141         return NULL;
142     }
143 
144     /*
145      * If not backed by host ram, there is no tag storage: access unchecked.
146      * This is probably a guest os bug though, so log it.
147      */
148     if (unlikely(flags & TLB_MMIO)) {
149         qemu_log_mask(LOG_GUEST_ERROR,
150                       "Page @ 0x%" PRIx64 " indicates Tagged Normal memory "
151                       "but is not backed by host ram\n", ptr);
152         return NULL;
153     }
154 
155     /*
156      * Remember these values across the second lookup below,
157      * which may invalidate this pointer via tlb resize.
158      */
159     ptr_paddr = full->phys_addr | (ptr & ~TARGET_PAGE_MASK);
160     attrs = full->attrs;
161     full = NULL;
162 
163     /*
164      * The Normal memory access can extend to the next page.  E.g. a single
165      * 8-byte access to the last byte of a page will check only the last
166      * tag on the first page.
167      * Any page access exception has priority over tag check exception.
168      */
169     in_page = -(ptr | TARGET_PAGE_MASK);
170     if (unlikely(ptr_size > in_page)) {
171         flags |= probe_access_full(env, ptr + in_page, 0, ptr_access,
172                                    ptr_mmu_idx, ra == 0, &host, &full, ra);
173         assert(!(flags & TLB_INVALID_MASK));
174     }
175 
176     /* Any debug exception has priority over a tag check exception. */
177     if (!probe && unlikely(flags & TLB_WATCHPOINT)) {
178         int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE;
179         assert(ra != 0);
180         cpu_check_watchpoint(env_cpu(env), ptr, ptr_size, attrs, wp, ra);
181     }
182 
183     /* Convert to the physical address in tag space.  */
184     tag_paddr = ptr_paddr >> (LOG2_TAG_GRANULE + 1);
185 
186     /* Look up the address in tag space. */
187     tag_asi = attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS;
188     tag_as = cpu_get_address_space(env_cpu(env), tag_asi);
189     mr = address_space_translate(tag_as, tag_paddr, &xlat, NULL,
190                                  tag_access == MMU_DATA_STORE, attrs);
191 
192     /*
193      * Note that @mr will never be NULL.  If there is nothing in the address
194      * space at @tag_paddr, the translation will return the unallocated memory
195      * region.  For our purposes, the result must be ram.
196      */
197     if (unlikely(!memory_region_is_ram(mr))) {
198         /* ??? Failure is a board configuration error. */
199         qemu_log_mask(LOG_UNIMP,
200                       "Tag Memory @ 0x%" HWADDR_PRIx " not found for "
201                       "Normal Memory @ 0x%" HWADDR_PRIx "\n",
202                       tag_paddr, ptr_paddr);
203         return NULL;
204     }
205 
206     /*
207      * Ensure the tag memory is dirty on write, for migration.
208      * Tag memory can never contain code or display memory (vga).
209      */
210     if (tag_access == MMU_DATA_STORE) {
211         ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat;
212         cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION);
213     }
214 
215     return memory_region_get_ram_ptr(mr) + xlat;
216 #endif
217 }
218 
219 static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
220                                    uint64_t ptr, MMUAccessType ptr_access,
221                                    int ptr_size, MMUAccessType tag_access,
222                                    uintptr_t ra)
223 {
224     return allocation_tag_mem_probe(env, ptr_mmu_idx, ptr, ptr_access,
225                                     ptr_size, tag_access, false, ra);
226 }
227 
228 uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
229 {
230     uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
231     int rrnd = extract32(env->cp15.gcr_el1, 16, 1);
232     int start = extract32(env->cp15.rgsr_el1, 0, 4);
233     int seed = extract32(env->cp15.rgsr_el1, 8, 16);
234     int offset, i, rtag;
235 
236     /*
237      * Our IMPDEF choice for GCR_EL1.RRND==1 is to continue to use the
238      * deterministic algorithm.  Except that with RRND==1 the kernel is
239      * not required to have set RGSR_EL1.SEED != 0, which is required for
240      * the deterministic algorithm to function.  So we force a non-zero
241      * SEED for that case.
242      */
243     if (unlikely(seed == 0) && rrnd) {
244         do {
245             Error *err = NULL;
246             uint16_t two;
247 
248             if (qemu_guest_getrandom(&two, sizeof(two), &err) < 0) {
249                 /*
250                  * Failed, for unknown reasons in the crypto subsystem.
251                  * Best we can do is log the reason and use a constant seed.
252                  */
253                 qemu_log_mask(LOG_UNIMP, "IRG: Crypto failure: %s\n",
254                               error_get_pretty(err));
255                 error_free(err);
256                 two = 1;
257             }
258             seed = two;
259         } while (seed == 0);
260     }
261 
262     /* RandomTag */
263     for (i = offset = 0; i < 4; ++i) {
264         /* NextRandomTagBit */
265         int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^
266                    extract32(seed, 2, 1) ^ extract32(seed, 0, 1));
267         seed = (top << 15) | (seed >> 1);
268         offset |= top << i;
269     }
270     rtag = choose_nonexcluded_tag(start, offset, exclude);
271     env->cp15.rgsr_el1 = rtag | (seed << 8);
272 
273     return address_with_allocation_tag(rn, rtag);
274 }
275 
276 uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr,
277                          int32_t offset, uint32_t tag_offset)
278 {
279     int start_tag = allocation_tag_from_addr(ptr);
280     uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16);
281     int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude);
282 
283     return address_with_allocation_tag(ptr + offset, rtag);
284 }
285 
286 static int load_tag1(uint64_t ptr, uint8_t *mem)
287 {
288     int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
289     return extract32(*mem, ofs, 4);
290 }
291 
292 uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
293 {
294     int mmu_idx = arm_env_mmu_index(env);
295     uint8_t *mem;
296     int rtag = 0;
297 
298     /* Trap if accessing an invalid page.  */
299     mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1,
300                              MMU_DATA_LOAD, GETPC());
301 
302     /* Load if page supports tags. */
303     if (mem) {
304         rtag = load_tag1(ptr, mem);
305     }
306 
307     return address_with_allocation_tag(xt, rtag);
308 }
309 
310 static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra)
311 {
312     if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) {
313         arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
314                                     arm_env_mmu_index(env), ra);
315         g_assert_not_reached();
316     }
317 }
318 
319 /* For use in a non-parallel context, store to the given nibble.  */
320 static void store_tag1(uint64_t ptr, uint8_t *mem, int tag)
321 {
322     int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
323     *mem = deposit32(*mem, ofs, 4, tag);
324 }
325 
326 /* For use in a parallel context, atomically store to the given nibble.  */
327 static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag)
328 {
329     int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
330     uint8_t old = qatomic_read(mem);
331 
332     while (1) {
333         uint8_t new = deposit32(old, ofs, 4, tag);
334         uint8_t cmp = qatomic_cmpxchg(mem, old, new);
335         if (likely(cmp == old)) {
336             return;
337         }
338         old = cmp;
339     }
340 }
341 
342 typedef void stg_store1(uint64_t, uint8_t *, int);
343 
344 static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt,
345                           uintptr_t ra, stg_store1 store1)
346 {
347     int mmu_idx = arm_env_mmu_index(env);
348     uint8_t *mem;
349 
350     check_tag_aligned(env, ptr, ra);
351 
352     /* Trap if accessing an invalid page.  */
353     mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE,
354                              MMU_DATA_STORE, ra);
355 
356     /* Store if page supports tags. */
357     if (mem) {
358         store1(ptr, mem, allocation_tag_from_addr(xt));
359     }
360 }
361 
362 void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
363 {
364     do_stg(env, ptr, xt, GETPC(), store_tag1);
365 }
366 
367 void HELPER(stg_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
368 {
369     do_stg(env, ptr, xt, GETPC(), store_tag1_parallel);
370 }
371 
372 void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr)
373 {
374     int mmu_idx = arm_env_mmu_index(env);
375     uintptr_t ra = GETPC();
376 
377     check_tag_aligned(env, ptr, ra);
378     probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
379 }
380 
381 static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt,
382                            uintptr_t ra, stg_store1 store1)
383 {
384     int mmu_idx = arm_env_mmu_index(env);
385     int tag = allocation_tag_from_addr(xt);
386     uint8_t *mem1, *mem2;
387 
388     check_tag_aligned(env, ptr, ra);
389 
390     /*
391      * Trap if accessing an invalid page(s).
392      * This takes priority over !allocation_tag_access_enabled.
393      */
394     if (ptr & TAG_GRANULE) {
395         /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */
396         mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
397                                   TAG_GRANULE, MMU_DATA_STORE, ra);
398         mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE,
399                                   MMU_DATA_STORE, TAG_GRANULE,
400                                   MMU_DATA_STORE, ra);
401 
402         /* Store if page(s) support tags. */
403         if (mem1) {
404             store1(TAG_GRANULE, mem1, tag);
405         }
406         if (mem2) {
407             store1(0, mem2, tag);
408         }
409     } else {
410         /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */
411         mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
412                                   2 * TAG_GRANULE, MMU_DATA_STORE, ra);
413         if (mem1) {
414             tag |= tag << 4;
415             qatomic_set(mem1, tag);
416         }
417     }
418 }
419 
420 void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt)
421 {
422     do_st2g(env, ptr, xt, GETPC(), store_tag1);
423 }
424 
425 void HELPER(st2g_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
426 {
427     do_st2g(env, ptr, xt, GETPC(), store_tag1_parallel);
428 }
429 
430 void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr)
431 {
432     int mmu_idx = arm_env_mmu_index(env);
433     uintptr_t ra = GETPC();
434     int in_page = -(ptr | TARGET_PAGE_MASK);
435 
436     check_tag_aligned(env, ptr, ra);
437 
438     if (likely(in_page >= 2 * TAG_GRANULE)) {
439         probe_write(env, ptr, 2 * TAG_GRANULE, mmu_idx, ra);
440     } else {
441         probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
442         probe_write(env, ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra);
443     }
444 }
445 
446 uint64_t HELPER(ldgm)(CPUARMState *env, uint64_t ptr)
447 {
448     int mmu_idx = arm_env_mmu_index(env);
449     uintptr_t ra = GETPC();
450     int gm_bs = env_archcpu(env)->gm_blocksize;
451     int gm_bs_bytes = 4 << gm_bs;
452     void *tag_mem;
453     uint64_t ret;
454     int shift;
455 
456     ptr = QEMU_ALIGN_DOWN(ptr, gm_bs_bytes);
457 
458     /* Trap if accessing an invalid page.  */
459     tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD,
460                                  gm_bs_bytes, MMU_DATA_LOAD, ra);
461 
462     /* The tag is squashed to zero if the page does not support tags.  */
463     if (!tag_mem) {
464         return 0;
465     }
466 
467     /*
468      * The ordering of elements within the word corresponds to
469      * a little-endian operation.  Computation of shift comes from
470      *
471      *     index = address<LOG2_TAG_GRANULE+3:LOG2_TAG_GRANULE>
472      *     data<index*4+3:index*4> = tag
473      *
474      * Because of the alignment of ptr above, BS=6 has shift=0.
475      * All memory operations are aligned.  Defer support for BS=2,
476      * requiring insertion or extraction of a nibble, until we
477      * support a cpu that requires it.
478      */
479     switch (gm_bs) {
480     case 3:
481         /* 32 bytes -> 2 tags -> 8 result bits */
482         ret = *(uint8_t *)tag_mem;
483         break;
484     case 4:
485         /* 64 bytes -> 4 tags -> 16 result bits */
486         ret = cpu_to_le16(*(uint16_t *)tag_mem);
487         break;
488     case 5:
489         /* 128 bytes -> 8 tags -> 32 result bits */
490         ret = cpu_to_le32(*(uint32_t *)tag_mem);
491         break;
492     case 6:
493         /* 256 bytes -> 16 tags -> 64 result bits */
494         return cpu_to_le64(*(uint64_t *)tag_mem);
495     default:
496         /*
497          * CPU configured with unsupported/invalid gm blocksize.
498          * This is detected early in arm_cpu_realizefn.
499          */
500         g_assert_not_reached();
501     }
502     shift = extract64(ptr, LOG2_TAG_GRANULE, 4) * 4;
503     return ret << shift;
504 }
505 
506 void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val)
507 {
508     int mmu_idx = arm_env_mmu_index(env);
509     uintptr_t ra = GETPC();
510     int gm_bs = env_archcpu(env)->gm_blocksize;
511     int gm_bs_bytes = 4 << gm_bs;
512     void *tag_mem;
513     int shift;
514 
515     ptr = QEMU_ALIGN_DOWN(ptr, gm_bs_bytes);
516 
517     /* Trap if accessing an invalid page.  */
518     tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
519                                  gm_bs_bytes, MMU_DATA_LOAD, ra);
520 
521     /*
522      * Tag store only happens if the page support tags,
523      * and if the OS has enabled access to the tags.
524      */
525     if (!tag_mem) {
526         return;
527     }
528 
529     /* See LDGM for comments on BS and on shift.  */
530     shift = extract64(ptr, LOG2_TAG_GRANULE, 4) * 4;
531     val >>= shift;
532     switch (gm_bs) {
533     case 3:
534         /* 32 bytes -> 2 tags -> 8 result bits */
535         *(uint8_t *)tag_mem = val;
536         break;
537     case 4:
538         /* 64 bytes -> 4 tags -> 16 result bits */
539         *(uint16_t *)tag_mem = cpu_to_le16(val);
540         break;
541     case 5:
542         /* 128 bytes -> 8 tags -> 32 result bits */
543         *(uint32_t *)tag_mem = cpu_to_le32(val);
544         break;
545     case 6:
546         /* 256 bytes -> 16 tags -> 64 result bits */
547         *(uint64_t *)tag_mem = cpu_to_le64(val);
548         break;
549     default:
550         /* cpu configured with unsupported gm blocksize. */
551         g_assert_not_reached();
552     }
553 }
554 
555 void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val)
556 {
557     uintptr_t ra = GETPC();
558     int mmu_idx = arm_env_mmu_index(env);
559     int log2_dcz_bytes, log2_tag_bytes;
560     intptr_t dcz_bytes, tag_bytes;
561     uint8_t *mem;
562 
563     /*
564      * In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1,
565      * i.e. 32 bytes, which is an unreasonably small dcz anyway,
566      * to make sure that we can access one complete tag byte here.
567      */
568     log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
569     log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
570     dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
571     tag_bytes = (intptr_t)1 << log2_tag_bytes;
572     ptr &= -dcz_bytes;
573 
574     mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes,
575                              MMU_DATA_STORE, ra);
576     if (mem) {
577         int tag_pair = (val & 0xf) * 0x11;
578         memset(mem, tag_pair, tag_bytes);
579     }
580 }
581 
582 static void mte_sync_check_fail(CPUARMState *env, uint32_t desc,
583                                 uint64_t dirty_ptr, uintptr_t ra)
584 {
585     int is_write, syn;
586 
587     env->exception.vaddress = dirty_ptr;
588 
589     is_write = FIELD_EX32(desc, MTEDESC, WRITE);
590     syn = syn_data_abort_no_iss(arm_current_el(env) != 0, 0, 0, 0, 0, is_write,
591                                 0x11);
592     raise_exception_ra(env, EXCP_DATA_ABORT, syn, exception_target_el(env), ra);
593     g_assert_not_reached();
594 }
595 
596 static void mte_async_check_fail(CPUARMState *env, uint64_t dirty_ptr,
597                                  uintptr_t ra, ARMMMUIdx arm_mmu_idx, int el)
598 {
599     int select;
600 
601     if (regime_has_2_ranges(arm_mmu_idx)) {
602         select = extract64(dirty_ptr, 55, 1);
603     } else {
604         select = 0;
605     }
606     env->cp15.tfsr_el[el] |= 1 << select;
607 #ifdef CONFIG_USER_ONLY
608     /*
609      * Stand in for a timer irq, setting _TIF_MTE_ASYNC_FAULT,
610      * which then sends a SIGSEGV when the thread is next scheduled.
611      * This cpu will return to the main loop at the end of the TB,
612      * which is rather sooner than "normal".  But the alternative
613      * is waiting until the next syscall.
614      */
615     qemu_cpu_kick(env_cpu(env));
616 #endif
617 }
618 
619 /* Record a tag check failure.  */
620 void mte_check_fail(CPUARMState *env, uint32_t desc,
621                     uint64_t dirty_ptr, uintptr_t ra)
622 {
623     int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
624     ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx);
625     int el, reg_el, tcf;
626     uint64_t sctlr;
627 
628     reg_el = regime_el(env, arm_mmu_idx);
629     sctlr = env->cp15.sctlr_el[reg_el];
630 
631     switch (arm_mmu_idx) {
632     case ARMMMUIdx_E10_0:
633     case ARMMMUIdx_E20_0:
634         el = 0;
635         tcf = extract64(sctlr, 38, 2);
636         break;
637     default:
638         el = reg_el;
639         tcf = extract64(sctlr, 40, 2);
640     }
641 
642     switch (tcf) {
643     case 1:
644         /* Tag check fail causes a synchronous exception. */
645         mte_sync_check_fail(env, desc, dirty_ptr, ra);
646         break;
647 
648     case 0:
649         /*
650          * Tag check fail does not affect the PE.
651          * We eliminate this case by not setting MTE_ACTIVE
652          * in tb_flags, so that we never make this runtime call.
653          */
654         g_assert_not_reached();
655 
656     case 2:
657         /* Tag check fail causes asynchronous flag set.  */
658         mte_async_check_fail(env, dirty_ptr, ra, arm_mmu_idx, el);
659         break;
660 
661     case 3:
662         /*
663          * Tag check fail causes asynchronous flag set for stores, or
664          * a synchronous exception for loads.
665          */
666         if (FIELD_EX32(desc, MTEDESC, WRITE)) {
667             mte_async_check_fail(env, dirty_ptr, ra, arm_mmu_idx, el);
668         } else {
669             mte_sync_check_fail(env, desc, dirty_ptr, ra);
670         }
671         break;
672     }
673 }
674 
675 /**
676  * checkN:
677  * @tag: tag memory to test
678  * @odd: true to begin testing at tags at odd nibble
679  * @cmp: the tag to compare against
680  * @count: number of tags to test
681  *
682  * Return the number of successful tests.
683  * Thus a return value < @count indicates a failure.
684  *
685  * A note about sizes: count is expected to be small.
686  *
687  * The most common use will be LDP/STP of two integer registers,
688  * which means 16 bytes of memory touching at most 2 tags, but
689  * often the access is aligned and thus just 1 tag.
690  *
691  * Using AdvSIMD LD/ST (multiple), one can access 64 bytes of memory,
692  * touching at most 5 tags.  SVE LDR/STR (vector) with the default
693  * vector length is also 64 bytes; the maximum architectural length
694  * is 256 bytes touching at most 9 tags.
695  *
696  * The loop below uses 7 logical operations and 1 memory operation
697  * per tag pair.  An implementation that loads an aligned word and
698  * uses masking to ignore adjacent tags requires 18 logical operations
699  * and thus does not begin to pay off until 6 tags.
700  * Which, according to the survey above, is unlikely to be common.
701  */
702 static int checkN(uint8_t *mem, int odd, int cmp, int count)
703 {
704     int n = 0, diff;
705 
706     /* Replicate the test tag and compare.  */
707     cmp *= 0x11;
708     diff = *mem++ ^ cmp;
709 
710     if (odd) {
711         goto start_odd;
712     }
713 
714     while (1) {
715         /* Test even tag. */
716         if (unlikely((diff) & 0x0f)) {
717             break;
718         }
719         if (++n == count) {
720             break;
721         }
722 
723     start_odd:
724         /* Test odd tag. */
725         if (unlikely((diff) & 0xf0)) {
726             break;
727         }
728         if (++n == count) {
729             break;
730         }
731 
732         diff = *mem++ ^ cmp;
733     }
734     return n;
735 }
736 
737 /**
738  * checkNrev:
739  * @tag: tag memory to test
740  * @odd: true to begin testing at tags at odd nibble
741  * @cmp: the tag to compare against
742  * @count: number of tags to test
743  *
744  * Return the number of successful tests.
745  * Thus a return value < @count indicates a failure.
746  *
747  * This is like checkN, but it runs backwards, checking the
748  * tags starting with @tag and then the tags preceding it.
749  * This is needed by the backwards-memory-copying operations.
750  */
751 static int checkNrev(uint8_t *mem, int odd, int cmp, int count)
752 {
753     int n = 0, diff;
754 
755     /* Replicate the test tag and compare.  */
756     cmp *= 0x11;
757     diff = *mem-- ^ cmp;
758 
759     if (!odd) {
760         goto start_even;
761     }
762 
763     while (1) {
764         /* Test odd tag. */
765         if (unlikely((diff) & 0xf0)) {
766             break;
767         }
768         if (++n == count) {
769             break;
770         }
771 
772     start_even:
773         /* Test even tag. */
774         if (unlikely((diff) & 0x0f)) {
775             break;
776         }
777         if (++n == count) {
778             break;
779         }
780 
781         diff = *mem-- ^ cmp;
782     }
783     return n;
784 }
785 
786 /**
787  * mte_probe_int() - helper for mte_probe and mte_check
788  * @env: CPU environment
789  * @desc: MTEDESC descriptor
790  * @ptr: virtual address of the base of the access
791  * @fault: return virtual address of the first check failure
792  *
793  * Internal routine for both mte_probe and mte_check.
794  * Return zero on failure, filling in *fault.
795  * Return negative on trivial success for tbi disabled.
796  * Return positive on success with tbi enabled.
797  */
798 static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
799                          uintptr_t ra, uint64_t *fault)
800 {
801     int mmu_idx, ptr_tag, bit55;
802     uint64_t ptr_last, prev_page, next_page;
803     uint64_t tag_first, tag_last;
804     uint32_t sizem1, tag_count, n, c;
805     uint8_t *mem1, *mem2;
806     MMUAccessType type;
807 
808     bit55 = extract64(ptr, 55, 1);
809     *fault = ptr;
810 
811     /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
812     if (unlikely(!tbi_check(desc, bit55))) {
813         return -1;
814     }
815 
816     ptr_tag = allocation_tag_from_addr(ptr);
817 
818     if (tcma_check(desc, bit55, ptr_tag)) {
819         return 1;
820     }
821 
822     mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
823     type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
824     sizem1 = FIELD_EX32(desc, MTEDESC, SIZEM1);
825 
826     /* Find the addr of the end of the access */
827     ptr_last = ptr + sizem1;
828 
829     /* Round the bounds to the tag granule, and compute the number of tags. */
830     tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
831     tag_last = QEMU_ALIGN_DOWN(ptr_last, TAG_GRANULE);
832     tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1;
833 
834     /* Locate the page boundaries. */
835     prev_page = ptr & TARGET_PAGE_MASK;
836     next_page = prev_page + TARGET_PAGE_SIZE;
837 
838     if (likely(tag_last - prev_page < TARGET_PAGE_SIZE)) {
839         /* Memory access stays on one page. */
840         mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, sizem1 + 1,
841                                   MMU_DATA_LOAD, ra);
842         if (!mem1) {
843             return 1;
844         }
845         /* Perform all of the comparisons. */
846         n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count);
847     } else {
848         /* Memory access crosses to next page. */
849         mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr,
850                                   MMU_DATA_LOAD, ra);
851 
852         mem2 = allocation_tag_mem(env, mmu_idx, next_page, type,
853                                   ptr_last - next_page + 1,
854                                   MMU_DATA_LOAD, ra);
855 
856         /*
857          * Perform all of the comparisons.
858          * Note the possible but unlikely case of the operation spanning
859          * two pages that do not both have tagging enabled.
860          */
861         n = c = (next_page - tag_first) / TAG_GRANULE;
862         if (mem1) {
863             n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, c);
864         }
865         if (n == c) {
866             if (!mem2) {
867                 return 1;
868             }
869             n += checkN(mem2, 0, ptr_tag, tag_count - c);
870         }
871     }
872 
873     if (likely(n == tag_count)) {
874         return 1;
875     }
876 
877     /*
878      * If we failed, we know which granule.  For the first granule, the
879      * failure address is @ptr, the first byte accessed.  Otherwise the
880      * failure address is the first byte of the nth granule.
881      */
882     if (n > 0) {
883         *fault = tag_first + n * TAG_GRANULE;
884     }
885     return 0;
886 }
887 
888 uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr, uintptr_t ra)
889 {
890     uint64_t fault;
891     int ret = mte_probe_int(env, desc, ptr, ra, &fault);
892 
893     if (unlikely(ret == 0)) {
894         mte_check_fail(env, desc, fault, ra);
895     } else if (ret < 0) {
896         return ptr;
897     }
898     return useronly_clean_ptr(ptr);
899 }
900 
901 uint64_t HELPER(mte_check)(CPUARMState *env, uint32_t desc, uint64_t ptr)
902 {
903     /*
904      * R_XCHFJ: Alignment check not caused by memory type is priority 1,
905      * higher than any translation fault.  When MTE is disabled, tcg
906      * performs the alignment check during the code generated for the
907      * memory access.  With MTE enabled, we must check this here before
908      * raising any translation fault in allocation_tag_mem.
909      */
910     unsigned align = FIELD_EX32(desc, MTEDESC, ALIGN);
911     if (unlikely(align)) {
912         align = (1u << align) - 1;
913         if (unlikely(ptr & align)) {
914             int idx = FIELD_EX32(desc, MTEDESC, MIDX);
915             bool w = FIELD_EX32(desc, MTEDESC, WRITE);
916             MMUAccessType type = w ? MMU_DATA_STORE : MMU_DATA_LOAD;
917             arm_cpu_do_unaligned_access(env_cpu(env), ptr, type, idx, GETPC());
918         }
919     }
920 
921     return mte_check(env, desc, ptr, GETPC());
922 }
923 
924 /*
925  * No-fault version of mte_check, to be used by SVE for MemSingleNF.
926  * Returns false if the access is Checked and the check failed.  This
927  * is only intended to probe the tag -- the validity of the page must
928  * be checked beforehand.
929  */
930 bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr)
931 {
932     uint64_t fault;
933     int ret = mte_probe_int(env, desc, ptr, 0, &fault);
934 
935     return ret != 0;
936 }
937 
938 /*
939  * Perform an MTE checked access for DC_ZVA.
940  */
941 uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr)
942 {
943     uintptr_t ra = GETPC();
944     int log2_dcz_bytes, log2_tag_bytes;
945     int mmu_idx, bit55;
946     intptr_t dcz_bytes, tag_bytes, i;
947     void *mem;
948     uint64_t ptr_tag, mem_tag, align_ptr;
949 
950     bit55 = extract64(ptr, 55, 1);
951 
952     /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
953     if (unlikely(!tbi_check(desc, bit55))) {
954         return ptr;
955     }
956 
957     ptr_tag = allocation_tag_from_addr(ptr);
958 
959     if (tcma_check(desc, bit55, ptr_tag)) {
960         goto done;
961     }
962 
963     /*
964      * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
965      * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
966      * sure that we can access one complete tag byte here.
967      */
968     log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
969     log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
970     dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
971     tag_bytes = (intptr_t)1 << log2_tag_bytes;
972     align_ptr = ptr & -dcz_bytes;
973 
974     /*
975      * Trap if accessing an invalid page.  DC_ZVA requires that we supply
976      * the original pointer for an invalid page.  But watchpoints require
977      * that we probe the actual space.  So do both.
978      */
979     mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
980     (void) probe_write(env, ptr, 1, mmu_idx, ra);
981     mem = allocation_tag_mem(env, mmu_idx, align_ptr, MMU_DATA_STORE,
982                              dcz_bytes, MMU_DATA_LOAD, ra);
983     if (!mem) {
984         goto done;
985     }
986 
987     /*
988      * Unlike the reasoning for checkN, DC_ZVA is always aligned, and thus
989      * it is quite easy to perform all of the comparisons at once without
990      * any extra masking.
991      *
992      * The most common zva block size is 64; some of the thunderx cpus use
993      * a block size of 128.  For user-only, aarch64_max_initfn will set the
994      * block size to 512.  Fill out the other cases for future-proofing.
995      *
996      * In order to be able to find the first miscompare later, we want the
997      * tag bytes to be in little-endian order.
998      */
999     switch (log2_tag_bytes) {
1000     case 0: /* zva_blocksize 32 */
1001         mem_tag = *(uint8_t *)mem;
1002         ptr_tag *= 0x11u;
1003         break;
1004     case 1: /* zva_blocksize 64 */
1005         mem_tag = cpu_to_le16(*(uint16_t *)mem);
1006         ptr_tag *= 0x1111u;
1007         break;
1008     case 2: /* zva_blocksize 128 */
1009         mem_tag = cpu_to_le32(*(uint32_t *)mem);
1010         ptr_tag *= 0x11111111u;
1011         break;
1012     case 3: /* zva_blocksize 256 */
1013         mem_tag = cpu_to_le64(*(uint64_t *)mem);
1014         ptr_tag *= 0x1111111111111111ull;
1015         break;
1016 
1017     default: /* zva_blocksize 512, 1024, 2048 */
1018         ptr_tag *= 0x1111111111111111ull;
1019         i = 0;
1020         do {
1021             mem_tag = cpu_to_le64(*(uint64_t *)(mem + i));
1022             if (unlikely(mem_tag != ptr_tag)) {
1023                 goto fail;
1024             }
1025             i += 8;
1026             align_ptr += 16 * TAG_GRANULE;
1027         } while (i < tag_bytes);
1028         goto done;
1029     }
1030 
1031     if (likely(mem_tag == ptr_tag)) {
1032         goto done;
1033     }
1034 
1035  fail:
1036     /* Locate the first nibble that differs. */
1037     i = ctz64(mem_tag ^ ptr_tag) >> 4;
1038     mte_check_fail(env, desc, align_ptr + i * TAG_GRANULE, ra);
1039 
1040  done:
1041     return useronly_clean_ptr(ptr);
1042 }
1043 
1044 uint64_t mte_mops_probe(CPUARMState *env, uint64_t ptr, uint64_t size,
1045                         uint32_t desc)
1046 {
1047     int mmu_idx, tag_count;
1048     uint64_t ptr_tag, tag_first, tag_last;
1049     void *mem;
1050     bool w = FIELD_EX32(desc, MTEDESC, WRITE);
1051     uint32_t n;
1052 
1053     mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
1054     /* True probe; this will never fault */
1055     mem = allocation_tag_mem_probe(env, mmu_idx, ptr,
1056                                    w ? MMU_DATA_STORE : MMU_DATA_LOAD,
1057                                    size, MMU_DATA_LOAD, true, 0);
1058     if (!mem) {
1059         return size;
1060     }
1061 
1062     /*
1063      * TODO: checkN() is not designed for checks of the size we expect
1064      * for FEAT_MOPS operations, so we should implement this differently.
1065      * Maybe we should do something like
1066      *   if (region start and size are aligned nicely) {
1067      *      do direct loads of 64 tag bits at a time;
1068      *   } else {
1069      *      call checkN()
1070      *   }
1071      */
1072     /* Round the bounds to the tag granule, and compute the number of tags. */
1073     ptr_tag = allocation_tag_from_addr(ptr);
1074     tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
1075     tag_last = QEMU_ALIGN_DOWN(ptr + size - 1, TAG_GRANULE);
1076     tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1;
1077     n = checkN(mem, ptr & TAG_GRANULE, ptr_tag, tag_count);
1078     if (likely(n == tag_count)) {
1079         return size;
1080     }
1081 
1082     /*
1083      * Failure; for the first granule, it's at @ptr. Otherwise
1084      * it's at the first byte of the nth granule. Calculate how
1085      * many bytes we can access without hitting that failure.
1086      */
1087     if (n == 0) {
1088         return 0;
1089     } else {
1090         return n * TAG_GRANULE - (ptr - tag_first);
1091     }
1092 }
1093 
1094 uint64_t mte_mops_probe_rev(CPUARMState *env, uint64_t ptr, uint64_t size,
1095                             uint32_t desc)
1096 {
1097     int mmu_idx, tag_count;
1098     uint64_t ptr_tag, tag_first, tag_last;
1099     void *mem;
1100     bool w = FIELD_EX32(desc, MTEDESC, WRITE);
1101     uint32_t n;
1102 
1103     mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
1104     /*
1105      * True probe; this will never fault. Note that our caller passes
1106      * us a pointer to the end of the region, but allocation_tag_mem_probe()
1107      * wants a pointer to the start. Because we know we don't span a page
1108      * boundary and that allocation_tag_mem_probe() doesn't otherwise care
1109      * about the size, pass in a size of 1 byte. This is simpler than
1110      * adjusting the ptr to point to the start of the region and then having
1111      * to adjust the returned 'mem' to get the end of the tag memory.
1112      */
1113     mem = allocation_tag_mem_probe(env, mmu_idx, ptr,
1114                                    w ? MMU_DATA_STORE : MMU_DATA_LOAD,
1115                                    1, MMU_DATA_LOAD, true, 0);
1116     if (!mem) {
1117         return size;
1118     }
1119 
1120     /*
1121      * TODO: checkNrev() is not designed for checks of the size we expect
1122      * for FEAT_MOPS operations, so we should implement this differently.
1123      * Maybe we should do something like
1124      *   if (region start and size are aligned nicely) {
1125      *      do direct loads of 64 tag bits at a time;
1126      *   } else {
1127      *      call checkN()
1128      *   }
1129      */
1130     /* Round the bounds to the tag granule, and compute the number of tags. */
1131     ptr_tag = allocation_tag_from_addr(ptr);
1132     tag_first = QEMU_ALIGN_DOWN(ptr - (size - 1), TAG_GRANULE);
1133     tag_last = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
1134     tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1;
1135     n = checkNrev(mem, ptr & TAG_GRANULE, ptr_tag, tag_count);
1136     if (likely(n == tag_count)) {
1137         return size;
1138     }
1139 
1140     /*
1141      * Failure; for the first granule, it's at @ptr. Otherwise
1142      * it's at the last byte of the nth granule. Calculate how
1143      * many bytes we can access without hitting that failure.
1144      */
1145     if (n == 0) {
1146         return 0;
1147     } else {
1148         return (n - 1) * TAG_GRANULE + ((ptr + 1) - tag_last);
1149     }
1150 }
1151 
1152 void mte_mops_set_tags(CPUARMState *env, uint64_t ptr, uint64_t size,
1153                        uint32_t desc)
1154 {
1155     int mmu_idx, tag_count;
1156     uint64_t ptr_tag;
1157     void *mem;
1158 
1159     if (!desc) {
1160         /* Tags not actually enabled */
1161         return;
1162     }
1163 
1164     mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
1165     /* True probe: this will never fault */
1166     mem = allocation_tag_mem_probe(env, mmu_idx, ptr, MMU_DATA_STORE, size,
1167                                    MMU_DATA_STORE, true, 0);
1168     if (!mem) {
1169         return;
1170     }
1171 
1172     /*
1173      * We know that ptr and size are both TAG_GRANULE aligned; store
1174      * the tag from the pointer value into the tag memory.
1175      */
1176     ptr_tag = allocation_tag_from_addr(ptr);
1177     tag_count = size / TAG_GRANULE;
1178     if (ptr & TAG_GRANULE) {
1179         /* Not 2*TAG_GRANULE-aligned: store tag to first nibble */
1180         store_tag1_parallel(TAG_GRANULE, mem, ptr_tag);
1181         mem++;
1182         tag_count--;
1183     }
1184     memset(mem, ptr_tag | (ptr_tag << 4), tag_count / 2);
1185     if (tag_count & 1) {
1186         /* Final trailing unaligned nibble */
1187         mem += tag_count / 2;
1188         store_tag1_parallel(0, mem, ptr_tag);
1189     }
1190 }
1191