1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5#include <linux/linkage.h> 6 7#include <asm/asm-uaccess.h> 8#include <asm/assembler.h> 9#include <asm/mte.h> 10#include <asm/page.h> 11#include <asm/sysreg.h> 12 13 .arch armv8.5-a+memtag 14 15/* 16 * multitag_transfer_size - set \reg to the block size that is accessed by the 17 * LDGM/STGM instructions. 18 */ 19 .macro multitag_transfer_size, reg, tmp 20 mrs_s \reg, SYS_GMID_EL1 21 ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH 22 mov \tmp, #4 23 lsl \reg, \tmp, \reg 24 .endm 25 26/* 27 * Clear the tags in a page 28 * x0 - address of the page to be cleared 29 */ 30SYM_FUNC_START(mte_clear_page_tags) 31 multitag_transfer_size x1, x2 321: stgm xzr, [x0] 33 add x0, x0, x1 34 tst x0, #(PAGE_SIZE - 1) 35 b.ne 1b 36 ret 37SYM_FUNC_END(mte_clear_page_tags) 38 39/* 40 * Zero the page and tags at the same time 41 * 42 * Parameters: 43 * x0 - address to the beginning of the page 44 */ 45SYM_FUNC_START(mte_zero_clear_page_tags) 46 and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag 47 mrs x1, dczid_el0 48 tbnz x1, #4, 2f // Branch if DC GZVA is prohibited 49 and w1, w1, #0xf 50 mov x2, #4 51 lsl x1, x2, x1 52 531: dc gzva, x0 54 add x0, x0, x1 55 tst x0, #(PAGE_SIZE - 1) 56 b.ne 1b 57 ret 58 592: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2) 60 tst x0, #(PAGE_SIZE - 1) 61 b.ne 2b 62 ret 63SYM_FUNC_END(mte_zero_clear_page_tags) 64 65/* 66 * Copy the tags from the source page to the destination one 67 * x0 - address of the destination page 68 * x1 - address of the source page 69 */ 70SYM_FUNC_START(mte_copy_page_tags) 71 mov x2, x0 72 mov x3, x1 73 multitag_transfer_size x5, x6 741: ldgm x4, [x3] 75 stgm x4, [x2] 76 add x2, x2, x5 77 add x3, x3, x5 78 tst x2, #(PAGE_SIZE - 1) 79 b.ne 1b 80 ret 81SYM_FUNC_END(mte_copy_page_tags) 82 83/* 84 * Read tags from a user buffer (one tag per byte) and set the corresponding 85 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. 86 * x0 - kernel address (to) 87 * x1 - user buffer (from) 88 * x2 - number of tags/bytes (n) 89 * Returns: 90 * x0 - number of tags read/set 91 */ 92SYM_FUNC_START(mte_copy_tags_from_user) 93 mov x3, x1 94 cbz x2, 2f 951: 96USER(2f, ldtrb w4, [x1]) 97 lsl x4, x4, #MTE_TAG_SHIFT 98 stg x4, [x0], #MTE_GRANULE_SIZE 99 add x1, x1, #1 100 subs x2, x2, #1 101 b.ne 1b 102 103 // exception handling and function return 1042: sub x0, x1, x3 // update the number of tags set 105 ret 106SYM_FUNC_END(mte_copy_tags_from_user) 107 108/* 109 * Get the tags from a kernel address range and write the tag values to the 110 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS. 111 * x0 - user buffer (to) 112 * x1 - kernel address (from) 113 * x2 - number of tags/bytes (n) 114 * Returns: 115 * x0 - number of tags read/set 116 */ 117SYM_FUNC_START(mte_copy_tags_to_user) 118 mov x3, x0 119 cbz x2, 2f 1201: 121 ldg x4, [x1] 122 ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE 123USER(2f, sttrb w4, [x0]) 124 add x0, x0, #1 125 add x1, x1, #MTE_GRANULE_SIZE 126 subs x2, x2, #1 127 b.ne 1b 128 129 // exception handling and function return 1302: sub x0, x0, x3 // update the number of tags copied 131 ret 132SYM_FUNC_END(mte_copy_tags_to_user) 133 134/* 135 * Save the tags in a page 136 * x0 - page address 137 * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes 138 */ 139SYM_FUNC_START(mte_save_page_tags) 140 multitag_transfer_size x7, x5 1411: 142 mov x2, #0 1432: 144 ldgm x5, [x0] 145 orr x2, x2, x5 146 add x0, x0, x7 147 tst x0, #0xFF // 16 tag values fit in a register, 148 b.ne 2b // which is 16*16=256 bytes 149 150 str x2, [x1], #8 151 152 tst x0, #(PAGE_SIZE - 1) 153 b.ne 1b 154 155 ret 156SYM_FUNC_END(mte_save_page_tags) 157 158/* 159 * Restore the tags in a page 160 * x0 - page address 161 * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes 162 */ 163SYM_FUNC_START(mte_restore_page_tags) 164 multitag_transfer_size x7, x5 1651: 166 ldr x2, [x1], #8 1672: 168 stgm x2, [x0] 169 add x0, x0, x7 170 tst x0, #0xFF 171 b.ne 2b 172 173 tst x0, #(PAGE_SIZE - 1) 174 b.ne 1b 175 176 ret 177SYM_FUNC_END(mte_restore_page_tags) 178