1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5#include <linux/linkage.h> 6 7#include <asm/asm-uaccess.h> 8#include <asm/assembler.h> 9#include <asm/mte.h> 10#include <asm/page.h> 11#include <asm/sysreg.h> 12 13 .arch armv8.5-a+memtag 14 15/* 16 * multitag_transfer_size - set \reg to the block size that is accessed by the 17 * LDGM/STGM instructions. 18 */ 19 .macro multitag_transfer_size, reg, tmp 20 mrs_s \reg, SYS_GMID_EL1 21 ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE 22 mov \tmp, #4 23 lsl \reg, \tmp, \reg 24 .endm 25 26/* 27 * Clear the tags in a page 28 * x0 - address of the page to be cleared 29 */ 30SYM_FUNC_START(mte_clear_page_tags) 31 multitag_transfer_size x1, x2 321: stgm xzr, [x0] 33 add x0, x0, x1 34 tst x0, #(PAGE_SIZE - 1) 35 b.ne 1b 36 ret 37SYM_FUNC_END(mte_clear_page_tags) 38 39/* 40 * Zero the page and tags at the same time 41 * 42 * Parameters: 43 * x0 - address to the beginning of the page 44 */ 45SYM_FUNC_START(mte_zero_clear_page_tags) 46 mrs x1, dczid_el0 47 and w1, w1, #0xf 48 mov x2, #4 49 lsl x1, x2, x1 50 and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag 51 521: dc gzva, x0 53 add x0, x0, x1 54 tst x0, #(PAGE_SIZE - 1) 55 b.ne 1b 56 ret 57SYM_FUNC_END(mte_zero_clear_page_tags) 58 59/* 60 * Copy the tags from the source page to the destination one 61 * x0 - address of the destination page 62 * x1 - address of the source page 63 */ 64SYM_FUNC_START(mte_copy_page_tags) 65 mov x2, x0 66 mov x3, x1 67 multitag_transfer_size x5, x6 681: ldgm x4, [x3] 69 stgm x4, [x2] 70 add x2, x2, x5 71 add x3, x3, x5 72 tst x2, #(PAGE_SIZE - 1) 73 b.ne 1b 74 ret 75SYM_FUNC_END(mte_copy_page_tags) 76 77/* 78 * Read tags from a user buffer (one tag per byte) and set the corresponding 79 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. 80 * x0 - kernel address (to) 81 * x1 - user buffer (from) 82 * x2 - number of tags/bytes (n) 83 * Returns: 84 * x0 - number of tags read/set 85 */ 86SYM_FUNC_START(mte_copy_tags_from_user) 87 mov x3, x1 88 cbz x2, 2f 891: 90 user_ldst 2f, ldtrb, w4, x1, 0 91 lsl x4, x4, #MTE_TAG_SHIFT 92 stg x4, [x0], #MTE_GRANULE_SIZE 93 add x1, x1, #1 94 subs x2, x2, #1 95 b.ne 1b 96 97 // exception handling and function return 982: sub x0, x1, x3 // update the number of tags set 99 ret 100SYM_FUNC_END(mte_copy_tags_from_user) 101 102/* 103 * Get the tags from a kernel address range and write the tag values to the 104 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS. 105 * x0 - user buffer (to) 106 * x1 - kernel address (from) 107 * x2 - number of tags/bytes (n) 108 * Returns: 109 * x0 - number of tags read/set 110 */ 111SYM_FUNC_START(mte_copy_tags_to_user) 112 mov x3, x0 113 cbz x2, 2f 1141: 115 ldg x4, [x1] 116 ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE 117 user_ldst 2f, sttrb, w4, x0, 0 118 add x0, x0, #1 119 add x1, x1, #MTE_GRANULE_SIZE 120 subs x2, x2, #1 121 b.ne 1b 122 123 // exception handling and function return 1242: sub x0, x0, x3 // update the number of tags copied 125 ret 126SYM_FUNC_END(mte_copy_tags_to_user) 127 128/* 129 * Save the tags in a page 130 * x0 - page address 131 * x1 - tag storage 132 */ 133SYM_FUNC_START(mte_save_page_tags) 134 multitag_transfer_size x7, x5 1351: 136 mov x2, #0 1372: 138 ldgm x5, [x0] 139 orr x2, x2, x5 140 add x0, x0, x7 141 tst x0, #0xFF // 16 tag values fit in a register, 142 b.ne 2b // which is 16*16=256 bytes 143 144 str x2, [x1], #8 145 146 tst x0, #(PAGE_SIZE - 1) 147 b.ne 1b 148 149 ret 150SYM_FUNC_END(mte_save_page_tags) 151 152/* 153 * Restore the tags in a page 154 * x0 - page address 155 * x1 - tag storage 156 */ 157SYM_FUNC_START(mte_restore_page_tags) 158 multitag_transfer_size x7, x5 1591: 160 ldr x2, [x1], #8 1612: 162 stgm x2, [x0] 163 add x0, x0, x7 164 tst x0, #0xFF 165 b.ne 2b 166 167 tst x0, #(PAGE_SIZE - 1) 168 b.ne 1b 169 170 ret 171SYM_FUNC_END(mte_restore_page_tags) 172