1 /* adi_64.c: support for ADI (Application Data Integrity) feature on 2 * sparc m7 and newer processors. This feature is also known as 3 * SSM (Silicon Secured Memory). 4 * 5 * Copyright (C) 2016 Oracle and/or its affiliates. All rights reserved. 6 * Author: Khalid Aziz (khalid.aziz@oracle.com) 7 * 8 * This work is licensed under the terms of the GNU GPL, version 2. 9 */ 10 #include <linux/init.h> 11 #include <linux/slab.h> 12 #include <linux/mm_types.h> 13 #include <asm/mdesc.h> 14 #include <asm/adi_64.h> 15 #include <asm/mmu_64.h> 16 #include <asm/pgtable_64.h> 17 18 /* Each page of storage for ADI tags can accommodate tags for 128 19 * pages. When ADI enabled pages are being swapped out, it would be 20 * prudent to allocate at least enough tag storage space to accommodate 21 * SWAPFILE_CLUSTER number of pages. Allocate enough tag storage to 22 * store tags for four SWAPFILE_CLUSTER pages to reduce need for 23 * further allocations for same vma. 24 */ 25 #define TAG_STORAGE_PAGES 8 26 27 struct adi_config adi_state; 28 EXPORT_SYMBOL(adi_state); 29 30 /* mdesc_adi_init() : Parse machine description provided by the 31 * hypervisor to detect ADI capabilities 32 * 33 * Hypervisor reports ADI capabilities of platform in "hwcap-list" property 34 * for "cpu" node. If the platform supports ADI, "hwcap-list" property 35 * contains the keyword "adp". If the platform supports ADI, "platform" 36 * node will contain "adp-blksz", "adp-nbits" and "ue-on-adp" properties 37 * to describe the ADI capabilities. 38 */ 39 void __init mdesc_adi_init(void) 40 { 41 struct mdesc_handle *hp = mdesc_grab(); 42 const char *prop; 43 u64 pn, *val; 44 int len; 45 46 if (!hp) 47 goto adi_not_found; 48 49 pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "cpu"); 50 if (pn == MDESC_NODE_NULL) 51 goto adi_not_found; 52 53 prop = mdesc_get_property(hp, pn, "hwcap-list", &len); 54 if (!prop) 55 goto adi_not_found; 56 57 /* 58 * Look for "adp" keyword in hwcap-list which would indicate 59 * ADI support 60 */ 61 adi_state.enabled = false; 62 while (len) { 63 int plen; 64 65 if (!strcmp(prop, "adp")) { 66 adi_state.enabled = true; 67 break; 68 } 69 70 plen = strlen(prop) + 1; 71 prop += plen; 72 len -= plen; 73 } 74 75 if (!adi_state.enabled) 76 goto adi_not_found; 77 78 /* Find the ADI properties in "platform" node. If all ADI 79 * properties are not found, ADI support is incomplete and 80 * do not enable ADI in the kernel. 81 */ 82 pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); 83 if (pn == MDESC_NODE_NULL) 84 goto adi_not_found; 85 86 val = (u64 *) mdesc_get_property(hp, pn, "adp-blksz", &len); 87 if (!val) 88 goto adi_not_found; 89 adi_state.caps.blksz = *val; 90 91 val = (u64 *) mdesc_get_property(hp, pn, "adp-nbits", &len); 92 if (!val) 93 goto adi_not_found; 94 adi_state.caps.nbits = *val; 95 96 val = (u64 *) mdesc_get_property(hp, pn, "ue-on-adp", &len); 97 if (!val) 98 goto adi_not_found; 99 adi_state.caps.ue_on_adi = *val; 100 101 /* Some of the code to support swapping ADI tags is written 102 * assumption that two ADI tags can fit inside one byte. If 103 * this assumption is broken by a future architecture change, 104 * that code will have to be revisited. If that were to happen, 105 * disable ADI support so we do not get unpredictable results 106 * with programs trying to use ADI and their pages getting 107 * swapped out 108 */ 109 if (adi_state.caps.nbits > 4) { 110 pr_warn("WARNING: ADI tag size >4 on this platform. Disabling AADI support\n"); 111 adi_state.enabled = false; 112 } 113 114 mdesc_release(hp); 115 return; 116 117 adi_not_found: 118 adi_state.enabled = false; 119 adi_state.caps.blksz = 0; 120 adi_state.caps.nbits = 0; 121 if (hp) 122 mdesc_release(hp); 123 } 124 125 tag_storage_desc_t *find_tag_store(struct mm_struct *mm, 126 struct vm_area_struct *vma, 127 unsigned long addr) 128 { 129 tag_storage_desc_t *tag_desc = NULL; 130 unsigned long i, max_desc, flags; 131 132 /* Check if this vma already has tag storage descriptor 133 * allocated for it. 134 */ 135 max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); 136 if (mm->context.tag_store) { 137 tag_desc = mm->context.tag_store; 138 spin_lock_irqsave(&mm->context.tag_lock, flags); 139 for (i = 0; i < max_desc; i++) { 140 if ((addr >= tag_desc->start) && 141 ((addr + PAGE_SIZE - 1) <= tag_desc->end)) 142 break; 143 tag_desc++; 144 } 145 spin_unlock_irqrestore(&mm->context.tag_lock, flags); 146 147 /* If no matching entries were found, this must be a 148 * freshly allocated page 149 */ 150 if (i >= max_desc) 151 tag_desc = NULL; 152 } 153 154 return tag_desc; 155 } 156 157 tag_storage_desc_t *alloc_tag_store(struct mm_struct *mm, 158 struct vm_area_struct *vma, 159 unsigned long addr) 160 { 161 unsigned char *tags; 162 unsigned long i, size, max_desc, flags; 163 tag_storage_desc_t *tag_desc, *open_desc; 164 unsigned long end_addr, hole_start, hole_end; 165 166 max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); 167 open_desc = NULL; 168 hole_start = 0; 169 hole_end = ULONG_MAX; 170 end_addr = addr + PAGE_SIZE - 1; 171 172 /* Check if this vma already has tag storage descriptor 173 * allocated for it. 174 */ 175 spin_lock_irqsave(&mm->context.tag_lock, flags); 176 if (mm->context.tag_store) { 177 tag_desc = mm->context.tag_store; 178 179 /* Look for a matching entry for this address. While doing 180 * that, look for the first open slot as well and find 181 * the hole in already allocated range where this request 182 * will fit in. 183 */ 184 for (i = 0; i < max_desc; i++) { 185 if (tag_desc->tag_users == 0) { 186 if (open_desc == NULL) 187 open_desc = tag_desc; 188 } else { 189 if ((addr >= tag_desc->start) && 190 (tag_desc->end >= (addr + PAGE_SIZE - 1))) { 191 tag_desc->tag_users++; 192 goto out; 193 } 194 } 195 if ((tag_desc->start > end_addr) && 196 (tag_desc->start < hole_end)) 197 hole_end = tag_desc->start; 198 if ((tag_desc->end < addr) && 199 (tag_desc->end > hole_start)) 200 hole_start = tag_desc->end; 201 tag_desc++; 202 } 203 204 } else { 205 size = sizeof(tag_storage_desc_t)*max_desc; 206 mm->context.tag_store = kzalloc(size, GFP_NOWAIT|__GFP_NOWARN); 207 if (mm->context.tag_store == NULL) { 208 tag_desc = NULL; 209 goto out; 210 } 211 tag_desc = mm->context.tag_store; 212 for (i = 0; i < max_desc; i++, tag_desc++) 213 tag_desc->tag_users = 0; 214 open_desc = mm->context.tag_store; 215 i = 0; 216 } 217 218 /* Check if we ran out of tag storage descriptors */ 219 if (open_desc == NULL) { 220 tag_desc = NULL; 221 goto out; 222 } 223 224 /* Mark this tag descriptor slot in use and then initialize it */ 225 tag_desc = open_desc; 226 tag_desc->tag_users = 1; 227 228 /* Tag storage has not been allocated for this vma and space 229 * is available in tag storage descriptor. Since this page is 230 * being swapped out, there is high probability subsequent pages 231 * in the VMA will be swapped out as well. Allocate pages to 232 * store tags for as many pages in this vma as possible but not 233 * more than TAG_STORAGE_PAGES. Each byte in tag space holds 234 * two ADI tags since each ADI tag is 4 bits. Each ADI tag 235 * covers adi_blksize() worth of addresses. Check if the hole is 236 * big enough to accommodate full address range for using 237 * TAG_STORAGE_PAGES number of tag pages. 238 */ 239 size = TAG_STORAGE_PAGES * PAGE_SIZE; 240 end_addr = addr + (size*2*adi_blksize()) - 1; 241 /* Check for overflow. If overflow occurs, allocate only one page */ 242 if (end_addr < addr) { 243 size = PAGE_SIZE; 244 end_addr = addr + (size*2*adi_blksize()) - 1; 245 /* If overflow happens with the minimum tag storage 246 * allocation as well, adjust ending address for this 247 * tag storage. 248 */ 249 if (end_addr < addr) 250 end_addr = ULONG_MAX; 251 } 252 if (hole_end < end_addr) { 253 /* Available hole is too small on the upper end of 254 * address. Can we expand the range towards the lower 255 * address and maximize use of this slot? 256 */ 257 unsigned long tmp_addr; 258 259 end_addr = hole_end - 1; 260 tmp_addr = end_addr - (size*2*adi_blksize()) + 1; 261 /* Check for underflow. If underflow occurs, allocate 262 * only one page for storing ADI tags 263 */ 264 if (tmp_addr > addr) { 265 size = PAGE_SIZE; 266 tmp_addr = end_addr - (size*2*adi_blksize()) - 1; 267 /* If underflow happens with the minimum tag storage 268 * allocation as well, adjust starting address for 269 * this tag storage. 270 */ 271 if (tmp_addr > addr) 272 tmp_addr = 0; 273 } 274 if (tmp_addr < hole_start) { 275 /* Available hole is restricted on lower address 276 * end as well 277 */ 278 tmp_addr = hole_start + 1; 279 } 280 addr = tmp_addr; 281 size = (end_addr + 1 - addr)/(2*adi_blksize()); 282 size = (size + (PAGE_SIZE-adi_blksize()))/PAGE_SIZE; 283 size = size * PAGE_SIZE; 284 } 285 tags = kzalloc(size, GFP_NOWAIT|__GFP_NOWARN); 286 if (tags == NULL) { 287 tag_desc->tag_users = 0; 288 tag_desc = NULL; 289 goto out; 290 } 291 tag_desc->start = addr; 292 tag_desc->tags = tags; 293 tag_desc->end = end_addr; 294 295 out: 296 spin_unlock_irqrestore(&mm->context.tag_lock, flags); 297 return tag_desc; 298 } 299 300 void del_tag_store(tag_storage_desc_t *tag_desc, struct mm_struct *mm) 301 { 302 unsigned long flags; 303 unsigned char *tags = NULL; 304 305 spin_lock_irqsave(&mm->context.tag_lock, flags); 306 tag_desc->tag_users--; 307 if (tag_desc->tag_users == 0) { 308 tag_desc->start = tag_desc->end = 0; 309 /* Do not free up the tag storage space allocated 310 * by the first descriptor. This is persistent 311 * emergency tag storage space for the task. 312 */ 313 if (tag_desc != mm->context.tag_store) { 314 tags = tag_desc->tags; 315 tag_desc->tags = NULL; 316 } 317 } 318 spin_unlock_irqrestore(&mm->context.tag_lock, flags); 319 kfree(tags); 320 } 321 322 #define tag_start(addr, tag_desc) \ 323 ((tag_desc)->tags + ((addr - (tag_desc)->start)/(2*adi_blksize()))) 324 325 /* Retrieve any saved ADI tags for the page being swapped back in and 326 * restore these tags to the newly allocated physical page. 327 */ 328 void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma, 329 unsigned long addr, pte_t pte) 330 { 331 unsigned char *tag; 332 tag_storage_desc_t *tag_desc; 333 unsigned long paddr, tmp, version1, version2; 334 335 /* Check if the swapped out page has an ADI version 336 * saved. If yes, restore version tag to the newly 337 * allocated page. 338 */ 339 tag_desc = find_tag_store(mm, vma, addr); 340 if (tag_desc == NULL) 341 return; 342 343 tag = tag_start(addr, tag_desc); 344 paddr = pte_val(pte) & _PAGE_PADDR_4V; 345 for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) { 346 version1 = (*tag) >> 4; 347 version2 = (*tag) & 0x0f; 348 *tag++ = 0; 349 asm volatile("stxa %0, [%1] %2\n\t" 350 : 351 : "r" (version1), "r" (tmp), 352 "i" (ASI_MCD_REAL)); 353 tmp += adi_blksize(); 354 asm volatile("stxa %0, [%1] %2\n\t" 355 : 356 : "r" (version2), "r" (tmp), 357 "i" (ASI_MCD_REAL)); 358 } 359 asm volatile("membar #Sync\n\t"); 360 361 /* Check and mark this tag space for release later if 362 * the swapped in page was the last user of tag space 363 */ 364 del_tag_store(tag_desc, mm); 365 } 366 367 /* A page is about to be swapped out. Save any ADI tags associated with 368 * this physical page so they can be restored later when the page is swapped 369 * back in. 370 */ 371 int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma, 372 unsigned long addr, pte_t oldpte) 373 { 374 unsigned char *tag; 375 tag_storage_desc_t *tag_desc; 376 unsigned long version1, version2, paddr, tmp; 377 378 tag_desc = alloc_tag_store(mm, vma, addr); 379 if (tag_desc == NULL) 380 return -1; 381 382 tag = tag_start(addr, tag_desc); 383 paddr = pte_val(oldpte) & _PAGE_PADDR_4V; 384 for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) { 385 asm volatile("ldxa [%1] %2, %0\n\t" 386 : "=r" (version1) 387 : "r" (tmp), "i" (ASI_MCD_REAL)); 388 tmp += adi_blksize(); 389 asm volatile("ldxa [%1] %2, %0\n\t" 390 : "=r" (version2) 391 : "r" (tmp), "i" (ASI_MCD_REAL)); 392 *tag = (version1 << 4) | version2; 393 tag++; 394 } 395 396 return 0; 397 } 398