1 /* 2 * Copyright (C) 2014-2016 Broadcom Corporation 3 * Copyright (c) 2017 Red Hat, Inc. 4 * Written by Prem Mallappa, Eric Auger 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * Author: Prem Mallappa <pmallapp@broadcom.com> 16 * 17 */ 18 19 #include "qemu/osdep.h" 20 #include "exec/address-spaces.h" 21 #include "trace.h" 22 #include "exec/target_page.h" 23 #include "hw/core/cpu.h" 24 #include "hw/qdev-properties.h" 25 #include "qapi/error.h" 26 #include "qemu/jhash.h" 27 #include "qemu/module.h" 28 29 #include "qemu/error-report.h" 30 #include "hw/arm/smmu-common.h" 31 #include "smmu-internal.h" 32 33 /* IOTLB Management */ 34 35 inline void smmu_iotlb_inv_all(SMMUState *s) 36 { 37 trace_smmu_iotlb_inv_all(); 38 g_hash_table_remove_all(s->iotlb); 39 } 40 41 static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, 42 gpointer user_data) 43 { 44 uint16_t asid = *(uint16_t *)user_data; 45 SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; 46 47 return iotlb_key->asid == asid; 48 } 49 50 inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) 51 { 52 SMMUIOTLBKey key = {.asid = asid, .iova = iova}; 53 54 trace_smmu_iotlb_inv_iova(asid, iova); 55 g_hash_table_remove(s->iotlb, &key); 56 } 57 58 inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) 59 { 60 trace_smmu_iotlb_inv_asid(asid); 61 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); 62 } 63 64 /* VMSAv8-64 Translation */ 65 66 /** 67 * get_pte - Get the content of a page table entry located at 68 * @base_addr[@index] 69 */ 70 static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, 71 SMMUPTWEventInfo *info) 72 { 73 int ret; 74 dma_addr_t addr = baseaddr + index * sizeof(*pte); 75 76 /* TODO: guarantee 64-bit single-copy atomicity */ 77 ret = dma_memory_read(&address_space_memory, addr, pte, sizeof(*pte)); 78 79 if (ret != MEMTX_OK) { 80 info->type = SMMU_PTW_ERR_WALK_EABT; 81 info->addr = addr; 82 return -EINVAL; 83 } 84 trace_smmu_get_pte(baseaddr, index, addr, *pte); 85 return 0; 86 } 87 88 /* VMSAv8-64 Translation Table Format Descriptor Decoding */ 89 90 /** 91 * get_page_pte_address - returns the L3 descriptor output address, 92 * ie. the page frame 93 * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format 94 */ 95 static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) 96 { 97 return PTE_ADDRESS(pte, granule_sz); 98 } 99 100 /** 101 * get_table_pte_address - return table descriptor output address, 102 * ie. address of next level table 103 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 104 */ 105 static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) 106 { 107 return PTE_ADDRESS(pte, granule_sz); 108 } 109 110 /** 111 * get_block_pte_address - return block descriptor output address and block size 112 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 113 */ 114 static inline hwaddr get_block_pte_address(uint64_t pte, int level, 115 int granule_sz, uint64_t *bsz) 116 { 117 int n = level_shift(level, granule_sz); 118 119 *bsz = 1ULL << n; 120 return PTE_ADDRESS(pte, n); 121 } 122 123 SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) 124 { 125 bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); 126 uint8_t tbi_byte = tbi * 8; 127 128 if (cfg->tt[0].tsz && 129 !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { 130 /* there is a ttbr0 region and we are in it (high bits all zero) */ 131 return &cfg->tt[0]; 132 } else if (cfg->tt[1].tsz && 133 !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { 134 /* there is a ttbr1 region and we are in it (high bits all one) */ 135 return &cfg->tt[1]; 136 } else if (!cfg->tt[0].tsz) { 137 /* ttbr0 region is "everything not in the ttbr1 region" */ 138 return &cfg->tt[0]; 139 } else if (!cfg->tt[1].tsz) { 140 /* ttbr1 region is "everything not in the ttbr0 region" */ 141 return &cfg->tt[1]; 142 } 143 /* in the gap between the two regions, this is a Translation fault */ 144 return NULL; 145 } 146 147 /** 148 * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA 149 * @cfg: translation config 150 * @iova: iova to translate 151 * @perm: access type 152 * @tlbe: IOMMUTLBEntry (out) 153 * @info: handle to an error info 154 * 155 * Return 0 on success, < 0 on error. In case of error, @info is filled 156 * and tlbe->perm is set to IOMMU_NONE. 157 * Upon success, @tlbe is filled with translated_addr and entry 158 * permission rights. 159 */ 160 static int smmu_ptw_64(SMMUTransCfg *cfg, 161 dma_addr_t iova, IOMMUAccessFlags perm, 162 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 163 { 164 dma_addr_t baseaddr, indexmask; 165 int stage = cfg->stage; 166 SMMUTransTableInfo *tt = select_tt(cfg, iova); 167 uint8_t level, granule_sz, inputsize, stride; 168 169 if (!tt || tt->disabled) { 170 info->type = SMMU_PTW_ERR_TRANSLATION; 171 goto error; 172 } 173 174 granule_sz = tt->granule_sz; 175 stride = granule_sz - 3; 176 inputsize = 64 - tt->tsz; 177 level = 4 - (inputsize - 4) / stride; 178 indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; 179 baseaddr = extract64(tt->ttb, 0, 48); 180 baseaddr &= ~indexmask; 181 182 tlbe->iova = iova; 183 tlbe->addr_mask = (1 << granule_sz) - 1; 184 185 while (level <= 3) { 186 uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); 187 uint64_t mask = subpage_size - 1; 188 uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); 189 uint64_t pte; 190 dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); 191 uint8_t ap; 192 193 if (get_pte(baseaddr, offset, &pte, info)) { 194 goto error; 195 } 196 trace_smmu_ptw_level(level, iova, subpage_size, 197 baseaddr, offset, pte); 198 199 if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { 200 trace_smmu_ptw_invalid_pte(stage, level, baseaddr, 201 pte_addr, offset, pte); 202 info->type = SMMU_PTW_ERR_TRANSLATION; 203 goto error; 204 } 205 206 if (is_page_pte(pte, level)) { 207 uint64_t gpa = get_page_pte_address(pte, granule_sz); 208 209 ap = PTE_AP(pte); 210 if (is_permission_fault(ap, perm)) { 211 info->type = SMMU_PTW_ERR_PERMISSION; 212 goto error; 213 } 214 215 tlbe->translated_addr = gpa + (iova & mask); 216 tlbe->perm = PTE_AP_TO_PERM(ap); 217 trace_smmu_ptw_page_pte(stage, level, iova, 218 baseaddr, pte_addr, pte, gpa); 219 return 0; 220 } 221 if (is_block_pte(pte, level)) { 222 uint64_t block_size; 223 hwaddr gpa = get_block_pte_address(pte, level, granule_sz, 224 &block_size); 225 226 ap = PTE_AP(pte); 227 if (is_permission_fault(ap, perm)) { 228 info->type = SMMU_PTW_ERR_PERMISSION; 229 goto error; 230 } 231 232 trace_smmu_ptw_block_pte(stage, level, baseaddr, 233 pte_addr, pte, iova, gpa, 234 block_size >> 20); 235 236 tlbe->translated_addr = gpa + (iova & mask); 237 tlbe->perm = PTE_AP_TO_PERM(ap); 238 return 0; 239 } 240 241 /* table pte */ 242 ap = PTE_APTABLE(pte); 243 244 if (is_permission_fault(ap, perm)) { 245 info->type = SMMU_PTW_ERR_PERMISSION; 246 goto error; 247 } 248 baseaddr = get_table_pte_address(pte, granule_sz); 249 level++; 250 } 251 252 info->type = SMMU_PTW_ERR_TRANSLATION; 253 254 error: 255 tlbe->perm = IOMMU_NONE; 256 return -EINVAL; 257 } 258 259 /** 260 * smmu_ptw - Walk the page tables for an IOVA, according to @cfg 261 * 262 * @cfg: translation configuration 263 * @iova: iova to translate 264 * @perm: tentative access type 265 * @tlbe: returned entry 266 * @info: ptw event handle 267 * 268 * return 0 on success 269 */ 270 inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, 271 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 272 { 273 if (!cfg->aa64) { 274 /* 275 * This code path is not entered as we check this while decoding 276 * the configuration data in the derived SMMU model. 277 */ 278 g_assert_not_reached(); 279 } 280 281 return smmu_ptw_64(cfg, iova, perm, tlbe, info); 282 } 283 284 /** 285 * The bus number is used for lookup when SID based invalidation occurs. 286 * In that case we lazily populate the SMMUPciBus array from the bus hash 287 * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus 288 * numbers may not be always initialized yet. 289 */ 290 SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) 291 { 292 SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; 293 GHashTableIter iter; 294 295 if (smmu_pci_bus) { 296 return smmu_pci_bus; 297 } 298 299 g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); 300 while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { 301 if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { 302 s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; 303 return smmu_pci_bus; 304 } 305 } 306 307 return NULL; 308 } 309 310 static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) 311 { 312 SMMUState *s = opaque; 313 SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); 314 SMMUDevice *sdev; 315 static unsigned int index; 316 317 if (!sbus) { 318 sbus = g_malloc0(sizeof(SMMUPciBus) + 319 sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); 320 sbus->bus = bus; 321 g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); 322 } 323 324 sdev = sbus->pbdev[devfn]; 325 if (!sdev) { 326 char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); 327 328 sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); 329 330 sdev->smmu = s; 331 sdev->bus = bus; 332 sdev->devfn = devfn; 333 334 memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), 335 s->mrtypename, 336 OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); 337 address_space_init(&sdev->as, 338 MEMORY_REGION(&sdev->iommu), name); 339 trace_smmu_add_mr(name); 340 g_free(name); 341 } 342 343 return &sdev->as; 344 } 345 346 IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) 347 { 348 uint8_t bus_n, devfn; 349 SMMUPciBus *smmu_bus; 350 SMMUDevice *smmu; 351 352 bus_n = PCI_BUS_NUM(sid); 353 smmu_bus = smmu_find_smmu_pcibus(s, bus_n); 354 if (smmu_bus) { 355 devfn = SMMU_PCI_DEVFN(sid); 356 smmu = smmu_bus->pbdev[devfn]; 357 if (smmu) { 358 return &smmu->iommu; 359 } 360 } 361 return NULL; 362 } 363 364 static guint smmu_iotlb_key_hash(gconstpointer v) 365 { 366 SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; 367 uint32_t a, b, c; 368 369 /* Jenkins hash */ 370 a = b = c = JHASH_INITVAL + sizeof(*key); 371 a += key->asid; 372 b += extract64(key->iova, 0, 32); 373 c += extract64(key->iova, 32, 32); 374 375 __jhash_mix(a, b, c); 376 __jhash_final(a, b, c); 377 378 return c; 379 } 380 381 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) 382 { 383 const SMMUIOTLBKey *k1 = v1; 384 const SMMUIOTLBKey *k2 = v2; 385 386 return (k1->asid == k2->asid) && (k1->iova == k2->iova); 387 } 388 389 /* Unmap the whole notifier's range */ 390 static void smmu_unmap_notifier_range(IOMMUNotifier *n) 391 { 392 IOMMUTLBEntry entry; 393 394 entry.target_as = &address_space_memory; 395 entry.iova = n->start; 396 entry.perm = IOMMU_NONE; 397 entry.addr_mask = n->end - n->start; 398 399 memory_region_notify_one(n, &entry); 400 } 401 402 /* Unmap all notifiers attached to @mr */ 403 inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) 404 { 405 IOMMUNotifier *n; 406 407 trace_smmu_inv_notifiers_mr(mr->parent_obj.name); 408 IOMMU_NOTIFIER_FOREACH(n, mr) { 409 smmu_unmap_notifier_range(n); 410 } 411 } 412 413 /* Unmap all notifiers of all mr's */ 414 void smmu_inv_notifiers_all(SMMUState *s) 415 { 416 SMMUDevice *sdev; 417 418 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { 419 smmu_inv_notifiers_mr(&sdev->iommu); 420 } 421 } 422 423 static void smmu_base_realize(DeviceState *dev, Error **errp) 424 { 425 SMMUState *s = ARM_SMMU(dev); 426 SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); 427 Error *local_err = NULL; 428 429 sbc->parent_realize(dev, &local_err); 430 if (local_err) { 431 error_propagate(errp, local_err); 432 return; 433 } 434 s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); 435 s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, 436 g_free, g_free); 437 s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); 438 439 if (s->primary_bus) { 440 pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); 441 } else { 442 error_setg(errp, "SMMU is not attached to any PCI bus!"); 443 } 444 } 445 446 static void smmu_base_reset(DeviceState *dev) 447 { 448 SMMUState *s = ARM_SMMU(dev); 449 450 g_hash_table_remove_all(s->configs); 451 g_hash_table_remove_all(s->iotlb); 452 } 453 454 static Property smmu_dev_properties[] = { 455 DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), 456 DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), 457 DEFINE_PROP_END_OF_LIST(), 458 }; 459 460 static void smmu_base_class_init(ObjectClass *klass, void *data) 461 { 462 DeviceClass *dc = DEVICE_CLASS(klass); 463 SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); 464 465 device_class_set_props(dc, smmu_dev_properties); 466 device_class_set_parent_realize(dc, smmu_base_realize, 467 &sbc->parent_realize); 468 dc->reset = smmu_base_reset; 469 } 470 471 static const TypeInfo smmu_base_info = { 472 .name = TYPE_ARM_SMMU, 473 .parent = TYPE_SYS_BUS_DEVICE, 474 .instance_size = sizeof(SMMUState), 475 .class_data = NULL, 476 .class_size = sizeof(SMMUBaseClass), 477 .class_init = smmu_base_class_init, 478 .abstract = true, 479 }; 480 481 static void smmu_base_register_types(void) 482 { 483 type_register_static(&smmu_base_info); 484 } 485 486 type_init(smmu_base_register_types) 487 488