1 /* 2 * Copyright (C) 2014-2016 Broadcom Corporation 3 * Copyright (c) 2017 Red Hat, Inc. 4 * Written by Prem Mallappa, Eric Auger 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * Author: Prem Mallappa <pmallapp@broadcom.com> 16 * 17 */ 18 19 #include "qemu/osdep.h" 20 #include "sysemu/sysemu.h" 21 #include "exec/address-spaces.h" 22 #include "trace.h" 23 #include "exec/target_page.h" 24 #include "qom/cpu.h" 25 #include "hw/qdev-properties.h" 26 #include "qapi/error.h" 27 #include "qemu/jhash.h" 28 #include "qemu/module.h" 29 30 #include "qemu/error-report.h" 31 #include "hw/arm/smmu-common.h" 32 #include "smmu-internal.h" 33 34 /* IOTLB Management */ 35 36 inline void smmu_iotlb_inv_all(SMMUState *s) 37 { 38 trace_smmu_iotlb_inv_all(); 39 g_hash_table_remove_all(s->iotlb); 40 } 41 42 static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, 43 gpointer user_data) 44 { 45 uint16_t asid = *(uint16_t *)user_data; 46 SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; 47 48 return iotlb_key->asid == asid; 49 } 50 51 inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) 52 { 53 SMMUIOTLBKey key = {.asid = asid, .iova = iova}; 54 55 trace_smmu_iotlb_inv_iova(asid, iova); 56 g_hash_table_remove(s->iotlb, &key); 57 } 58 59 inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) 60 { 61 trace_smmu_iotlb_inv_asid(asid); 62 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); 63 } 64 65 /* VMSAv8-64 Translation */ 66 67 /** 68 * get_pte - Get the content of a page table entry located at 69 * @base_addr[@index] 70 */ 71 static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, 72 SMMUPTWEventInfo *info) 73 { 74 int ret; 75 dma_addr_t addr = baseaddr + index * sizeof(*pte); 76 77 /* TODO: guarantee 64-bit single-copy atomicity */ 78 ret = dma_memory_read(&address_space_memory, addr, 79 (uint8_t *)pte, sizeof(*pte)); 80 81 if (ret != MEMTX_OK) { 82 info->type = SMMU_PTW_ERR_WALK_EABT; 83 info->addr = addr; 84 return -EINVAL; 85 } 86 trace_smmu_get_pte(baseaddr, index, addr, *pte); 87 return 0; 88 } 89 90 /* VMSAv8-64 Translation Table Format Descriptor Decoding */ 91 92 /** 93 * get_page_pte_address - returns the L3 descriptor output address, 94 * ie. the page frame 95 * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format 96 */ 97 static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) 98 { 99 return PTE_ADDRESS(pte, granule_sz); 100 } 101 102 /** 103 * get_table_pte_address - return table descriptor output address, 104 * ie. address of next level table 105 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 106 */ 107 static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) 108 { 109 return PTE_ADDRESS(pte, granule_sz); 110 } 111 112 /** 113 * get_block_pte_address - return block descriptor output address and block size 114 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 115 */ 116 static inline hwaddr get_block_pte_address(uint64_t pte, int level, 117 int granule_sz, uint64_t *bsz) 118 { 119 int n = level_shift(level, granule_sz); 120 121 *bsz = 1ULL << n; 122 return PTE_ADDRESS(pte, n); 123 } 124 125 SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) 126 { 127 bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); 128 uint8_t tbi_byte = tbi * 8; 129 130 if (cfg->tt[0].tsz && 131 !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { 132 /* there is a ttbr0 region and we are in it (high bits all zero) */ 133 return &cfg->tt[0]; 134 } else if (cfg->tt[1].tsz && 135 !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { 136 /* there is a ttbr1 region and we are in it (high bits all one) */ 137 return &cfg->tt[1]; 138 } else if (!cfg->tt[0].tsz) { 139 /* ttbr0 region is "everything not in the ttbr1 region" */ 140 return &cfg->tt[0]; 141 } else if (!cfg->tt[1].tsz) { 142 /* ttbr1 region is "everything not in the ttbr0 region" */ 143 return &cfg->tt[1]; 144 } 145 /* in the gap between the two regions, this is a Translation fault */ 146 return NULL; 147 } 148 149 /** 150 * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA 151 * @cfg: translation config 152 * @iova: iova to translate 153 * @perm: access type 154 * @tlbe: IOMMUTLBEntry (out) 155 * @info: handle to an error info 156 * 157 * Return 0 on success, < 0 on error. In case of error, @info is filled 158 * and tlbe->perm is set to IOMMU_NONE. 159 * Upon success, @tlbe is filled with translated_addr and entry 160 * permission rights. 161 */ 162 static int smmu_ptw_64(SMMUTransCfg *cfg, 163 dma_addr_t iova, IOMMUAccessFlags perm, 164 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 165 { 166 dma_addr_t baseaddr, indexmask; 167 int stage = cfg->stage; 168 SMMUTransTableInfo *tt = select_tt(cfg, iova); 169 uint8_t level, granule_sz, inputsize, stride; 170 171 if (!tt || tt->disabled) { 172 info->type = SMMU_PTW_ERR_TRANSLATION; 173 goto error; 174 } 175 176 granule_sz = tt->granule_sz; 177 stride = granule_sz - 3; 178 inputsize = 64 - tt->tsz; 179 level = 4 - (inputsize - 4) / stride; 180 indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; 181 baseaddr = extract64(tt->ttb, 0, 48); 182 baseaddr &= ~indexmask; 183 184 tlbe->iova = iova; 185 tlbe->addr_mask = (1 << granule_sz) - 1; 186 187 while (level <= 3) { 188 uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); 189 uint64_t mask = subpage_size - 1; 190 uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); 191 uint64_t pte; 192 dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); 193 uint8_t ap; 194 195 if (get_pte(baseaddr, offset, &pte, info)) { 196 goto error; 197 } 198 trace_smmu_ptw_level(level, iova, subpage_size, 199 baseaddr, offset, pte); 200 201 if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { 202 trace_smmu_ptw_invalid_pte(stage, level, baseaddr, 203 pte_addr, offset, pte); 204 info->type = SMMU_PTW_ERR_TRANSLATION; 205 goto error; 206 } 207 208 if (is_page_pte(pte, level)) { 209 uint64_t gpa = get_page_pte_address(pte, granule_sz); 210 211 ap = PTE_AP(pte); 212 if (is_permission_fault(ap, perm)) { 213 info->type = SMMU_PTW_ERR_PERMISSION; 214 goto error; 215 } 216 217 tlbe->translated_addr = gpa + (iova & mask); 218 tlbe->perm = PTE_AP_TO_PERM(ap); 219 trace_smmu_ptw_page_pte(stage, level, iova, 220 baseaddr, pte_addr, pte, gpa); 221 return 0; 222 } 223 if (is_block_pte(pte, level)) { 224 uint64_t block_size; 225 hwaddr gpa = get_block_pte_address(pte, level, granule_sz, 226 &block_size); 227 228 ap = PTE_AP(pte); 229 if (is_permission_fault(ap, perm)) { 230 info->type = SMMU_PTW_ERR_PERMISSION; 231 goto error; 232 } 233 234 trace_smmu_ptw_block_pte(stage, level, baseaddr, 235 pte_addr, pte, iova, gpa, 236 block_size >> 20); 237 238 tlbe->translated_addr = gpa + (iova & mask); 239 tlbe->perm = PTE_AP_TO_PERM(ap); 240 return 0; 241 } 242 243 /* table pte */ 244 ap = PTE_APTABLE(pte); 245 246 if (is_permission_fault(ap, perm)) { 247 info->type = SMMU_PTW_ERR_PERMISSION; 248 goto error; 249 } 250 baseaddr = get_table_pte_address(pte, granule_sz); 251 level++; 252 } 253 254 info->type = SMMU_PTW_ERR_TRANSLATION; 255 256 error: 257 tlbe->perm = IOMMU_NONE; 258 return -EINVAL; 259 } 260 261 /** 262 * smmu_ptw - Walk the page tables for an IOVA, according to @cfg 263 * 264 * @cfg: translation configuration 265 * @iova: iova to translate 266 * @perm: tentative access type 267 * @tlbe: returned entry 268 * @info: ptw event handle 269 * 270 * return 0 on success 271 */ 272 inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, 273 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 274 { 275 if (!cfg->aa64) { 276 /* 277 * This code path is not entered as we check this while decoding 278 * the configuration data in the derived SMMU model. 279 */ 280 g_assert_not_reached(); 281 } 282 283 return smmu_ptw_64(cfg, iova, perm, tlbe, info); 284 } 285 286 /** 287 * The bus number is used for lookup when SID based invalidation occurs. 288 * In that case we lazily populate the SMMUPciBus array from the bus hash 289 * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus 290 * numbers may not be always initialized yet. 291 */ 292 SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) 293 { 294 SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; 295 296 if (!smmu_pci_bus) { 297 GHashTableIter iter; 298 299 g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); 300 while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { 301 if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { 302 s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; 303 return smmu_pci_bus; 304 } 305 } 306 } 307 return smmu_pci_bus; 308 } 309 310 static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) 311 { 312 SMMUState *s = opaque; 313 SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); 314 SMMUDevice *sdev; 315 static unsigned int index; 316 317 if (!sbus) { 318 sbus = g_malloc0(sizeof(SMMUPciBus) + 319 sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); 320 sbus->bus = bus; 321 g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); 322 } 323 324 sdev = sbus->pbdev[devfn]; 325 if (!sdev) { 326 char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); 327 328 sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); 329 330 sdev->smmu = s; 331 sdev->bus = bus; 332 sdev->devfn = devfn; 333 334 memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), 335 s->mrtypename, 336 OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); 337 address_space_init(&sdev->as, 338 MEMORY_REGION(&sdev->iommu), name); 339 trace_smmu_add_mr(name); 340 g_free(name); 341 } 342 343 return &sdev->as; 344 } 345 346 IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) 347 { 348 uint8_t bus_n, devfn; 349 SMMUPciBus *smmu_bus; 350 SMMUDevice *smmu; 351 352 bus_n = PCI_BUS_NUM(sid); 353 smmu_bus = smmu_find_smmu_pcibus(s, bus_n); 354 if (smmu_bus) { 355 devfn = SMMU_PCI_DEVFN(sid); 356 smmu = smmu_bus->pbdev[devfn]; 357 if (smmu) { 358 return &smmu->iommu; 359 } 360 } 361 return NULL; 362 } 363 364 static guint smmu_iotlb_key_hash(gconstpointer v) 365 { 366 SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; 367 uint32_t a, b, c; 368 369 /* Jenkins hash */ 370 a = b = c = JHASH_INITVAL + sizeof(*key); 371 a += key->asid; 372 b += extract64(key->iova, 0, 32); 373 c += extract64(key->iova, 32, 32); 374 375 __jhash_mix(a, b, c); 376 __jhash_final(a, b, c); 377 378 return c; 379 } 380 381 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) 382 { 383 const SMMUIOTLBKey *k1 = v1; 384 const SMMUIOTLBKey *k2 = v2; 385 386 return (k1->asid == k2->asid) && (k1->iova == k2->iova); 387 } 388 389 /* Unmap the whole notifier's range */ 390 static void smmu_unmap_notifier_range(IOMMUNotifier *n) 391 { 392 IOMMUTLBEntry entry; 393 394 entry.target_as = &address_space_memory; 395 entry.iova = n->start; 396 entry.perm = IOMMU_NONE; 397 entry.addr_mask = n->end - n->start; 398 399 memory_region_notify_one(n, &entry); 400 } 401 402 /* Unmap all notifiers attached to @mr */ 403 inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) 404 { 405 IOMMUNotifier *n; 406 407 trace_smmu_inv_notifiers_mr(mr->parent_obj.name); 408 IOMMU_NOTIFIER_FOREACH(n, mr) { 409 smmu_unmap_notifier_range(n); 410 } 411 } 412 413 /* Unmap all notifiers of all mr's */ 414 void smmu_inv_notifiers_all(SMMUState *s) 415 { 416 SMMUDevice *sdev; 417 418 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { 419 smmu_inv_notifiers_mr(&sdev->iommu); 420 } 421 } 422 423 static void smmu_base_realize(DeviceState *dev, Error **errp) 424 { 425 SMMUState *s = ARM_SMMU(dev); 426 SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); 427 Error *local_err = NULL; 428 429 sbc->parent_realize(dev, &local_err); 430 if (local_err) { 431 error_propagate(errp, local_err); 432 return; 433 } 434 s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); 435 s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, 436 g_free, g_free); 437 s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); 438 439 if (s->primary_bus) { 440 pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); 441 } else { 442 error_setg(errp, "SMMU is not attached to any PCI bus!"); 443 } 444 } 445 446 static void smmu_base_reset(DeviceState *dev) 447 { 448 SMMUState *s = ARM_SMMU(dev); 449 450 g_hash_table_remove_all(s->configs); 451 g_hash_table_remove_all(s->iotlb); 452 } 453 454 static Property smmu_dev_properties[] = { 455 DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), 456 DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), 457 DEFINE_PROP_END_OF_LIST(), 458 }; 459 460 static void smmu_base_class_init(ObjectClass *klass, void *data) 461 { 462 DeviceClass *dc = DEVICE_CLASS(klass); 463 SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); 464 465 dc->props = smmu_dev_properties; 466 device_class_set_parent_realize(dc, smmu_base_realize, 467 &sbc->parent_realize); 468 dc->reset = smmu_base_reset; 469 } 470 471 static const TypeInfo smmu_base_info = { 472 .name = TYPE_ARM_SMMU, 473 .parent = TYPE_SYS_BUS_DEVICE, 474 .instance_size = sizeof(SMMUState), 475 .class_data = NULL, 476 .class_size = sizeof(SMMUBaseClass), 477 .class_init = smmu_base_class_init, 478 .abstract = true, 479 }; 480 481 static void smmu_base_register_types(void) 482 { 483 type_register_static(&smmu_base_info); 484 } 485 486 type_init(smmu_base_register_types) 487 488