1 /* 2 * Copyright (C) 2014-2016 Broadcom Corporation 3 * Copyright (c) 2017 Red Hat, Inc. 4 * Written by Prem Mallappa, Eric Auger 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * Author: Prem Mallappa <pmallapp@broadcom.com> 16 * 17 */ 18 19 #include "qemu/osdep.h" 20 #include "exec/address-spaces.h" 21 #include "trace.h" 22 #include "exec/target_page.h" 23 #include "hw/core/cpu.h" 24 #include "hw/qdev-properties.h" 25 #include "qapi/error.h" 26 #include "qemu/jhash.h" 27 #include "qemu/module.h" 28 29 #include "qemu/error-report.h" 30 #include "hw/arm/smmu-common.h" 31 #include "smmu-internal.h" 32 33 /* IOTLB Management */ 34 35 inline void smmu_iotlb_inv_all(SMMUState *s) 36 { 37 trace_smmu_iotlb_inv_all(); 38 g_hash_table_remove_all(s->iotlb); 39 } 40 41 static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, 42 gpointer user_data) 43 { 44 uint16_t asid = *(uint16_t *)user_data; 45 SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; 46 47 return iotlb_key->asid == asid; 48 } 49 50 inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) 51 { 52 SMMUIOTLBKey key = {.asid = asid, .iova = iova}; 53 54 trace_smmu_iotlb_inv_iova(asid, iova); 55 g_hash_table_remove(s->iotlb, &key); 56 } 57 58 inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) 59 { 60 trace_smmu_iotlb_inv_asid(asid); 61 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); 62 } 63 64 /* VMSAv8-64 Translation */ 65 66 /** 67 * get_pte - Get the content of a page table entry located at 68 * @base_addr[@index] 69 */ 70 static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, 71 SMMUPTWEventInfo *info) 72 { 73 int ret; 74 dma_addr_t addr = baseaddr + index * sizeof(*pte); 75 76 /* TODO: guarantee 64-bit single-copy atomicity */ 77 ret = dma_memory_read(&address_space_memory, addr, pte, sizeof(*pte)); 78 79 if (ret != MEMTX_OK) { 80 info->type = SMMU_PTW_ERR_WALK_EABT; 81 info->addr = addr; 82 return -EINVAL; 83 } 84 trace_smmu_get_pte(baseaddr, index, addr, *pte); 85 return 0; 86 } 87 88 /* VMSAv8-64 Translation Table Format Descriptor Decoding */ 89 90 /** 91 * get_page_pte_address - returns the L3 descriptor output address, 92 * ie. the page frame 93 * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format 94 */ 95 static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) 96 { 97 return PTE_ADDRESS(pte, granule_sz); 98 } 99 100 /** 101 * get_table_pte_address - return table descriptor output address, 102 * ie. address of next level table 103 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 104 */ 105 static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) 106 { 107 return PTE_ADDRESS(pte, granule_sz); 108 } 109 110 /** 111 * get_block_pte_address - return block descriptor output address and block size 112 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 113 */ 114 static inline hwaddr get_block_pte_address(uint64_t pte, int level, 115 int granule_sz, uint64_t *bsz) 116 { 117 int n = level_shift(level, granule_sz); 118 119 *bsz = 1ULL << n; 120 return PTE_ADDRESS(pte, n); 121 } 122 123 SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) 124 { 125 bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); 126 uint8_t tbi_byte = tbi * 8; 127 128 if (cfg->tt[0].tsz && 129 !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { 130 /* there is a ttbr0 region and we are in it (high bits all zero) */ 131 return &cfg->tt[0]; 132 } else if (cfg->tt[1].tsz && 133 !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { 134 /* there is a ttbr1 region and we are in it (high bits all one) */ 135 return &cfg->tt[1]; 136 } else if (!cfg->tt[0].tsz) { 137 /* ttbr0 region is "everything not in the ttbr1 region" */ 138 return &cfg->tt[0]; 139 } else if (!cfg->tt[1].tsz) { 140 /* ttbr1 region is "everything not in the ttbr0 region" */ 141 return &cfg->tt[1]; 142 } 143 /* in the gap between the two regions, this is a Translation fault */ 144 return NULL; 145 } 146 147 /** 148 * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA 149 * @cfg: translation config 150 * @iova: iova to translate 151 * @perm: access type 152 * @tlbe: IOMMUTLBEntry (out) 153 * @info: handle to an error info 154 * 155 * Return 0 on success, < 0 on error. In case of error, @info is filled 156 * and tlbe->perm is set to IOMMU_NONE. 157 * Upon success, @tlbe is filled with translated_addr and entry 158 * permission rights. 159 */ 160 static int smmu_ptw_64(SMMUTransCfg *cfg, 161 dma_addr_t iova, IOMMUAccessFlags perm, 162 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 163 { 164 dma_addr_t baseaddr, indexmask; 165 int stage = cfg->stage; 166 SMMUTransTableInfo *tt = select_tt(cfg, iova); 167 uint8_t level, granule_sz, inputsize, stride; 168 169 if (!tt || tt->disabled) { 170 info->type = SMMU_PTW_ERR_TRANSLATION; 171 goto error; 172 } 173 174 granule_sz = tt->granule_sz; 175 stride = granule_sz - 3; 176 inputsize = 64 - tt->tsz; 177 level = 4 - (inputsize - 4) / stride; 178 indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; 179 baseaddr = extract64(tt->ttb, 0, 48); 180 baseaddr &= ~indexmask; 181 182 tlbe->iova = iova; 183 tlbe->addr_mask = (1 << granule_sz) - 1; 184 185 while (level <= 3) { 186 uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); 187 uint64_t mask = subpage_size - 1; 188 uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); 189 uint64_t pte; 190 dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); 191 uint8_t ap; 192 193 if (get_pte(baseaddr, offset, &pte, info)) { 194 goto error; 195 } 196 trace_smmu_ptw_level(level, iova, subpage_size, 197 baseaddr, offset, pte); 198 199 if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { 200 trace_smmu_ptw_invalid_pte(stage, level, baseaddr, 201 pte_addr, offset, pte); 202 info->type = SMMU_PTW_ERR_TRANSLATION; 203 goto error; 204 } 205 206 if (is_page_pte(pte, level)) { 207 uint64_t gpa = get_page_pte_address(pte, granule_sz); 208 209 ap = PTE_AP(pte); 210 if (is_permission_fault(ap, perm)) { 211 info->type = SMMU_PTW_ERR_PERMISSION; 212 goto error; 213 } 214 215 tlbe->translated_addr = gpa + (iova & mask); 216 tlbe->perm = PTE_AP_TO_PERM(ap); 217 trace_smmu_ptw_page_pte(stage, level, iova, 218 baseaddr, pte_addr, pte, gpa); 219 return 0; 220 } 221 if (is_block_pte(pte, level)) { 222 uint64_t block_size; 223 hwaddr gpa = get_block_pte_address(pte, level, granule_sz, 224 &block_size); 225 226 ap = PTE_AP(pte); 227 if (is_permission_fault(ap, perm)) { 228 info->type = SMMU_PTW_ERR_PERMISSION; 229 goto error; 230 } 231 232 trace_smmu_ptw_block_pte(stage, level, baseaddr, 233 pte_addr, pte, iova, gpa, 234 block_size >> 20); 235 236 tlbe->translated_addr = gpa + (iova & mask); 237 tlbe->perm = PTE_AP_TO_PERM(ap); 238 return 0; 239 } 240 241 /* table pte */ 242 ap = PTE_APTABLE(pte); 243 244 if (is_permission_fault(ap, perm)) { 245 info->type = SMMU_PTW_ERR_PERMISSION; 246 goto error; 247 } 248 baseaddr = get_table_pte_address(pte, granule_sz); 249 level++; 250 } 251 252 info->type = SMMU_PTW_ERR_TRANSLATION; 253 254 error: 255 tlbe->perm = IOMMU_NONE; 256 return -EINVAL; 257 } 258 259 /** 260 * smmu_ptw - Walk the page tables for an IOVA, according to @cfg 261 * 262 * @cfg: translation configuration 263 * @iova: iova to translate 264 * @perm: tentative access type 265 * @tlbe: returned entry 266 * @info: ptw event handle 267 * 268 * return 0 on success 269 */ 270 inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, 271 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 272 { 273 if (!cfg->aa64) { 274 /* 275 * This code path is not entered as we check this while decoding 276 * the configuration data in the derived SMMU model. 277 */ 278 g_assert_not_reached(); 279 } 280 281 return smmu_ptw_64(cfg, iova, perm, tlbe, info); 282 } 283 284 /** 285 * The bus number is used for lookup when SID based invalidation occurs. 286 * In that case we lazily populate the SMMUPciBus array from the bus hash 287 * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus 288 * numbers may not be always initialized yet. 289 */ 290 SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) 291 { 292 SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; 293 294 if (!smmu_pci_bus) { 295 GHashTableIter iter; 296 297 g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); 298 while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { 299 if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { 300 s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; 301 return smmu_pci_bus; 302 } 303 } 304 } 305 return smmu_pci_bus; 306 } 307 308 static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) 309 { 310 SMMUState *s = opaque; 311 SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); 312 SMMUDevice *sdev; 313 static unsigned int index; 314 315 if (!sbus) { 316 sbus = g_malloc0(sizeof(SMMUPciBus) + 317 sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); 318 sbus->bus = bus; 319 g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); 320 } 321 322 sdev = sbus->pbdev[devfn]; 323 if (!sdev) { 324 char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); 325 326 sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); 327 328 sdev->smmu = s; 329 sdev->bus = bus; 330 sdev->devfn = devfn; 331 332 memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), 333 s->mrtypename, 334 OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); 335 address_space_init(&sdev->as, 336 MEMORY_REGION(&sdev->iommu), name); 337 trace_smmu_add_mr(name); 338 g_free(name); 339 } 340 341 return &sdev->as; 342 } 343 344 IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) 345 { 346 uint8_t bus_n, devfn; 347 SMMUPciBus *smmu_bus; 348 SMMUDevice *smmu; 349 350 bus_n = PCI_BUS_NUM(sid); 351 smmu_bus = smmu_find_smmu_pcibus(s, bus_n); 352 if (smmu_bus) { 353 devfn = SMMU_PCI_DEVFN(sid); 354 smmu = smmu_bus->pbdev[devfn]; 355 if (smmu) { 356 return &smmu->iommu; 357 } 358 } 359 return NULL; 360 } 361 362 static guint smmu_iotlb_key_hash(gconstpointer v) 363 { 364 SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; 365 uint32_t a, b, c; 366 367 /* Jenkins hash */ 368 a = b = c = JHASH_INITVAL + sizeof(*key); 369 a += key->asid; 370 b += extract64(key->iova, 0, 32); 371 c += extract64(key->iova, 32, 32); 372 373 __jhash_mix(a, b, c); 374 __jhash_final(a, b, c); 375 376 return c; 377 } 378 379 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) 380 { 381 const SMMUIOTLBKey *k1 = v1; 382 const SMMUIOTLBKey *k2 = v2; 383 384 return (k1->asid == k2->asid) && (k1->iova == k2->iova); 385 } 386 387 /* Unmap the whole notifier's range */ 388 static void smmu_unmap_notifier_range(IOMMUNotifier *n) 389 { 390 IOMMUTLBEntry entry; 391 392 entry.target_as = &address_space_memory; 393 entry.iova = n->start; 394 entry.perm = IOMMU_NONE; 395 entry.addr_mask = n->end - n->start; 396 397 memory_region_notify_one(n, &entry); 398 } 399 400 /* Unmap all notifiers attached to @mr */ 401 inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) 402 { 403 IOMMUNotifier *n; 404 405 trace_smmu_inv_notifiers_mr(mr->parent_obj.name); 406 IOMMU_NOTIFIER_FOREACH(n, mr) { 407 smmu_unmap_notifier_range(n); 408 } 409 } 410 411 /* Unmap all notifiers of all mr's */ 412 void smmu_inv_notifiers_all(SMMUState *s) 413 { 414 SMMUDevice *sdev; 415 416 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { 417 smmu_inv_notifiers_mr(&sdev->iommu); 418 } 419 } 420 421 static void smmu_base_realize(DeviceState *dev, Error **errp) 422 { 423 SMMUState *s = ARM_SMMU(dev); 424 SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); 425 Error *local_err = NULL; 426 427 sbc->parent_realize(dev, &local_err); 428 if (local_err) { 429 error_propagate(errp, local_err); 430 return; 431 } 432 s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); 433 s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, 434 g_free, g_free); 435 s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); 436 437 if (s->primary_bus) { 438 pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); 439 } else { 440 error_setg(errp, "SMMU is not attached to any PCI bus!"); 441 } 442 } 443 444 static void smmu_base_reset(DeviceState *dev) 445 { 446 SMMUState *s = ARM_SMMU(dev); 447 448 g_hash_table_remove_all(s->configs); 449 g_hash_table_remove_all(s->iotlb); 450 } 451 452 static Property smmu_dev_properties[] = { 453 DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), 454 DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), 455 DEFINE_PROP_END_OF_LIST(), 456 }; 457 458 static void smmu_base_class_init(ObjectClass *klass, void *data) 459 { 460 DeviceClass *dc = DEVICE_CLASS(klass); 461 SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); 462 463 device_class_set_props(dc, smmu_dev_properties); 464 device_class_set_parent_realize(dc, smmu_base_realize, 465 &sbc->parent_realize); 466 dc->reset = smmu_base_reset; 467 } 468 469 static const TypeInfo smmu_base_info = { 470 .name = TYPE_ARM_SMMU, 471 .parent = TYPE_SYS_BUS_DEVICE, 472 .instance_size = sizeof(SMMUState), 473 .class_data = NULL, 474 .class_size = sizeof(SMMUBaseClass), 475 .class_init = smmu_base_class_init, 476 .abstract = true, 477 }; 478 479 static void smmu_base_register_types(void) 480 { 481 type_register_static(&smmu_base_info); 482 } 483 484 type_init(smmu_base_register_types) 485 486