1 /* 2 * Copyright (C) 2014-2016 Broadcom Corporation 3 * Copyright (c) 2017 Red Hat, Inc. 4 * Written by Prem Mallappa, Eric Auger 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * Author: Prem Mallappa <pmallapp@broadcom.com> 16 * 17 */ 18 19 #include "qemu/osdep.h" 20 #include "exec/address-spaces.h" 21 #include "trace.h" 22 #include "exec/target_page.h" 23 #include "hw/core/cpu.h" 24 #include "hw/qdev-properties.h" 25 #include "qapi/error.h" 26 #include "qemu/jhash.h" 27 #include "qemu/module.h" 28 29 #include "qemu/error-report.h" 30 #include "hw/arm/smmu-common.h" 31 #include "smmu-internal.h" 32 33 /* IOTLB Management */ 34 35 inline void smmu_iotlb_inv_all(SMMUState *s) 36 { 37 trace_smmu_iotlb_inv_all(); 38 g_hash_table_remove_all(s->iotlb); 39 } 40 41 static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, 42 gpointer user_data) 43 { 44 uint16_t asid = *(uint16_t *)user_data; 45 SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; 46 47 return iotlb_key->asid == asid; 48 } 49 50 inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) 51 { 52 SMMUIOTLBKey key = {.asid = asid, .iova = iova}; 53 54 trace_smmu_iotlb_inv_iova(asid, iova); 55 g_hash_table_remove(s->iotlb, &key); 56 } 57 58 inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) 59 { 60 trace_smmu_iotlb_inv_asid(asid); 61 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); 62 } 63 64 /* VMSAv8-64 Translation */ 65 66 /** 67 * get_pte - Get the content of a page table entry located at 68 * @base_addr[@index] 69 */ 70 static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, 71 SMMUPTWEventInfo *info) 72 { 73 int ret; 74 dma_addr_t addr = baseaddr + index * sizeof(*pte); 75 76 /* TODO: guarantee 64-bit single-copy atomicity */ 77 ret = dma_memory_read(&address_space_memory, addr, 78 (uint8_t *)pte, sizeof(*pte)); 79 80 if (ret != MEMTX_OK) { 81 info->type = SMMU_PTW_ERR_WALK_EABT; 82 info->addr = addr; 83 return -EINVAL; 84 } 85 trace_smmu_get_pte(baseaddr, index, addr, *pte); 86 return 0; 87 } 88 89 /* VMSAv8-64 Translation Table Format Descriptor Decoding */ 90 91 /** 92 * get_page_pte_address - returns the L3 descriptor output address, 93 * ie. the page frame 94 * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format 95 */ 96 static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) 97 { 98 return PTE_ADDRESS(pte, granule_sz); 99 } 100 101 /** 102 * get_table_pte_address - return table descriptor output address, 103 * ie. address of next level table 104 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 105 */ 106 static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) 107 { 108 return PTE_ADDRESS(pte, granule_sz); 109 } 110 111 /** 112 * get_block_pte_address - return block descriptor output address and block size 113 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats 114 */ 115 static inline hwaddr get_block_pte_address(uint64_t pte, int level, 116 int granule_sz, uint64_t *bsz) 117 { 118 int n = level_shift(level, granule_sz); 119 120 *bsz = 1ULL << n; 121 return PTE_ADDRESS(pte, n); 122 } 123 124 SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) 125 { 126 bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); 127 uint8_t tbi_byte = tbi * 8; 128 129 if (cfg->tt[0].tsz && 130 !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { 131 /* there is a ttbr0 region and we are in it (high bits all zero) */ 132 return &cfg->tt[0]; 133 } else if (cfg->tt[1].tsz && 134 !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { 135 /* there is a ttbr1 region and we are in it (high bits all one) */ 136 return &cfg->tt[1]; 137 } else if (!cfg->tt[0].tsz) { 138 /* ttbr0 region is "everything not in the ttbr1 region" */ 139 return &cfg->tt[0]; 140 } else if (!cfg->tt[1].tsz) { 141 /* ttbr1 region is "everything not in the ttbr0 region" */ 142 return &cfg->tt[1]; 143 } 144 /* in the gap between the two regions, this is a Translation fault */ 145 return NULL; 146 } 147 148 /** 149 * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA 150 * @cfg: translation config 151 * @iova: iova to translate 152 * @perm: access type 153 * @tlbe: IOMMUTLBEntry (out) 154 * @info: handle to an error info 155 * 156 * Return 0 on success, < 0 on error. In case of error, @info is filled 157 * and tlbe->perm is set to IOMMU_NONE. 158 * Upon success, @tlbe is filled with translated_addr and entry 159 * permission rights. 160 */ 161 static int smmu_ptw_64(SMMUTransCfg *cfg, 162 dma_addr_t iova, IOMMUAccessFlags perm, 163 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 164 { 165 dma_addr_t baseaddr, indexmask; 166 int stage = cfg->stage; 167 SMMUTransTableInfo *tt = select_tt(cfg, iova); 168 uint8_t level, granule_sz, inputsize, stride; 169 170 if (!tt || tt->disabled) { 171 info->type = SMMU_PTW_ERR_TRANSLATION; 172 goto error; 173 } 174 175 granule_sz = tt->granule_sz; 176 stride = granule_sz - 3; 177 inputsize = 64 - tt->tsz; 178 level = 4 - (inputsize - 4) / stride; 179 indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; 180 baseaddr = extract64(tt->ttb, 0, 48); 181 baseaddr &= ~indexmask; 182 183 tlbe->iova = iova; 184 tlbe->addr_mask = (1 << granule_sz) - 1; 185 186 while (level <= 3) { 187 uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); 188 uint64_t mask = subpage_size - 1; 189 uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); 190 uint64_t pte; 191 dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); 192 uint8_t ap; 193 194 if (get_pte(baseaddr, offset, &pte, info)) { 195 goto error; 196 } 197 trace_smmu_ptw_level(level, iova, subpage_size, 198 baseaddr, offset, pte); 199 200 if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { 201 trace_smmu_ptw_invalid_pte(stage, level, baseaddr, 202 pte_addr, offset, pte); 203 info->type = SMMU_PTW_ERR_TRANSLATION; 204 goto error; 205 } 206 207 if (is_page_pte(pte, level)) { 208 uint64_t gpa = get_page_pte_address(pte, granule_sz); 209 210 ap = PTE_AP(pte); 211 if (is_permission_fault(ap, perm)) { 212 info->type = SMMU_PTW_ERR_PERMISSION; 213 goto error; 214 } 215 216 tlbe->translated_addr = gpa + (iova & mask); 217 tlbe->perm = PTE_AP_TO_PERM(ap); 218 trace_smmu_ptw_page_pte(stage, level, iova, 219 baseaddr, pte_addr, pte, gpa); 220 return 0; 221 } 222 if (is_block_pte(pte, level)) { 223 uint64_t block_size; 224 hwaddr gpa = get_block_pte_address(pte, level, granule_sz, 225 &block_size); 226 227 ap = PTE_AP(pte); 228 if (is_permission_fault(ap, perm)) { 229 info->type = SMMU_PTW_ERR_PERMISSION; 230 goto error; 231 } 232 233 trace_smmu_ptw_block_pte(stage, level, baseaddr, 234 pte_addr, pte, iova, gpa, 235 block_size >> 20); 236 237 tlbe->translated_addr = gpa + (iova & mask); 238 tlbe->perm = PTE_AP_TO_PERM(ap); 239 return 0; 240 } 241 242 /* table pte */ 243 ap = PTE_APTABLE(pte); 244 245 if (is_permission_fault(ap, perm)) { 246 info->type = SMMU_PTW_ERR_PERMISSION; 247 goto error; 248 } 249 baseaddr = get_table_pte_address(pte, granule_sz); 250 level++; 251 } 252 253 info->type = SMMU_PTW_ERR_TRANSLATION; 254 255 error: 256 tlbe->perm = IOMMU_NONE; 257 return -EINVAL; 258 } 259 260 /** 261 * smmu_ptw - Walk the page tables for an IOVA, according to @cfg 262 * 263 * @cfg: translation configuration 264 * @iova: iova to translate 265 * @perm: tentative access type 266 * @tlbe: returned entry 267 * @info: ptw event handle 268 * 269 * return 0 on success 270 */ 271 inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, 272 IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) 273 { 274 if (!cfg->aa64) { 275 /* 276 * This code path is not entered as we check this while decoding 277 * the configuration data in the derived SMMU model. 278 */ 279 g_assert_not_reached(); 280 } 281 282 return smmu_ptw_64(cfg, iova, perm, tlbe, info); 283 } 284 285 /** 286 * The bus number is used for lookup when SID based invalidation occurs. 287 * In that case we lazily populate the SMMUPciBus array from the bus hash 288 * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus 289 * numbers may not be always initialized yet. 290 */ 291 SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) 292 { 293 SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; 294 295 if (!smmu_pci_bus) { 296 GHashTableIter iter; 297 298 g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); 299 while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { 300 if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { 301 s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; 302 return smmu_pci_bus; 303 } 304 } 305 } 306 return smmu_pci_bus; 307 } 308 309 static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) 310 { 311 SMMUState *s = opaque; 312 SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); 313 SMMUDevice *sdev; 314 static unsigned int index; 315 316 if (!sbus) { 317 sbus = g_malloc0(sizeof(SMMUPciBus) + 318 sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); 319 sbus->bus = bus; 320 g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); 321 } 322 323 sdev = sbus->pbdev[devfn]; 324 if (!sdev) { 325 char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); 326 327 sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); 328 329 sdev->smmu = s; 330 sdev->bus = bus; 331 sdev->devfn = devfn; 332 333 memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), 334 s->mrtypename, 335 OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); 336 address_space_init(&sdev->as, 337 MEMORY_REGION(&sdev->iommu), name); 338 trace_smmu_add_mr(name); 339 g_free(name); 340 } 341 342 return &sdev->as; 343 } 344 345 IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) 346 { 347 uint8_t bus_n, devfn; 348 SMMUPciBus *smmu_bus; 349 SMMUDevice *smmu; 350 351 bus_n = PCI_BUS_NUM(sid); 352 smmu_bus = smmu_find_smmu_pcibus(s, bus_n); 353 if (smmu_bus) { 354 devfn = SMMU_PCI_DEVFN(sid); 355 smmu = smmu_bus->pbdev[devfn]; 356 if (smmu) { 357 return &smmu->iommu; 358 } 359 } 360 return NULL; 361 } 362 363 static guint smmu_iotlb_key_hash(gconstpointer v) 364 { 365 SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; 366 uint32_t a, b, c; 367 368 /* Jenkins hash */ 369 a = b = c = JHASH_INITVAL + sizeof(*key); 370 a += key->asid; 371 b += extract64(key->iova, 0, 32); 372 c += extract64(key->iova, 32, 32); 373 374 __jhash_mix(a, b, c); 375 __jhash_final(a, b, c); 376 377 return c; 378 } 379 380 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) 381 { 382 const SMMUIOTLBKey *k1 = v1; 383 const SMMUIOTLBKey *k2 = v2; 384 385 return (k1->asid == k2->asid) && (k1->iova == k2->iova); 386 } 387 388 /* Unmap the whole notifier's range */ 389 static void smmu_unmap_notifier_range(IOMMUNotifier *n) 390 { 391 IOMMUTLBEntry entry; 392 393 entry.target_as = &address_space_memory; 394 entry.iova = n->start; 395 entry.perm = IOMMU_NONE; 396 entry.addr_mask = n->end - n->start; 397 398 memory_region_notify_one(n, &entry); 399 } 400 401 /* Unmap all notifiers attached to @mr */ 402 inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) 403 { 404 IOMMUNotifier *n; 405 406 trace_smmu_inv_notifiers_mr(mr->parent_obj.name); 407 IOMMU_NOTIFIER_FOREACH(n, mr) { 408 smmu_unmap_notifier_range(n); 409 } 410 } 411 412 /* Unmap all notifiers of all mr's */ 413 void smmu_inv_notifiers_all(SMMUState *s) 414 { 415 SMMUDevice *sdev; 416 417 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { 418 smmu_inv_notifiers_mr(&sdev->iommu); 419 } 420 } 421 422 static void smmu_base_realize(DeviceState *dev, Error **errp) 423 { 424 SMMUState *s = ARM_SMMU(dev); 425 SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); 426 Error *local_err = NULL; 427 428 sbc->parent_realize(dev, &local_err); 429 if (local_err) { 430 error_propagate(errp, local_err); 431 return; 432 } 433 s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); 434 s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, 435 g_free, g_free); 436 s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); 437 438 if (s->primary_bus) { 439 pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); 440 } else { 441 error_setg(errp, "SMMU is not attached to any PCI bus!"); 442 } 443 } 444 445 static void smmu_base_reset(DeviceState *dev) 446 { 447 SMMUState *s = ARM_SMMU(dev); 448 449 g_hash_table_remove_all(s->configs); 450 g_hash_table_remove_all(s->iotlb); 451 } 452 453 static Property smmu_dev_properties[] = { 454 DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0), 455 DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *), 456 DEFINE_PROP_END_OF_LIST(), 457 }; 458 459 static void smmu_base_class_init(ObjectClass *klass, void *data) 460 { 461 DeviceClass *dc = DEVICE_CLASS(klass); 462 SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); 463 464 device_class_set_props(dc, smmu_dev_properties); 465 device_class_set_parent_realize(dc, smmu_base_realize, 466 &sbc->parent_realize); 467 dc->reset = smmu_base_reset; 468 } 469 470 static const TypeInfo smmu_base_info = { 471 .name = TYPE_ARM_SMMU, 472 .parent = TYPE_SYS_BUS_DEVICE, 473 .instance_size = sizeof(SMMUState), 474 .class_data = NULL, 475 .class_size = sizeof(SMMUBaseClass), 476 .class_init = smmu_base_class_init, 477 .abstract = true, 478 }; 479 480 static void smmu_base_register_types(void) 481 { 482 type_register_static(&smmu_base_info); 483 } 484 485 type_init(smmu_base_register_types) 486 487