Lines Matching +full:smmu +full:- +full:v1
2 * Copyright (C) 2014-2016 Broadcom Corporation
23 #include "hw/qdev-properties.h"
28 #include "qemu/error-report.h"
29 #include "hw/arm/smmu-common.h"
30 #include "smmu-internal.h"
41 a += key->asid + key->vmid + key->level + key->tg; in smmu_iotlb_key_hash()
42 b += extract64(key->iova, 0, 32); in smmu_iotlb_key_hash()
43 c += extract64(key->iova, 32, 32); in smmu_iotlb_key_hash()
51 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) in smmu_iotlb_key_equal() argument
53 SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2; in smmu_iotlb_key_equal()
55 return (k1->asid == k2->asid) && (k1->iova == k2->iova) && in smmu_iotlb_key_equal()
56 (k1->level == k2->level) && (k1->tg == k2->tg) && in smmu_iotlb_key_equal()
57 (k1->vmid == k2->vmid); in smmu_iotlb_key_equal()
74 uint8_t tg = (tt->granule_sz - 10) / 2; in smmu_iotlb_lookup_all_levels()
75 uint8_t inputsize = 64 - tt->tsz; in smmu_iotlb_lookup_all_levels()
76 uint8_t stride = tt->granule_sz - 3; in smmu_iotlb_lookup_all_levels()
77 uint8_t level = 4 - (inputsize - 4) / stride; in smmu_iotlb_lookup_all_levels()
81 uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz); in smmu_iotlb_lookup_all_levels()
82 uint64_t mask = subpage_size - 1; in smmu_iotlb_lookup_all_levels()
85 key = smmu_get_iotlb_key(cfg->asid, cfg->s2cfg.vmid, in smmu_iotlb_lookup_all_levels()
87 entry = g_hash_table_lookup(bs->iotlb, &key); in smmu_iotlb_lookup_all_levels()
97 * smmu_iotlb_lookup - Look up for a TLB entry.
98 * @bs: SMMU state which includes the TLB instance
118 if (!entry && (cfg->stage == SMMU_NESTED) && in smmu_iotlb_lookup()
119 (cfg->s2cfg.granule_sz != tt->granule_sz)) { in smmu_iotlb_lookup()
120 tt->granule_sz = cfg->s2cfg.granule_sz; in smmu_iotlb_lookup()
125 cfg->iotlb_hits++; in smmu_iotlb_lookup()
126 trace_smmu_iotlb_lookup_hit(cfg->asid, cfg->s2cfg.vmid, iova, in smmu_iotlb_lookup()
127 cfg->iotlb_hits, cfg->iotlb_misses, in smmu_iotlb_lookup()
128 100 * cfg->iotlb_hits / in smmu_iotlb_lookup()
129 (cfg->iotlb_hits + cfg->iotlb_misses)); in smmu_iotlb_lookup()
131 cfg->iotlb_misses++; in smmu_iotlb_lookup()
132 trace_smmu_iotlb_lookup_miss(cfg->asid, cfg->s2cfg.vmid, iova, in smmu_iotlb_lookup()
133 cfg->iotlb_hits, cfg->iotlb_misses, in smmu_iotlb_lookup()
134 100 * cfg->iotlb_hits / in smmu_iotlb_lookup()
135 (cfg->iotlb_hits + cfg->iotlb_misses)); in smmu_iotlb_lookup()
143 uint8_t tg = (new->granule - 10) / 2; in smmu_iotlb_insert()
145 if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) { in smmu_iotlb_insert()
149 *key = smmu_get_iotlb_key(cfg->asid, cfg->s2cfg.vmid, new->entry.iova, in smmu_iotlb_insert()
150 tg, new->level); in smmu_iotlb_insert()
151 trace_smmu_iotlb_insert(cfg->asid, cfg->s2cfg.vmid, new->entry.iova, in smmu_iotlb_insert()
152 tg, new->level); in smmu_iotlb_insert()
153 g_hash_table_insert(bs->iotlb, key, new); in smmu_iotlb_insert()
159 g_hash_table_remove_all(s->iotlb); in smmu_iotlb_inv_all()
168 return (SMMU_IOTLB_ASID(*iotlb_key) == info->asid) && in smmu_hash_remove_by_asid_vmid()
169 (SMMU_IOTLB_VMID(*iotlb_key) == info->vmid); in smmu_hash_remove_by_asid_vmid()
195 IOMMUTLBEntry *entry = &iter->entry; in smmu_hash_remove_by_asid_vmid_iova()
199 if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) { in smmu_hash_remove_by_asid_vmid_iova()
202 if (info->vmid >= 0 && info->vmid != SMMU_IOTLB_VMID(iotlb_key)) { in smmu_hash_remove_by_asid_vmid_iova()
205 return ((info->iova & ~entry->addr_mask) == entry->iova) || in smmu_hash_remove_by_asid_vmid_iova()
206 ((entry->iova & ~info->mask) == info->iova); in smmu_hash_remove_by_asid_vmid_iova()
213 IOMMUTLBEntry *entry = &iter->entry; in smmu_hash_remove_by_vmid_ipa()
218 /* This is a stage-1 address. */ in smmu_hash_remove_by_vmid_ipa()
221 if (info->vmid != SMMU_IOTLB_VMID(iotlb_key)) { in smmu_hash_remove_by_vmid_ipa()
224 return ((info->iova & ~entry->addr_mask) == entry->iova) || in smmu_hash_remove_by_vmid_ipa()
225 ((entry->iova & ~info->mask) == info->iova); in smmu_hash_remove_by_vmid_ipa()
237 if (g_hash_table_remove(s->iotlb, &key)) { in smmu_iotlb_inv_iova()
249 .mask = (num_pages * 1 << granule) - 1}; in smmu_iotlb_inv_iova()
251 g_hash_table_foreach_remove(s->iotlb, in smmu_iotlb_inv_iova()
257 * Similar to smmu_iotlb_inv_iova(), but for Stage-2, ASID is always -1,
258 * in Stage-1 invalidation ASID = -1, means don't care.
264 int asid = -1; in smmu_iotlb_inv_ipa()
269 if (g_hash_table_remove(s->iotlb, &key)) { in smmu_iotlb_inv_ipa()
277 .mask = (num_pages << granule) - 1}; in smmu_iotlb_inv_ipa()
279 g_hash_table_foreach_remove(s->iotlb, in smmu_iotlb_inv_ipa()
292 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid_vmid, &info); in smmu_iotlb_inv_asid_vmid()
298 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_vmid, &vmid); in smmu_iotlb_inv_vmid()
304 g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_vmid_s1, &vmid); in smmu_iotlb_inv_vmid_s1()
307 /* VMSAv8-64 Translation */
310 * get_pte - Get the content of a page table entry located at
319 /* TODO: guarantee 64-bit single-copy atomicity */ in get_pte()
323 info->type = SMMU_PTW_ERR_WALK_EABT; in get_pte()
324 info->addr = addr; in get_pte()
325 return -EINVAL; in get_pte()
331 /* VMSAv8-64 Translation Table Format Descriptor Decoding */
334 * get_page_pte_address - returns the L3 descriptor output address,
336 * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format
344 * get_table_pte_address - return table descriptor output address,
346 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats
354 * get_block_pte_address - return block descriptor output address and block size
355 * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats
368 bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); in select_tt()
371 if (cfg->tt[0].tsz && in select_tt()
372 !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { in select_tt()
374 return &cfg->tt[0]; in select_tt()
375 } else if (cfg->tt[1].tsz && in select_tt()
376 sextract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte) == -1) { in select_tt()
378 return &cfg->tt[1]; in select_tt()
379 } else if (!cfg->tt[0].tsz) { in select_tt()
381 return &cfg->tt[0]; in select_tt()
382 } else if (!cfg->tt[1].tsz) { in select_tt()
384 return &cfg->tt[1]; in select_tt()
390 /* Translate stage-1 table address using stage-2 page table. */
404 asid = cfg->asid; in translate_table_addr_ipa()
405 cfg->stage = SMMU_STAGE_2; in translate_table_addr_ipa()
406 cfg->asid = -1; in translate_table_addr_ipa()
408 cfg->asid = asid; in translate_table_addr_ipa()
409 cfg->stage = SMMU_NESTED; in translate_table_addr_ipa()
416 info->stage = SMMU_STAGE_2; in translate_table_addr_ipa()
417 info->addr = addr; in translate_table_addr_ipa()
418 info->is_ipa_descriptor = true; in translate_table_addr_ipa()
419 return -EINVAL; in translate_table_addr_ipa()
423 * smmu_ptw_64_s1 - VMSAv8-64 Walk of the page tables for a given IOVA
424 * @bs: smmu state which includes TLB instance
432 * and tlbe->perm is set to IOMMU_NONE.
441 SMMUStage stage = cfg->stage; in smmu_ptw_64_s1()
445 if (!tt || tt->disabled) { in smmu_ptw_64_s1()
446 info->type = SMMU_PTW_ERR_TRANSLATION; in smmu_ptw_64_s1()
450 granule_sz = tt->granule_sz; in smmu_ptw_64_s1()
452 inputsize = 64 - tt->tsz; in smmu_ptw_64_s1()
453 level = 4 - (inputsize - 4) / stride; in smmu_ptw_64_s1()
456 baseaddr = extract64(tt->ttb, 0, cfg->oas); in smmu_ptw_64_s1()
461 uint64_t mask = subpage_size - 1; in smmu_ptw_64_s1()
482 if (is_permission_fault(ap, perm) && !tt->had) { in smmu_ptw_64_s1()
483 info->type = SMMU_PTW_ERR_PERMISSION; in smmu_ptw_64_s1()
487 if (cfg->stage == SMMU_NESTED) { in smmu_ptw_64_s1()
513 if (!PTE_AF(pte) && !cfg->affd) { in smmu_ptw_64_s1()
514 info->type = SMMU_PTW_ERR_ACCESS; in smmu_ptw_64_s1()
520 info->type = SMMU_PTW_ERR_PERMISSION; in smmu_ptw_64_s1()
529 if (gpa >= (1ULL << cfg->oas)) { in smmu_ptw_64_s1()
530 info->type = SMMU_PTW_ERR_ADDR_SIZE; in smmu_ptw_64_s1()
534 tlbe->entry.translated_addr = gpa; in smmu_ptw_64_s1()
535 tlbe->entry.iova = iova & ~mask; in smmu_ptw_64_s1()
536 tlbe->entry.addr_mask = mask; in smmu_ptw_64_s1()
537 tlbe->parent_perm = PTE_AP_TO_PERM(ap); in smmu_ptw_64_s1()
538 tlbe->entry.perm = tlbe->parent_perm; in smmu_ptw_64_s1()
539 tlbe->level = level; in smmu_ptw_64_s1()
540 tlbe->granule = granule_sz; in smmu_ptw_64_s1()
543 info->type = SMMU_PTW_ERR_TRANSLATION; in smmu_ptw_64_s1()
546 info->stage = SMMU_STAGE_1; in smmu_ptw_64_s1()
547 tlbe->entry.perm = IOMMU_NONE; in smmu_ptw_64_s1()
548 return -EINVAL; in smmu_ptw_64_s1()
552 * smmu_ptw_64_s2 - VMSAv8-64 Walk of the page tables for a given ipa
553 * for stage-2.
561 * and tlbe->perm is set to IOMMU_NONE.
570 int granule_sz = cfg->s2cfg.granule_sz; in smmu_ptw_64_s2()
571 /* ARM DDI0487I.a: Table D8-7. */ in smmu_ptw_64_s2()
572 int inputsize = 64 - cfg->s2cfg.tsz; in smmu_ptw_64_s2()
573 int level = get_start_level(cfg->s2cfg.sl0, granule_sz); in smmu_ptw_64_s2()
580 uint64_t baseaddr = extract64(cfg->s2cfg.vttb, 0, cfg->s2cfg.eff_ps) + in smmu_ptw_64_s2()
591 info->type = SMMU_PTW_ERR_TRANSLATION; in smmu_ptw_64_s2()
597 uint64_t mask = subpage_size - 1; in smmu_ptw_64_s2()
636 if (!PTE_AF(pte) && !cfg->s2cfg.affd) { in smmu_ptw_64_s2()
637 info->type = SMMU_PTW_ERR_ACCESS; in smmu_ptw_64_s2()
643 info->type = SMMU_PTW_ERR_PERMISSION; in smmu_ptw_64_s2()
651 if (gpa >= (1ULL << cfg->s2cfg.eff_ps)) { in smmu_ptw_64_s2()
652 info->type = SMMU_PTW_ERR_ADDR_SIZE; in smmu_ptw_64_s2()
656 tlbe->entry.translated_addr = gpa; in smmu_ptw_64_s2()
657 tlbe->entry.iova = ipa & ~mask; in smmu_ptw_64_s2()
658 tlbe->entry.addr_mask = mask; in smmu_ptw_64_s2()
659 tlbe->parent_perm = s2ap; in smmu_ptw_64_s2()
660 tlbe->entry.perm = tlbe->parent_perm; in smmu_ptw_64_s2()
661 tlbe->level = level; in smmu_ptw_64_s2()
662 tlbe->granule = granule_sz; in smmu_ptw_64_s2()
665 info->type = SMMU_PTW_ERR_TRANSLATION; in smmu_ptw_64_s2()
668 info->addr = ipa; in smmu_ptw_64_s2()
670 info->stage = SMMU_STAGE_2; in smmu_ptw_64_s2()
671 tlbe->entry.perm = IOMMU_NONE; in smmu_ptw_64_s2()
672 return -EINVAL; in smmu_ptw_64_s2()
682 if (tlbe_s2->entry.addr_mask < tlbe->entry.addr_mask) { in combine_tlb()
683 tlbe->entry.addr_mask = tlbe_s2->entry.addr_mask; in combine_tlb()
684 tlbe->granule = tlbe_s2->granule; in combine_tlb()
685 tlbe->level = tlbe_s2->level; in combine_tlb()
688 tlbe->entry.translated_addr = CACHED_ENTRY_TO_ADDR(tlbe_s2, in combine_tlb()
689 tlbe->entry.translated_addr); in combine_tlb()
691 tlbe->entry.iova = iova & ~tlbe->entry.addr_mask; in combine_tlb()
693 tlbe->parent_perm = tlbe_s2->entry.perm; in combine_tlb()
698 * smmu_ptw - Walk the page tables for an IOVA, according to @cfg
700 * @bs: smmu state which includes TLB instance
716 if (cfg->stage == SMMU_STAGE_1) { in smmu_ptw()
718 } else if (cfg->stage == SMMU_STAGE_2) { in smmu_ptw()
725 if (iova >= (1ULL << cfg->oas)) { in smmu_ptw()
726 info->type = SMMU_PTW_ERR_ADDR_SIZE; in smmu_ptw()
727 info->stage = SMMU_STAGE_1; in smmu_ptw()
728 tlbe->entry.perm = IOMMU_NONE; in smmu_ptw()
729 return -EINVAL; in smmu_ptw()
764 if (cfg->stage == SMMU_STAGE_2) { in smmu_translate()
766 tt_combined.granule_sz = cfg->s2cfg.granule_sz; in smmu_translate()
767 tt_combined.tsz = cfg->s2cfg.tsz; in smmu_translate()
772 info->type = SMMU_PTW_ERR_TRANSLATION; in smmu_translate()
773 info->stage = SMMU_STAGE_1; in smmu_translate()
776 tt_combined.granule_sz = tt->granule_sz; in smmu_translate()
777 tt_combined.tsz = tt->tsz; in smmu_translate()
782 if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & in smmu_translate()
783 cached_entry->parent_perm & IOMMU_WO)) { in smmu_translate()
784 info->type = SMMU_PTW_ERR_PERMISSION; in smmu_translate()
785 info->stage = !(cached_entry->entry.perm & IOMMU_WO) ? in smmu_translate()
811 SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; in smmu_find_smmu_pcibus()
818 g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); in smmu_find_smmu_pcibus()
820 if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { in smmu_find_smmu_pcibus()
821 s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; in smmu_find_smmu_pcibus()
832 SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); in smmu_find_add_as()
839 sbus->bus = bus; in smmu_find_add_as()
840 g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); in smmu_find_add_as()
843 sdev = sbus->pbdev[devfn]; in smmu_find_add_as()
845 char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++); in smmu_find_add_as()
847 sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); in smmu_find_add_as()
849 sdev->smmu = s; in smmu_find_add_as()
850 sdev->bus = bus; in smmu_find_add_as()
851 sdev->devfn = devfn; in smmu_find_add_as()
853 memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), in smmu_find_add_as()
854 s->mrtypename, in smmu_find_add_as()
856 address_space_init(&sdev->as, in smmu_find_add_as()
857 MEMORY_REGION(&sdev->iommu), name); in smmu_find_add_as()
862 return &sdev->as; in smmu_find_add_as()
878 return smmu_bus->pbdev[devfn]; in smmu_find_sdev()
888 trace_smmu_inv_notifiers_mr(mr->parent_obj.name); in smmu_inv_notifiers_mr()
899 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { in smmu_inv_notifiers_all()
900 smmu_inv_notifiers_mr(&sdev->iommu); in smmu_inv_notifiers_all()
910 sbc->parent_realize(dev, &local_err); in smmu_base_realize()
915 s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); in smmu_base_realize()
916 s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, in smmu_base_realize()
918 s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); in smmu_base_realize()
920 if (s->primary_bus) { in smmu_base_realize()
921 pci_setup_iommu(s->primary_bus, &smmu_ops, s); in smmu_base_realize()
923 error_setg(errp, "SMMU is not attached to any PCI bus!"); in smmu_base_realize()
931 memset(s->smmu_pcibus_by_bus_num, 0, sizeof(s->smmu_pcibus_by_bus_num)); in smmu_base_reset_hold()
933 g_hash_table_remove_all(s->configs); in smmu_base_reset_hold()
934 g_hash_table_remove_all(s->iotlb); in smmu_base_reset_hold()
939 DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus,
952 &sbc->parent_realize); in smmu_base_class_init()
953 rc->phases.hold = smmu_base_reset_hold; in smmu_base_class_init()