15 #include <linux/iommu.h>
21 #include "iommu.h"
25  * Intel IOMMU system wide PASID name space:
29 int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid)  in vcmd_alloc_pasid()  argument
36 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
37 	dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC);  in vcmd_alloc_pasid()
38 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_alloc_pasid()
40 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
48 		pr_info("IOMMU: %s: No PASID available\n", iommu->name);  in vcmd_alloc_pasid()
53 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_alloc_pasid()
54 			iommu->name, status_code);  in vcmd_alloc_pasid()
60 void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid)  in vcmd_free_pasid()  argument
66 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_free_pasid()
67 	dmar_writeq(iommu->reg + DMAR_VCMD_REG,  in vcmd_free_pasid()
69 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_free_pasid()
71 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_free_pasid()
78 		pr_info("IOMMU: %s: Invalid PASID\n", iommu->name);  in vcmd_free_pasid()
81 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_free_pasid()
82 			iommu->name, status_code);  in vcmd_free_pasid()
119 	pages = alloc_pages_node(info->iommu->node,  in intel_pasid_alloc_table()
131 	if (!ecap_coherent(info->iommu->ecap))  in intel_pasid_alloc_table()
206 		entries = alloc_pgtable_page(info->iommu->node, GFP_ATOMIC);  in intel_pasid_get_entry()
221 		if (!ecap_coherent(info->iommu->ecap)) {  in intel_pasid_get_entry()
406 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,  in pasid_cache_invalidation_with_pasid()  argument
417 	qi_submit_sync(iommu, &desc, 1, 0);  in pasid_cache_invalidation_with_pasid()
421 devtlb_invalidation_with_pasid(struct intel_iommu *iommu,  in devtlb_invalidation_with_pasid()  argument
445 		qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
447 		qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
450 void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_tear_down_entry()  argument
456 	spin_lock(&iommu->lock);  in intel_pasid_tear_down_entry()
459 		spin_unlock(&iommu->lock);  in intel_pasid_tear_down_entry()
466 	spin_unlock(&iommu->lock);  in intel_pasid_tear_down_entry()
468 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_tear_down_entry()
471 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
474 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_tear_down_entry()
476 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);  in intel_pasid_tear_down_entry()
479 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_tear_down_entry()
480 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_tear_down_entry()
487 static void pasid_flush_caches(struct intel_iommu *iommu,  in pasid_flush_caches()  argument
491 	if (!ecap_coherent(iommu->ecap))  in pasid_flush_caches()
494 	if (cap_caching_mode(iommu->cap)) {  in pasid_flush_caches()
495 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in pasid_flush_caches()
496 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in pasid_flush_caches()
498 		iommu_flush_write_buffer(iommu);  in pasid_flush_caches()
506 int intel_pasid_setup_first_level(struct intel_iommu *iommu,  in intel_pasid_setup_first_level()  argument
512 	if (!ecap_flts(iommu->ecap)) {  in intel_pasid_setup_first_level()
514 		       iommu->name);  in intel_pasid_setup_first_level()
518 	if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {  in intel_pasid_setup_first_level()
520 		       iommu->name);  in intel_pasid_setup_first_level()
524 	spin_lock(&iommu->lock);  in intel_pasid_setup_first_level()
527 		spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
532 		spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
548 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_first_level()
549 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_first_level()
555 	spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
557 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_first_level()
563  * Skip top levels of page tables for iommu which has less agaw
567 				  struct intel_iommu *iommu,  in iommu_skip_agaw()  argument
572 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {  in iommu_skip_agaw()
584 int intel_pasid_setup_second_level(struct intel_iommu *iommu,  in intel_pasid_setup_second_level()  argument
598 	if (!ecap_slts(iommu->ecap)) {  in intel_pasid_setup_second_level()
600 		       iommu->name);  in intel_pasid_setup_second_level()
605 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_second_level()
612 	did = domain_id_iommu(domain, iommu);  in intel_pasid_setup_second_level()
614 	spin_lock(&iommu->lock);  in intel_pasid_setup_second_level()
617 		spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
622 		spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
632 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_second_level()
635 	spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
637 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_second_level()
645 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,  in intel_pasid_setup_pass_through()  argument
652 	spin_lock(&iommu->lock);  in intel_pasid_setup_pass_through()
655 		spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
660 		spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
666 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_pass_through()
669 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_pass_through()
671 	spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
673 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_pass_through()
681 void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,  in intel_pasid_setup_page_snoop_control()  argument
687 	spin_lock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
690 		spin_unlock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
696 	spin_unlock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
698 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_page_snoop_control()
712 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_page_snoop_control()
713 	qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_setup_page_snoop_control()
716 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_setup_page_snoop_control()
717 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_setup_page_snoop_control()