72 	struct ipmmu_vmsa_device *mmu;  member
150 static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu)  in ipmmu_is_root()  argument
152 	return mmu->root == mmu;  in ipmmu_is_root()
157 	struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);  in __ipmmu_check_device()  local
160 	if (ipmmu_is_root(mmu))  in __ipmmu_check_device()
161 		*rootp = mmu;  in __ipmmu_check_device()
178 static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)  in ipmmu_read()  argument
180 	return ioread32(mmu->base + offset);  in ipmmu_read()
183 static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,  in ipmmu_write()  argument
186 	iowrite32(data, mmu->base + offset);  in ipmmu_write()
189 static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_reg()  argument
192 	unsigned int base = mmu->features->ctx_offset_base;  in ipmmu_ctx_reg()
197 	return base + context_id * mmu->features->ctx_offset_stride + reg;  in ipmmu_ctx_reg()
200 static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_read()  argument
203 	return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg));  in ipmmu_ctx_read()
206 static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_write()  argument
209 	ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data);  in ipmmu_ctx_write()
215 	return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);  in ipmmu_ctx_read_root()
221 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_root()
227 	if (domain->mmu != domain->mmu->root)  in ipmmu_ctx_write_all()
228 		ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
230 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
233 static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg)  in ipmmu_utlb_reg()  argument
235 	return mmu->features->utlb_offset_base + reg;  in ipmmu_utlb_reg()
238 static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_imuasid_write()  argument
241 	ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data);  in ipmmu_imuasid_write()
244 static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_imuctr_write()  argument
247 	ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data);  in ipmmu_imuctr_write()
262 		dev_err_ratelimited(domain->mmu->dev,  in ipmmu_tlb_sync()
283 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_enable()  local
291 	ipmmu_imuasid_write(mmu, utlb, 0);  in ipmmu_utlb_enable()
293 	ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |  in ipmmu_utlb_enable()
295 	mmu->utlb_ctx[utlb] = domain->context_id;  in ipmmu_utlb_enable()
320 static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,  in ipmmu_domain_allocate_context()  argument
326 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_domain_allocate_context()
328 	ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx);  in ipmmu_domain_allocate_context()
329 	if (ret != mmu->num_ctx) {  in ipmmu_domain_allocate_context()
330 		mmu->domains[ret] = domain;  in ipmmu_domain_allocate_context()
331 		set_bit(ret, mmu->ctx);  in ipmmu_domain_allocate_context()
335 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_domain_allocate_context()
340 static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,  in ipmmu_domain_free_context()  argument
345 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_domain_free_context()
347 	clear_bit(context_id, mmu->ctx);  in ipmmu_domain_free_context()
348 	mmu->domains[context_id] = NULL;  in ipmmu_domain_free_context()
350 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_domain_free_context()
368 	if (domain->mmu->features->twobit_imttbcr_sl0)  in ipmmu_domain_setup_context()
373 	if (domain->mmu->features->cache_snoop)  in ipmmu_domain_setup_context()
384 	if (domain->mmu->features->setup_imbuscr)  in ipmmu_domain_setup_context()
433 	domain->cfg.iommu_dev = domain->mmu->root->dev;  in ipmmu_domain_init_context()
438 	ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);  in ipmmu_domain_init_context()
447 		ipmmu_domain_free_context(domain->mmu->root,  in ipmmu_domain_init_context()
458 	if (!domain->mmu)  in ipmmu_domain_destroy_context()
469 	ipmmu_domain_free_context(domain->mmu->root, domain->context_id);  in ipmmu_domain_destroy_context()
479 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_domain_irq()  local
501 		dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n",  in ipmmu_domain_irq()
504 		dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n",  in ipmmu_domain_irq()
516 	if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))  in ipmmu_domain_irq()
519 	dev_err_ratelimited(mmu->dev,  in ipmmu_domain_irq()
528 	struct ipmmu_vmsa_device *mmu = dev;  in ipmmu_irq()  local
533 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_irq()
538 	for (i = 0; i < mmu->num_ctx; i++) {  in ipmmu_irq()
539 		if (!mmu->domains[i])  in ipmmu_irq()
541 		if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED)  in ipmmu_irq()
545 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_irq()
587 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_attach_device()  local
592 	if (!mmu) {  in ipmmu_attach_device()
599 	if (!domain->mmu) {  in ipmmu_attach_device()
601 		domain->mmu = mmu;  in ipmmu_attach_device()
605 			domain->mmu = NULL;  in ipmmu_attach_device()
610 	} else if (domain->mmu != mmu) {  in ipmmu_attach_device()
653 	if (domain->mmu)  in ipmmu_flush_iotlb_all()
754 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_init_arm_mapping()  local
766 	if (!mmu->mapping) {  in ipmmu_init_arm_mapping()
772 			dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");  in ipmmu_init_arm_mapping()
777 		mmu->mapping = mapping;  in ipmmu_init_arm_mapping()
781 	ret = arm_iommu_attach_device(dev, mmu->mapping);  in ipmmu_init_arm_mapping()
790 	if (mmu->mapping)  in ipmmu_init_arm_mapping()
791 		arm_iommu_release_mapping(mmu->mapping);  in ipmmu_init_arm_mapping()
798 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_probe_device()  local
803 	if (!mmu)  in ipmmu_probe_device()
806 	return &mmu->iommu;  in ipmmu_probe_device()
823 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_release_device()  local
829 		ipmmu_imuctr_write(mmu, utlb, 0);  in ipmmu_release_device()
830 		mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;  in ipmmu_release_device()
833 	arm_iommu_release_mapping(mmu->mapping);  in ipmmu_release_device()
838 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_find_group()  local
841 	if (mmu->group)  in ipmmu_find_group()
842 		return iommu_group_ref_get(mmu->group);  in ipmmu_find_group()
846 		mmu->group = group;  in ipmmu_find_group()
875 static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)  in ipmmu_device_reset()  argument
880 	for (i = 0; i < mmu->num_ctx; ++i)  in ipmmu_device_reset()
881 		ipmmu_ctx_write(mmu, i, IMCTR, 0);  in ipmmu_device_reset()
979 	struct ipmmu_vmsa_device *mmu;  in ipmmu_probe()  local
984 	mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);  in ipmmu_probe()
985 	if (!mmu) {  in ipmmu_probe()
990 	mmu->dev = &pdev->dev;  in ipmmu_probe()
991 	spin_lock_init(&mmu->lock);  in ipmmu_probe()
992 	bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);  in ipmmu_probe()
993 	mmu->features = of_device_get_match_data(&pdev->dev);  in ipmmu_probe()
994 	memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);  in ipmmu_probe()
1001 	mmu->base = devm_ioremap_resource(&pdev->dev, res);  in ipmmu_probe()
1002 	if (IS_ERR(mmu->base))  in ipmmu_probe()
1003 		return PTR_ERR(mmu->base);  in ipmmu_probe()
1017 	if (mmu->features->use_ns_alias_offset)  in ipmmu_probe()
1018 		mmu->base += IM_NS_ALIAS_OFFSET;  in ipmmu_probe()
1020 	mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts);  in ipmmu_probe()
1026 	if (!mmu->features->has_cache_leaf_nodes ||  in ipmmu_probe()
1028 		mmu->root = mmu;  in ipmmu_probe()
1030 		mmu->root = ipmmu_find_root();  in ipmmu_probe()
1035 	if (!mmu->root)  in ipmmu_probe()
1039 	if (ipmmu_is_root(mmu)) {  in ipmmu_probe()
1045 				       dev_name(&pdev->dev), mmu);  in ipmmu_probe()
1051 		ipmmu_device_reset(mmu);  in ipmmu_probe()
1053 		if (mmu->features->reserved_context) {  in ipmmu_probe()
1055 			set_bit(0, mmu->ctx);  in ipmmu_probe()
1064 	if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) {  in ipmmu_probe()
1065 		ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,  in ipmmu_probe()
1070 		ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev);  in ipmmu_probe()
1081 	platform_set_drvdata(pdev, mmu);  in ipmmu_probe()
1088 	struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);  in ipmmu_remove()  local
1090 	iommu_device_sysfs_remove(&mmu->iommu);  in ipmmu_remove()
1091 	iommu_device_unregister(&mmu->iommu);  in ipmmu_remove()
1093 	arm_iommu_release_mapping(mmu->mapping);  in ipmmu_remove()
1095 	ipmmu_device_reset(mmu);  in ipmmu_remove()
1101 	struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);  in ipmmu_resume_noirq()  local
1105 	if (ipmmu_is_root(mmu)) {  in ipmmu_resume_noirq()
1106 		ipmmu_device_reset(mmu);  in ipmmu_resume_noirq()
1108 		for (i = 0; i < mmu->num_ctx; i++) {  in ipmmu_resume_noirq()
1109 			if (!mmu->domains[i])  in ipmmu_resume_noirq()
1112 			ipmmu_domain_setup_context(mmu->domains[i]);  in ipmmu_resume_noirq()
1117 	for (i = 0; i < mmu->features->num_utlbs; i++) {  in ipmmu_resume_noirq()
1118 		if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID)  in ipmmu_resume_noirq()
1121 		ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i);  in ipmmu_resume_noirq()