277 	struct tegra_smmu_as *as;  in tegra_smmu_domain_alloc()  local
282 	as = kzalloc(sizeof(*as), GFP_KERNEL);  in tegra_smmu_domain_alloc()
283 	if (!as)  in tegra_smmu_domain_alloc()
286 	as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;  in tegra_smmu_domain_alloc()
288 	as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);  in tegra_smmu_domain_alloc()
289 	if (!as->pd) {  in tegra_smmu_domain_alloc()
290 		kfree(as);  in tegra_smmu_domain_alloc()
294 	as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);  in tegra_smmu_domain_alloc()
295 	if (!as->count) {  in tegra_smmu_domain_alloc()
296 		__free_page(as->pd);  in tegra_smmu_domain_alloc()
297 		kfree(as);  in tegra_smmu_domain_alloc()
301 	as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);  in tegra_smmu_domain_alloc()
302 	if (!as->pts) {  in tegra_smmu_domain_alloc()
303 		kfree(as->count);  in tegra_smmu_domain_alloc()
304 		__free_page(as->pd);  in tegra_smmu_domain_alloc()
305 		kfree(as);  in tegra_smmu_domain_alloc()
309 	spin_lock_init(&as->lock);  in tegra_smmu_domain_alloc()
312 	as->domain.geometry.aperture_start = 0;  in tegra_smmu_domain_alloc()
313 	as->domain.geometry.aperture_end = 0xffffffff;  in tegra_smmu_domain_alloc()
314 	as->domain.geometry.force_aperture = true;  in tegra_smmu_domain_alloc()
316 	return &as->domain;  in tegra_smmu_domain_alloc()
321 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_domain_free()  local
325 	WARN_ON_ONCE(as->use_count);  in tegra_smmu_domain_free()
326 	kfree(as->count);  in tegra_smmu_domain_free()
327 	kfree(as->pts);  in tegra_smmu_domain_free()
328 	kfree(as);  in tegra_smmu_domain_free()
409 				 struct tegra_smmu_as *as)  in tegra_smmu_as_prepare()  argument
416 	if (as->use_count > 0) {  in tegra_smmu_as_prepare()
417 		as->use_count++;  in tegra_smmu_as_prepare()
421 	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,  in tegra_smmu_as_prepare()
423 	if (dma_mapping_error(smmu->dev, as->pd_dma)) {  in tegra_smmu_as_prepare()
429 	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {  in tegra_smmu_as_prepare()
434 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
438 	smmu_flush_ptc(smmu, as->pd_dma, 0);  in tegra_smmu_as_prepare()
439 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
441 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
442 	value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);  in tegra_smmu_as_prepare()
446 	as->smmu = smmu;  in tegra_smmu_as_prepare()
447 	as->use_count++;  in tegra_smmu_as_prepare()
454 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_prepare()
462 				    struct tegra_smmu_as *as)  in tegra_smmu_as_unprepare()  argument
466 	if (--as->use_count > 0) {  in tegra_smmu_as_unprepare()
471 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
473 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_unprepare()
475 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
485 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_attach_dev()  local
493 		err = tegra_smmu_as_prepare(smmu, as);  in tegra_smmu_attach_dev()
497 		tegra_smmu_enable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
507 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
508 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_attach_dev()
518 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_set_platform_dma()  local
519 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_platform_dma()
526 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_set_platform_dma()
527 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_set_platform_dma()
531 static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,  in tegra_smmu_set_pde()  argument
535 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pde()
536 	u32 *pd = page_address(as->pd);  in tegra_smmu_set_pde()
543 	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,  in tegra_smmu_set_pde()
547 	smmu_flush_ptc(smmu, as->pd_dma, offset);  in tegra_smmu_set_pde()
548 	smmu_flush_tlb_section(smmu, as->id, iova);  in tegra_smmu_set_pde()
559 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,  in tegra_smmu_pte_lookup()  argument
563 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_lookup()
567 	pt_page = as->pts[pd_index];  in tegra_smmu_pte_lookup()
571 	pd = page_address(as->pd);  in tegra_smmu_pte_lookup()
577 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,  in as_get_pte()  argument
581 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()
583 	if (!as->pts[pde]) {  in as_get_pte()
600 		as->pts[pde] = page;  in as_get_pte()
602 		tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |  in as_get_pte()
607 		u32 *pd = page_address(as->pd);  in as_get_pte()
612 	return tegra_smmu_pte_offset(as->pts[pde], iova);  in as_get_pte()
615 static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)  in tegra_smmu_pte_get_use()  argument
619 	as->count[pd_index]++;  in tegra_smmu_pte_get_use()
622 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)  in tegra_smmu_pte_put_use()  argument
625 	struct page *page = as->pts[pde];  in tegra_smmu_pte_put_use()
631 	if (--as->count[pde] == 0) {  in tegra_smmu_pte_put_use()
632 		struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_put_use()
633 		u32 *pd = page_address(as->pd);  in tegra_smmu_pte_put_use()
636 		tegra_smmu_set_pde(as, iova, 0);  in tegra_smmu_pte_put_use()
640 		as->pts[pde] = NULL;  in tegra_smmu_pte_put_use()
644 static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,  in tegra_smmu_set_pte()  argument
647 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pte()
655 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_set_pte()
659 static struct page *as_get_pde_page(struct tegra_smmu_as *as,  in as_get_pde_page()  argument
664 	struct page *page = as->pts[pde];  in as_get_pde_page()
676 		spin_unlock_irqrestore(&as->lock, *flags);  in as_get_pde_page()
681 		spin_lock_irqsave(&as->lock, *flags);  in as_get_pde_page()
688 	if (as->pts[pde]) {  in as_get_pde_page()
692 		page = as->pts[pde];  in as_get_pde_page()
703 	struct tegra_smmu_as *as = to_smmu_as(domain);  in __tegra_smmu_map()  local
709 	page = as_get_pde_page(as, iova, gfp, flags);  in __tegra_smmu_map()
713 	pte = as_get_pte(as, iova, &pte_dma, page);  in __tegra_smmu_map()
719 		tegra_smmu_pte_get_use(as, iova);  in __tegra_smmu_map()
729 	tegra_smmu_set_pte(as, iova, pte, pte_dma,  in __tegra_smmu_map()
739 	struct tegra_smmu_as *as = to_smmu_as(domain);  in __tegra_smmu_unmap()  local
743 	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);  in __tegra_smmu_unmap()
747 	tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);  in __tegra_smmu_unmap()
748 	tegra_smmu_pte_put_use(as, iova);  in __tegra_smmu_unmap()
756 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_map()  local
760 	spin_lock_irqsave(&as->lock, flags);  in tegra_smmu_map()
762 	spin_unlock_irqrestore(&as->lock, flags);  in tegra_smmu_map()
770 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_unmap()  local
773 	spin_lock_irqsave(&as->lock, flags);  in tegra_smmu_unmap()
775 	spin_unlock_irqrestore(&as->lock, flags);  in tegra_smmu_unmap()
783 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_iova_to_phys()  local
788 	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);  in tegra_smmu_iova_to_phys()
792 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
955 	 * the SMMU parent device is the same as the MC, so the reference count  in tegra_smmu_of_xlate()