131 static size_t get_align_offset(struct pci_epf_mhi *epf_mhi, u64 addr)  in get_align_offset()  argument
133 	return addr & (epf_mhi->epc_features->align -1);  in get_align_offset()
140 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in __pci_epf_mhi_alloc_map()  local
141 	struct pci_epf *epf = epf_mhi->epf;  in __pci_epf_mhi_alloc_map()
166 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_alloc_map()  local
167 	size_t offset = get_align_offset(epf_mhi, pci_addr);  in pci_epf_mhi_alloc_map()
178 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in __pci_epf_mhi_unmap_free()  local
179 	struct pci_epf *epf = epf_mhi->epf;  in __pci_epf_mhi_unmap_free()
191 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_unmap_free()  local
192 	size_t offset = get_align_offset(epf_mhi, pci_addr);  in pci_epf_mhi_unmap_free()
200 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_raise_irq()  local
201 	struct pci_epf *epf = epf_mhi->epf;  in pci_epf_mhi_raise_irq()
215 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_iatu_read()  local
216 	size_t offset = get_align_offset(epf_mhi, buf_info->host_addr);  in pci_epf_mhi_iatu_read()
221 	mutex_lock(&epf_mhi->lock);  in pci_epf_mhi_iatu_read()
226 		mutex_unlock(&epf_mhi->lock);  in pci_epf_mhi_iatu_read()
235 	mutex_unlock(&epf_mhi->lock);  in pci_epf_mhi_iatu_read()
243 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_iatu_write()  local
244 	size_t offset = get_align_offset(epf_mhi, buf_info->host_addr);  in pci_epf_mhi_iatu_write()
249 	mutex_lock(&epf_mhi->lock);  in pci_epf_mhi_iatu_write()
254 		mutex_unlock(&epf_mhi->lock);  in pci_epf_mhi_iatu_write()
263 	mutex_unlock(&epf_mhi->lock);  in pci_epf_mhi_iatu_write()
276 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_edma_read()  local
277 	struct device *dma_dev = epf_mhi->epf->epc->dev.parent;  in pci_epf_mhi_edma_read()
278 	struct dma_chan *chan = epf_mhi->dma_chan_rx;  in pci_epf_mhi_edma_read()
279 	struct device *dev = &epf_mhi->epf->dev;  in pci_epf_mhi_edma_read()
290 	mutex_lock(&epf_mhi->lock);  in pci_epf_mhi_edma_read()
339 	mutex_unlock(&epf_mhi->lock);  in pci_epf_mhi_edma_read()
347 	struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);  in pci_epf_mhi_edma_write()  local
348 	struct device *dma_dev = epf_mhi->epf->epc->dev.parent;  in pci_epf_mhi_edma_write()
349 	struct dma_chan *chan = epf_mhi->dma_chan_tx;  in pci_epf_mhi_edma_write()
350 	struct device *dev = &epf_mhi->epf->dev;  in pci_epf_mhi_edma_write()
361 	mutex_lock(&epf_mhi->lock);  in pci_epf_mhi_edma_write()
410 	mutex_unlock(&epf_mhi->lock);  in pci_epf_mhi_edma_write()
432 static int pci_epf_mhi_dma_init(struct pci_epf_mhi *epf_mhi)  in pci_epf_mhi_dma_init()  argument
434 	struct device *dma_dev = epf_mhi->epf->epc->dev.parent;  in pci_epf_mhi_dma_init()
435 	struct device *dev = &epf_mhi->epf->dev;  in pci_epf_mhi_dma_init()
444 	epf_mhi->dma_chan_tx = dma_request_channel(mask, pci_epf_mhi_filter,  in pci_epf_mhi_dma_init()
446 	if (IS_ERR_OR_NULL(epf_mhi->dma_chan_tx)) {  in pci_epf_mhi_dma_init()
452 	epf_mhi->dma_chan_rx = dma_request_channel(mask, pci_epf_mhi_filter,  in pci_epf_mhi_dma_init()
454 	if (IS_ERR_OR_NULL(epf_mhi->dma_chan_rx)) {  in pci_epf_mhi_dma_init()
456 		dma_release_channel(epf_mhi->dma_chan_tx);  in pci_epf_mhi_dma_init()
457 		epf_mhi->dma_chan_tx = NULL;  in pci_epf_mhi_dma_init()
464 static void pci_epf_mhi_dma_deinit(struct pci_epf_mhi *epf_mhi)  in pci_epf_mhi_dma_deinit()  argument
466 	dma_release_channel(epf_mhi->dma_chan_tx);  in pci_epf_mhi_dma_deinit()
467 	dma_release_channel(epf_mhi->dma_chan_rx);  in pci_epf_mhi_dma_deinit()
468 	epf_mhi->dma_chan_tx = NULL;  in pci_epf_mhi_dma_deinit()
469 	epf_mhi->dma_chan_rx = NULL;  in pci_epf_mhi_dma_deinit()
474 	struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);  in pci_epf_mhi_core_init()  local
475 	const struct pci_epf_mhi_ep_info *info = epf_mhi->info;  in pci_epf_mhi_core_init()
481 	epf_bar->phys_addr = epf_mhi->mmio_phys;  in pci_epf_mhi_core_init()
482 	epf_bar->size = epf_mhi->mmio_size;  in pci_epf_mhi_core_init()
505 	epf_mhi->epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);  in pci_epf_mhi_core_init()
506 	if (!epf_mhi->epc_features)  in pci_epf_mhi_core_init()
514 	struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);  in pci_epf_mhi_link_up()  local
515 	const struct pci_epf_mhi_ep_info *info = epf_mhi->info;  in pci_epf_mhi_link_up()
516 	struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;  in pci_epf_mhi_link_up()
522 		ret = pci_epf_mhi_dma_init(epf_mhi);  in pci_epf_mhi_link_up()
529 	mhi_cntrl->mmio = epf_mhi->mmio;  in pci_epf_mhi_link_up()
530 	mhi_cntrl->irq = epf_mhi->irq;  in pci_epf_mhi_link_up()
551 			pci_epf_mhi_dma_deinit(epf_mhi);  in pci_epf_mhi_link_up()
560 	struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);  in pci_epf_mhi_link_down()  local
561 	const struct pci_epf_mhi_ep_info *info = epf_mhi->info;  in pci_epf_mhi_link_down()
562 	struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;  in pci_epf_mhi_link_down()
567 			pci_epf_mhi_dma_deinit(epf_mhi);  in pci_epf_mhi_link_down()
576 	struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);  in pci_epf_mhi_bme()  local
577 	const struct pci_epf_mhi_ep_info *info = epf_mhi->info;  in pci_epf_mhi_bme()
578 	struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;  in pci_epf_mhi_bme()
591 				pci_epf_mhi_dma_deinit(epf_mhi);  in pci_epf_mhi_bme()
601 	struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);  in pci_epf_mhi_bind()  local
609 	epf_mhi->mmio_phys = res->start;  in pci_epf_mhi_bind()
610 	epf_mhi->mmio_size = resource_size(res);  in pci_epf_mhi_bind()
612 	epf_mhi->mmio = ioremap(epf_mhi->mmio_phys, epf_mhi->mmio_size);  in pci_epf_mhi_bind()
613 	if (!epf_mhi->mmio)  in pci_epf_mhi_bind()
618 		iounmap(epf_mhi->mmio);  in pci_epf_mhi_bind()
622 	epf_mhi->irq = ret;  in pci_epf_mhi_bind()
629 	struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);  in pci_epf_mhi_unbind()  local
630 	const struct pci_epf_mhi_ep_info *info = epf_mhi->info;  in pci_epf_mhi_unbind()
632 	struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;  in pci_epf_mhi_unbind()
643 			pci_epf_mhi_dma_deinit(epf_mhi);  in pci_epf_mhi_unbind()
647 	iounmap(epf_mhi->mmio);  in pci_epf_mhi_unbind()
663 	struct pci_epf_mhi *epf_mhi;  in pci_epf_mhi_probe()  local
666 	epf_mhi = devm_kzalloc(dev, sizeof(*epf_mhi), GFP_KERNEL);  in pci_epf_mhi_probe()
667 	if (!epf_mhi)  in pci_epf_mhi_probe()
671 	epf_mhi->info = info;  in pci_epf_mhi_probe()
672 	epf_mhi->epf = epf;  in pci_epf_mhi_probe()
676 	mutex_init(&epf_mhi->lock);  in pci_epf_mhi_probe()
678 	epf_set_drvdata(epf, epf_mhi);  in pci_epf_mhi_probe()