191 static inline u32 bm_in(struct bm_portal *p, u32 offset)  in bm_in()  argument
193 	return ioread32be(p->addr.ci + offset);  in bm_in()
196 static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)  in bm_out()  argument
198 	iowrite32be(val, p->addr.ci + offset);  in bm_out()
202 static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)  in bm_cl_invalidate()  argument
204 	dpaa_invalidate(p->addr.ce + offset);  in bm_cl_invalidate()
207 static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)  in bm_cl_touch_ro()  argument
209 	dpaa_touch_ro(p->addr.ce + offset);  in bm_cl_touch_ro()
212 static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)  in bm_ce_in()  argument
214 	return be32_to_cpu(*(p->addr.ce_be + (offset/4)));  in bm_ce_in()
218 	struct bm_portal p;  member
253 static u32 poll_portal_slow(struct bman_portal *p, u32 is);
257 	struct bman_portal *p = ptr;  in portal_isr()  local
258 	struct bm_portal *portal = &p->p;  in portal_isr()
259 	u32 clear = p->irq_sources;  in portal_isr()
260 	u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources;  in portal_isr()
265 	clear |= poll_portal_slow(p, is);  in portal_isr()
276 static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)  in rcr_carryclear()  argument
278 	uintptr_t addr = (uintptr_t)p;  in rcr_carryclear()
529 	struct bm_portal *p;  in bman_create_portal()  local
532 	p = &portal->p;  in bman_create_portal()
538 	p->addr.ce = c->addr_virt_ce;  in bman_create_portal()
539 	p->addr.ce_be = c->addr_virt_ce;  in bman_create_portal()
540 	p->addr.ci = c->addr_virt_ci;  in bman_create_portal()
541 	if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {  in bman_create_portal()
545 	if (bm_mc_init(p)) {  in bman_create_portal()
553 	bm_isr_bscn_disable(p);  in bman_create_portal()
556 	bm_out(p, BM_REG_ISDR, 0xffffffff);  in bman_create_portal()
558 	bm_out(p, BM_REG_IER, 0);  in bman_create_portal()
559 	bm_out(p, BM_REG_ISR, 0xffffffff);  in bman_create_portal()
570 	ret = bm_rcr_get_fill(p);  in bman_create_portal()
578 	bm_out(p, BM_REG_ISDR, 0);  in bman_create_portal()
579 	bm_out(p, BM_REG_IIR, 0);  in bman_create_portal()
587 	bm_mc_finish(p);  in bman_create_portal()
589 	bm_rcr_finish(p);  in bman_create_portal()
611 static u32 poll_portal_slow(struct bman_portal *p, u32 is)  in poll_portal_slow()  argument
616 		bm_rcr_cce_update(&p->p);  in poll_portal_slow()
617 		bm_rcr_set_ithresh(&p->p, 0);  in poll_portal_slow()
618 		bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI);  in poll_portal_slow()
627 int bman_p_irqsource_add(struct bman_portal *p, u32 bits)  in bman_p_irqsource_add()  argument
632 	p->irq_sources |= bits & BM_PIRQ_VISIBLE;  in bman_p_irqsource_add()
633 	bm_out(&p->p, BM_REG_IER, p->irq_sources);  in bman_p_irqsource_add()
645 	struct bman_portal *p = get_affine_portal();  in bm_shutdown_pool()  local
648 		bm_cmd = bm_mc_start(&p->p);  in bm_shutdown_pool()
650 		bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);  in bm_shutdown_pool()
651 		if (!bm_mc_result_timeout(&p->p, &bm_res)) {  in bm_shutdown_pool()
730 static void update_rcr_ci(struct bman_portal *p, int avail)  in update_rcr_ci()  argument
733 		bm_rcr_cce_prefetch(&p->p);  in update_rcr_ci()
735 		bm_rcr_cce_update(&p->p);  in update_rcr_ci()
740 	struct bman_portal *p;  in bman_release()  local
749 		p = get_affine_portal();  in bman_release()
751 		avail = bm_rcr_get_avail(&p->p);  in bman_release()
753 			update_rcr_ci(p, avail);  in bman_release()
754 		r = bm_rcr_start(&p->p);  in bman_release()
766 	p = get_affine_portal();  in bman_release()
777 	bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |  in bman_release()
788 	struct bman_portal *p = get_affine_portal();  in bman_acquire()  local
795 	mcc = bm_mc_start(&p->p);  in bman_acquire()
797 	bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |  in bman_acquire()
799 	if (!bm_mc_result_timeout(&p->p, &mcr)) {  in bman_acquire()