104 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)  in queue_has_space()  argument
108 	prod = Q_IDX(q, q->prod);  in queue_has_space()
109 	cons = Q_IDX(q, q->cons);  in queue_has_space()
111 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))  in queue_has_space()
112 		space = (1 << q->max_n_shift) - (prod - cons);  in queue_has_space()
119 static bool queue_full(struct arm_smmu_ll_queue *q)  in queue_full()  argument
121 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_full()
122 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);  in queue_full()
125 static bool queue_empty(struct arm_smmu_ll_queue *q)  in queue_empty()  argument
127 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_empty()
128 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);  in queue_empty()
131 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)  in queue_consumed()  argument
133 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&  in queue_consumed()
134 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||  in queue_consumed()
135 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&  in queue_consumed()
136 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));  in queue_consumed()
139 static void queue_sync_cons_out(struct arm_smmu_queue *q)  in queue_sync_cons_out()  argument
146 	writel_relaxed(q->llq.cons, q->cons_reg);  in queue_sync_cons_out()
149 static void queue_inc_cons(struct arm_smmu_ll_queue *q)  in queue_inc_cons()  argument
151 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;  in queue_inc_cons()
152 	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);  in queue_inc_cons()
155 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)  in queue_sync_cons_ovf()  argument
157 	struct arm_smmu_ll_queue *llq = &q->llq;  in queue_sync_cons_ovf()
164 	queue_sync_cons_out(q);  in queue_sync_cons_ovf()
167 static int queue_sync_prod_in(struct arm_smmu_queue *q)  in queue_sync_prod_in()  argument
177 	prod = readl(q->prod_reg);  in queue_sync_prod_in()
179 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))  in queue_sync_prod_in()
182 	q->llq.prod = prod;  in queue_sync_prod_in()
186 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)  in queue_inc_prod_n()  argument
188 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;  in queue_inc_prod_n()
189 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);  in queue_inc_prod_n()
235 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)  in queue_remove_raw()  argument
237 	if (queue_empty(&q->llq))  in queue_remove_raw()
240 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);  in queue_remove_raw()
241 	queue_inc_cons(&q->llq);  in queue_remove_raw()
242 	queue_sync_cons_out(q);  in queue_remove_raw()
354 					 struct arm_smmu_queue *q, u32 prod)  in arm_smmu_cmdq_build_sync_cmd()  argument
365 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *  in arm_smmu_cmdq_build_sync_cmd()
366 				   q->ent_dwords * 8;  in arm_smmu_cmdq_build_sync_cmd()
373 				     struct arm_smmu_queue *q)  in __arm_smmu_cmdq_skip_err()  argument
384 	u32 cons = readl_relaxed(q->cons_reg);  in __arm_smmu_cmdq_skip_err()
416 	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);  in __arm_smmu_cmdq_skip_err()
424 	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);  in __arm_smmu_cmdq_skip_err()
429 	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);  in arm_smmu_cmdq_skip_err()
537 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in __arm_smmu_cmdq_poll_set_valid_map()
606 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_poll_until_not_full()
608 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
614 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
634 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));  in __arm_smmu_cmdq_poll_until_msi()
661 	llq->val = READ_ONCE(cmdq->q.llq.val);  in __arm_smmu_cmdq_poll_until_consumed()
696 		llq->cons = readl(cmdq->q.cons_reg);  in __arm_smmu_cmdq_poll_until_consumed()
716 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in arm_smmu_cmdq_write_entries()
724 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_write_entries()
755 	llq.max_n_shift = cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_issue_cmdlist()
759 	llq.val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_issue_cmdlist()
774 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);  in arm_smmu_cmdq_issue_cmdlist()
791 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);  in arm_smmu_cmdq_issue_cmdlist()
792 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_issue_cmdlist()
814 						   &cmdq->q.llq.atomic.prod);  in arm_smmu_cmdq_issue_cmdlist()
828 		writel_relaxed(prod, cmdq->q.prod_reg);  in arm_smmu_cmdq_issue_cmdlist()
846 					    readl_relaxed(cmdq->q.prod_reg),  in arm_smmu_cmdq_issue_cmdlist()
847 					    readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_issue_cmdlist()
855 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);  in arm_smmu_cmdq_issue_cmdlist()
1567 	struct arm_smmu_queue *q = &smmu->evtq.q;  in arm_smmu_evtq_thread()  local
1568 	struct arm_smmu_ll_queue *llq = &q->llq;  in arm_smmu_evtq_thread()
1574 		while (!queue_remove_raw(q, evt)) {  in arm_smmu_evtq_thread()
1593 		if (queue_sync_prod_in(q) == -EOVERFLOW)  in arm_smmu_evtq_thread()
1598 	queue_sync_cons_ovf(q);  in arm_smmu_evtq_thread()
1643 	struct arm_smmu_queue *q = &smmu->priq.q;  in arm_smmu_priq_thread()  local
1644 	struct arm_smmu_ll_queue *llq = &q->llq;  in arm_smmu_priq_thread()
1648 		while (!queue_remove_raw(q, evt))  in arm_smmu_priq_thread()
1651 		if (queue_sync_prod_in(q) == -EOVERFLOW)  in arm_smmu_priq_thread()
1656 	queue_sync_cons_ovf(q);  in arm_smmu_priq_thread()
2891 				   struct arm_smmu_queue *q,  in arm_smmu_init_one_queue()  argument
2900 		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;  in arm_smmu_init_one_queue()
2901 		q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,  in arm_smmu_init_one_queue()
2903 		if (q->base || qsz < PAGE_SIZE)  in arm_smmu_init_one_queue()
2906 		q->llq.max_n_shift--;  in arm_smmu_init_one_queue()
2909 	if (!q->base) {  in arm_smmu_init_one_queue()
2916 	if (!WARN_ON(q->base_dma & (qsz - 1))) {  in arm_smmu_init_one_queue()
2918 			 1 << q->llq.max_n_shift, name);  in arm_smmu_init_one_queue()
2921 	q->prod_reg	= page + prod_off;  in arm_smmu_init_one_queue()
2922 	q->cons_reg	= page + cons_off;  in arm_smmu_init_one_queue()
2923 	q->ent_dwords	= dwords;  in arm_smmu_init_one_queue()
2925 	q->q_base  = Q_BASE_RWA;  in arm_smmu_init_one_queue()
2926 	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;  in arm_smmu_init_one_queue()
2927 	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);  in arm_smmu_init_one_queue()
2929 	q->llq.prod = q->llq.cons = 0;  in arm_smmu_init_one_queue()
2936 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_init()
2954 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,  in arm_smmu_init_queues()
2965 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,  in arm_smmu_init_queues()
2982 	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,  in arm_smmu_init_queues()
3191 	smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);  in arm_smmu_setup_msis()
3193 	smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);  in arm_smmu_setup_msis()
3206 	irq = smmu->evtq.q.irq;  in arm_smmu_setup_unique_irqs()
3229 		irq = smmu->priq.q.irq;  in arm_smmu_setup_unique_irqs()
3339 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);  in arm_smmu_device_reset()
3340 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
3341 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);  in arm_smmu_device_reset()
3365 	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);  in arm_smmu_device_reset()
3366 	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);  in arm_smmu_device_reset()
3367 	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);  in arm_smmu_device_reset()
3379 		writeq_relaxed(smmu->priq.q.q_base,  in arm_smmu_device_reset()
3381 		writel_relaxed(smmu->priq.q.llq.prod,  in arm_smmu_device_reset()
3383 		writel_relaxed(smmu->priq.q.llq.cons,  in arm_smmu_device_reset()
3581 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3583 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {  in arm_smmu_device_hw_probe()
3595 	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3597 	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3860 			smmu->evtq.q.irq = irq;  in arm_smmu_device_probe()
3864 			smmu->priq.q.irq = irq;  in arm_smmu_device_probe()