161 	struct apple_nvme_queue *q;  member
208 static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q)  in queue_to_apple_nvme()  argument
210 	if (q->is_adminq)  in queue_to_apple_nvme()
211 		return container_of(q, struct apple_nvme, adminq);  in queue_to_apple_nvme()
213 	return container_of(q, struct apple_nvme, ioq);  in queue_to_apple_nvme()
216 static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q)  in apple_nvme_queue_depth()  argument
218 	if (q->is_adminq)  in apple_nvme_queue_depth()
273 static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag)  in apple_nvmmu_inval()  argument
275 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvmmu_inval()
283 static void apple_nvme_submit_cmd(struct apple_nvme_queue *q,  in apple_nvme_submit_cmd()  argument
286 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvme_submit_cmd()
288 	struct apple_nvmmu_tcb *tcb = &q->tcbs[tag];  in apple_nvme_submit_cmd()
301 	memcpy(&q->sqes[tag], cmd, sizeof(*cmd));  in apple_nvme_submit_cmd()
314 	writel(tag, q->sq_db);  in apple_nvme_submit_cmd()
528 	iod->nents = blk_rq_map_sg(req->q, req, iod->sg);  in apple_nvme_map_data()
552 	struct apple_nvme *anv = queue_to_apple_nvme(iod->q);  in apple_nvme_unmap_rq()
569 static inline bool apple_nvme_cqe_pending(struct apple_nvme_queue *q)  in apple_nvme_cqe_pending()  argument
571 	struct nvme_completion *hcqe = &q->cqes[q->cq_head];  in apple_nvme_cqe_pending()
573 	return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == q->cq_phase;  in apple_nvme_cqe_pending()
577 apple_nvme_queue_tagset(struct apple_nvme *anv, struct apple_nvme_queue *q)  in apple_nvme_queue_tagset()  argument
579 	if (q->is_adminq)  in apple_nvme_queue_tagset()
585 static inline void apple_nvme_handle_cqe(struct apple_nvme_queue *q,  in apple_nvme_handle_cqe()  argument
588 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvme_handle_cqe()
589 	struct nvme_completion *cqe = &q->cqes[idx];  in apple_nvme_handle_cqe()
593 	apple_nvmmu_inval(q, command_id);  in apple_nvme_handle_cqe()
595 	req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id);  in apple_nvme_handle_cqe()
607 static inline void apple_nvme_update_cq_head(struct apple_nvme_queue *q)  in apple_nvme_update_cq_head()  argument
609 	u32 tmp = q->cq_head + 1;  in apple_nvme_update_cq_head()
611 	if (tmp == apple_nvme_queue_depth(q)) {  in apple_nvme_update_cq_head()
612 		q->cq_head = 0;  in apple_nvme_update_cq_head()
613 		q->cq_phase ^= 1;  in apple_nvme_update_cq_head()
615 		q->cq_head = tmp;  in apple_nvme_update_cq_head()
619 static bool apple_nvme_poll_cq(struct apple_nvme_queue *q,  in apple_nvme_poll_cq()  argument
624 	while (apple_nvme_cqe_pending(q)) {  in apple_nvme_poll_cq()
632 		apple_nvme_handle_cqe(q, iob, q->cq_head);  in apple_nvme_poll_cq()
633 		apple_nvme_update_cq_head(q);  in apple_nvme_poll_cq()
637 		writel(q->cq_head, q->cq_db);  in apple_nvme_poll_cq()
642 static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force)  in apple_nvme_handle_cq()  argument
647 	if (!READ_ONCE(q->enabled) && !force)  in apple_nvme_handle_cq()
650 	found = apple_nvme_poll_cq(q, &iob);  in apple_nvme_handle_cq()
736 	struct apple_nvme_queue *q = hctx->driver_data;  in apple_nvme_queue_rq()  local
737 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvme_queue_rq()
750 	if (unlikely(!READ_ONCE(q->enabled)))  in apple_nvme_queue_rq()
767 	apple_nvme_submit_cmd(q, cmnd);  in apple_nvme_queue_rq()
786 	struct apple_nvme_queue *q = set->driver_data;  in apple_nvme_init_request()  local
787 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvme_init_request()
791 	iod->q = q;  in apple_nvme_init_request()
879 	struct apple_nvme_queue *q = iod->q;  in apple_nvme_timeout()  local
880 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvme_timeout()
901 			 req->tag, q->is_adminq);  in apple_nvme_timeout()
914 		apple_nvme_handle_cq(q, false);  in apple_nvme_timeout()
919 				 req->tag, q->is_adminq);  in apple_nvme_timeout()
929 		 req->tag, q->is_adminq);  in apple_nvme_timeout()
939 	struct apple_nvme_queue *q = hctx->driver_data;  in apple_nvme_poll()  local
940 	struct apple_nvme *anv = queue_to_apple_nvme(q);  in apple_nvme_poll()
945 	found = apple_nvme_poll_cq(q, iob);  in apple_nvme_poll()
968 static void apple_nvme_init_queue(struct apple_nvme_queue *q)  in apple_nvme_init_queue()  argument
970 	unsigned int depth = apple_nvme_queue_depth(q);  in apple_nvme_init_queue()
972 	q->cq_head = 0;  in apple_nvme_init_queue()
973 	q->cq_phase = 1;  in apple_nvme_init_queue()
974 	memset(q->tcbs, 0,  in apple_nvme_init_queue()
976 	memset(q->cqes, 0, depth * sizeof(struct nvme_completion));  in apple_nvme_init_queue()
977 	WRITE_ONCE(q->enabled, true);  in apple_nvme_init_queue()
1295 				  struct apple_nvme_queue *q)  in apple_nvme_queue_alloc()  argument
1297 	unsigned int depth = apple_nvme_queue_depth(q);  in apple_nvme_queue_alloc()
1299 	q->cqes = dmam_alloc_coherent(anv->dev,  in apple_nvme_queue_alloc()
1301 				      &q->cq_dma_addr, GFP_KERNEL);  in apple_nvme_queue_alloc()
1302 	if (!q->cqes)  in apple_nvme_queue_alloc()
1305 	q->sqes = dmam_alloc_coherent(anv->dev,  in apple_nvme_queue_alloc()
1307 				      &q->sq_dma_addr, GFP_KERNEL);  in apple_nvme_queue_alloc()
1308 	if (!q->sqes)  in apple_nvme_queue_alloc()
1315 	q->tcbs = dmam_alloc_coherent(anv->dev,  in apple_nvme_queue_alloc()
1318 				      &q->tcb_dma_addr, GFP_KERNEL);  in apple_nvme_queue_alloc()
1319 	if (!q->tcbs)  in apple_nvme_queue_alloc()
1326 	q->cq_phase = 1;  in apple_nvme_queue_alloc()