337 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_iod_list()  local
339 	return (void **)(iod->sg + blk_rq_nr_phys_segments(req));  in apple_nvme_iod_list()
345 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_free_prps()  local
346 	dma_addr_t dma_addr = iod->first_dma;  in apple_nvme_free_prps()
349 	for (i = 0; i < iod->npages; i++) {  in apple_nvme_free_prps()
360 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_unmap_data()  local
362 	if (iod->dma_len) {  in apple_nvme_unmap_data()
363 		dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len,  in apple_nvme_unmap_data()
368 	WARN_ON_ONCE(!iod->nents);  in apple_nvme_unmap_data()
370 	dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));  in apple_nvme_unmap_data()
371 	if (iod->npages == 0)  in apple_nvme_unmap_data()
373 			      iod->first_dma);  in apple_nvme_unmap_data()
376 	mempool_free(iod->sg, anv->iod_mempool);  in apple_nvme_unmap_data()
397 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_setup_prps()  local
400 	struct scatterlist *sg = iod->sg;  in apple_nvme_setup_prps()
411 		iod->first_dma = 0;  in apple_nvme_setup_prps()
425 		iod->first_dma = dma_addr;  in apple_nvme_setup_prps()
432 		iod->npages = 0;  in apple_nvme_setup_prps()
435 		iod->npages = 1;  in apple_nvme_setup_prps()
440 		iod->first_dma = dma_addr;  in apple_nvme_setup_prps()
441 		iod->npages = -1;  in apple_nvme_setup_prps()
445 	iod->first_dma = prp_dma;  in apple_nvme_setup_prps()
454 			list[iod->npages++] = prp_list;  in apple_nvme_setup_prps()
474 	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));  in apple_nvme_setup_prps()
475 	cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);  in apple_nvme_setup_prps()
481 	WARN(DO_ONCE(apple_nvme_print_sgl, iod->sg, iod->nents),  in apple_nvme_setup_prps()
483 	     iod->nents);  in apple_nvme_setup_prps()
492 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_setup_prp_simple()  local
496 	iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0);  in apple_nvme_setup_prp_simple()
497 	if (dma_mapping_error(anv->dev, iod->first_dma))  in apple_nvme_setup_prp_simple()
499 	iod->dma_len = bv->bv_len;  in apple_nvme_setup_prp_simple()
501 	cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);  in apple_nvme_setup_prp_simple()
503 		cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);  in apple_nvme_setup_prp_simple()
511 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_map_data()  local
523 	iod->dma_len = 0;  in apple_nvme_map_data()
524 	iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC);  in apple_nvme_map_data()
525 	if (!iod->sg)  in apple_nvme_map_data()
527 	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));  in apple_nvme_map_data()
528 	iod->nents = blk_rq_map_sg(req->q, req, iod->sg);  in apple_nvme_map_data()
529 	if (!iod->nents)  in apple_nvme_map_data()
532 	nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents,  in apple_nvme_map_data()
543 	dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));  in apple_nvme_map_data()
545 	mempool_free(iod->sg, anv->iod_mempool);  in apple_nvme_map_data()
551 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_unmap_rq()  local
552 	struct apple_nvme *anv = queue_to_apple_nvme(iod->q);  in apple_nvme_unmap_rq()
739 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_queue_rq()  local
740 	struct nvme_command *cmnd = &iod->cmd;  in apple_nvme_queue_rq()
743 	iod->npages = -1;  in apple_nvme_queue_rq()
744 	iod->nents = 0;  in apple_nvme_queue_rq()
788 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_init_request()  local
791 	iod->q = q;  in apple_nvme_init_request()
793 	nreq->cmd = &iod->cmd;  in apple_nvme_init_request()
878 	struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);  in apple_nvme_timeout()  local
879 	struct apple_nvme_queue *q = iod->q;  in apple_nvme_timeout()