81 	void			*pdu;  member
127 	void			*pdu;  member
213 	return req->pdu;  in nvme_tcp_req_cmd_pdu()
219 	return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) -  in nvme_tcp_req_data_pdu()
410 		void *pdu, size_t len)  in nvme_tcp_hdgst()  argument
414 	sg_init_one(&sg, pdu, len);  in nvme_tcp_hdgst()
415 	ahash_request_set_crypt(hash, &sg, pdu + len, len);  in nvme_tcp_hdgst()
420 		void *pdu, size_t pdu_len)  in nvme_tcp_verify_hdgst()  argument
422 	struct nvme_tcp_hdr *hdr = pdu;  in nvme_tcp_verify_hdgst()
433 	recv_digest = *(__le32 *)(pdu + hdr->hlen);  in nvme_tcp_verify_hdgst()
434 	nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);  in nvme_tcp_verify_hdgst()
435 	exp_digest = *(__le32 *)(pdu + hdr->hlen);  in nvme_tcp_verify_hdgst()
446 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)  in nvme_tcp_check_ddgst()  argument
448 	struct nvme_tcp_hdr *hdr = pdu;  in nvme_tcp_check_ddgst()
471 	page_frag_free(req->pdu);  in nvme_tcp_exit_request()
480 	struct nvme_tcp_cmd_pdu *pdu;  in nvme_tcp_init_request()  local
485 	req->pdu = page_frag_alloc(&queue->pf_cache,  in nvme_tcp_init_request()
488 	if (!req->pdu)  in nvme_tcp_init_request()
491 	pdu = req->pdu;  in nvme_tcp_init_request()
494 	nvme_req(rq)->cmd = &pdu->cmd;  in nvme_tcp_init_request()
572 		struct nvme_tcp_data_pdu *pdu)  in nvme_tcp_handle_c2h_data()  argument
576 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_handle_c2h_data()
580 			pdu->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_handle_c2h_data()
591 	queue->data_remaining = le32_to_cpu(pdu->data_length);  in nvme_tcp_handle_c2h_data()
593 	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&  in nvme_tcp_handle_c2h_data()
594 	    unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {  in nvme_tcp_handle_c2h_data()
606 		struct nvme_tcp_rsp_pdu *pdu)  in nvme_tcp_handle_comp()  argument
608 	struct nvme_completion *cqe = &pdu->cqe;  in nvme_tcp_handle_comp()
662 		struct nvme_tcp_r2t_pdu *pdu)  in nvme_tcp_handle_r2t()  argument
666 	u32 r2t_length = le32_to_cpu(pdu->r2t_length);  in nvme_tcp_handle_r2t()
667 	u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);  in nvme_tcp_handle_r2t()
669 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_handle_r2t()
673 			pdu->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_handle_r2t()
702 	req->ttag = pdu->ttag;  in nvme_tcp_handle_r2t()
714 	char *pdu = queue->pdu;  in nvme_tcp_recv_pdu()  local
719 		&pdu[queue->pdu_offset], rcv_len);  in nvme_tcp_recv_pdu()
730 	hdr = queue->pdu;  in nvme_tcp_recv_pdu()
732 		ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);  in nvme_tcp_recv_pdu()
739 		ret = nvme_tcp_check_ddgst(queue, queue->pdu);  in nvme_tcp_recv_pdu()
746 		return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
749 		return nvme_tcp_handle_comp(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
752 		return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
771 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;  in nvme_tcp_recv_data()  local
773 		nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_recv_data()
827 			if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {  in nvme_tcp_recv_data()
842 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;  in nvme_tcp_recv_ddgst()  local
860 					pdu->command_id);  in nvme_tcp_recv_ddgst()
871 	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {  in nvme_tcp_recv_ddgst()
873 					pdu->command_id);  in nvme_tcp_recv_ddgst()
1059 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);  in nvme_tcp_try_send_cmd_pdu()  local
1064 	int len = sizeof(*pdu) + hdgst - req->offset;  in nvme_tcp_try_send_cmd_pdu()
1073 		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvme_tcp_try_send_cmd_pdu()
1075 	bvec_set_virt(&bvec, (void *)pdu + req->offset, len);  in nvme_tcp_try_send_cmd_pdu()
1100 	struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req);  in nvme_tcp_try_send_data_pdu()  local
1104 	int len = sizeof(*pdu) - req->offset + hdgst;  in nvme_tcp_try_send_data_pdu()
1108 		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));  in nvme_tcp_try_send_data_pdu()
1113 	bvec_set_virt(&bvec, (void *)pdu + req->offset, len);  in nvme_tcp_try_send_data_pdu()
1302 	page_frag_free(async->pdu);  in nvme_tcp_free_async_req()
1311 	async->pdu = page_frag_alloc(&queue->pf_cache,  in nvme_tcp_alloc_async_req()
1314 	if (!async->pdu)  in nvme_tcp_alloc_async_req()
1344 	kfree(queue->pdu);  in nvme_tcp_free_queue()
1610 	queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);  in nvme_tcp_alloc_queue()
1611 	if (!queue->pdu) {  in nvme_tcp_alloc_queue()
1638 	kfree(queue->pdu);  in nvme_tcp_alloc_queue()
1728 	if (to_tcp_ctrl(ctrl)->async_req.pdu) {  in nvme_tcp_free_admin_queue()
1731 		to_tcp_ctrl(ctrl)->async_req.pdu = NULL;  in nvme_tcp_free_admin_queue()
2238 	struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;  in nvme_tcp_submit_async_event()  local
2239 	struct nvme_command *cmd = &pdu->cmd;  in nvme_tcp_submit_async_event()
2242 	memset(pdu, 0, sizeof(*pdu));  in nvme_tcp_submit_async_event()
2243 	pdu->hdr.type = nvme_tcp_cmd;  in nvme_tcp_submit_async_event()
2245 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvme_tcp_submit_async_event()
2246 	pdu->hdr.hlen = sizeof(*pdu);  in nvme_tcp_submit_async_event()
2247 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);  in nvme_tcp_submit_async_event()
2275 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);  in nvme_tcp_timeout()  local
2276 	u8 opc = pdu->cmd.common.opcode, fctype = pdu->cmd.fabrics.fctype;  in nvme_tcp_timeout()
2281 		nvme_tcp_queue_id(req->queue), nvme_cid(rq), pdu->hdr.type,  in nvme_tcp_timeout()
2314 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);  in nvme_tcp_map_data()  local
2315 	struct nvme_command *c = &pdu->cmd;  in nvme_tcp_map_data()
2334 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);  in nvme_tcp_setup_cmd_pdu()  local
2360 	pdu->hdr.type = nvme_tcp_cmd;  in nvme_tcp_setup_cmd_pdu()
2361 	pdu->hdr.flags = 0;  in nvme_tcp_setup_cmd_pdu()
2363 		pdu->hdr.flags |= NVME_TCP_F_HDGST;  in nvme_tcp_setup_cmd_pdu()
2365 		pdu->hdr.flags |= NVME_TCP_F_DDGST;  in nvme_tcp_setup_cmd_pdu()
2368 	pdu->hdr.hlen = sizeof(*pdu);  in nvme_tcp_setup_cmd_pdu()
2369 	pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;  in nvme_tcp_setup_cmd_pdu()
2370 	pdu->hdr.plen =  in nvme_tcp_setup_cmd_pdu()
2371 		cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);  in nvme_tcp_setup_cmd_pdu()