nvme/host/tcp.c

81 	void			*pdu;
127 	void			*pdu;
213 	return req->pdu;
218 	/* use the pdu space in the back for the data pdu */
219 	return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) -
410 		void *pdu, size_t len)
414 	sg_init_one(&sg, pdu, len);
415 	ahash_request_set_crypt(hash, &sg, pdu + len, len);
420 		void *pdu, size_t pdu_len)
422 	struct nvme_tcp_hdr *hdr = pdu;
433 	recv_digest = *(__le32 *)(pdu + hdr->hlen);
434 	nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
435 	exp_digest = *(__le32 *)(pdu + hdr->hlen);
446 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
448 	struct nvme_tcp_hdr *hdr = pdu;
471 	page_frag_free(req->pdu);
480 	struct nvme_tcp_cmd_pdu *pdu;
485 	req->pdu = page_frag_alloc(&queue->pf_cache,
488 	if (!req->pdu)
491 	pdu = req->pdu;
494 	nvme_req(rq)->cmd = &pdu->cmd;
572 		struct nvme_tcp_data_pdu *pdu)
576 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
580 			pdu->command_id, nvme_tcp_queue_id(queue));
591 	queue->data_remaining = le32_to_cpu(pdu->data_length);
593 	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
594 	    unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
606 		struct nvme_tcp_rsp_pdu *pdu)
608 	struct nvme_completion *cqe = &pdu->cqe;
662 		struct nvme_tcp_r2t_pdu *pdu)
666 	u32 r2t_length = le32_to_cpu(pdu->r2t_length);
667 	u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
669 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
673 			pdu->command_id, nvme_tcp_queue_id(queue));
702 	req->ttag = pdu->ttag;
711 		struct nvme_tcp_term_pdu *pdu)
715 	u32 plen = le32_to_cpu(pdu->hdr.plen);
734 	fes = le16_to_cpu(pdu->fes);
748 	char *pdu = queue->pdu;
753 		&pdu[queue->pdu_offset], rcv_len);
764 	hdr = queue->pdu;
770 		nvme_tcp_handle_c2h_term(queue, (void *)queue->pdu);
775 		ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
782 		ret = nvme_tcp_check_ddgst(queue, queue->pdu);
789 		return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
792 		return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
795 		return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
798 			"unsupported pdu type (%d)\n", hdr->type);
814 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
816 		nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
870 			if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
885 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
903 					pdu->command_id);
914 	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
916 					pdu->command_id);
1102 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
1107 	int len = sizeof(*pdu) + hdgst - req->offset;
1116 		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1118 	bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1143 	struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req);
1147 	int len = sizeof(*pdu) - req->offset + hdgst;
1151 		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1156 	bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1345 	page_frag_free(async->pdu);
1354 	async->pdu = page_frag_alloc(&queue->pf_cache,
1357 	if (!async->pdu)
1387 	kfree(queue->pdu);
1446 		pr_err("queue %d: bad pdu length returned %d\n",
1653 	queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1654 	if (!queue->pdu) {
1681 	kfree(queue->pdu);
1771 	if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1774 		to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
2281 	struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2282 	struct nvme_command *cmd = &pdu->cmd;
2285 	memset(pdu, 0, sizeof(*pdu));
2286 	pdu->hdr.type = nvme_tcp_cmd;
2288 		pdu->hdr.flags |= NVME_TCP_F_HDGST;
2289 	pdu->hdr.hlen = sizeof(*pdu);
2290 	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2318 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2319 	u8 opc = pdu->cmd.common.opcode, fctype = pdu->cmd.fabrics.fctype;
2324 		nvme_tcp_queue_id(req->queue), nvme_cid(rq), pdu->hdr.type,
2357 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2358 	struct nvme_command *c = &pdu->cmd;
2377 	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2403 	pdu->hdr.type = nvme_tcp_cmd;
2404 	pdu->hdr.flags = 0;
2406 		pdu->hdr.flags |= NVME_TCP_F_HDGST;
2408 		pdu->hdr.flags |= NVME_TCP_F_DDGST;
2411 	pdu->hdr.hlen = sizeof(*pdu);
2412 	pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2413 	pdu->hdr.plen =
2414 		cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);