1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 /* Copyright 2021 Marvell. All rights reserved. */
3 
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/list.h>
8 #include <linux/mm.h>
9 #include <linux/types.h>
10 #include <asm/byteorder.h>
11 #include <linux/qed/common_hsi.h>
12 #include <linux/qed/storage_common.h>
13 #include <linux/qed/nvmetcp_common.h>
14 #include <linux/qed/qed_nvmetcp_if.h>
15 #include "qed_nvmetcp_fw_funcs.h"
16 
17 #define NVMETCP_NUM_SGES_IN_CACHE 0x4
18 
19 bool nvmetcp_is_slow_sgl(u16 num_sges, bool small_mid_sge)
20 {
21 	return (num_sges > SCSI_NUM_SGES_SLOW_SGL_THR && small_mid_sge);
22 }
23 
24 void init_scsi_sgl_context(struct scsi_sgl_params *ctx_sgl_params,
25 			   struct scsi_cached_sges *ctx_data_desc,
26 			   struct storage_sgl_task_params *sgl_params)
27 {
28 	u8 num_sges_to_init = (u8)(sgl_params->num_sges > NVMETCP_NUM_SGES_IN_CACHE ?
29 				   NVMETCP_NUM_SGES_IN_CACHE : sgl_params->num_sges);
30 	u8 sge_index;
31 
32 	/* sgl params */
33 	ctx_sgl_params->sgl_addr.lo = cpu_to_le32(sgl_params->sgl_phys_addr.lo);
34 	ctx_sgl_params->sgl_addr.hi = cpu_to_le32(sgl_params->sgl_phys_addr.hi);
35 	ctx_sgl_params->sgl_total_length = cpu_to_le32(sgl_params->total_buffer_size);
36 	ctx_sgl_params->sgl_num_sges = cpu_to_le16(sgl_params->num_sges);
37 
38 	for (sge_index = 0; sge_index < num_sges_to_init; sge_index++) {
39 		ctx_data_desc->sge[sge_index].sge_addr.lo =
40 			cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.lo);
41 		ctx_data_desc->sge[sge_index].sge_addr.hi =
42 			cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.hi);
43 		ctx_data_desc->sge[sge_index].sge_len =
44 			cpu_to_le32(sgl_params->sgl[sge_index].sge_len);
45 	}
46 }
47 
48 static inline u32 calc_rw_task_size(struct nvmetcp_task_params *task_params,
49 				    enum nvmetcp_task_type task_type)
50 {
51 	u32 io_size;
52 
53 	if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE)
54 		io_size = task_params->tx_io_size;
55 	else
56 		io_size = task_params->rx_io_size;
57 
58 	if (unlikely(!io_size))
59 		return 0;
60 
61 	return io_size;
62 }
63 
64 static inline void init_sqe(struct nvmetcp_task_params *task_params,
65 			    struct storage_sgl_task_params *sgl_task_params,
66 			    enum nvmetcp_task_type task_type)
67 {
68 	if (!task_params->sqe)
69 		return;
70 
71 	memset(task_params->sqe, 0, sizeof(*task_params->sqe));
72 	task_params->sqe->task_id = cpu_to_le16(task_params->itid);
73 
74 	switch (task_type) {
75 	case NVMETCP_TASK_TYPE_HOST_WRITE: {
76 		u32 buf_size = 0;
77 		u32 num_sges = 0;
78 
79 		SET_FIELD(task_params->sqe->contlen_cdbsize,
80 			  NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
81 		SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
82 			  NVMETCP_WQE_TYPE_NORMAL);
83 		if (task_params->tx_io_size) {
84 			if (task_params->send_write_incapsule)
85 				buf_size = calc_rw_task_size(task_params, task_type);
86 
87 			if (nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
88 						sgl_task_params->small_mid_sge))
89 				num_sges = NVMETCP_WQE_NUM_SGES_SLOWIO;
90 			else
91 				num_sges = min((u16)sgl_task_params->num_sges,
92 					       (u16)SCSI_NUM_SGES_SLOW_SGL_THR);
93 		}
94 		SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES, num_sges);
95 		SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN, buf_size);
96 	} break;
97 
98 	case NVMETCP_TASK_TYPE_HOST_READ: {
99 		SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
100 			  NVMETCP_WQE_TYPE_NORMAL);
101 		SET_FIELD(task_params->sqe->contlen_cdbsize,
102 			  NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
103 	} break;
104 
105 	case NVMETCP_TASK_TYPE_INIT_CONN_REQUEST: {
106 		SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
107 			  NVMETCP_WQE_TYPE_MIDDLE_PATH);
108 
109 		if (task_params->tx_io_size) {
110 			SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN,
111 				  task_params->tx_io_size);
112 			SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES,
113 				  min((u16)sgl_task_params->num_sges,
114 				      (u16)SCSI_NUM_SGES_SLOW_SGL_THR));
115 		}
116 	} break;
117 
118 	case NVMETCP_TASK_TYPE_CLEANUP:
119 		SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
120 			  NVMETCP_WQE_TYPE_TASK_CLEANUP);
121 
122 	default:
123 		break;
124 	}
125 }
126 
127 /* The following function initializes of NVMeTCP task params */
128 static inline void
129 init_nvmetcp_task_params(struct e5_nvmetcp_task_context *context,
130 			 struct nvmetcp_task_params *task_params,
131 			 enum nvmetcp_task_type task_type)
132 {
133 	context->ystorm_st_context.state.cccid = task_params->host_cccid;
134 	SET_FIELD(context->ustorm_st_context.error_flags, USTORM_NVMETCP_TASK_ST_CTX_NVME_TCP, 1);
135 	context->ustorm_st_context.nvme_tcp_opaque_lo = cpu_to_le32(task_params->opq.lo);
136 	context->ustorm_st_context.nvme_tcp_opaque_hi = cpu_to_le32(task_params->opq.hi);
137 }
138 
139 /* The following function initializes default values to all tasks */
140 static inline void
141 init_default_nvmetcp_task(struct nvmetcp_task_params *task_params,
142 			  void *pdu_header, void *nvme_cmd,
143 			  enum nvmetcp_task_type task_type)
144 {
145 	struct e5_nvmetcp_task_context *context = task_params->context;
146 	const u8 val_byte = context->mstorm_ag_context.cdu_validation;
147 	u8 dw_index;
148 
149 	memset(context, 0, sizeof(*context));
150 	init_nvmetcp_task_params(context, task_params,
151 				 (enum nvmetcp_task_type)task_type);
152 
153 	/* Swapping requirements used below, will be removed in future FW versions */
154 	if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE ||
155 	    task_type == NVMETCP_TASK_TYPE_HOST_READ) {
156 		for (dw_index = 0;
157 		     dw_index < QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
158 		     dw_index++)
159 			context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
160 				cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
161 
162 		for (dw_index = QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
163 		     dw_index < QED_NVMETCP_CMD_HDR_SIZE / sizeof(u32);
164 		     dw_index++)
165 			context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
166 				cpu_to_le32(__swab32(((u32 *)nvme_cmd)[dw_index - 2]));
167 	} else {
168 		for (dw_index = 0;
169 		     dw_index < QED_NVMETCP_NON_IO_HDR_SIZE / sizeof(u32);
170 		     dw_index++)
171 			context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
172 				cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
173 	}
174 
175 	/* M-Storm Context: */
176 	context->mstorm_ag_context.cdu_validation = val_byte;
177 	context->mstorm_st_context.task_type = (u8)(task_type);
178 	context->mstorm_ag_context.task_cid = cpu_to_le16(task_params->conn_icid);
179 
180 	/* Ustorm Context: */
181 	SET_FIELD(context->ustorm_ag_context.flags1, E5_USTORM_NVMETCP_TASK_AG_CTX_R2T2RECV, 1);
182 	context->ustorm_st_context.task_type = (u8)(task_type);
183 	context->ustorm_st_context.cq_rss_number = task_params->cq_rss_number;
184 	context->ustorm_ag_context.icid = cpu_to_le16(task_params->conn_icid);
185 }
186 
187 /* The following function initializes the U-Storm Task Contexts */
188 static inline void
189 init_ustorm_task_contexts(struct ustorm_nvmetcp_task_st_ctx *ustorm_st_context,
190 			  struct e5_ustorm_nvmetcp_task_ag_ctx *ustorm_ag_context,
191 			  u32 remaining_recv_len,
192 			  u32 expected_data_transfer_len, u8 num_sges,
193 			  bool tx_dif_conn_err_en)
194 {
195 	/* Remaining data to be received in bytes. Used in validations*/
196 	ustorm_st_context->rem_rcv_len = cpu_to_le32(remaining_recv_len);
197 	ustorm_ag_context->exp_data_acked = cpu_to_le32(expected_data_transfer_len);
198 	ustorm_st_context->exp_data_transfer_len = cpu_to_le32(expected_data_transfer_len);
199 	SET_FIELD(ustorm_st_context->reg1_map, REG1_NUM_SGES, num_sges);
200 	SET_FIELD(ustorm_ag_context->flags2, E5_USTORM_NVMETCP_TASK_AG_CTX_DIF_ERROR_CF_EN,
201 		  tx_dif_conn_err_en ? 1 : 0);
202 }
203 
204 /* The following function initializes Local Completion Contexts: */
205 static inline void
206 set_local_completion_context(struct e5_nvmetcp_task_context *context)
207 {
208 	SET_FIELD(context->ystorm_st_context.state.flags,
209 		  YSTORM_NVMETCP_TASK_STATE_LOCAL_COMP, 1);
210 	SET_FIELD(context->ustorm_st_context.flags,
211 		  USTORM_NVMETCP_TASK_ST_CTX_LOCAL_COMP, 1);
212 }
213 
214 /* Common Fastpath task init function: */
215 static inline void
216 init_rw_nvmetcp_task(struct nvmetcp_task_params *task_params,
217 		     enum nvmetcp_task_type task_type,
218 		     void *pdu_header, void *nvme_cmd,
219 		     struct storage_sgl_task_params *sgl_task_params)
220 {
221 	struct e5_nvmetcp_task_context *context = task_params->context;
222 	u32 task_size = calc_rw_task_size(task_params, task_type);
223 	bool slow_io = false;
224 	u8 num_sges = 0;
225 
226 	init_default_nvmetcp_task(task_params, pdu_header, nvme_cmd, task_type);
227 
228 	/* Tx/Rx: */
229 	if (task_params->tx_io_size) {
230 		/* if data to transmit: */
231 		init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
232 				      &context->ystorm_st_context.state.data_desc,
233 				      sgl_task_params);
234 		slow_io = nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
235 					      sgl_task_params->small_mid_sge);
236 		num_sges =
237 			(u8)(!slow_io ? min((u32)sgl_task_params->num_sges,
238 					    (u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
239 					    NVMETCP_WQE_NUM_SGES_SLOWIO);
240 		if (slow_io) {
241 			SET_FIELD(context->ystorm_st_context.state.flags,
242 				  YSTORM_NVMETCP_TASK_STATE_SLOW_IO, 1);
243 		}
244 	} else if (task_params->rx_io_size) {
245 		/* if data to receive: */
246 		init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
247 				      &context->mstorm_st_context.data_desc,
248 				      sgl_task_params);
249 		num_sges =
250 			(u8)(!nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
251 						  sgl_task_params->small_mid_sge) ?
252 						  min((u32)sgl_task_params->num_sges,
253 						      (u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
254 						      NVMETCP_WQE_NUM_SGES_SLOWIO);
255 		context->mstorm_st_context.rem_task_size = cpu_to_le32(task_size);
256 	}
257 
258 	/* Ustorm context: */
259 	init_ustorm_task_contexts(&context->ustorm_st_context,
260 				  &context->ustorm_ag_context,
261 				  /* Remaining Receive length is the Task Size */
262 				  task_size,
263 				  /* The size of the transmitted task */
264 				  task_size,
265 				  /* num_sges */
266 				  num_sges,
267 				  false);
268 
269 	/* Set exp_data_acked */
270 	if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE) {
271 		if (task_params->send_write_incapsule)
272 			context->ustorm_ag_context.exp_data_acked = task_size;
273 		else
274 			context->ustorm_ag_context.exp_data_acked = 0;
275 	} else if (task_type == NVMETCP_TASK_TYPE_HOST_READ) {
276 		context->ustorm_ag_context.exp_data_acked = 0;
277 	}
278 
279 	context->ustorm_ag_context.exp_cont_len = 0;
280 	init_sqe(task_params, sgl_task_params, task_type);
281 }
282 
283 static void
284 init_common_initiator_read_task(struct nvmetcp_task_params *task_params,
285 				struct nvme_tcp_cmd_pdu *cmd_pdu_header,
286 				struct nvme_command *nvme_cmd,
287 				struct storage_sgl_task_params *sgl_task_params)
288 {
289 	init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_READ,
290 			     cmd_pdu_header, nvme_cmd, sgl_task_params);
291 }
292 
293 void init_nvmetcp_host_read_task(struct nvmetcp_task_params *task_params,
294 				 struct nvme_tcp_cmd_pdu *cmd_pdu_header,
295 				 struct nvme_command *nvme_cmd,
296 				 struct storage_sgl_task_params *sgl_task_params)
297 {
298 	init_common_initiator_read_task(task_params, (void *)cmd_pdu_header,
299 					(void *)nvme_cmd, sgl_task_params);
300 }
301 
302 static void
303 init_common_initiator_write_task(struct nvmetcp_task_params *task_params,
304 				 struct nvme_tcp_cmd_pdu *cmd_pdu_header,
305 				 struct nvme_command *nvme_cmd,
306 				 struct storage_sgl_task_params *sgl_task_params)
307 {
308 	init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_WRITE,
309 			     cmd_pdu_header, nvme_cmd, sgl_task_params);
310 }
311 
312 void init_nvmetcp_host_write_task(struct nvmetcp_task_params *task_params,
313 				  struct nvme_tcp_cmd_pdu *cmd_pdu_header,
314 				  struct nvme_command *nvme_cmd,
315 				  struct storage_sgl_task_params *sgl_task_params)
316 {
317 	init_common_initiator_write_task(task_params, (void *)cmd_pdu_header,
318 					 (void *)nvme_cmd, sgl_task_params);
319 }
320 
321 static void
322 init_common_login_request_task(struct nvmetcp_task_params *task_params,
323 			       void *login_req_pdu_header,
324 			       struct storage_sgl_task_params *tx_sgl_task_params,
325 			       struct storage_sgl_task_params *rx_sgl_task_params)
326 {
327 	struct e5_nvmetcp_task_context *context = task_params->context;
328 
329 	init_default_nvmetcp_task(task_params, (void *)login_req_pdu_header, NULL,
330 				  NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
331 
332 	/* Ustorm Context: */
333 	init_ustorm_task_contexts(&context->ustorm_st_context,
334 				  &context->ustorm_ag_context,
335 
336 				  /* Remaining Receive length is the Task Size */
337 				  task_params->rx_io_size ?
338 				  rx_sgl_task_params->total_buffer_size : 0,
339 
340 				  /* The size of the transmitted task */
341 				  task_params->tx_io_size ?
342 				  tx_sgl_task_params->total_buffer_size : 0,
343 				  0, /* num_sges */
344 				  0); /* tx_dif_conn_err_en */
345 
346 	/* SGL context: */
347 	if (task_params->tx_io_size)
348 		init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
349 				      &context->ystorm_st_context.state.data_desc,
350 				      tx_sgl_task_params);
351 	if (task_params->rx_io_size)
352 		init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
353 				      &context->mstorm_st_context.data_desc,
354 				      rx_sgl_task_params);
355 
356 	context->mstorm_st_context.rem_task_size =
357 		cpu_to_le32(task_params->rx_io_size ?
358 				 rx_sgl_task_params->total_buffer_size : 0);
359 	init_sqe(task_params, tx_sgl_task_params, NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
360 }
361 
362 /* The following function initializes Login task in Host mode: */
363 void init_nvmetcp_init_conn_req_task(struct nvmetcp_task_params *task_params,
364 				     struct nvme_tcp_icreq_pdu *init_conn_req_pdu_hdr,
365 				     struct storage_sgl_task_params *tx_sgl_task_params,
366 				     struct storage_sgl_task_params *rx_sgl_task_params)
367 {
368 	init_common_login_request_task(task_params, init_conn_req_pdu_hdr,
369 				       tx_sgl_task_params, rx_sgl_task_params);
370 }
371 
372 void init_cleanup_task_nvmetcp(struct nvmetcp_task_params *task_params)
373 {
374 	init_sqe(task_params, NULL, NVMETCP_TASK_TYPE_CLEANUP);
375 }
376