1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 /* QLogic qed NIC Driver
3  * Copyright (c) 2015-2017  QLogic Corporation
4  * Copyright (c) 2019-2020 Marvell International Ltd.
5  */
6 
7 #include <linux/types.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/kernel.h>
10 #include <linux/list.h>
11 #include <linux/pci.h>
12 #include <linux/slab.h>
13 #include <linux/string.h>
14 #include "qed.h"
15 #include "qed_iscsi.h"
16 #include "qed_ll2.h"
17 #include "qed_ooo.h"
18 #include "qed_cxt.h"
19 #include "qed_nvmetcp.h"
20 static struct qed_ooo_archipelago
21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn,
22 			  struct qed_ooo_info
23 			  *p_ooo_info,
24 			  u32 cid)
25 {
26 	u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
27 	struct qed_ooo_archipelago *p_archipelago;
28 
29 	if (unlikely(idx >= p_ooo_info->max_num_archipelagos))
30 		return NULL;
31 
32 	p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
33 
34 	if (unlikely(list_empty(&p_archipelago->isles_list)))
35 		return NULL;
36 
37 	return p_archipelago;
38 }
39 
40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn,
41 					      struct qed_ooo_info *p_ooo_info,
42 					      u32 cid, u8 isle)
43 {
44 	struct qed_ooo_archipelago *p_archipelago = NULL;
45 	struct qed_ooo_isle *p_isle = NULL;
46 	u8 the_num_of_isle = 1;
47 
48 	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
49 	if (unlikely(!p_archipelago)) {
50 		DP_NOTICE(p_hwfn,
51 			  "Connection %d is not found in OOO list\n", cid);
52 		return NULL;
53 	}
54 
55 	list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) {
56 		if (the_num_of_isle == isle)
57 			return p_isle;
58 		the_num_of_isle++;
59 	}
60 
61 	return NULL;
62 }
63 
64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn,
65 				struct qed_ooo_info *p_ooo_info,
66 				struct ooo_opaque *p_cqe)
67 {
68 	struct qed_ooo_history *p_history = &p_ooo_info->ooo_history;
69 
70 	if (p_history->head_idx == p_history->num_of_cqes)
71 		p_history->head_idx = 0;
72 	p_history->p_cqes[p_history->head_idx] = *p_cqe;
73 	p_history->head_idx++;
74 }
75 
76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn)
77 {
78 	u16 max_num_archipelagos = 0, cid_base;
79 	struct qed_ooo_info *p_ooo_info;
80 	enum protocol_type proto;
81 	u16 max_num_isles = 0;
82 	u32 i;
83 
84 	switch (p_hwfn->hw_info.personality) {
85 	case QED_PCI_ISCSI:
86 	case QED_PCI_NVMETCP:
87 		proto = PROTOCOLID_TCP_ULP;
88 		break;
89 	case QED_PCI_ETH_RDMA:
90 	case QED_PCI_ETH_IWARP:
91 		proto = PROTOCOLID_IWARP;
92 		break;
93 	default:
94 		DP_NOTICE(p_hwfn,
95 			  "Failed to allocate qed_ooo_info: unknown personality\n");
96 		return -EINVAL;
97 	}
98 
99 	max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto,
100 								NULL);
101 	max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos;
102 	cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto);
103 
104 	if (!max_num_archipelagos) {
105 		DP_NOTICE(p_hwfn,
106 			  "Failed to allocate qed_ooo_info: unknown amount of connections\n");
107 		return -EINVAL;
108 	}
109 
110 	p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL);
111 	if (!p_ooo_info)
112 		return -ENOMEM;
113 
114 	p_ooo_info->cid_base = cid_base;
115 	p_ooo_info->max_num_archipelagos = max_num_archipelagos;
116 
117 	INIT_LIST_HEAD(&p_ooo_info->free_buffers_list);
118 	INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list);
119 	INIT_LIST_HEAD(&p_ooo_info->free_isles_list);
120 
121 	p_ooo_info->p_isles_mem = kcalloc(max_num_isles,
122 					  sizeof(struct qed_ooo_isle),
123 					  GFP_KERNEL);
124 	if (!p_ooo_info->p_isles_mem)
125 		goto no_isles_mem;
126 
127 	for (i = 0; i < max_num_isles; i++) {
128 		INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list);
129 		list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry,
130 			      &p_ooo_info->free_isles_list);
131 	}
132 
133 	p_ooo_info->p_archipelagos_mem =
134 				kcalloc(max_num_archipelagos,
135 					sizeof(struct qed_ooo_archipelago),
136 					GFP_KERNEL);
137 	if (!p_ooo_info->p_archipelagos_mem)
138 		goto no_archipelagos_mem;
139 
140 	for (i = 0; i < max_num_archipelagos; i++)
141 		INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list);
142 
143 	p_ooo_info->ooo_history.p_cqes =
144 				kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES,
145 					sizeof(struct ooo_opaque),
146 					GFP_KERNEL);
147 	if (!p_ooo_info->ooo_history.p_cqes)
148 		goto no_history_mem;
149 
150 	p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES;
151 
152 	p_hwfn->p_ooo_info = p_ooo_info;
153 	return 0;
154 
155 no_history_mem:
156 	kfree(p_ooo_info->p_archipelagos_mem);
157 no_archipelagos_mem:
158 	kfree(p_ooo_info->p_isles_mem);
159 no_isles_mem:
160 	kfree(p_ooo_info);
161 	return -ENOMEM;
162 }
163 
164 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn,
165 				      struct qed_ooo_info *p_ooo_info, u32 cid)
166 {
167 	struct qed_ooo_archipelago *p_archipelago;
168 	struct qed_ooo_buffer *p_buffer;
169 	struct qed_ooo_isle *p_isle;
170 
171 	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
172 	if (!p_archipelago)
173 		return;
174 
175 	while (!list_empty(&p_archipelago->isles_list)) {
176 		p_isle = list_first_entry(&p_archipelago->isles_list,
177 					  struct qed_ooo_isle, list_entry);
178 
179 		list_del(&p_isle->list_entry);
180 
181 		while (!list_empty(&p_isle->buffers_list)) {
182 			p_buffer = list_first_entry(&p_isle->buffers_list,
183 						    struct qed_ooo_buffer,
184 						    list_entry);
185 
186 			if (!p_buffer)
187 				break;
188 
189 			list_move_tail(&p_buffer->list_entry,
190 				       &p_ooo_info->free_buffers_list);
191 		}
192 		list_add_tail(&p_isle->list_entry,
193 			      &p_ooo_info->free_isles_list);
194 	}
195 }
196 
197 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn,
198 			       struct qed_ooo_info *p_ooo_info)
199 {
200 	struct qed_ooo_archipelago *p_archipelago;
201 	struct qed_ooo_buffer *p_buffer;
202 	struct qed_ooo_isle *p_isle;
203 	u32 i;
204 
205 	for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
206 		p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
207 
208 		while (!list_empty(&p_archipelago->isles_list)) {
209 			p_isle = list_first_entry(&p_archipelago->isles_list,
210 						  struct qed_ooo_isle,
211 						  list_entry);
212 
213 			list_del(&p_isle->list_entry);
214 
215 			while (!list_empty(&p_isle->buffers_list)) {
216 				p_buffer =
217 				    list_first_entry(&p_isle->buffers_list,
218 						     struct qed_ooo_buffer,
219 						     list_entry);
220 
221 				if (!p_buffer)
222 					break;
223 
224 				list_move_tail(&p_buffer->list_entry,
225 					       &p_ooo_info->free_buffers_list);
226 			}
227 			list_add_tail(&p_isle->list_entry,
228 				      &p_ooo_info->free_isles_list);
229 		}
230 	}
231 	if (!list_empty(&p_ooo_info->ready_buffers_list))
232 		list_splice_tail_init(&p_ooo_info->ready_buffers_list,
233 				      &p_ooo_info->free_buffers_list);
234 }
235 
236 void qed_ooo_setup(struct qed_hwfn *p_hwfn)
237 {
238 	qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
239 	memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0,
240 	       p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
241 	       sizeof(struct ooo_opaque));
242 	p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
243 }
244 
245 void qed_ooo_free(struct qed_hwfn *p_hwfn)
246 {
247 	struct qed_ooo_info *p_ooo_info  = p_hwfn->p_ooo_info;
248 	struct qed_ooo_buffer *p_buffer;
249 
250 	if (!p_ooo_info)
251 		return;
252 
253 	qed_ooo_release_all_isles(p_hwfn, p_ooo_info);
254 	while (!list_empty(&p_ooo_info->free_buffers_list)) {
255 		p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
256 					    struct qed_ooo_buffer, list_entry);
257 
258 		if (!p_buffer)
259 			break;
260 
261 		list_del(&p_buffer->list_entry);
262 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
263 				  p_buffer->rx_buffer_size,
264 				  p_buffer->rx_buffer_virt_addr,
265 				  p_buffer->rx_buffer_phys_addr);
266 		kfree(p_buffer);
267 	}
268 
269 	kfree(p_ooo_info->p_isles_mem);
270 	kfree(p_ooo_info->p_archipelagos_mem);
271 	kfree(p_ooo_info->ooo_history.p_cqes);
272 	kfree(p_ooo_info);
273 	p_hwfn->p_ooo_info = NULL;
274 }
275 
276 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn,
277 			     struct qed_ooo_info *p_ooo_info,
278 			     struct qed_ooo_buffer *p_buffer)
279 {
280 	list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list);
281 }
282 
283 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn,
284 					       struct qed_ooo_info *p_ooo_info)
285 {
286 	struct qed_ooo_buffer *p_buffer = NULL;
287 
288 	if (!list_empty(&p_ooo_info->free_buffers_list)) {
289 		p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
290 					    struct qed_ooo_buffer, list_entry);
291 
292 		list_del(&p_buffer->list_entry);
293 	}
294 
295 	return p_buffer;
296 }
297 
298 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn,
299 			      struct qed_ooo_info *p_ooo_info,
300 			      struct qed_ooo_buffer *p_buffer, u8 on_tail)
301 {
302 	if (on_tail)
303 		list_add_tail(&p_buffer->list_entry,
304 			      &p_ooo_info->ready_buffers_list);
305 	else
306 		list_add(&p_buffer->list_entry,
307 			 &p_ooo_info->ready_buffers_list);
308 }
309 
310 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn,
311 						struct qed_ooo_info *p_ooo_info)
312 {
313 	struct qed_ooo_buffer *p_buffer = NULL;
314 
315 	if (!list_empty(&p_ooo_info->ready_buffers_list)) {
316 		p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list,
317 					    struct qed_ooo_buffer, list_entry);
318 
319 		list_del(&p_buffer->list_entry);
320 	}
321 
322 	return p_buffer;
323 }
324 
325 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn,
326 			  struct qed_ooo_info *p_ooo_info,
327 			  u32 cid, u8 drop_isle, u8 drop_size)
328 {
329 	struct qed_ooo_isle *p_isle = NULL;
330 	u8 isle_idx;
331 
332 	for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
333 		p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle);
334 		if (!p_isle) {
335 			DP_NOTICE(p_hwfn,
336 				  "Isle %d is not found(cid %d)\n",
337 				  drop_isle, cid);
338 			return;
339 		}
340 		if (list_empty(&p_isle->buffers_list))
341 			DP_NOTICE(p_hwfn,
342 				  "Isle %d is empty(cid %d)\n", drop_isle, cid);
343 		else
344 			list_splice_tail_init(&p_isle->buffers_list,
345 					      &p_ooo_info->free_buffers_list);
346 
347 		list_del(&p_isle->list_entry);
348 		p_ooo_info->cur_isles_number--;
349 		list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list);
350 	}
351 }
352 
353 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn,
354 			  struct qed_ooo_info *p_ooo_info,
355 			  u32 cid, u8 ooo_isle,
356 			  struct qed_ooo_buffer *p_buffer)
357 {
358 	struct qed_ooo_archipelago *p_archipelago = NULL;
359 	struct qed_ooo_isle *p_prev_isle = NULL;
360 	struct qed_ooo_isle *p_isle = NULL;
361 
362 	if (ooo_isle > 1) {
363 		p_prev_isle = qed_ooo_seek_isle(p_hwfn,
364 						p_ooo_info, cid, ooo_isle - 1);
365 		if (unlikely(!p_prev_isle)) {
366 			DP_NOTICE(p_hwfn,
367 				  "Isle %d is not found(cid %d)\n",
368 				  ooo_isle - 1, cid);
369 			return;
370 		}
371 	}
372 	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
373 	if (unlikely(!p_archipelago && ooo_isle != 1)) {
374 		DP_NOTICE(p_hwfn,
375 			  "Connection %d is not found in OOO list\n", cid);
376 		return;
377 	}
378 
379 	if (!list_empty(&p_ooo_info->free_isles_list)) {
380 		p_isle = list_first_entry(&p_ooo_info->free_isles_list,
381 					  struct qed_ooo_isle, list_entry);
382 
383 		list_del(&p_isle->list_entry);
384 		if (unlikely(!list_empty(&p_isle->buffers_list))) {
385 			DP_NOTICE(p_hwfn, "Free isle is not empty\n");
386 			INIT_LIST_HEAD(&p_isle->buffers_list);
387 		}
388 	} else {
389 		DP_NOTICE(p_hwfn, "No more free isles\n");
390 		return;
391 	}
392 
393 	if (!p_archipelago) {
394 		u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
395 
396 		p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
397 	}
398 
399 	list_add(&p_buffer->list_entry, &p_isle->buffers_list);
400 	p_ooo_info->cur_isles_number++;
401 	p_ooo_info->gen_isles_number++;
402 
403 	if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
404 		p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
405 
406 	if (!p_prev_isle)
407 		list_add(&p_isle->list_entry, &p_archipelago->isles_list);
408 	else
409 		list_add(&p_isle->list_entry, &p_prev_isle->list_entry);
410 }
411 
412 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn,
413 			    struct qed_ooo_info *p_ooo_info,
414 			    u32 cid,
415 			    u8 ooo_isle,
416 			    struct qed_ooo_buffer *p_buffer, u8 buffer_side)
417 {
418 	struct qed_ooo_isle *p_isle = NULL;
419 
420 	p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
421 	if (unlikely(!p_isle)) {
422 		DP_NOTICE(p_hwfn,
423 			  "Isle %d is not found(cid %d)\n", ooo_isle, cid);
424 		return;
425 	}
426 
427 	if (unlikely(buffer_side == QED_OOO_LEFT_BUF))
428 		list_add(&p_buffer->list_entry, &p_isle->buffers_list);
429 	else
430 		list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list);
431 }
432 
433 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn,
434 			struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle)
435 {
436 	struct qed_ooo_isle *p_right_isle = NULL;
437 	struct qed_ooo_isle *p_left_isle = NULL;
438 
439 	p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
440 					 left_isle + 1);
441 	if (unlikely(!p_right_isle)) {
442 		DP_NOTICE(p_hwfn,
443 			  "Right isle %d is not found(cid %d)\n",
444 			  left_isle + 1, cid);
445 		return;
446 	}
447 
448 	list_del(&p_right_isle->list_entry);
449 	p_ooo_info->cur_isles_number--;
450 	if (left_isle) {
451 		p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
452 						left_isle);
453 		if (unlikely(!p_left_isle)) {
454 			DP_NOTICE(p_hwfn,
455 				  "Left isle %d is not found(cid %d)\n",
456 				  left_isle, cid);
457 			return;
458 		}
459 		list_splice_tail_init(&p_right_isle->buffers_list,
460 				      &p_left_isle->buffers_list);
461 	} else {
462 		list_splice_tail_init(&p_right_isle->buffers_list,
463 				      &p_ooo_info->ready_buffers_list);
464 	}
465 	list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list);
466 }
467