1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 /* QLogic qed NIC Driver
3  * Copyright (c) 2015-2017  QLogic Corporation
4  * Copyright (c) 2019-2020 Marvell International Ltd.
5  */
6 
7 #include <linux/types.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/kernel.h>
10 #include <linux/list.h>
11 #include <linux/pci.h>
12 #include <linux/slab.h>
13 #include <linux/string.h>
14 #include "qed.h"
15 #include "qed_iscsi.h"
16 #include "qed_ll2.h"
17 #include "qed_ooo.h"
18 #include "qed_cxt.h"
19 
20 static struct qed_ooo_archipelago
21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn,
22 			  struct qed_ooo_info
23 			  *p_ooo_info,
24 			  u32 cid)
25 {
26 	u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
27 	struct qed_ooo_archipelago *p_archipelago;
28 
29 	if (idx >= p_ooo_info->max_num_archipelagos)
30 		return NULL;
31 
32 	p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
33 
34 	if (list_empty(&p_archipelago->isles_list))
35 		return NULL;
36 
37 	return p_archipelago;
38 }
39 
40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn,
41 					      struct qed_ooo_info *p_ooo_info,
42 					      u32 cid, u8 isle)
43 {
44 	struct qed_ooo_archipelago *p_archipelago = NULL;
45 	struct qed_ooo_isle *p_isle = NULL;
46 	u8 the_num_of_isle = 1;
47 
48 	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
49 	if (!p_archipelago) {
50 		DP_NOTICE(p_hwfn,
51 			  "Connection %d is not found in OOO list\n", cid);
52 		return NULL;
53 	}
54 
55 	list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) {
56 		if (the_num_of_isle == isle)
57 			return p_isle;
58 		the_num_of_isle++;
59 	}
60 
61 	return NULL;
62 }
63 
64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn,
65 				struct qed_ooo_info *p_ooo_info,
66 				struct ooo_opaque *p_cqe)
67 {
68 	struct qed_ooo_history *p_history = &p_ooo_info->ooo_history;
69 
70 	if (p_history->head_idx == p_history->num_of_cqes)
71 		p_history->head_idx = 0;
72 	p_history->p_cqes[p_history->head_idx] = *p_cqe;
73 	p_history->head_idx++;
74 }
75 
76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn)
77 {
78 	u16 max_num_archipelagos = 0, cid_base;
79 	struct qed_ooo_info *p_ooo_info;
80 	enum protocol_type proto;
81 	u16 max_num_isles = 0;
82 	u32 i;
83 
84 	switch (p_hwfn->hw_info.personality) {
85 	case QED_PCI_ISCSI:
86 		proto = PROTOCOLID_ISCSI;
87 		break;
88 	case QED_PCI_ETH_RDMA:
89 	case QED_PCI_ETH_IWARP:
90 		proto = PROTOCOLID_IWARP;
91 		break;
92 	default:
93 		DP_NOTICE(p_hwfn,
94 			  "Failed to allocate qed_ooo_info: unknown personality\n");
95 		return -EINVAL;
96 	}
97 
98 	max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto,
99 								NULL);
100 	max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos;
101 	cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto);
102 
103 	if (!max_num_archipelagos) {
104 		DP_NOTICE(p_hwfn,
105 			  "Failed to allocate qed_ooo_info: unknown amount of connections\n");
106 		return -EINVAL;
107 	}
108 
109 	p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL);
110 	if (!p_ooo_info)
111 		return -ENOMEM;
112 
113 	p_ooo_info->cid_base = cid_base;
114 	p_ooo_info->max_num_archipelagos = max_num_archipelagos;
115 
116 	INIT_LIST_HEAD(&p_ooo_info->free_buffers_list);
117 	INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list);
118 	INIT_LIST_HEAD(&p_ooo_info->free_isles_list);
119 
120 	p_ooo_info->p_isles_mem = kcalloc(max_num_isles,
121 					  sizeof(struct qed_ooo_isle),
122 					  GFP_KERNEL);
123 	if (!p_ooo_info->p_isles_mem)
124 		goto no_isles_mem;
125 
126 	for (i = 0; i < max_num_isles; i++) {
127 		INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list);
128 		list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry,
129 			      &p_ooo_info->free_isles_list);
130 	}
131 
132 	p_ooo_info->p_archipelagos_mem =
133 				kcalloc(max_num_archipelagos,
134 					sizeof(struct qed_ooo_archipelago),
135 					GFP_KERNEL);
136 	if (!p_ooo_info->p_archipelagos_mem)
137 		goto no_archipelagos_mem;
138 
139 	for (i = 0; i < max_num_archipelagos; i++)
140 		INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list);
141 
142 	p_ooo_info->ooo_history.p_cqes =
143 				kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES,
144 					sizeof(struct ooo_opaque),
145 					GFP_KERNEL);
146 	if (!p_ooo_info->ooo_history.p_cqes)
147 		goto no_history_mem;
148 
149 	p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES;
150 
151 	p_hwfn->p_ooo_info = p_ooo_info;
152 	return 0;
153 
154 no_history_mem:
155 	kfree(p_ooo_info->p_archipelagos_mem);
156 no_archipelagos_mem:
157 	kfree(p_ooo_info->p_isles_mem);
158 no_isles_mem:
159 	kfree(p_ooo_info);
160 	return -ENOMEM;
161 }
162 
163 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn,
164 				      struct qed_ooo_info *p_ooo_info, u32 cid)
165 {
166 	struct qed_ooo_archipelago *p_archipelago;
167 	struct qed_ooo_buffer *p_buffer;
168 	struct qed_ooo_isle *p_isle;
169 
170 	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
171 	if (!p_archipelago)
172 		return;
173 
174 	while (!list_empty(&p_archipelago->isles_list)) {
175 		p_isle = list_first_entry(&p_archipelago->isles_list,
176 					  struct qed_ooo_isle, list_entry);
177 
178 		list_del(&p_isle->list_entry);
179 
180 		while (!list_empty(&p_isle->buffers_list)) {
181 			p_buffer = list_first_entry(&p_isle->buffers_list,
182 						    struct qed_ooo_buffer,
183 						    list_entry);
184 
185 			if (!p_buffer)
186 				break;
187 
188 			list_move_tail(&p_buffer->list_entry,
189 				       &p_ooo_info->free_buffers_list);
190 		}
191 		list_add_tail(&p_isle->list_entry,
192 			      &p_ooo_info->free_isles_list);
193 	}
194 }
195 
196 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn,
197 			       struct qed_ooo_info *p_ooo_info)
198 {
199 	struct qed_ooo_archipelago *p_archipelago;
200 	struct qed_ooo_buffer *p_buffer;
201 	struct qed_ooo_isle *p_isle;
202 	u32 i;
203 
204 	for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
205 		p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
206 
207 		while (!list_empty(&p_archipelago->isles_list)) {
208 			p_isle = list_first_entry(&p_archipelago->isles_list,
209 						  struct qed_ooo_isle,
210 						  list_entry);
211 
212 			list_del(&p_isle->list_entry);
213 
214 			while (!list_empty(&p_isle->buffers_list)) {
215 				p_buffer =
216 				    list_first_entry(&p_isle->buffers_list,
217 						     struct qed_ooo_buffer,
218 						     list_entry);
219 
220 				if (!p_buffer)
221 					break;
222 
223 				list_move_tail(&p_buffer->list_entry,
224 					       &p_ooo_info->free_buffers_list);
225 			}
226 			list_add_tail(&p_isle->list_entry,
227 				      &p_ooo_info->free_isles_list);
228 		}
229 	}
230 	if (!list_empty(&p_ooo_info->ready_buffers_list))
231 		list_splice_tail_init(&p_ooo_info->ready_buffers_list,
232 				      &p_ooo_info->free_buffers_list);
233 }
234 
235 void qed_ooo_setup(struct qed_hwfn *p_hwfn)
236 {
237 	qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
238 	memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0,
239 	       p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
240 	       sizeof(struct ooo_opaque));
241 	p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
242 }
243 
244 void qed_ooo_free(struct qed_hwfn *p_hwfn)
245 {
246 	struct qed_ooo_info *p_ooo_info  = p_hwfn->p_ooo_info;
247 	struct qed_ooo_buffer *p_buffer;
248 
249 	if (!p_ooo_info)
250 		return;
251 
252 	qed_ooo_release_all_isles(p_hwfn, p_ooo_info);
253 	while (!list_empty(&p_ooo_info->free_buffers_list)) {
254 		p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
255 					    struct qed_ooo_buffer, list_entry);
256 
257 		if (!p_buffer)
258 			break;
259 
260 		list_del(&p_buffer->list_entry);
261 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
262 				  p_buffer->rx_buffer_size,
263 				  p_buffer->rx_buffer_virt_addr,
264 				  p_buffer->rx_buffer_phys_addr);
265 		kfree(p_buffer);
266 	}
267 
268 	kfree(p_ooo_info->p_isles_mem);
269 	kfree(p_ooo_info->p_archipelagos_mem);
270 	kfree(p_ooo_info->ooo_history.p_cqes);
271 	kfree(p_ooo_info);
272 	p_hwfn->p_ooo_info = NULL;
273 }
274 
275 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn,
276 			     struct qed_ooo_info *p_ooo_info,
277 			     struct qed_ooo_buffer *p_buffer)
278 {
279 	list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list);
280 }
281 
282 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn,
283 					       struct qed_ooo_info *p_ooo_info)
284 {
285 	struct qed_ooo_buffer *p_buffer = NULL;
286 
287 	if (!list_empty(&p_ooo_info->free_buffers_list)) {
288 		p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
289 					    struct qed_ooo_buffer, list_entry);
290 
291 		list_del(&p_buffer->list_entry);
292 	}
293 
294 	return p_buffer;
295 }
296 
297 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn,
298 			      struct qed_ooo_info *p_ooo_info,
299 			      struct qed_ooo_buffer *p_buffer, u8 on_tail)
300 {
301 	if (on_tail)
302 		list_add_tail(&p_buffer->list_entry,
303 			      &p_ooo_info->ready_buffers_list);
304 	else
305 		list_add(&p_buffer->list_entry,
306 			 &p_ooo_info->ready_buffers_list);
307 }
308 
309 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn,
310 						struct qed_ooo_info *p_ooo_info)
311 {
312 	struct qed_ooo_buffer *p_buffer = NULL;
313 
314 	if (!list_empty(&p_ooo_info->ready_buffers_list)) {
315 		p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list,
316 					    struct qed_ooo_buffer, list_entry);
317 
318 		list_del(&p_buffer->list_entry);
319 	}
320 
321 	return p_buffer;
322 }
323 
324 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn,
325 			  struct qed_ooo_info *p_ooo_info,
326 			  u32 cid, u8 drop_isle, u8 drop_size)
327 {
328 	struct qed_ooo_isle *p_isle = NULL;
329 	u8 isle_idx;
330 
331 	for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
332 		p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle);
333 		if (!p_isle) {
334 			DP_NOTICE(p_hwfn,
335 				  "Isle %d is not found(cid %d)\n",
336 				  drop_isle, cid);
337 			return;
338 		}
339 		if (list_empty(&p_isle->buffers_list))
340 			DP_NOTICE(p_hwfn,
341 				  "Isle %d is empty(cid %d)\n", drop_isle, cid);
342 		else
343 			list_splice_tail_init(&p_isle->buffers_list,
344 					      &p_ooo_info->free_buffers_list);
345 
346 		list_del(&p_isle->list_entry);
347 		p_ooo_info->cur_isles_number--;
348 		list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list);
349 	}
350 }
351 
352 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn,
353 			  struct qed_ooo_info *p_ooo_info,
354 			  u32 cid, u8 ooo_isle,
355 			  struct qed_ooo_buffer *p_buffer)
356 {
357 	struct qed_ooo_archipelago *p_archipelago = NULL;
358 	struct qed_ooo_isle *p_prev_isle = NULL;
359 	struct qed_ooo_isle *p_isle = NULL;
360 
361 	if (ooo_isle > 1) {
362 		p_prev_isle = qed_ooo_seek_isle(p_hwfn,
363 						p_ooo_info, cid, ooo_isle - 1);
364 		if (!p_prev_isle) {
365 			DP_NOTICE(p_hwfn,
366 				  "Isle %d is not found(cid %d)\n",
367 				  ooo_isle - 1, cid);
368 			return;
369 		}
370 	}
371 	p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
372 	if (!p_archipelago && (ooo_isle != 1)) {
373 		DP_NOTICE(p_hwfn,
374 			  "Connection %d is not found in OOO list\n", cid);
375 		return;
376 	}
377 
378 	if (!list_empty(&p_ooo_info->free_isles_list)) {
379 		p_isle = list_first_entry(&p_ooo_info->free_isles_list,
380 					  struct qed_ooo_isle, list_entry);
381 
382 		list_del(&p_isle->list_entry);
383 		if (!list_empty(&p_isle->buffers_list)) {
384 			DP_NOTICE(p_hwfn, "Free isle is not empty\n");
385 			INIT_LIST_HEAD(&p_isle->buffers_list);
386 		}
387 	} else {
388 		DP_NOTICE(p_hwfn, "No more free isles\n");
389 		return;
390 	}
391 
392 	if (!p_archipelago) {
393 		u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
394 
395 		p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
396 	}
397 
398 	list_add(&p_buffer->list_entry, &p_isle->buffers_list);
399 	p_ooo_info->cur_isles_number++;
400 	p_ooo_info->gen_isles_number++;
401 
402 	if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
403 		p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
404 
405 	if (!p_prev_isle)
406 		list_add(&p_isle->list_entry, &p_archipelago->isles_list);
407 	else
408 		list_add(&p_isle->list_entry, &p_prev_isle->list_entry);
409 }
410 
411 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn,
412 			    struct qed_ooo_info *p_ooo_info,
413 			    u32 cid,
414 			    u8 ooo_isle,
415 			    struct qed_ooo_buffer *p_buffer, u8 buffer_side)
416 {
417 	struct qed_ooo_isle *p_isle = NULL;
418 
419 	p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
420 	if (!p_isle) {
421 		DP_NOTICE(p_hwfn,
422 			  "Isle %d is not found(cid %d)\n", ooo_isle, cid);
423 		return;
424 	}
425 
426 	if (buffer_side == QED_OOO_LEFT_BUF)
427 		list_add(&p_buffer->list_entry, &p_isle->buffers_list);
428 	else
429 		list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list);
430 }
431 
432 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn,
433 			struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle)
434 {
435 	struct qed_ooo_isle *p_right_isle = NULL;
436 	struct qed_ooo_isle *p_left_isle = NULL;
437 
438 	p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
439 					 left_isle + 1);
440 	if (!p_right_isle) {
441 		DP_NOTICE(p_hwfn,
442 			  "Right isle %d is not found(cid %d)\n",
443 			  left_isle + 1, cid);
444 		return;
445 	}
446 
447 	list_del(&p_right_isle->list_entry);
448 	p_ooo_info->cur_isles_number--;
449 	if (left_isle) {
450 		p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
451 						left_isle);
452 		if (!p_left_isle) {
453 			DP_NOTICE(p_hwfn,
454 				  "Left isle %d is not found(cid %d)\n",
455 				  left_isle, cid);
456 			return;
457 		}
458 		list_splice_tail_init(&p_right_isle->buffers_list,
459 				      &p_left_isle->buffers_list);
460 	} else {
461 		list_splice_tail_init(&p_right_isle->buffers_list,
462 				      &p_ooo_info->ready_buffers_list);
463 	}
464 	list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list);
465 }
466