1 /* Broadcom NetXtreme-C/E network driver.
2  *
3  * Copyright (c) 2016-2018 Broadcom Limited
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  */
9 
10 #include <linux/module.h>
11 
12 #include <linux/kernel.h>
13 #include <linux/errno.h>
14 #include <linux/interrupt.h>
15 #include <linux/pci.h>
16 #include <linux/netdevice.h>
17 #include <linux/rtnetlink.h>
18 #include <linux/bitops.h>
19 #include <linux/irq.h>
20 #include <asm/byteorder.h>
21 #include <linux/bitmap.h>
22 
23 #include "bnxt_hsi.h"
24 #include "bnxt.h"
25 #include "bnxt_ulp.h"
26 
27 static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id,
28 			     struct bnxt_ulp_ops *ulp_ops, void *handle)
29 {
30 	struct net_device *dev = edev->net;
31 	struct bnxt *bp = netdev_priv(dev);
32 	struct bnxt_ulp *ulp;
33 
34 	ASSERT_RTNL();
35 	if (ulp_id >= BNXT_MAX_ULP)
36 		return -EINVAL;
37 
38 	ulp = &edev->ulp_tbl[ulp_id];
39 	if (rcu_access_pointer(ulp->ulp_ops)) {
40 		netdev_err(bp->dev, "ulp id %d already registered\n", ulp_id);
41 		return -EBUSY;
42 	}
43 	if (ulp_id == BNXT_ROCE_ULP) {
44 		unsigned int max_stat_ctxs;
45 
46 		max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);
47 		if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS ||
48 		    bp->cp_nr_rings == max_stat_ctxs)
49 			return -ENOMEM;
50 	}
51 
52 	atomic_set(&ulp->ref_count, 0);
53 	ulp->handle = handle;
54 	rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
55 
56 	if (ulp_id == BNXT_ROCE_ULP) {
57 		if (test_bit(BNXT_STATE_OPEN, &bp->state))
58 			bnxt_hwrm_vnic_cfg(bp, 0);
59 	}
60 
61 	return 0;
62 }
63 
64 static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id)
65 {
66 	struct net_device *dev = edev->net;
67 	struct bnxt *bp = netdev_priv(dev);
68 	struct bnxt_ulp *ulp;
69 	int i = 0;
70 
71 	ASSERT_RTNL();
72 	if (ulp_id >= BNXT_MAX_ULP)
73 		return -EINVAL;
74 
75 	ulp = &edev->ulp_tbl[ulp_id];
76 	if (!rcu_access_pointer(ulp->ulp_ops)) {
77 		netdev_err(bp->dev, "ulp id %d not registered\n", ulp_id);
78 		return -EINVAL;
79 	}
80 	if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested)
81 		edev->en_ops->bnxt_free_msix(edev, ulp_id);
82 
83 	if (ulp->max_async_event_id)
84 		bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true);
85 
86 	RCU_INIT_POINTER(ulp->ulp_ops, NULL);
87 	synchronize_rcu();
88 	ulp->max_async_event_id = 0;
89 	ulp->async_events_bmap = NULL;
90 	while (atomic_read(&ulp->ref_count) != 0 && i < 10) {
91 		msleep(100);
92 		i++;
93 	}
94 	return 0;
95 }
96 
97 static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)
98 {
99 	struct bnxt_en_dev *edev = bp->edev;
100 	int num_msix, idx, i;
101 
102 	num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested;
103 	idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base;
104 	for (i = 0; i < num_msix; i++) {
105 		ent[i].vector = bp->irq_tbl[idx + i].vector;
106 		ent[i].ring_idx = idx + i;
107 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
108 			ent[i].db_offset = DB_PF_OFFSET_P5;
109 			if (BNXT_VF(bp))
110 				ent[i].db_offset = DB_VF_OFFSET_P5;
111 		} else {
112 			ent[i].db_offset = (idx + i) * 0x80;
113 		}
114 	}
115 }
116 
117 static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id,
118 			      struct bnxt_msix_entry *ent, int num_msix)
119 {
120 	struct net_device *dev = edev->net;
121 	struct bnxt *bp = netdev_priv(dev);
122 	struct bnxt_hw_resc *hw_resc;
123 	int max_idx, max_cp_rings;
124 	int avail_msix, idx;
125 	int total_vecs;
126 	int rc = 0;
127 
128 	ASSERT_RTNL();
129 	if (ulp_id != BNXT_ROCE_ULP)
130 		return -EINVAL;
131 
132 	if (!(bp->flags & BNXT_FLAG_USING_MSIX))
133 		return -ENODEV;
134 
135 	if (edev->ulp_tbl[ulp_id].msix_requested)
136 		return -EAGAIN;
137 
138 	max_cp_rings = bnxt_get_max_func_cp_rings(bp);
139 	avail_msix = bnxt_get_avail_msix(bp, num_msix);
140 	if (!avail_msix)
141 		return -ENOMEM;
142 	if (avail_msix > num_msix)
143 		avail_msix = num_msix;
144 
145 	if (BNXT_NEW_RM(bp)) {
146 		idx = bp->cp_nr_rings;
147 	} else {
148 		max_idx = min_t(int, bp->total_irqs, max_cp_rings);
149 		idx = max_idx - avail_msix;
150 	}
151 	edev->ulp_tbl[ulp_id].msix_base = idx;
152 	edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
153 	hw_resc = &bp->hw_resc;
154 	total_vecs = idx + avail_msix;
155 	if (bp->total_irqs < total_vecs ||
156 	    (BNXT_NEW_RM(bp) && hw_resc->resv_irqs < total_vecs)) {
157 		if (netif_running(dev)) {
158 			bnxt_close_nic(bp, true, false);
159 			rc = bnxt_open_nic(bp, true, false);
160 		} else {
161 			rc = bnxt_reserve_rings(bp, true);
162 		}
163 	}
164 	if (rc) {
165 		edev->ulp_tbl[ulp_id].msix_requested = 0;
166 		return -EAGAIN;
167 	}
168 
169 	if (BNXT_NEW_RM(bp)) {
170 		int resv_msix;
171 
172 		resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings;
173 		avail_msix = min_t(int, resv_msix, avail_msix);
174 		edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
175 	}
176 	bnxt_fill_msix_vecs(bp, ent);
177 	edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED;
178 	return avail_msix;
179 }
180 
181 static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id)
182 {
183 	struct net_device *dev = edev->net;
184 	struct bnxt *bp = netdev_priv(dev);
185 
186 	ASSERT_RTNL();
187 	if (ulp_id != BNXT_ROCE_ULP)
188 		return -EINVAL;
189 
190 	if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
191 		return 0;
192 
193 	edev->ulp_tbl[ulp_id].msix_requested = 0;
194 	edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
195 	if (netif_running(dev) && !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) {
196 		bnxt_close_nic(bp, true, false);
197 		bnxt_open_nic(bp, true, false);
198 	}
199 	return 0;
200 }
201 
202 int bnxt_get_ulp_msix_num(struct bnxt *bp)
203 {
204 	if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
205 		struct bnxt_en_dev *edev = bp->edev;
206 
207 		return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested;
208 	}
209 	return 0;
210 }
211 
212 int bnxt_get_ulp_msix_base(struct bnxt *bp)
213 {
214 	if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
215 		struct bnxt_en_dev *edev = bp->edev;
216 
217 		if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested)
218 			return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base;
219 	}
220 	return 0;
221 }
222 
223 int bnxt_get_ulp_stat_ctxs(struct bnxt *bp)
224 {
225 	if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
226 		struct bnxt_en_dev *edev = bp->edev;
227 
228 		if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested)
229 			return BNXT_MIN_ROCE_STAT_CTXS;
230 	}
231 
232 	return 0;
233 }
234 
235 static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id,
236 			 struct bnxt_fw_msg *fw_msg)
237 {
238 	struct net_device *dev = edev->net;
239 	struct bnxt *bp = netdev_priv(dev);
240 	struct input *req;
241 	int rc;
242 
243 	if (ulp_id != BNXT_ROCE_ULP && bp->fw_reset_state)
244 		return -EBUSY;
245 
246 	mutex_lock(&bp->hwrm_cmd_lock);
247 	req = fw_msg->msg;
248 	req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
249 	rc = _hwrm_send_message(bp, fw_msg->msg, fw_msg->msg_len,
250 				fw_msg->timeout);
251 	if (!rc) {
252 		struct output *resp = bp->hwrm_cmd_resp_addr;
253 		u32 len = le16_to_cpu(resp->resp_len);
254 
255 		if (fw_msg->resp_max_len < len)
256 			len = fw_msg->resp_max_len;
257 
258 		memcpy(fw_msg->resp, resp, len);
259 	}
260 	mutex_unlock(&bp->hwrm_cmd_lock);
261 	return rc;
262 }
263 
264 static void bnxt_ulp_get(struct bnxt_ulp *ulp)
265 {
266 	atomic_inc(&ulp->ref_count);
267 }
268 
269 static void bnxt_ulp_put(struct bnxt_ulp *ulp)
270 {
271 	atomic_dec(&ulp->ref_count);
272 }
273 
274 void bnxt_ulp_stop(struct bnxt *bp)
275 {
276 	struct bnxt_en_dev *edev = bp->edev;
277 	struct bnxt_ulp_ops *ops;
278 	int i;
279 
280 	if (!edev)
281 		return;
282 
283 	edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;
284 	for (i = 0; i < BNXT_MAX_ULP; i++) {
285 		struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
286 
287 		ops = rtnl_dereference(ulp->ulp_ops);
288 		if (!ops || !ops->ulp_stop)
289 			continue;
290 		ops->ulp_stop(ulp->handle);
291 	}
292 }
293 
294 void bnxt_ulp_start(struct bnxt *bp, int err)
295 {
296 	struct bnxt_en_dev *edev = bp->edev;
297 	struct bnxt_ulp_ops *ops;
298 	int i;
299 
300 	if (!edev)
301 		return;
302 
303 	edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED;
304 
305 	if (err)
306 		return;
307 
308 	for (i = 0; i < BNXT_MAX_ULP; i++) {
309 		struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
310 
311 		ops = rtnl_dereference(ulp->ulp_ops);
312 		if (!ops || !ops->ulp_start)
313 			continue;
314 		ops->ulp_start(ulp->handle);
315 	}
316 }
317 
318 void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs)
319 {
320 	struct bnxt_en_dev *edev = bp->edev;
321 	struct bnxt_ulp_ops *ops;
322 	int i;
323 
324 	if (!edev)
325 		return;
326 
327 	for (i = 0; i < BNXT_MAX_ULP; i++) {
328 		struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
329 
330 		rcu_read_lock();
331 		ops = rcu_dereference(ulp->ulp_ops);
332 		if (!ops || !ops->ulp_sriov_config) {
333 			rcu_read_unlock();
334 			continue;
335 		}
336 		bnxt_ulp_get(ulp);
337 		rcu_read_unlock();
338 		ops->ulp_sriov_config(ulp->handle, num_vfs);
339 		bnxt_ulp_put(ulp);
340 	}
341 }
342 
343 void bnxt_ulp_shutdown(struct bnxt *bp)
344 {
345 	struct bnxt_en_dev *edev = bp->edev;
346 	struct bnxt_ulp_ops *ops;
347 	int i;
348 
349 	if (!edev)
350 		return;
351 
352 	for (i = 0; i < BNXT_MAX_ULP; i++) {
353 		struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
354 
355 		ops = rtnl_dereference(ulp->ulp_ops);
356 		if (!ops || !ops->ulp_shutdown)
357 			continue;
358 		ops->ulp_shutdown(ulp->handle);
359 	}
360 }
361 
362 void bnxt_ulp_irq_stop(struct bnxt *bp)
363 {
364 	struct bnxt_en_dev *edev = bp->edev;
365 	struct bnxt_ulp_ops *ops;
366 
367 	if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
368 		return;
369 
370 	if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
371 		struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP];
372 
373 		if (!ulp->msix_requested)
374 			return;
375 
376 		ops = rtnl_dereference(ulp->ulp_ops);
377 		if (!ops || !ops->ulp_irq_stop)
378 			return;
379 		ops->ulp_irq_stop(ulp->handle);
380 	}
381 }
382 
383 void bnxt_ulp_irq_restart(struct bnxt *bp, int err)
384 {
385 	struct bnxt_en_dev *edev = bp->edev;
386 	struct bnxt_ulp_ops *ops;
387 
388 	if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
389 		return;
390 
391 	if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
392 		struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP];
393 		struct bnxt_msix_entry *ent = NULL;
394 
395 		if (!ulp->msix_requested)
396 			return;
397 
398 		ops = rtnl_dereference(ulp->ulp_ops);
399 		if (!ops || !ops->ulp_irq_restart)
400 			return;
401 
402 		if (!err) {
403 			ent = kcalloc(ulp->msix_requested, sizeof(*ent),
404 				      GFP_KERNEL);
405 			if (!ent)
406 				return;
407 			bnxt_fill_msix_vecs(bp, ent);
408 		}
409 		ops->ulp_irq_restart(ulp->handle, ent);
410 		kfree(ent);
411 	}
412 }
413 
414 void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl)
415 {
416 	u16 event_id = le16_to_cpu(cmpl->event_id);
417 	struct bnxt_en_dev *edev = bp->edev;
418 	struct bnxt_ulp_ops *ops;
419 	int i;
420 
421 	if (!edev)
422 		return;
423 
424 	rcu_read_lock();
425 	for (i = 0; i < BNXT_MAX_ULP; i++) {
426 		struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
427 
428 		ops = rcu_dereference(ulp->ulp_ops);
429 		if (!ops || !ops->ulp_async_notifier)
430 			continue;
431 		if (!ulp->async_events_bmap ||
432 		    event_id > ulp->max_async_event_id)
433 			continue;
434 
435 		/* Read max_async_event_id first before testing the bitmap. */
436 		smp_rmb();
437 		if (test_bit(event_id, ulp->async_events_bmap))
438 			ops->ulp_async_notifier(ulp->handle, cmpl);
439 	}
440 	rcu_read_unlock();
441 }
442 
443 static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id,
444 				      unsigned long *events_bmap, u16 max_id)
445 {
446 	struct net_device *dev = edev->net;
447 	struct bnxt *bp = netdev_priv(dev);
448 	struct bnxt_ulp *ulp;
449 
450 	if (ulp_id >= BNXT_MAX_ULP)
451 		return -EINVAL;
452 
453 	ulp = &edev->ulp_tbl[ulp_id];
454 	ulp->async_events_bmap = events_bmap;
455 	/* Make sure bnxt_ulp_async_events() sees this order */
456 	smp_wmb();
457 	ulp->max_async_event_id = max_id;
458 	bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true);
459 	return 0;
460 }
461 
462 static const struct bnxt_en_ops bnxt_en_ops_tbl = {
463 	.bnxt_register_device	= bnxt_register_dev,
464 	.bnxt_unregister_device	= bnxt_unregister_dev,
465 	.bnxt_request_msix	= bnxt_req_msix_vecs,
466 	.bnxt_free_msix		= bnxt_free_msix_vecs,
467 	.bnxt_send_fw_msg	= bnxt_send_msg,
468 	.bnxt_register_fw_async_events	= bnxt_register_async_events,
469 };
470 
471 struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev)
472 {
473 	struct bnxt *bp = netdev_priv(dev);
474 	struct bnxt_en_dev *edev;
475 
476 	edev = bp->edev;
477 	if (!edev) {
478 		edev = kzalloc(sizeof(*edev), GFP_KERNEL);
479 		if (!edev)
480 			return ERR_PTR(-ENOMEM);
481 		edev->en_ops = &bnxt_en_ops_tbl;
482 		if (bp->flags & BNXT_FLAG_ROCEV1_CAP)
483 			edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;
484 		if (bp->flags & BNXT_FLAG_ROCEV2_CAP)
485 			edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
486 		edev->net = dev;
487 		edev->pdev = bp->pdev;
488 		edev->l2_db_size = bp->db_size;
489 		edev->l2_db_size_nc = bp->db_size;
490 		bp->edev = edev;
491 	}
492 	return bp->edev;
493 }
494