main.c (4a6cc7a44e98a0460bd094b68c75f0705fdc450a) main.c (5c99eaecb1fce76e86cf74020624e36fbb63c3bf)
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 36 unchanged lines hidden (view full) ---

45#include <linux/sched/mm.h>
46#include <linux/sched/task.h>
47#include <linux/delay.h>
48#include <rdma/ib_user_verbs.h>
49#include <rdma/ib_addr.h>
50#include <rdma/ib_cache.h>
51#include <linux/mlx5/port.h>
52#include <linux/mlx5/vport.h>
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 36 unchanged lines hidden (view full) ---

45#include <linux/sched/mm.h>
46#include <linux/sched/task.h>
47#include <linux/delay.h>
48#include <rdma/ib_user_verbs.h>
49#include <rdma/ib_addr.h>
50#include <rdma/ib_cache.h>
51#include <linux/mlx5/port.h>
52#include <linux/mlx5/vport.h>
53#include <linux/mlx5/fs.h>
53#include <linux/list.h>
54#include <rdma/ib_smi.h>
55#include <rdma/ib_umem.h>
56#include <linux/in.h>
57#include <linux/etherdevice.h>
54#include <linux/list.h>
55#include <rdma/ib_smi.h>
56#include <rdma/ib_umem.h>
57#include <linux/in.h>
58#include <linux/etherdevice.h>
58#include <linux/mlx5/fs.h>
59#include <linux/mlx5/vport.h>
60#include "mlx5_ib.h"
61#include "cmd.h"
59#include "mlx5_ib.h"
60#include "cmd.h"
62#include <linux/mlx5/vport.h>
63
64#define DRIVER_NAME "mlx5_ib"
65#define DRIVER_VERSION "5.0-0"
66
67MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
68MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
69MODULE_LICENSE("Dual BSD/GPL");
70
71static char mlx5_version[] =
72 DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
73 DRIVER_VERSION "\n";
74
61
62#define DRIVER_NAME "mlx5_ib"
63#define DRIVER_VERSION "5.0-0"
64
65MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
66MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
67MODULE_LICENSE("Dual BSD/GPL");
68
69static char mlx5_version[] =
70 DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
71 DRIVER_VERSION "\n";
72
73struct mlx5_ib_event_work {
74 struct work_struct work;
75 struct mlx5_core_dev *dev;
76 void *context;
77 enum mlx5_dev_event event;
78 unsigned long param;
79};
80
75enum {
76 MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
77};
78
81enum {
82 MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
83};
84
85static struct workqueue_struct *mlx5_ib_event_wq;
86static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
87static LIST_HEAD(mlx5_ib_dev_list);
88/*
89 * This mutex should be held when accessing either of the above lists
90 */
91static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
92
93struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
94{
95 struct mlx5_ib_dev *dev;
96
97 mutex_lock(&mlx5_ib_multiport_mutex);
98 dev = mpi->ibdev;
99 mutex_unlock(&mlx5_ib_multiport_mutex);
100 return dev;
101}
102
79static enum rdma_link_layer
80mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
81{
82 switch (port_type_cap) {
83 case MLX5_CAP_PORT_TYPE_IB:
84 return IB_LINK_LAYER_INFINIBAND;
85 case MLX5_CAP_PORT_TYPE_ETH:
86 return IB_LINK_LAYER_ETHERNET;

--- 23 unchanged lines hidden (view full) ---

110 if (!ret)
111 *state = attr.state;
112 return ret;
113}
114
115static int mlx5_netdev_event(struct notifier_block *this,
116 unsigned long event, void *ptr)
117{
103static enum rdma_link_layer
104mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
105{
106 switch (port_type_cap) {
107 case MLX5_CAP_PORT_TYPE_IB:
108 return IB_LINK_LAYER_INFINIBAND;
109 case MLX5_CAP_PORT_TYPE_ETH:
110 return IB_LINK_LAYER_ETHERNET;

--- 23 unchanged lines hidden (view full) ---

134 if (!ret)
135 *state = attr.state;
136 return ret;
137}
138
139static int mlx5_netdev_event(struct notifier_block *this,
140 unsigned long event, void *ptr)
141{
142 struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
118 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
143 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
119 struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
120 roce.nb);
144 u8 port_num = roce->native_port_num;
145 struct mlx5_core_dev *mdev;
146 struct mlx5_ib_dev *ibdev;
121
147
148 ibdev = roce->dev;
149 mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
150 if (!mdev)
151 return NOTIFY_DONE;
152
122 switch (event) {
123 case NETDEV_REGISTER:
124 case NETDEV_UNREGISTER:
153 switch (event) {
154 case NETDEV_REGISTER:
155 case NETDEV_UNREGISTER:
125 write_lock(&ibdev->roce.netdev_lock);
126 if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
127 ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ?
128 NULL : ndev;
129 write_unlock(&ibdev->roce.netdev_lock);
156 write_lock(&roce->netdev_lock);
157
158 if (ndev->dev.parent == &mdev->pdev->dev)
159 roce->netdev = (event == NETDEV_UNREGISTER) ?
160 NULL : ndev;
161 write_unlock(&roce->netdev_lock);
130 break;
131
132 case NETDEV_CHANGE:
133 case NETDEV_UP:
134 case NETDEV_DOWN: {
162 break;
163
164 case NETDEV_CHANGE:
165 case NETDEV_UP:
166 case NETDEV_DOWN: {
135 struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
167 struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);
136 struct net_device *upper = NULL;
137
138 if (lag_ndev) {
139 upper = netdev_master_upper_dev_get(lag_ndev);
140 dev_put(lag_ndev);
141 }
142
168 struct net_device *upper = NULL;
169
170 if (lag_ndev) {
171 upper = netdev_master_upper_dev_get(lag_ndev);
172 dev_put(lag_ndev);
173 }
174
143 if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
175 if ((upper == ndev || (!upper && ndev == roce->netdev))
144 && ibdev->ib_active) {
145 struct ib_event ibev = { };
146 enum ib_port_state port_state;
147
176 && ibdev->ib_active) {
177 struct ib_event ibev = { };
178 enum ib_port_state port_state;
179
148 if (get_port_state(&ibdev->ib_dev, 1, &port_state))
149 return NOTIFY_DONE;
180 if (get_port_state(&ibdev->ib_dev, port_num,
181 &port_state))
182 goto done;
150
183
151 if (ibdev->roce.last_port_state == port_state)
152 return NOTIFY_DONE;
184 if (roce->last_port_state == port_state)
185 goto done;
153
186
154 ibdev->roce.last_port_state = port_state;
187 roce->last_port_state = port_state;
155 ibev.device = &ibdev->ib_dev;
156 if (port_state == IB_PORT_DOWN)
157 ibev.event = IB_EVENT_PORT_ERR;
158 else if (port_state == IB_PORT_ACTIVE)
159 ibev.event = IB_EVENT_PORT_ACTIVE;
160 else
188 ibev.device = &ibdev->ib_dev;
189 if (port_state == IB_PORT_DOWN)
190 ibev.event = IB_EVENT_PORT_ERR;
191 else if (port_state == IB_PORT_ACTIVE)
192 ibev.event = IB_EVENT_PORT_ACTIVE;
193 else
161 return NOTIFY_DONE;
194 goto done;
162
195
163 ibev.element.port_num = 1;
196 ibev.element.port_num = port_num;
164 ib_dispatch_event(&ibev);
165 }
166 break;
167 }
168
169 default:
170 break;
171 }
197 ib_dispatch_event(&ibev);
198 }
199 break;
200 }
201
202 default:
203 break;
204 }
172
205done:
206 mlx5_ib_put_native_port_mdev(ibdev, port_num);
173 return NOTIFY_DONE;
174}
175
176static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
177 u8 port_num)
178{
179 struct mlx5_ib_dev *ibdev = to_mdev(device);
180 struct net_device *ndev;
207 return NOTIFY_DONE;
208}
209
210static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
211 u8 port_num)
212{
213 struct mlx5_ib_dev *ibdev = to_mdev(device);
214 struct net_device *ndev;
215 struct mlx5_core_dev *mdev;
181
216
182 ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
217 mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
218 if (!mdev)
219 return NULL;
220
221 ndev = mlx5_lag_get_roce_netdev(mdev);
183 if (ndev)
222 if (ndev)
184 return ndev;
223 goto out;
185
186 /* Ensure ndev does not disappear before we invoke dev_hold()
187 */
224
225 /* Ensure ndev does not disappear before we invoke dev_hold()
226 */
188 read_lock(&ibdev->roce.netdev_lock);
189 ndev = ibdev->roce.netdev;
227 read_lock(&ibdev->roce[port_num - 1].netdev_lock);
228 ndev = ibdev->roce[port_num - 1].netdev;
190 if (ndev)
191 dev_hold(ndev);
229 if (ndev)
230 dev_hold(ndev);
192 read_unlock(&ibdev->roce.netdev_lock);
231 read_unlock(&ibdev->roce[port_num - 1].netdev_lock);
193
232
233out:
234 mlx5_ib_put_native_port_mdev(ibdev, port_num);
194 return ndev;
195}
196
235 return ndev;
236}
237
238struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
239 u8 ib_port_num,
240 u8 *native_port_num)
241{
242 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
243 ib_port_num);
244 struct mlx5_core_dev *mdev = NULL;
245 struct mlx5_ib_multiport_info *mpi;
246 struct mlx5_ib_port *port;
247
248 if (native_port_num)
249 *native_port_num = 1;
250
251 if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
252 return ibdev->mdev;
253
254 port = &ibdev->port[ib_port_num - 1];
255 if (!port)
256 return NULL;
257
258 spin_lock(&port->mp.mpi_lock);
259 mpi = ibdev->port[ib_port_num - 1].mp.mpi;
260 if (mpi && !mpi->unaffiliate) {
261 mdev = mpi->mdev;
262 /* If it's the master no need to refcount, it'll exist
263 * as long as the ib_dev exists.
264 */
265 if (!mpi->is_master)
266 mpi->mdev_refcnt++;
267 }
268 spin_unlock(&port->mp.mpi_lock);
269
270 return mdev;
271}
272
273void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u8 port_num)
274{
275 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
276 port_num);
277 struct mlx5_ib_multiport_info *mpi;
278 struct mlx5_ib_port *port;
279
280 if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
281 return;
282
283 port = &ibdev->port[port_num - 1];
284
285 spin_lock(&port->mp.mpi_lock);
286 mpi = ibdev->port[port_num - 1].mp.mpi;
287 if (mpi->is_master)
288 goto out;
289
290 mpi->mdev_refcnt--;
291 if (mpi->unaffiliate)
292 complete(&mpi->unref_comp);
293out:
294 spin_unlock(&port->mp.mpi_lock);
295}
296
197static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
198 u8 *active_width)
199{
200 switch (eth_proto_oper) {
201 case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
202 case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
203 case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
204 case MLX5E_PROT_MASK(MLX5E_1000BASE_T):

--- 46 unchanged lines hidden (view full) ---

251
252 return 0;
253}
254
255static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
256 struct ib_port_attr *props)
257{
258 struct mlx5_ib_dev *dev = to_mdev(device);
297static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
298 u8 *active_width)
299{
300 switch (eth_proto_oper) {
301 case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
302 case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
303 case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
304 case MLX5E_PROT_MASK(MLX5E_1000BASE_T):

--- 46 unchanged lines hidden (view full) ---

351
352 return 0;
353}
354
355static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
356 struct ib_port_attr *props)
357{
358 struct mlx5_ib_dev *dev = to_mdev(device);
259 struct mlx5_core_dev *mdev = dev->mdev;
359 struct mlx5_core_dev *mdev;
260 struct net_device *ndev, *upper;
261 enum ib_mtu ndev_ib_mtu;
360 struct net_device *ndev, *upper;
361 enum ib_mtu ndev_ib_mtu;
362 bool put_mdev = true;
262 u16 qkey_viol_cntr;
263 u32 eth_prot_oper;
363 u16 qkey_viol_cntr;
364 u32 eth_prot_oper;
365 u8 mdev_port_num;
264 int err;
265
366 int err;
367
368 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
369 if (!mdev) {
370 /* This means the port isn't affiliated yet. Get the
371 * info for the master port instead.
372 */
373 put_mdev = false;
374 mdev = dev->mdev;
375 mdev_port_num = 1;
376 port_num = 1;
377 }
378
266 /* Possible bad flows are checked before filling out props so in case
267 * of an error it will still be zeroed out.
268 */
379 /* Possible bad flows are checked before filling out props so in case
380 * of an error it will still be zeroed out.
381 */
269 err = mlx5_query_port_eth_proto_oper(mdev, &eth_prot_oper, port_num);
382 err = mlx5_query_port_eth_proto_oper(mdev, &eth_prot_oper,
383 mdev_port_num);
270 if (err)
384 if (err)
271 return err;
385 goto out;
272
273 translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
274 &props->active_width);
275
276 props->port_cap_flags |= IB_PORT_CM_SUP;
277 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
278
279 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
280 roce_address_table_size);
281 props->max_mtu = IB_MTU_4096;
282 props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
283 props->pkey_tbl_len = 1;
284 props->state = IB_PORT_DOWN;
285 props->phys_state = 3;
286
386
387 translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
388 &props->active_width);
389
390 props->port_cap_flags |= IB_PORT_CM_SUP;
391 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
392
393 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
394 roce_address_table_size);
395 props->max_mtu = IB_MTU_4096;
396 props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
397 props->pkey_tbl_len = 1;
398 props->state = IB_PORT_DOWN;
399 props->phys_state = 3;
400
287 mlx5_query_nic_vport_qkey_viol_cntr(dev->mdev, &qkey_viol_cntr);
401 mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
288 props->qkey_viol_cntr = qkey_viol_cntr;
289
402 props->qkey_viol_cntr = qkey_viol_cntr;
403
404 /* If this is a stub query for an unaffiliated port stop here */
405 if (!put_mdev)
406 goto out;
407
290 ndev = mlx5_ib_get_netdev(device, port_num);
291 if (!ndev)
408 ndev = mlx5_ib_get_netdev(device, port_num);
409 if (!ndev)
292 return 0;
410 goto out;
293
294 if (mlx5_lag_is_active(dev->mdev)) {
295 rcu_read_lock();
296 upper = netdev_master_upper_dev_get_rcu(ndev);
297 if (upper) {
298 dev_put(ndev);
299 ndev = upper;
300 dev_hold(ndev);

--- 6 unchanged lines hidden (view full) ---

307 props->phys_state = 5;
308 }
309
310 ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
311
312 dev_put(ndev);
313
314 props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
411
412 if (mlx5_lag_is_active(dev->mdev)) {
413 rcu_read_lock();
414 upper = netdev_master_upper_dev_get_rcu(ndev);
415 if (upper) {
416 dev_put(ndev);
417 ndev = upper;
418 dev_hold(ndev);

--- 6 unchanged lines hidden (view full) ---

425 props->phys_state = 5;
426 }
427
428 ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
429
430 dev_put(ndev);
431
432 props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
315 return 0;
433out:
434 if (put_mdev)
435 mlx5_ib_put_native_port_mdev(dev, port_num);
436 return err;
316}
317
318static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
319 unsigned int index, const union ib_gid *gid,
320 const struct ib_gid_attr *attr)
321{
322 enum ib_gid_type gid_type = IB_GID_TYPE_IB;
323 u8 roce_version = 0;

--- 25 unchanged lines hidden (view full) ---

349 break;
350
351 default:
352 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
353 }
354
355 return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
356 roce_l3_type, gid->raw, mac, vlan,
437}
438
439static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
440 unsigned int index, const union ib_gid *gid,
441 const struct ib_gid_attr *attr)
442{
443 enum ib_gid_type gid_type = IB_GID_TYPE_IB;
444 u8 roce_version = 0;

--- 25 unchanged lines hidden (view full) ---

470 break;
471
472 default:
473 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
474 }
475
476 return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
477 roce_l3_type, gid->raw, mac, vlan,
357 vlan_id);
478 vlan_id, port_num);
358}
359
360static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
361 unsigned int index, const union ib_gid *gid,
362 const struct ib_gid_attr *attr,
363 __always_unused void **context)
364{
365 return set_roce_addr(to_mdev(device), port_num, index, gid, attr);

--- 67 unchanged lines hidden (view full) ---

433 if (mlx5_ib_port_link_layer(ibdev, 1) ==
434 IB_LINK_LAYER_ETHERNET)
435 return MLX5_VPORT_ACCESS_METHOD_NIC;
436
437 return MLX5_VPORT_ACCESS_METHOD_HCA;
438}
439
440static void get_atomic_caps(struct mlx5_ib_dev *dev,
479}
480
481static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
482 unsigned int index, const union ib_gid *gid,
483 const struct ib_gid_attr *attr,
484 __always_unused void **context)
485{
486 return set_roce_addr(to_mdev(device), port_num, index, gid, attr);

--- 67 unchanged lines hidden (view full) ---

554 if (mlx5_ib_port_link_layer(ibdev, 1) ==
555 IB_LINK_LAYER_ETHERNET)
556 return MLX5_VPORT_ACCESS_METHOD_NIC;
557
558 return MLX5_VPORT_ACCESS_METHOD_HCA;
559}
560
561static void get_atomic_caps(struct mlx5_ib_dev *dev,
562 u8 atomic_size_qp,
441 struct ib_device_attr *props)
442{
443 u8 tmp;
444 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
563 struct ib_device_attr *props)
564{
565 u8 tmp;
566 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
445 u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
446 u8 atomic_req_8B_endianness_mode =
447 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
448
449 /* Check if HW supports 8 bytes standard atomic operations and capable
450 * of host endianness respond
451 */
452 tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
453 if (((atomic_operations & tmp) == tmp) &&
454 (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
455 (atomic_req_8B_endianness_mode)) {
456 props->atomic_cap = IB_ATOMIC_HCA;
457 } else {
458 props->atomic_cap = IB_ATOMIC_NONE;
459 }
460}
461
567 u8 atomic_req_8B_endianness_mode =
568 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
569
570 /* Check if HW supports 8 bytes standard atomic operations and capable
571 * of host endianness respond
572 */
573 tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
574 if (((atomic_operations & tmp) == tmp) &&
575 (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
576 (atomic_req_8B_endianness_mode)) {
577 props->atomic_cap = IB_ATOMIC_HCA;
578 } else {
579 props->atomic_cap = IB_ATOMIC_NONE;
580 }
581}
582
583static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
584 struct ib_device_attr *props)
585{
586 u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
587
588 get_atomic_caps(dev, atomic_size_qp, props);
589}
590
591static void get_atomic_caps_dc(struct mlx5_ib_dev *dev,
592 struct ib_device_attr *props)
593{
594 u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
595
596 get_atomic_caps(dev, atomic_size_qp, props);
597}
598
599bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev)
600{
601 struct ib_device_attr props = {};
602
603 get_atomic_caps_dc(dev, &props);
604 return (props.atomic_cap == IB_ATOMIC_HCA) ? true : false;
605}
462static int mlx5_query_system_image_guid(struct ib_device *ibdev,
463 __be64 *sys_image_guid)
464{
465 struct mlx5_ib_dev *dev = to_mdev(ibdev);
466 struct mlx5_core_dev *mdev = dev->mdev;
467 u64 tmp;
468 int err;
469

--- 112 unchanged lines hidden (view full) ---

582{
583 struct mlx5_ib_dev *dev = to_mdev(ibdev);
584 struct mlx5_core_dev *mdev = dev->mdev;
585 int err = -ENOMEM;
586 int max_sq_desc;
587 int max_rq_sg;
588 int max_sq_sg;
589 u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
606static int mlx5_query_system_image_guid(struct ib_device *ibdev,
607 __be64 *sys_image_guid)
608{
609 struct mlx5_ib_dev *dev = to_mdev(ibdev);
610 struct mlx5_core_dev *mdev = dev->mdev;
611 u64 tmp;
612 int err;
613

--- 112 unchanged lines hidden (view full) ---

726{
727 struct mlx5_ib_dev *dev = to_mdev(ibdev);
728 struct mlx5_core_dev *mdev = dev->mdev;
729 int err = -ENOMEM;
730 int max_sq_desc;
731 int max_rq_sg;
732 int max_sq_sg;
733 u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
734 bool raw_support = !mlx5_core_mp_enabled(mdev);
590 struct mlx5_ib_query_device_resp resp = {};
591 size_t resp_len;
592 u64 max_tso;
593
594 resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
595 if (uhw->outlen && uhw->outlen < resp_len)
596 return -EINVAL;
597 else

--- 47 unchanged lines hidden (view full) ---

645 IB_PROT_T10DIF_TYPE_2 |
646 IB_PROT_T10DIF_TYPE_3;
647 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
648 IB_GUARD_T10DIF_CSUM;
649 }
650 if (MLX5_CAP_GEN(mdev, block_lb_mc))
651 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
652
735 struct mlx5_ib_query_device_resp resp = {};
736 size_t resp_len;
737 u64 max_tso;
738
739 resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
740 if (uhw->outlen && uhw->outlen < resp_len)
741 return -EINVAL;
742 else

--- 47 unchanged lines hidden (view full) ---

790 IB_PROT_T10DIF_TYPE_2 |
791 IB_PROT_T10DIF_TYPE_3;
792 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
793 IB_GUARD_T10DIF_CSUM;
794 }
795 if (MLX5_CAP_GEN(mdev, block_lb_mc))
796 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
797
653 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads)) {
798 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
654 if (MLX5_CAP_ETH(mdev, csum_cap)) {
655 /* Legacy bit to support old userspace libraries */
656 props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
657 props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
658 }
659
660 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
661 props->raw_packet_caps |=

--- 15 unchanged lines hidden (view full) ---

677 resp.rss_caps.rx_hash_fields_mask =
678 MLX5_RX_HASH_SRC_IPV4 |
679 MLX5_RX_HASH_DST_IPV4 |
680 MLX5_RX_HASH_SRC_IPV6 |
681 MLX5_RX_HASH_DST_IPV6 |
682 MLX5_RX_HASH_SRC_PORT_TCP |
683 MLX5_RX_HASH_DST_PORT_TCP |
684 MLX5_RX_HASH_SRC_PORT_UDP |
799 if (MLX5_CAP_ETH(mdev, csum_cap)) {
800 /* Legacy bit to support old userspace libraries */
801 props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
802 props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
803 }
804
805 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
806 props->raw_packet_caps |=

--- 15 unchanged lines hidden (view full) ---

822 resp.rss_caps.rx_hash_fields_mask =
823 MLX5_RX_HASH_SRC_IPV4 |
824 MLX5_RX_HASH_DST_IPV4 |
825 MLX5_RX_HASH_SRC_IPV6 |
826 MLX5_RX_HASH_DST_IPV6 |
827 MLX5_RX_HASH_SRC_PORT_TCP |
828 MLX5_RX_HASH_DST_PORT_TCP |
829 MLX5_RX_HASH_SRC_PORT_UDP |
685 MLX5_RX_HASH_DST_PORT_UDP;
830 MLX5_RX_HASH_DST_PORT_UDP |
831 MLX5_RX_HASH_INNER;
686 resp.response_length += sizeof(resp.rss_caps);
687 }
688 } else {
689 if (field_avail(typeof(resp), tso_caps, uhw->outlen))
690 resp.response_length += sizeof(resp.tso_caps);
691 if (field_avail(typeof(resp), rss_caps, uhw->outlen))
692 resp.response_length += sizeof(resp.rss_caps);
693 }
694
695 if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
696 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
697 props->device_cap_flags |= IB_DEVICE_UD_TSO;
698 }
699
700 if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
832 resp.response_length += sizeof(resp.rss_caps);
833 }
834 } else {
835 if (field_avail(typeof(resp), tso_caps, uhw->outlen))
836 resp.response_length += sizeof(resp.tso_caps);
837 if (field_avail(typeof(resp), rss_caps, uhw->outlen))
838 resp.response_length += sizeof(resp.rss_caps);
839 }
840
841 if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
842 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
843 props->device_cap_flags |= IB_DEVICE_UD_TSO;
844 }
845
846 if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
701 MLX5_CAP_GEN(dev->mdev, general_notification_event))
847 MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
848 raw_support)
702 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
703
704 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
705 MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
706 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
707
708 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
849 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
850
851 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
852 MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
853 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
854
855 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
709 MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
856 MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
857 raw_support) {
710 /* Legacy bit to support old userspace libraries */
711 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
712 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
713 }
714
715 if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
716 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
717

--- 23 unchanged lines hidden (view full) ---

741 props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
742 props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
743 props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
744 props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
745 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
746 props->max_srq_sge = max_rq_sg - 1;
747 props->max_fast_reg_page_list_len =
748 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
858 /* Legacy bit to support old userspace libraries */
859 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
860 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
861 }
862
863 if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
864 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
865

--- 23 unchanged lines hidden (view full) ---

889 props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
890 props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
891 props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
892 props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
893 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
894 props->max_srq_sge = max_rq_sg - 1;
895 props->max_fast_reg_page_list_len =
896 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
749 get_atomic_caps(dev, props);
897 get_atomic_caps_qp(dev, props);
750 props->masked_atomic_cap = IB_ATOMIC_NONE;
751 props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
752 props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
753 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
754 props->max_mcast_grp;
755 props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
756 props->max_ah = INT_MAX;
757 props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);

--- 7 unchanged lines hidden (view full) ---

765
766 if (MLX5_CAP_GEN(mdev, cd))
767 props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
768
769 if (!mlx5_core_is_pf(mdev))
770 props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
771
772 if (mlx5_ib_port_link_layer(ibdev, 1) ==
898 props->masked_atomic_cap = IB_ATOMIC_NONE;
899 props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
900 props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
901 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
902 props->max_mcast_grp;
903 props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
904 props->max_ah = INT_MAX;
905 props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);

--- 7 unchanged lines hidden (view full) ---

913
914 if (MLX5_CAP_GEN(mdev, cd))
915 props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
916
917 if (!mlx5_core_is_pf(mdev))
918 props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
919
920 if (mlx5_ib_port_link_layer(ibdev, 1) ==
773 IB_LINK_LAYER_ETHERNET) {
921 IB_LINK_LAYER_ETHERNET && raw_support) {
774 props->rss_caps.max_rwq_indirection_tables =
775 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
776 props->rss_caps.max_rwq_indirection_table_size =
777 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
778 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
779 props->max_wq_type_rq =
780 1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
781 }

--- 20 unchanged lines hidden (view full) ---

802 MLX5_CAP_GEN(dev->mdev, cqe_compression) ?
803 MLX5_CAP_GEN(dev->mdev, cqe_compression_max_num) : 0;
804 resp.cqe_comp_caps.supported_format =
805 MLX5_IB_CQE_RES_FORMAT_HASH |
806 MLX5_IB_CQE_RES_FORMAT_CSUM;
807 resp.response_length += sizeof(resp.cqe_comp_caps);
808 }
809
922 props->rss_caps.max_rwq_indirection_tables =
923 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
924 props->rss_caps.max_rwq_indirection_table_size =
925 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
926 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
927 props->max_wq_type_rq =
928 1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
929 }

--- 20 unchanged lines hidden (view full) ---

950 MLX5_CAP_GEN(dev->mdev, cqe_compression) ?
951 MLX5_CAP_GEN(dev->mdev, cqe_compression_max_num) : 0;
952 resp.cqe_comp_caps.supported_format =
953 MLX5_IB_CQE_RES_FORMAT_HASH |
954 MLX5_IB_CQE_RES_FORMAT_CSUM;
955 resp.response_length += sizeof(resp.cqe_comp_caps);
956 }
957
810 if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen)) {
958 if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen) &&
959 raw_support) {
811 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
812 MLX5_CAP_GEN(mdev, qos)) {
813 resp.packet_pacing_caps.qp_rate_limit_max =
814 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
815 resp.packet_pacing_caps.qp_rate_limit_min =
816 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
817 resp.packet_pacing_caps.supported_qpts |=
818 1 << IB_QPT_RAW_PACKET;

--- 42 unchanged lines hidden (view full) ---

861 MLX5_IB_SW_PARSING_LSO;
862
863 if (resp.sw_parsing_caps.sw_parsing_offloads)
864 resp.sw_parsing_caps.supported_qpts =
865 BIT(IB_QPT_RAW_PACKET);
866 }
867 }
868
960 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
961 MLX5_CAP_GEN(mdev, qos)) {
962 resp.packet_pacing_caps.qp_rate_limit_max =
963 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
964 resp.packet_pacing_caps.qp_rate_limit_min =
965 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
966 resp.packet_pacing_caps.supported_qpts |=
967 1 << IB_QPT_RAW_PACKET;

--- 42 unchanged lines hidden (view full) ---

1010 MLX5_IB_SW_PARSING_LSO;
1011
1012 if (resp.sw_parsing_caps.sw_parsing_offloads)
1013 resp.sw_parsing_caps.supported_qpts =
1014 BIT(IB_QPT_RAW_PACKET);
1015 }
1016 }
1017
869 if (field_avail(typeof(resp), striding_rq_caps, uhw->outlen)) {
1018 if (field_avail(typeof(resp), striding_rq_caps, uhw->outlen) &&
1019 raw_support) {
870 resp.response_length += sizeof(resp.striding_rq_caps);
871 if (MLX5_CAP_GEN(mdev, striding_rq)) {
872 resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
873 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
874 resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
875 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
876 resp.striding_rq_caps.min_single_wqe_log_num_of_strides =
877 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;

--- 214 unchanged lines hidden (view full) ---

1092 ret = mlx5_query_port_roce(ibdev, port, props);
1093 break;
1094
1095 default:
1096 ret = -EINVAL;
1097 }
1098
1099 if (!ret && props) {
1020 resp.response_length += sizeof(resp.striding_rq_caps);
1021 if (MLX5_CAP_GEN(mdev, striding_rq)) {
1022 resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1023 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1024 resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1025 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1026 resp.striding_rq_caps.min_single_wqe_log_num_of_strides =
1027 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;

--- 214 unchanged lines hidden (view full) ---

1242 ret = mlx5_query_port_roce(ibdev, port, props);
1243 break;
1244
1245 default:
1246 ret = -EINVAL;
1247 }
1248
1249 if (!ret && props) {
1100 count = mlx5_core_reserved_gids_count(to_mdev(ibdev)->mdev);
1250 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1251 struct mlx5_core_dev *mdev;
1252 bool put_mdev = true;
1253
1254 mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1255 if (!mdev) {
1256 /* If the port isn't affiliated yet query the master.
1257 * The master and slave will have the same values.
1258 */
1259 mdev = dev->mdev;
1260 port = 1;
1261 put_mdev = false;
1262 }
1263 count = mlx5_core_reserved_gids_count(mdev);
1264 if (put_mdev)
1265 mlx5_ib_put_native_port_mdev(dev, port);
1101 props->gid_tbl_len -= count;
1102 }
1103 return ret;
1104}
1105
1106static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
1107 union ib_gid *gid)
1108{

--- 8 unchanged lines hidden (view full) ---

1117 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1118
1119 default:
1120 return -EINVAL;
1121 }
1122
1123}
1124
1266 props->gid_tbl_len -= count;
1267 }
1268 return ret;
1269}
1270
1271static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
1272 union ib_gid *gid)
1273{

--- 8 unchanged lines hidden (view full) ---

1282 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1283
1284 default:
1285 return -EINVAL;
1286 }
1287
1288}
1289
1125static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1126 u16 *pkey)
1290static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port,
1291 u16 index, u16 *pkey)
1127{
1128 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1292{
1293 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1129 struct mlx5_core_dev *mdev = dev->mdev;
1294 struct mlx5_core_dev *mdev;
1295 bool put_mdev = true;
1296 u8 mdev_port_num;
1297 int err;
1130
1298
1299 mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1300 if (!mdev) {
1301 /* The port isn't affiliated yet, get the PKey from the master
1302 * port. For RoCE the PKey tables will be the same.
1303 */
1304 put_mdev = false;
1305 mdev = dev->mdev;
1306 mdev_port_num = 1;
1307 }
1308
1309 err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1310 index, pkey);
1311 if (put_mdev)
1312 mlx5_ib_put_native_port_mdev(dev, port);
1313
1314 return err;
1315}
1316
1317static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1318 u16 *pkey)
1319{
1131 switch (mlx5_get_vport_access_method(ibdev)) {
1132 case MLX5_VPORT_ACCESS_METHOD_MAD:
1133 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1134
1135 case MLX5_VPORT_ACCESS_METHOD_HCA:
1136 case MLX5_VPORT_ACCESS_METHOD_NIC:
1320 switch (mlx5_get_vport_access_method(ibdev)) {
1321 case MLX5_VPORT_ACCESS_METHOD_MAD:
1322 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1323
1324 case MLX5_VPORT_ACCESS_METHOD_HCA:
1325 case MLX5_VPORT_ACCESS_METHOD_NIC:
1137 return mlx5_query_hca_vport_pkey(mdev, 0, port, 0, index,
1138 pkey);
1326 return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1139 default:
1140 return -EINVAL;
1141 }
1142}
1143
1144static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1145 struct ib_device_modify *props)
1146{

--- 22 unchanged lines hidden (view full) ---

1169
1170 return err;
1171}
1172
1173static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,
1174 u32 value)
1175{
1176 struct mlx5_hca_vport_context ctx = {};
1327 default:
1328 return -EINVAL;
1329 }
1330}
1331
1332static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1333 struct ib_device_modify *props)
1334{

--- 22 unchanged lines hidden (view full) ---

1357
1358 return err;
1359}
1360
1361static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,
1362 u32 value)
1363{
1364 struct mlx5_hca_vport_context ctx = {};
1365 struct mlx5_core_dev *mdev;
1366 u8 mdev_port_num;
1177 int err;
1178
1367 int err;
1368
1179 err = mlx5_query_hca_vport_context(dev->mdev, 0,
1180 port_num, 0, &ctx);
1369 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1370 if (!mdev)
1371 return -ENODEV;
1372
1373 err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1181 if (err)
1374 if (err)
1182 return err;
1375 goto out;
1183
1184 if (~ctx.cap_mask1_perm & mask) {
1185 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1186 mask, ctx.cap_mask1_perm);
1376
1377 if (~ctx.cap_mask1_perm & mask) {
1378 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1379 mask, ctx.cap_mask1_perm);
1187 return -EINVAL;
1380 err = -EINVAL;
1381 goto out;
1188 }
1189
1190 ctx.cap_mask1 = value;
1191 ctx.cap_mask1_perm = mask;
1382 }
1383
1384 ctx.cap_mask1 = value;
1385 ctx.cap_mask1_perm = mask;
1192 err = mlx5_core_modify_hca_vport_context(dev->mdev, 0,
1193 port_num, 0, &ctx);
1386 err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1387 0, &ctx);
1194
1388
1389out:
1390 mlx5_ib_put_native_port_mdev(dev, port_num);
1391
1195 return err;
1196}
1197
1198static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
1199 struct ib_port_modify *props)
1200{
1201 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1202 struct ib_port_attr attr;

--- 33 unchanged lines hidden (view full) ---

1236}
1237
1238static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1239{
1240 mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1241 caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1242}
1243
1392 return err;
1393}
1394
1395static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
1396 struct ib_port_modify *props)
1397{
1398 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1399 struct ib_port_attr attr;

--- 33 unchanged lines hidden (view full) ---

1433}
1434
1435static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1436{
1437 mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1438 caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1439}
1440
1441static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1442{
1443 /* Large page with non 4k uar support might limit the dynamic size */
1444 if (uars_per_sys_page == 1 && PAGE_SIZE > 4096)
1445 return MLX5_MIN_DYN_BFREGS;
1446
1447 return MLX5_MAX_DYN_BFREGS;
1448}
1449
1244static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1245 struct mlx5_ib_alloc_ucontext_req_v2 *req,
1450static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1451 struct mlx5_ib_alloc_ucontext_req_v2 *req,
1246 u32 *num_sys_pages)
1452 struct mlx5_bfreg_info *bfregi)
1247{
1248 int uars_per_sys_page;
1249 int bfregs_per_sys_page;
1250 int ref_bfregs = req->total_num_bfregs;
1251
1252 if (req->total_num_bfregs == 0)
1253 return -EINVAL;
1254
1255 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1256 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1257
1258 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1259 return -ENOMEM;
1260
1261 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1262 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1453{
1454 int uars_per_sys_page;
1455 int bfregs_per_sys_page;
1456 int ref_bfregs = req->total_num_bfregs;
1457
1458 if (req->total_num_bfregs == 0)
1459 return -EINVAL;
1460
1461 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1462 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1463
1464 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1465 return -ENOMEM;
1466
1467 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1468 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1469 /* This holds the required static allocation asked by the user */
1263 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1470 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1264 *num_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1265
1266 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1267 return -EINVAL;
1268
1471 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1472 return -EINVAL;
1473
1269 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, using %d sys pages\n",
1474 bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1475 bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1476 bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1477 bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1478
1479 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1270 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1271 lib_uar_4k ? "yes" : "no", ref_bfregs,
1480 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1481 lib_uar_4k ? "yes" : "no", ref_bfregs,
1272 req->total_num_bfregs, *num_sys_pages);
1482 req->total_num_bfregs, bfregi->total_num_bfregs,
1483 bfregi->num_sys_pages);
1273
1274 return 0;
1275}
1276
1277static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1278{
1279 struct mlx5_bfreg_info *bfregi;
1280 int err;
1281 int i;
1282
1283 bfregi = &context->bfregi;
1484
1485 return 0;
1486}
1487
1488static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1489{
1490 struct mlx5_bfreg_info *bfregi;
1491 int err;
1492 int i;
1493
1494 bfregi = &context->bfregi;
1284 for (i = 0; i < bfregi->num_sys_pages; i++) {
1495 for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1285 err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1286 if (err)
1287 goto error;
1288
1289 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1290 }
1496 err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1497 if (err)
1498 goto error;
1499
1500 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1501 }
1502
1503 for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1504 bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1505
1291 return 0;
1292
1293error:
1294 for (--i; i >= 0; i--)
1295 if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1296 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1297
1298 return err;
1299}
1300
1301static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1302{
1303 struct mlx5_bfreg_info *bfregi;
1304 int err;
1305 int i;
1306
1307 bfregi = &context->bfregi;
1308 for (i = 0; i < bfregi->num_sys_pages; i++) {
1506 return 0;
1507
1508error:
1509 for (--i; i >= 0; i--)
1510 if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1511 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1512
1513 return err;
1514}
1515
1516static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1517{
1518 struct mlx5_bfreg_info *bfregi;
1519 int err;
1520 int i;
1521
1522 bfregi = &context->bfregi;
1523 for (i = 0; i < bfregi->num_sys_pages; i++) {
1309 err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1310 if (err) {
1311 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1312 return err;
1524 if (i < bfregi->num_static_sys_pages ||
1525 bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX) {
1526 err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1527 if (err) {
1528 mlx5_ib_warn(dev, "failed to free uar %d, err=%d\n", i, err);
1529 return err;
1530 }
1313 }
1314 }
1531 }
1532 }
1533
1315 return 0;
1316}
1317
1318static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn)
1319{
1320 int err;
1321
1322 err = mlx5_core_alloc_transport_domain(dev->mdev, tdn);

--- 32 unchanged lines hidden (view full) ---

1355}
1356
1357static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1358 struct ib_udata *udata)
1359{
1360 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1361 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1362 struct mlx5_ib_alloc_ucontext_resp resp = {};
1534 return 0;
1535}
1536
1537static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn)
1538{
1539 int err;
1540
1541 err = mlx5_core_alloc_transport_domain(dev->mdev, tdn);

--- 32 unchanged lines hidden (view full) ---

1574}
1575
1576static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1577 struct ib_udata *udata)
1578{
1579 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1580 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1581 struct mlx5_ib_alloc_ucontext_resp resp = {};
1582 struct mlx5_core_dev *mdev = dev->mdev;
1363 struct mlx5_ib_ucontext *context;
1364 struct mlx5_bfreg_info *bfregi;
1365 int ver;
1366 int err;
1367 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1368 max_cqe_version);
1369 bool lib_uar_4k;
1370

--- 44 unchanged lines hidden (view full) ---

1415 context = kzalloc(sizeof(*context), GFP_KERNEL);
1416 if (!context)
1417 return ERR_PTR(-ENOMEM);
1418
1419 lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1420 bfregi = &context->bfregi;
1421
1422 /* updates req->total_num_bfregs */
1583 struct mlx5_ib_ucontext *context;
1584 struct mlx5_bfreg_info *bfregi;
1585 int ver;
1586 int err;
1587 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1588 max_cqe_version);
1589 bool lib_uar_4k;
1590

--- 44 unchanged lines hidden (view full) ---

1635 context = kzalloc(sizeof(*context), GFP_KERNEL);
1636 if (!context)
1637 return ERR_PTR(-ENOMEM);
1638
1639 lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1640 bfregi = &context->bfregi;
1641
1642 /* updates req->total_num_bfregs */
1423 err = calc_total_bfregs(dev, lib_uar_4k, &req, &bfregi->num_sys_pages);
1643 err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1424 if (err)
1425 goto out_ctx;
1426
1427 mutex_init(&bfregi->lock);
1428 bfregi->lib_uar_4k = lib_uar_4k;
1644 if (err)
1645 goto out_ctx;
1646
1647 mutex_init(&bfregi->lock);
1648 bfregi->lib_uar_4k = lib_uar_4k;
1429 bfregi->count = kcalloc(req.total_num_bfregs, sizeof(*bfregi->count),
1649 bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
1430 GFP_KERNEL);
1431 if (!bfregi->count) {
1432 err = -ENOMEM;
1433 goto out_ctx;
1434 }
1435
1436 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1437 sizeof(*bfregi->sys_pages),

--- 25 unchanged lines hidden (view full) ---

1463 }
1464
1465 INIT_LIST_HEAD(&context->vma_private_list);
1466 mutex_init(&context->vma_private_list_mutex);
1467 INIT_LIST_HEAD(&context->db_page_list);
1468 mutex_init(&context->db_page_mutex);
1469
1470 resp.tot_bfregs = req.total_num_bfregs;
1650 GFP_KERNEL);
1651 if (!bfregi->count) {
1652 err = -ENOMEM;
1653 goto out_ctx;
1654 }
1655
1656 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1657 sizeof(*bfregi->sys_pages),

--- 25 unchanged lines hidden (view full) ---

1683 }
1684
1685 INIT_LIST_HEAD(&context->vma_private_list);
1686 mutex_init(&context->vma_private_list_mutex);
1687 INIT_LIST_HEAD(&context->db_page_list);
1688 mutex_init(&context->db_page_mutex);
1689
1690 resp.tot_bfregs = req.total_num_bfregs;
1471 resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
1691 resp.num_ports = dev->num_ports;
1472
1473 if (field_avail(typeof(resp), cqe_version, udata->outlen))
1474 resp.response_length += sizeof(resp.cqe_version);
1475
1476 if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
1477 resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1478 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1479 resp.response_length += sizeof(resp.cmds_supp_uhw);
1480 }
1481
1482 if (field_avail(typeof(resp), eth_min_inline, udata->outlen)) {
1483 if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1484 mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
1485 resp.eth_min_inline++;
1486 }
1487 resp.response_length += sizeof(resp.eth_min_inline);
1488 }
1489
1692
1693 if (field_avail(typeof(resp), cqe_version, udata->outlen))
1694 resp.response_length += sizeof(resp.cqe_version);
1695
1696 if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
1697 resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1698 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1699 resp.response_length += sizeof(resp.cmds_supp_uhw);
1700 }
1701
1702 if (field_avail(typeof(resp), eth_min_inline, udata->outlen)) {
1703 if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1704 mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
1705 resp.eth_min_inline++;
1706 }
1707 resp.response_length += sizeof(resp.eth_min_inline);
1708 }
1709
1710 if (field_avail(typeof(resp), clock_info_versions, udata->outlen)) {
1711 if (mdev->clock_info)
1712 resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1713 resp.response_length += sizeof(resp.clock_info_versions);
1714 }
1715
1490 /*
1491 * We don't want to expose information from the PCI bar that is located
1492 * after 4096 bytes, so if the arch only supports larger pages, let's
1493 * pretend we don't support reading the HCA's core clock. This is also
1494 * forced by mmap function.
1495 */
1496 if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
1497 if (PAGE_SIZE <= 4096) {
1498 resp.comp_mask |=
1499 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1500 resp.hca_core_clock_offset =
1501 offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1502 }
1716 /*
1717 * We don't want to expose information from the PCI bar that is located
1718 * after 4096 bytes, so if the arch only supports larger pages, let's
1719 * pretend we don't support reading the HCA's core clock. This is also
1720 * forced by mmap function.
1721 */
1722 if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
1723 if (PAGE_SIZE <= 4096) {
1724 resp.comp_mask |=
1725 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1726 resp.hca_core_clock_offset =
1727 offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1728 }
1503 resp.response_length += sizeof(resp.hca_core_clock_offset) +
1504 sizeof(resp.reserved2);
1729 resp.response_length += sizeof(resp.hca_core_clock_offset);
1505 }
1506
1507 if (field_avail(typeof(resp), log_uar_size, udata->outlen))
1508 resp.response_length += sizeof(resp.log_uar_size);
1509
1510 if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
1511 resp.response_length += sizeof(resp.num_uars_per_page);
1512
1730 }
1731
1732 if (field_avail(typeof(resp), log_uar_size, udata->outlen))
1733 resp.response_length += sizeof(resp.log_uar_size);
1734
1735 if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
1736 resp.response_length += sizeof(resp.num_uars_per_page);
1737
1738 if (field_avail(typeof(resp), num_dyn_bfregs, udata->outlen)) {
1739 resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
1740 resp.response_length += sizeof(resp.num_dyn_bfregs);
1741 }
1742
1513 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1514 if (err)
1515 goto out_td;
1516
1517 bfregi->ver = ver;
1518 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1519 context->cqe_version = resp.cqe_version;
1520 context->lib_caps = req.lib_caps;

--- 38 unchanged lines hidden (view full) ---

1559 kfree(bfregi->sys_pages);
1560 kfree(bfregi->count);
1561 kfree(context);
1562
1563 return 0;
1564}
1565
1566static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1743 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1744 if (err)
1745 goto out_td;
1746
1747 bfregi->ver = ver;
1748 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1749 context->cqe_version = resp.cqe_version;
1750 context->lib_caps = req.lib_caps;

--- 38 unchanged lines hidden (view full) ---

1789 kfree(bfregi->sys_pages);
1790 kfree(bfregi->count);
1791 kfree(context);
1792
1793 return 0;
1794}
1795
1796static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1567 struct mlx5_bfreg_info *bfregi,
1568 int idx)
1797 int uar_idx)
1569{
1570 int fw_uars_per_page;
1571
1572 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1573
1798{
1799 int fw_uars_per_page;
1800
1801 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1802
1574 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) +
1575 bfregi->sys_pages[idx] / fw_uars_per_page;
1803 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
1576}
1577
1578static int get_command(unsigned long offset)
1579{
1580 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1581}
1582
1583static int get_arg(unsigned long offset)
1584{
1585 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
1586}
1587
1588static int get_index(unsigned long offset)
1589{
1590 return get_arg(offset);
1591}
1592
1804}
1805
1806static int get_command(unsigned long offset)
1807{
1808 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1809}
1810
1811static int get_arg(unsigned long offset)
1812{
1813 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
1814}
1815
1816static int get_index(unsigned long offset)
1817{
1818 return get_arg(offset);
1819}
1820
1821/* Index resides in an extra byte to enable larger values than 255 */
1822static int get_extended_index(unsigned long offset)
1823{
1824 return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
1825}
1826
1593static void mlx5_ib_vma_open(struct vm_area_struct *area)
1594{
1595 /* vma_open is called when a new VMA is created on top of our VMA. This
1596 * is done through either mremap flow or split_vma (usually due to
1597 * mlock, madvise, munmap, etc.) We do not support a clone of the VMA,
1598 * as this VMA is strongly hardware related. Therefore we set the
1599 * vm_ops of the newly created/cloned VMA to NULL, to prevent it from
1600 * calling us again and trying to do incorrect actions. We assume that

--- 125 unchanged lines hidden (view full) ---

1726 return "best effort WC";
1727 case MLX5_IB_MMAP_NC_PAGE:
1728 return "NC";
1729 default:
1730 return NULL;
1731 }
1732}
1733
1827static void mlx5_ib_vma_open(struct vm_area_struct *area)
1828{
1829 /* vma_open is called when a new VMA is created on top of our VMA. This
1830 * is done through either mremap flow or split_vma (usually due to
1831 * mlock, madvise, munmap, etc.) We do not support a clone of the VMA,
1832 * as this VMA is strongly hardware related. Therefore we set the
1833 * vm_ops of the newly created/cloned VMA to NULL, to prevent it from
1834 * calling us again and trying to do incorrect actions. We assume that

--- 125 unchanged lines hidden (view full) ---

1960 return "best effort WC";
1961 case MLX5_IB_MMAP_NC_PAGE:
1962 return "NC";
1963 default:
1964 return NULL;
1965 }
1966}
1967
1968static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
1969 struct vm_area_struct *vma,
1970 struct mlx5_ib_ucontext *context)
1971{
1972 phys_addr_t pfn;
1973 int err;
1974
1975 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1976 return -EINVAL;
1977
1978 if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
1979 return -EOPNOTSUPP;
1980
1981 if (vma->vm_flags & VM_WRITE)
1982 return -EPERM;
1983
1984 if (!dev->mdev->clock_info_page)
1985 return -EOPNOTSUPP;
1986
1987 pfn = page_to_pfn(dev->mdev->clock_info_page);
1988 err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
1989 vma->vm_page_prot);
1990 if (err)
1991 return err;
1992
1993 mlx5_ib_dbg(dev, "mapped clock info at 0x%lx, PA 0x%llx\n",
1994 vma->vm_start,
1995 (unsigned long long)pfn << PAGE_SHIFT);
1996
1997 return mlx5_ib_set_vma_data(vma, context);
1998}
1999
1734static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
1735 struct vm_area_struct *vma,
1736 struct mlx5_ib_ucontext *context)
1737{
1738 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1739 int err;
1740 unsigned long idx;
1741 phys_addr_t pfn, pa;
1742 pgprot_t prot;
2000static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2001 struct vm_area_struct *vma,
2002 struct mlx5_ib_ucontext *context)
2003{
2004 struct mlx5_bfreg_info *bfregi = &context->bfregi;
2005 int err;
2006 unsigned long idx;
2007 phys_addr_t pfn, pa;
2008 pgprot_t prot;
1743 int uars_per_page;
2009 u32 bfreg_dyn_idx = 0;
2010 u32 uar_index;
2011 int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2012 int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2013 bfregi->num_static_sys_pages;
1744
1745 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1746 return -EINVAL;
1747
2014
2015 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2016 return -EINVAL;
2017
1748 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
1749 idx = get_index(vma->vm_pgoff);
1750 if (idx % uars_per_page ||
1751 idx * uars_per_page >= bfregi->num_sys_pages) {
1752 mlx5_ib_warn(dev, "invalid uar index %lu\n", idx);
2018 if (dyn_uar)
2019 idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2020 else
2021 idx = get_index(vma->vm_pgoff);
2022
2023 if (idx >= max_valid_idx) {
2024 mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2025 idx, max_valid_idx);
1753 return -EINVAL;
1754 }
1755
1756 switch (cmd) {
1757 case MLX5_IB_MMAP_WC_PAGE:
2026 return -EINVAL;
2027 }
2028
2029 switch (cmd) {
2030 case MLX5_IB_MMAP_WC_PAGE:
2031 case MLX5_IB_MMAP_ALLOC_WC:
1758/* Some architectures don't support WC memory */
1759#if defined(CONFIG_X86)
1760 if (!pat_enabled())
1761 return -EPERM;
1762#elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
1763 return -EPERM;
1764#endif
1765 /* fall through */
1766 case MLX5_IB_MMAP_REGULAR_PAGE:
1767 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
1768 prot = pgprot_writecombine(vma->vm_page_prot);
1769 break;
1770 case MLX5_IB_MMAP_NC_PAGE:
1771 prot = pgprot_noncached(vma->vm_page_prot);
1772 break;
1773 default:
1774 return -EINVAL;
1775 }
1776
2032/* Some architectures don't support WC memory */
2033#if defined(CONFIG_X86)
2034 if (!pat_enabled())
2035 return -EPERM;
2036#elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
2037 return -EPERM;
2038#endif
2039 /* fall through */
2040 case MLX5_IB_MMAP_REGULAR_PAGE:
2041 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2042 prot = pgprot_writecombine(vma->vm_page_prot);
2043 break;
2044 case MLX5_IB_MMAP_NC_PAGE:
2045 prot = pgprot_noncached(vma->vm_page_prot);
2046 break;
2047 default:
2048 return -EINVAL;
2049 }
2050
1777 pfn = uar_index2pfn(dev, bfregi, idx);
2051 if (dyn_uar) {
2052 int uars_per_page;
2053
2054 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2055 bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2056 if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2057 mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2058 bfreg_dyn_idx, bfregi->total_num_bfregs);
2059 return -EINVAL;
2060 }
2061
2062 mutex_lock(&bfregi->lock);
2063 /* Fail if uar already allocated, first bfreg index of each
2064 * page holds its count.
2065 */
2066 if (bfregi->count[bfreg_dyn_idx]) {
2067 mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2068 mutex_unlock(&bfregi->lock);
2069 return -EINVAL;
2070 }
2071
2072 bfregi->count[bfreg_dyn_idx]++;
2073 mutex_unlock(&bfregi->lock);
2074
2075 err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index);
2076 if (err) {
2077 mlx5_ib_warn(dev, "UAR alloc failed\n");
2078 goto free_bfreg;
2079 }
2080 } else {
2081 uar_index = bfregi->sys_pages[idx];
2082 }
2083
2084 pfn = uar_index2pfn(dev, uar_index);
1778 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
1779
1780 vma->vm_page_prot = prot;
1781 err = io_remap_pfn_range(vma, vma->vm_start, pfn,
1782 PAGE_SIZE, vma->vm_page_prot);
1783 if (err) {
1784 mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
1785 err, vma->vm_start, &pfn, mmap_cmd2str(cmd));
2085 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2086
2087 vma->vm_page_prot = prot;
2088 err = io_remap_pfn_range(vma, vma->vm_start, pfn,
2089 PAGE_SIZE, vma->vm_page_prot);
2090 if (err) {
2091 mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
2092 err, vma->vm_start, &pfn, mmap_cmd2str(cmd));
1786 return -EAGAIN;
2093 err = -EAGAIN;
2094 goto err;
1787 }
1788
1789 pa = pfn << PAGE_SHIFT;
1790 mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA %pa\n", mmap_cmd2str(cmd),
1791 vma->vm_start, &pa);
1792
2095 }
2096
2097 pa = pfn << PAGE_SHIFT;
2098 mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA %pa\n", mmap_cmd2str(cmd),
2099 vma->vm_start, &pa);
2100
1793 return mlx5_ib_set_vma_data(vma, context);
2101 err = mlx5_ib_set_vma_data(vma, context);
2102 if (err)
2103 goto err;
2104
2105 if (dyn_uar)
2106 bfregi->sys_pages[idx] = uar_index;
2107 return 0;
2108
2109err:
2110 if (!dyn_uar)
2111 return err;
2112
2113 mlx5_cmd_free_uar(dev->mdev, idx);
2114
2115free_bfreg:
2116 mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2117
2118 return err;
1794}
1795
1796static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
1797{
1798 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1799 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1800 unsigned long command;
1801 phys_addr_t pfn;
1802
1803 command = get_command(vma->vm_pgoff);
1804 switch (command) {
1805 case MLX5_IB_MMAP_WC_PAGE:
1806 case MLX5_IB_MMAP_NC_PAGE:
1807 case MLX5_IB_MMAP_REGULAR_PAGE:
2119}
2120
2121static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2122{
2123 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2124 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2125 unsigned long command;
2126 phys_addr_t pfn;
2127
2128 command = get_command(vma->vm_pgoff);
2129 switch (command) {
2130 case MLX5_IB_MMAP_WC_PAGE:
2131 case MLX5_IB_MMAP_NC_PAGE:
2132 case MLX5_IB_MMAP_REGULAR_PAGE:
2133 case MLX5_IB_MMAP_ALLOC_WC:
1808 return uar_mmap(dev, command, vma, context);
1809
1810 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
1811 return -ENOSYS;
1812
1813 case MLX5_IB_MMAP_CORE_CLOCK:
1814 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1815 return -EINVAL;

--- 12 unchanged lines hidden (view full) ---

1828 if (io_remap_pfn_range(vma, vma->vm_start, pfn,
1829 PAGE_SIZE, vma->vm_page_prot))
1830 return -EAGAIN;
1831
1832 mlx5_ib_dbg(dev, "mapped internal timer at 0x%lx, PA 0x%llx\n",
1833 vma->vm_start,
1834 (unsigned long long)pfn << PAGE_SHIFT);
1835 break;
2134 return uar_mmap(dev, command, vma, context);
2135
2136 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2137 return -ENOSYS;
2138
2139 case MLX5_IB_MMAP_CORE_CLOCK:
2140 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2141 return -EINVAL;

--- 12 unchanged lines hidden (view full) ---

2154 if (io_remap_pfn_range(vma, vma->vm_start, pfn,
2155 PAGE_SIZE, vma->vm_page_prot))
2156 return -EAGAIN;
2157
2158 mlx5_ib_dbg(dev, "mapped internal timer at 0x%lx, PA 0x%llx\n",
2159 vma->vm_start,
2160 (unsigned long long)pfn << PAGE_SHIFT);
2161 break;
2162 case MLX5_IB_MMAP_CLOCK_INFO:
2163 return mlx5_ib_mmap_clock_info_page(dev, vma, context);
1836
1837 default:
1838 return -EINVAL;
1839 }
1840
1841 return 0;
1842}
1843

--- 812 unchanged lines hidden (view full) ---

2656 struct mlx5_ib_flow_prio *ft_prio;
2657 int err;
2658 int underlay_qpn;
2659
2660 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO)
2661 return ERR_PTR(-ENOMEM);
2662
2663 if (domain != IB_FLOW_DOMAIN_USER ||
2164
2165 default:
2166 return -EINVAL;
2167 }
2168
2169 return 0;
2170}
2171

--- 812 unchanged lines hidden (view full) ---

2984 struct mlx5_ib_flow_prio *ft_prio;
2985 int err;
2986 int underlay_qpn;
2987
2988 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO)
2989 return ERR_PTR(-ENOMEM);
2990
2991 if (domain != IB_FLOW_DOMAIN_USER ||
2664 flow_attr->port > MLX5_CAP_GEN(dev->mdev, num_ports) ||
2992 flow_attr->port > dev->num_ports ||
2665 (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP))
2666 return ERR_PTR(-EINVAL);
2667
2668 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
2669 if (!dst)
2670 return ERR_PTR(-ENOMEM);
2671
2672 mutex_lock(&dev->flow_db.lock);

--- 248 unchanged lines hidden (view full) ---

2921 if (err) {
2922 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2923 delay_drop->timeout);
2924 delay_drop->activate = false;
2925 }
2926 mutex_unlock(&delay_drop->lock);
2927}
2928
2993 (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP))
2994 return ERR_PTR(-EINVAL);
2995
2996 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
2997 if (!dst)
2998 return ERR_PTR(-ENOMEM);
2999
3000 mutex_lock(&dev->flow_db.lock);

--- 248 unchanged lines hidden (view full) ---

3249 if (err) {
3250 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
3251 delay_drop->timeout);
3252 delay_drop->activate = false;
3253 }
3254 mutex_unlock(&delay_drop->lock);
3255}
3256
2929static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
2930 enum mlx5_dev_event event, unsigned long param)
3257static void mlx5_ib_handle_event(struct work_struct *_work)
2931{
3258{
2932 struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
3259 struct mlx5_ib_event_work *work =
3260 container_of(_work, struct mlx5_ib_event_work, work);
3261 struct mlx5_ib_dev *ibdev;
2933 struct ib_event ibev;
2934 bool fatal = false;
2935 u8 port = 0;
2936
3262 struct ib_event ibev;
3263 bool fatal = false;
3264 u8 port = 0;
3265
2937 switch (event) {
3266 if (mlx5_core_is_mp_slave(work->dev)) {
3267 ibdev = mlx5_ib_get_ibdev_from_mpi(work->context);
3268 if (!ibdev)
3269 goto out;
3270 } else {
3271 ibdev = work->context;
3272 }
3273
3274 switch (work->event) {
2938 case MLX5_DEV_EVENT_SYS_ERROR:
2939 ibev.event = IB_EVENT_DEVICE_FATAL;
2940 mlx5_ib_handle_internal_error(ibdev);
2941 fatal = true;
2942 break;
2943
2944 case MLX5_DEV_EVENT_PORT_UP:
2945 case MLX5_DEV_EVENT_PORT_DOWN:
2946 case MLX5_DEV_EVENT_PORT_INITIALIZED:
3275 case MLX5_DEV_EVENT_SYS_ERROR:
3276 ibev.event = IB_EVENT_DEVICE_FATAL;
3277 mlx5_ib_handle_internal_error(ibdev);
3278 fatal = true;
3279 break;
3280
3281 case MLX5_DEV_EVENT_PORT_UP:
3282 case MLX5_DEV_EVENT_PORT_DOWN:
3283 case MLX5_DEV_EVENT_PORT_INITIALIZED:
2947 port = (u8)param;
3284 port = (u8)work->param;
2948
2949 /* In RoCE, port up/down events are handled in
2950 * mlx5_netdev_event().
2951 */
2952 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2953 IB_LINK_LAYER_ETHERNET)
3285
3286 /* In RoCE, port up/down events are handled in
3287 * mlx5_netdev_event().
3288 */
3289 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
3290 IB_LINK_LAYER_ETHERNET)
2954 return;
3291 goto out;
2955
3292
2956 ibev.event = (event == MLX5_DEV_EVENT_PORT_UP) ?
3293 ibev.event = (work->event == MLX5_DEV_EVENT_PORT_UP) ?
2957 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2958 break;
2959
2960 case MLX5_DEV_EVENT_LID_CHANGE:
2961 ibev.event = IB_EVENT_LID_CHANGE;
3294 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
3295 break;
3296
3297 case MLX5_DEV_EVENT_LID_CHANGE:
3298 ibev.event = IB_EVENT_LID_CHANGE;
2962 port = (u8)param;
3299 port = (u8)work->param;
2963 break;
2964
2965 case MLX5_DEV_EVENT_PKEY_CHANGE:
2966 ibev.event = IB_EVENT_PKEY_CHANGE;
3300 break;
3301
3302 case MLX5_DEV_EVENT_PKEY_CHANGE:
3303 ibev.event = IB_EVENT_PKEY_CHANGE;
2967 port = (u8)param;
3304 port = (u8)work->param;
2968
2969 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2970 break;
2971
2972 case MLX5_DEV_EVENT_GUID_CHANGE:
2973 ibev.event = IB_EVENT_GID_CHANGE;
3305
3306 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
3307 break;
3308
3309 case MLX5_DEV_EVENT_GUID_CHANGE:
3310 ibev.event = IB_EVENT_GID_CHANGE;
2974 port = (u8)param;
3311 port = (u8)work->param;
2975 break;
2976
2977 case MLX5_DEV_EVENT_CLIENT_REREG:
2978 ibev.event = IB_EVENT_CLIENT_REREGISTER;
3312 break;
3313
3314 case MLX5_DEV_EVENT_CLIENT_REREG:
3315 ibev.event = IB_EVENT_CLIENT_REREGISTER;
2979 port = (u8)param;
3316 port = (u8)work->param;
2980 break;
2981 case MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT:
2982 schedule_work(&ibdev->delay_drop.delay_drop_work);
2983 goto out;
2984 default:
2985 goto out;
2986 }
2987

--- 5 unchanged lines hidden (view full) ---

2993 goto out;
2994 }
2995
2996 if (ibdev->ib_active)
2997 ib_dispatch_event(&ibev);
2998
2999 if (fatal)
3000 ibdev->ib_active = false;
3317 break;
3318 case MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT:
3319 schedule_work(&ibdev->delay_drop.delay_drop_work);
3320 goto out;
3321 default:
3322 goto out;
3323 }
3324

--- 5 unchanged lines hidden (view full) ---

3330 goto out;
3331 }
3332
3333 if (ibdev->ib_active)
3334 ib_dispatch_event(&ibev);
3335
3336 if (fatal)
3337 ibdev->ib_active = false;
3001
3002out:
3338out:
3003 return;
3339 kfree(work);
3004}
3005
3340}
3341
3342static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
3343 enum mlx5_dev_event event, unsigned long param)
3344{
3345 struct mlx5_ib_event_work *work;
3346
3347 work = kmalloc(sizeof(*work), GFP_ATOMIC);
3348 if (work) {
3349 INIT_WORK(&work->work, mlx5_ib_handle_event);
3350 work->dev = dev;
3351 work->param = param;
3352 work->context = context;
3353 work->event = event;
3354
3355 queue_work(mlx5_ib_event_wq, &work->work);
3356 return;
3357 }
3358
3359 dev_warn(&dev->pdev->dev, "%s: mlx5_dev_event: %d, with param: %lu dropped, couldn't allocate memory.\n",
3360 __func__, event, param);
3361}
3362
3006static int set_has_smi_cap(struct mlx5_ib_dev *dev)
3007{
3008 struct mlx5_hca_vport_context vport_ctx;
3009 int err;
3010 int port;
3011
3363static int set_has_smi_cap(struct mlx5_ib_dev *dev)
3364{
3365 struct mlx5_hca_vport_context vport_ctx;
3366 int err;
3367 int port;
3368
3012 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
3369 for (port = 1; port <= dev->num_ports; port++) {
3013 dev->mdev->port_caps[port - 1].has_smi = false;
3014 if (MLX5_CAP_GEN(dev->mdev, port_type) ==
3015 MLX5_CAP_PORT_TYPE_IB) {
3016 if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
3017 err = mlx5_query_hca_vport_context(dev->mdev, 0,
3018 port, 0,
3019 &vport_ctx);
3020 if (err) {

--- 10 unchanged lines hidden (view full) ---

3031 }
3032 return 0;
3033}
3034
3035static void get_ext_port_caps(struct mlx5_ib_dev *dev)
3036{
3037 int port;
3038
3370 dev->mdev->port_caps[port - 1].has_smi = false;
3371 if (MLX5_CAP_GEN(dev->mdev, port_type) ==
3372 MLX5_CAP_PORT_TYPE_IB) {
3373 if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
3374 err = mlx5_query_hca_vport_context(dev->mdev, 0,
3375 port, 0,
3376 &vport_ctx);
3377 if (err) {

--- 10 unchanged lines hidden (view full) ---

3388 }
3389 return 0;
3390}
3391
3392static void get_ext_port_caps(struct mlx5_ib_dev *dev)
3393{
3394 int port;
3395
3039 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++)
3396 for (port = 1; port <= dev->num_ports; port++)
3040 mlx5_query_ext_port_caps(dev, port);
3041}
3042
3397 mlx5_query_ext_port_caps(dev, port);
3398}
3399
3043static int get_port_caps(struct mlx5_ib_dev *dev)
3400static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
3044{
3045 struct ib_device_attr *dprops = NULL;
3046 struct ib_port_attr *pprops = NULL;
3047 int err = -ENOMEM;
3401{
3402 struct ib_device_attr *dprops = NULL;
3403 struct ib_port_attr *pprops = NULL;
3404 int err = -ENOMEM;
3048 int port;
3049 struct ib_udata uhw = {.inlen = 0, .outlen = 0};
3050
3051 pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
3052 if (!pprops)
3053 goto out;
3054
3055 dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
3056 if (!dprops)

--- 4 unchanged lines hidden (view full) ---

3061 goto out;
3062
3063 err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
3064 if (err) {
3065 mlx5_ib_warn(dev, "query_device failed %d\n", err);
3066 goto out;
3067 }
3068
3405 struct ib_udata uhw = {.inlen = 0, .outlen = 0};
3406
3407 pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
3408 if (!pprops)
3409 goto out;
3410
3411 dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
3412 if (!dprops)

--- 4 unchanged lines hidden (view full) ---

3417 goto out;
3418
3419 err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
3420 if (err) {
3421 mlx5_ib_warn(dev, "query_device failed %d\n", err);
3422 goto out;
3423 }
3424
3069 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
3070 memset(pprops, 0, sizeof(*pprops));
3071 err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
3072 if (err) {
3073 mlx5_ib_warn(dev, "query_port %d failed %d\n",
3074 port, err);
3075 break;
3076 }
3077 dev->mdev->port_caps[port - 1].pkey_table_len =
3078 dprops->max_pkeys;
3079 dev->mdev->port_caps[port - 1].gid_table_len =
3080 pprops->gid_tbl_len;
3081 mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
3082 dprops->max_pkeys, pprops->gid_tbl_len);
3425 memset(pprops, 0, sizeof(*pprops));
3426 err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
3427 if (err) {
3428 mlx5_ib_warn(dev, "query_port %d failed %d\n",
3429 port, err);
3430 goto out;
3083 }
3084
3431 }
3432
3433 dev->mdev->port_caps[port - 1].pkey_table_len =
3434 dprops->max_pkeys;
3435 dev->mdev->port_caps[port - 1].gid_table_len =
3436 pprops->gid_tbl_len;
3437 mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
3438 port, dprops->max_pkeys, pprops->gid_tbl_len);
3439
3085out:
3086 kfree(pprops);
3087 kfree(dprops);
3088
3089 return err;
3090}
3091
3092static void destroy_umrc_res(struct mlx5_ib_dev *dev)

--- 273 unchanged lines hidden (view full) ---

3366}
3367
3368static u32 get_core_cap_flags(struct ib_device *ibdev)
3369{
3370 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3371 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3372 u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3373 u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3440out:
3441 kfree(pprops);
3442 kfree(dprops);
3443
3444 return err;
3445}
3446
3447static void destroy_umrc_res(struct mlx5_ib_dev *dev)

--- 273 unchanged lines hidden (view full) ---

3721}
3722
3723static u32 get_core_cap_flags(struct ib_device *ibdev)
3724{
3725 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3726 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3727 u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3728 u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3729 bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
3374 u32 ret = 0;
3375
3376 if (ll == IB_LINK_LAYER_INFINIBAND)
3377 return RDMA_CORE_PORT_IBA_IB;
3378
3730 u32 ret = 0;
3731
3732 if (ll == IB_LINK_LAYER_INFINIBAND)
3733 return RDMA_CORE_PORT_IBA_IB;
3734
3379 ret = RDMA_CORE_PORT_RAW_PACKET;
3735 if (raw_support)
3736 ret = RDMA_CORE_PORT_RAW_PACKET;
3380
3381 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
3382 return ret;
3383
3384 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
3385 return ret;
3386
3387 if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)

--- 73 unchanged lines hidden (view full) ---

3461 if (dev->flow_db.lag_demux_ft) {
3462 mlx5_destroy_flow_table(dev->flow_db.lag_demux_ft);
3463 dev->flow_db.lag_demux_ft = NULL;
3464
3465 mlx5_cmd_destroy_vport_lag(mdev);
3466 }
3467}
3468
3737
3738 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
3739 return ret;
3740
3741 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
3742 return ret;
3743
3744 if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)

--- 73 unchanged lines hidden (view full) ---

3818 if (dev->flow_db.lag_demux_ft) {
3819 mlx5_destroy_flow_table(dev->flow_db.lag_demux_ft);
3820 dev->flow_db.lag_demux_ft = NULL;
3821
3822 mlx5_cmd_destroy_vport_lag(mdev);
3823 }
3824}
3825
3469static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev)
3826static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
3470{
3471 int err;
3472
3827{
3828 int err;
3829
3473 dev->roce.nb.notifier_call = mlx5_netdev_event;
3474 err = register_netdevice_notifier(&dev->roce.nb);
3830 dev->roce[port_num].nb.notifier_call = mlx5_netdev_event;
3831 err = register_netdevice_notifier(&dev->roce[port_num].nb);
3475 if (err) {
3832 if (err) {
3476 dev->roce.nb.notifier_call = NULL;
3833 dev->roce[port_num].nb.notifier_call = NULL;
3477 return err;
3478 }
3479
3480 return 0;
3481}
3482
3834 return err;
3835 }
3836
3837 return 0;
3838}
3839
3483static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev)
3840static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
3484{
3841{
3485 if (dev->roce.nb.notifier_call) {
3486 unregister_netdevice_notifier(&dev->roce.nb);
3487 dev->roce.nb.notifier_call = NULL;
3842 if (dev->roce[port_num].nb.notifier_call) {
3843 unregister_netdevice_notifier(&dev->roce[port_num].nb);
3844 dev->roce[port_num].nb.notifier_call = NULL;
3488 }
3489}
3490
3845 }
3846}
3847
3491static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3848static int mlx5_enable_eth(struct mlx5_ib_dev *dev, u8 port_num)
3492{
3493 int err;
3494
3849{
3850 int err;
3851
3495 err = mlx5_add_netdev_notifier(dev);
3852 err = mlx5_add_netdev_notifier(dev, port_num);
3496 if (err)
3497 return err;
3498
3499 if (MLX5_CAP_GEN(dev->mdev, roce)) {
3500 err = mlx5_nic_vport_enable_roce(dev->mdev);
3501 if (err)
3502 goto err_unregister_netdevice_notifier;
3503 }

--- 4 unchanged lines hidden (view full) ---

3508
3509 return 0;
3510
3511err_disable_roce:
3512 if (MLX5_CAP_GEN(dev->mdev, roce))
3513 mlx5_nic_vport_disable_roce(dev->mdev);
3514
3515err_unregister_netdevice_notifier:
3853 if (err)
3854 return err;
3855
3856 if (MLX5_CAP_GEN(dev->mdev, roce)) {
3857 err = mlx5_nic_vport_enable_roce(dev->mdev);
3858 if (err)
3859 goto err_unregister_netdevice_notifier;
3860 }

--- 4 unchanged lines hidden (view full) ---

3865
3866 return 0;
3867
3868err_disable_roce:
3869 if (MLX5_CAP_GEN(dev->mdev, roce))
3870 mlx5_nic_vport_disable_roce(dev->mdev);
3871
3872err_unregister_netdevice_notifier:
3516 mlx5_remove_netdev_notifier(dev);
3873 mlx5_remove_netdev_notifier(dev, port_num);
3517 return err;
3518}
3519
3520static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3521{
3522 mlx5_eth_lag_cleanup(dev);
3523 if (MLX5_CAP_GEN(dev->mdev, roce))
3524 mlx5_nic_vport_disable_roce(dev->mdev);

--- 45 unchanged lines hidden (view full) ---

3570 INIT_Q_COUNTER(req_remote_access_errors),
3571 INIT_Q_COUNTER(resp_remote_access_errors),
3572 INIT_Q_COUNTER(resp_cqe_flush_error),
3573 INIT_Q_COUNTER(req_cqe_flush_error),
3574};
3575
3576static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
3577{
3874 return err;
3875}
3876
3877static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3878{
3879 mlx5_eth_lag_cleanup(dev);
3880 if (MLX5_CAP_GEN(dev->mdev, roce))
3881 mlx5_nic_vport_disable_roce(dev->mdev);

--- 45 unchanged lines hidden (view full) ---

3927 INIT_Q_COUNTER(req_remote_access_errors),
3928 INIT_Q_COUNTER(resp_remote_access_errors),
3929 INIT_Q_COUNTER(resp_cqe_flush_error),
3930 INIT_Q_COUNTER(req_cqe_flush_error),
3931};
3932
3933static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
3934{
3578 unsigned int i;
3935 int i;
3579
3580 for (i = 0; i < dev->num_ports; i++) {
3936
3937 for (i = 0; i < dev->num_ports; i++) {
3581 mlx5_core_dealloc_q_counter(dev->mdev,
3582 dev->port[i].cnts.set_id);
3938 if (dev->port[i].cnts.set_id)
3939 mlx5_core_dealloc_q_counter(dev->mdev,
3940 dev->port[i].cnts.set_id);
3583 kfree(dev->port[i].cnts.names);
3584 kfree(dev->port[i].cnts.offsets);
3585 }
3586}
3587
3588static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
3589 struct mlx5_ib_counters *cnts)
3590{

--- 25 unchanged lines hidden (view full) ---

3616 sizeof(cnts->offsets), GFP_KERNEL);
3617 if (!cnts->offsets)
3618 goto err_names;
3619
3620 return 0;
3621
3622err_names:
3623 kfree(cnts->names);
3941 kfree(dev->port[i].cnts.names);
3942 kfree(dev->port[i].cnts.offsets);
3943 }
3944}
3945
3946static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
3947 struct mlx5_ib_counters *cnts)
3948{

--- 25 unchanged lines hidden (view full) ---

3974 sizeof(cnts->offsets), GFP_KERNEL);
3975 if (!cnts->offsets)
3976 goto err_names;
3977
3978 return 0;
3979
3980err_names:
3981 kfree(cnts->names);
3982 cnts->names = NULL;
3624 return -ENOMEM;
3625}
3626
3627static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
3628 const char **names,
3629 size_t *offsets)
3630{
3631 int i;

--- 30 unchanged lines hidden (view full) ---

3662 names[j] = cong_cnts[i].name;
3663 offsets[j] = cong_cnts[i].offset;
3664 }
3665 }
3666}
3667
3668static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
3669{
3983 return -ENOMEM;
3984}
3985
3986static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
3987 const char **names,
3988 size_t *offsets)
3989{
3990 int i;

--- 30 unchanged lines hidden (view full) ---

4021 names[j] = cong_cnts[i].name;
4022 offsets[j] = cong_cnts[i].offset;
4023 }
4024 }
4025}
4026
4027static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
4028{
4029 int err = 0;
3670 int i;
4030 int i;
3671 int ret;
3672
3673 for (i = 0; i < dev->num_ports; i++) {
4031
4032 for (i = 0; i < dev->num_ports; i++) {
3674 struct mlx5_ib_port *port = &dev->port[i];
4033 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
4034 if (err)
4035 goto err_alloc;
3675
4036
3676 ret = mlx5_core_alloc_q_counter(dev->mdev,
3677 &port->cnts.set_id);
3678 if (ret) {
4037 mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
4038 dev->port[i].cnts.offsets);
4039
4040 err = mlx5_core_alloc_q_counter(dev->mdev,
4041 &dev->port[i].cnts.set_id);
4042 if (err) {
3679 mlx5_ib_warn(dev,
3680 "couldn't allocate queue counter for port %d, err %d\n",
4043 mlx5_ib_warn(dev,
4044 "couldn't allocate queue counter for port %d, err %d\n",
3681 i + 1, ret);
3682 goto dealloc_counters;
4045 i + 1, err);
4046 goto err_alloc;
3683 }
4047 }
3684
3685 ret = __mlx5_ib_alloc_counters(dev, &port->cnts);
3686 if (ret)
3687 goto dealloc_counters;
3688
3689 mlx5_ib_fill_counters(dev, port->cnts.names,
3690 port->cnts.offsets);
4048 dev->port[i].cnts.set_id_valid = true;
3691 }
3692
3693 return 0;
3694
4049 }
4050
4051 return 0;
4052
3695dealloc_counters:
3696 while (--i >= 0)
3697 mlx5_core_dealloc_q_counter(dev->mdev,
3698 dev->port[i].cnts.set_id);
3699
3700 return ret;
4053err_alloc:
4054 mlx5_ib_dealloc_counters(dev);
4055 return err;
3701}
3702
3703static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
3704 u8 port_num)
3705{
3706 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3707 struct mlx5_ib_port *port = &dev->port[port_num - 1];
3708
3709 /* We support only per port stats */
3710 if (port_num == 0)
3711 return NULL;
3712
3713 return rdma_alloc_hw_stats_struct(port->cnts.names,
3714 port->cnts.num_q_counters +
3715 port->cnts.num_cong_counters,
3716 RDMA_HW_STATS_DEFAULT_LIFESPAN);
3717}
3718
4056}
4057
4058static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
4059 u8 port_num)
4060{
4061 struct mlx5_ib_dev *dev = to_mdev(ibdev);
4062 struct mlx5_ib_port *port = &dev->port[port_num - 1];
4063
4064 /* We support only per port stats */
4065 if (port_num == 0)
4066 return NULL;
4067
4068 return rdma_alloc_hw_stats_struct(port->cnts.names,
4069 port->cnts.num_q_counters +
4070 port->cnts.num_cong_counters,
4071 RDMA_HW_STATS_DEFAULT_LIFESPAN);
4072}
4073
3719static int mlx5_ib_query_q_counters(struct mlx5_ib_dev *dev,
4074static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
3720 struct mlx5_ib_port *port,
3721 struct rdma_hw_stats *stats)
3722{
3723 int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
3724 void *out;
3725 __be32 val;
3726 int ret, i;
3727
3728 out = kvzalloc(outlen, GFP_KERNEL);
3729 if (!out)
3730 return -ENOMEM;
3731
4075 struct mlx5_ib_port *port,
4076 struct rdma_hw_stats *stats)
4077{
4078 int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
4079 void *out;
4080 __be32 val;
4081 int ret, i;
4082
4083 out = kvzalloc(outlen, GFP_KERNEL);
4084 if (!out)
4085 return -ENOMEM;
4086
3732 ret = mlx5_core_query_q_counter(dev->mdev,
4087 ret = mlx5_core_query_q_counter(mdev,
3733 port->cnts.set_id, 0,
3734 out, outlen);
3735 if (ret)
3736 goto free;
3737
3738 for (i = 0; i < port->cnts.num_q_counters; i++) {
3739 val = *(__be32 *)(out + port->cnts.offsets[i]);
3740 stats->value[i] = (u64)be32_to_cpu(val);

--- 5 unchanged lines hidden (view full) ---

3746}
3747
3748static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
3749 struct rdma_hw_stats *stats,
3750 u8 port_num, int index)
3751{
3752 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3753 struct mlx5_ib_port *port = &dev->port[port_num - 1];
4088 port->cnts.set_id, 0,
4089 out, outlen);
4090 if (ret)
4091 goto free;
4092
4093 for (i = 0; i < port->cnts.num_q_counters; i++) {
4094 val = *(__be32 *)(out + port->cnts.offsets[i]);
4095 stats->value[i] = (u64)be32_to_cpu(val);

--- 5 unchanged lines hidden (view full) ---

4101}
4102
4103static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
4104 struct rdma_hw_stats *stats,
4105 u8 port_num, int index)
4106{
4107 struct mlx5_ib_dev *dev = to_mdev(ibdev);
4108 struct mlx5_ib_port *port = &dev->port[port_num - 1];
4109 struct mlx5_core_dev *mdev;
3754 int ret, num_counters;
4110 int ret, num_counters;
4111 u8 mdev_port_num;
3755
3756 if (!stats)
3757 return -EINVAL;
3758
4112
4113 if (!stats)
4114 return -EINVAL;
4115
3759 ret = mlx5_ib_query_q_counters(dev, port, stats);
4116 num_counters = port->cnts.num_q_counters + port->cnts.num_cong_counters;
4117
4118 /* q_counters are per IB device, query the master mdev */
4119 ret = mlx5_ib_query_q_counters(dev->mdev, port, stats);
3760 if (ret)
3761 return ret;
4120 if (ret)
4121 return ret;
3762 num_counters = port->cnts.num_q_counters;
3763
3764 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
4122
4123 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
4124 mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
4125 &mdev_port_num);
4126 if (!mdev) {
4127 /* If port is not affiliated yet, its in down state
4128 * which doesn't have any counters yet, so it would be
4129 * zero. So no need to read from the HCA.
4130 */
4131 goto done;
4132 }
3765 ret = mlx5_lag_query_cong_counters(dev->mdev,
3766 stats->value +
3767 port->cnts.num_q_counters,
3768 port->cnts.num_cong_counters,
3769 port->cnts.offsets +
3770 port->cnts.num_q_counters);
4133 ret = mlx5_lag_query_cong_counters(dev->mdev,
4134 stats->value +
4135 port->cnts.num_q_counters,
4136 port->cnts.num_cong_counters,
4137 port->cnts.offsets +
4138 port->cnts.num_q_counters);
4139
4140 mlx5_ib_put_native_port_mdev(dev, port_num);
3771 if (ret)
3772 return ret;
4141 if (ret)
4142 return ret;
3773 num_counters += port->cnts.num_cong_counters;
3774 }
3775
4143 }
4144
4145done:
3776 return num_counters;
3777}
3778
3779static void mlx5_ib_free_rdma_netdev(struct net_device *netdev)
3780{
3781 return mlx5_rdma_netdev_free(netdev);
3782}
3783

--- 145 unchanged lines hidden (view full) ---

3929static const struct cpumask *
3930mlx5_ib_get_vector_affinity(struct ib_device *ibdev, int comp_vector)
3931{
3932 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3933
3934 return mlx5_get_vector_affinity(dev->mdev, comp_vector);
3935}
3936
4146 return num_counters;
4147}
4148
4149static void mlx5_ib_free_rdma_netdev(struct net_device *netdev)
4150{
4151 return mlx5_rdma_netdev_free(netdev);
4152}
4153

--- 145 unchanged lines hidden (view full) ---

4299static const struct cpumask *
4300mlx5_ib_get_vector_affinity(struct ib_device *ibdev, int comp_vector)
4301{
4302 struct mlx5_ib_dev *dev = to_mdev(ibdev);
4303
4304 return mlx5_get_vector_affinity(dev->mdev, comp_vector);
4305}
4306
3937static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
4307/* The mlx5_ib_multiport_mutex should be held when calling this function */
4308static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
4309 struct mlx5_ib_multiport_info *mpi)
3938{
4310{
3939 struct mlx5_ib_dev *dev;
3940 enum rdma_link_layer ll;
3941 int port_type_cap;
3942 const char *name;
4311 u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
4312 struct mlx5_ib_port *port = &ibdev->port[port_num];
4313 int comps;
3943 int err;
3944 int i;
3945
4314 int err;
4315 int i;
4316
3946 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
3947 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4317 mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3948
4318
3949 printk_once(KERN_INFO "%s", mlx5_version);
4319 spin_lock(&port->mp.mpi_lock);
4320 if (!mpi->ibdev) {
4321 spin_unlock(&port->mp.mpi_lock);
4322 return;
4323 }
4324 mpi->ibdev = NULL;
3950
4325
3951 dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
3952 if (!dev)
3953 return NULL;
4326 spin_unlock(&port->mp.mpi_lock);
4327 mlx5_remove_netdev_notifier(ibdev, port_num);
4328 spin_lock(&port->mp.mpi_lock);
3954
4329
3955 dev->mdev = mdev;
4330 comps = mpi->mdev_refcnt;
4331 if (comps) {
4332 mpi->unaffiliate = true;
4333 init_completion(&mpi->unref_comp);
4334 spin_unlock(&port->mp.mpi_lock);
3956
4335
3957 dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port),
4336 for (i = 0; i < comps; i++)
4337 wait_for_completion(&mpi->unref_comp);
4338
4339 spin_lock(&port->mp.mpi_lock);
4340 mpi->unaffiliate = false;
4341 }
4342
4343 port->mp.mpi = NULL;
4344
4345 list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
4346
4347 spin_unlock(&port->mp.mpi_lock);
4348
4349 err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
4350
4351 mlx5_ib_dbg(ibdev, "unaffiliated port %d\n", port_num + 1);
4352 /* Log an error, still needed to cleanup the pointers and add
4353 * it back to the list.
4354 */
4355 if (err)
4356 mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
4357 port_num + 1);
4358
4359 ibdev->roce[port_num].last_port_state = IB_PORT_DOWN;
4360}
4361
4362/* The mlx5_ib_multiport_mutex should be held when calling this function */
4363static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
4364 struct mlx5_ib_multiport_info *mpi)
4365{
4366 u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
4367 int err;
4368
4369 spin_lock(&ibdev->port[port_num].mp.mpi_lock);
4370 if (ibdev->port[port_num].mp.mpi) {
4371 mlx5_ib_warn(ibdev, "port %d already affiliated.\n",
4372 port_num + 1);
4373 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
4374 return false;
4375 }
4376
4377 ibdev->port[port_num].mp.mpi = mpi;
4378 mpi->ibdev = ibdev;
4379 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
4380
4381 err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
4382 if (err)
4383 goto unbind;
4384
4385 err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi->mdev));
4386 if (err)
4387 goto unbind;
4388
4389 err = mlx5_add_netdev_notifier(ibdev, port_num);
4390 if (err) {
4391 mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
4392 port_num + 1);
4393 goto unbind;
4394 }
4395
4396 err = mlx5_ib_init_cong_debugfs(ibdev, port_num);
4397 if (err)
4398 goto unbind;
4399
4400 return true;
4401
4402unbind:
4403 mlx5_ib_unbind_slave_port(ibdev, mpi);
4404 return false;
4405}
4406
4407static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
4408{
4409 int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4410 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
4411 port_num + 1);
4412 struct mlx5_ib_multiport_info *mpi;
4413 int err;
4414 int i;
4415
4416 if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
4417 return 0;
4418
4419 err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
4420 &dev->sys_image_guid);
4421 if (err)
4422 return err;
4423
4424 err = mlx5_nic_vport_enable_roce(dev->mdev);
4425 if (err)
4426 return err;
4427
4428 mutex_lock(&mlx5_ib_multiport_mutex);
4429 for (i = 0; i < dev->num_ports; i++) {
4430 bool bound = false;
4431
4432 /* build a stub multiport info struct for the native port. */
4433 if (i == port_num) {
4434 mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4435 if (!mpi) {
4436 mutex_unlock(&mlx5_ib_multiport_mutex);
4437 mlx5_nic_vport_disable_roce(dev->mdev);
4438 return -ENOMEM;
4439 }
4440
4441 mpi->is_master = true;
4442 mpi->mdev = dev->mdev;
4443 mpi->sys_image_guid = dev->sys_image_guid;
4444 dev->port[i].mp.mpi = mpi;
4445 mpi->ibdev = dev;
4446 mpi = NULL;
4447 continue;
4448 }
4449
4450 list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
4451 list) {
4452 if (dev->sys_image_guid == mpi->sys_image_guid &&
4453 (mlx5_core_native_port_num(mpi->mdev) - 1) == i) {
4454 bound = mlx5_ib_bind_slave_port(dev, mpi);
4455 }
4456
4457 if (bound) {
4458 dev_dbg(&mpi->mdev->pdev->dev, "removing port from unaffiliated list.\n");
4459 mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
4460 list_del(&mpi->list);
4461 break;
4462 }
4463 }
4464 if (!bound) {
4465 get_port_caps(dev, i + 1);
4466 mlx5_ib_dbg(dev, "no free port found for port %d\n",
4467 i + 1);
4468 }
4469 }
4470
4471 list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
4472 mutex_unlock(&mlx5_ib_multiport_mutex);
4473 return err;
4474}
4475
4476static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
4477{
4478 int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4479 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
4480 port_num + 1);
4481 int i;
4482
4483 if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
4484 return;
4485
4486 mutex_lock(&mlx5_ib_multiport_mutex);
4487 for (i = 0; i < dev->num_ports; i++) {
4488 if (dev->port[i].mp.mpi) {
4489 /* Destroy the native port stub */
4490 if (i == port_num) {
4491 kfree(dev->port[i].mp.mpi);
4492 dev->port[i].mp.mpi = NULL;
4493 } else {
4494 mlx5_ib_dbg(dev, "unbinding port_num: %d\n", i + 1);
4495 mlx5_ib_unbind_slave_port(dev, dev->port[i].mp.mpi);
4496 }
4497 }
4498 }
4499
4500 mlx5_ib_dbg(dev, "removing from devlist\n");
4501 list_del(&dev->ib_dev_list);
4502 mutex_unlock(&mlx5_ib_multiport_mutex);
4503
4504 mlx5_nic_vport_disable_roce(dev->mdev);
4505}
4506
4507static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
4508{
4509 mlx5_ib_cleanup_multiport_master(dev);
4510#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4511 cleanup_srcu_struct(&dev->mr_srcu);
4512#endif
4513 kfree(dev->port);
4514}
4515
4516static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
4517{
4518 struct mlx5_core_dev *mdev = dev->mdev;
4519 const char *name;
4520 int err;
4521 int i;
4522
4523 dev->port = kcalloc(dev->num_ports, sizeof(*dev->port),
3958 GFP_KERNEL);
3959 if (!dev->port)
4524 GFP_KERNEL);
4525 if (!dev->port)
3960 goto err_dealloc;
4526 return -ENOMEM;
3961
4527
3962 rwlock_init(&dev->roce.netdev_lock);
3963 err = get_port_caps(dev);
4528 for (i = 0; i < dev->num_ports; i++) {
4529 spin_lock_init(&dev->port[i].mp.mpi_lock);
4530 rwlock_init(&dev->roce[i].netdev_lock);
4531 }
4532
4533 err = mlx5_ib_init_multiport_master(dev);
3964 if (err)
3965 goto err_free_port;
3966
4534 if (err)
4535 goto err_free_port;
4536
4537 if (!mlx5_core_mp_enabled(mdev)) {
4538 int i;
4539
4540 for (i = 1; i <= dev->num_ports; i++) {
4541 err = get_port_caps(dev, i);
4542 if (err)
4543 break;
4544 }
4545 } else {
4546 err = get_port_caps(dev, mlx5_core_native_port_num(mdev));
4547 }
4548 if (err)
4549 goto err_mp;
4550
3967 if (mlx5_use_mad_ifc(dev))
3968 get_ext_port_caps(dev);
3969
3970 if (!mlx5_lag_is_active(mdev))
3971 name = "mlx5_%d";
3972 else
3973 name = "mlx5_bond_%d";
3974
3975 strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
3976 dev->ib_dev.owner = THIS_MODULE;
3977 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
3978 dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
4551 if (mlx5_use_mad_ifc(dev))
4552 get_ext_port_caps(dev);
4553
4554 if (!mlx5_lag_is_active(mdev))
4555 name = "mlx5_%d";
4556 else
4557 name = "mlx5_bond_%d";
4558
4559 strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
4560 dev->ib_dev.owner = THIS_MODULE;
4561 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
4562 dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
3979 dev->num_ports = MLX5_CAP_GEN(mdev, num_ports);
3980 dev->ib_dev.phys_port_cnt = dev->num_ports;
4563 dev->ib_dev.phys_port_cnt = dev->num_ports;
3981 dev->ib_dev.num_comp_vectors =
3982 dev->mdev->priv.eq_table.num_comp_vectors;
3983 dev->ib_dev.dev.parent = &mdev->pdev->dev;
3984
4564 dev->ib_dev.num_comp_vectors =
4565 dev->mdev->priv.eq_table.num_comp_vectors;
4566 dev->ib_dev.dev.parent = &mdev->pdev->dev;
4567
4568 mutex_init(&dev->flow_db.lock);
4569 mutex_init(&dev->cap_mask_mutex);
4570 INIT_LIST_HEAD(&dev->qp_list);
4571 spin_lock_init(&dev->reset_flow_resource_lock);
4572
4573#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4574 err = init_srcu_struct(&dev->mr_srcu);
4575 if (err)
4576 goto err_free_port;
4577#endif
4578
4579 return 0;
4580err_mp:
4581 mlx5_ib_cleanup_multiport_master(dev);
4582
4583err_free_port:
4584 kfree(dev->port);
4585
4586 return -ENOMEM;
4587}
4588
4589static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
4590{
4591 struct mlx5_core_dev *mdev = dev->mdev;
4592 int err;
4593
3985 dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
3986 dev->ib_dev.uverbs_cmd_mask =
3987 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
3988 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
3989 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
3990 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
3991 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
3992 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |

--- 22 unchanged lines hidden (view full) ---

4015 (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) |
4016 (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP) |
4017 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP) |
4018 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ);
4019
4020 dev->ib_dev.query_device = mlx5_ib_query_device;
4021 dev->ib_dev.query_port = mlx5_ib_query_port;
4022 dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer;
4594 dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
4595 dev->ib_dev.uverbs_cmd_mask =
4596 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
4597 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
4598 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
4599 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
4600 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
4601 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |

--- 22 unchanged lines hidden (view full) ---

4624 (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) |
4625 (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP) |
4626 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP) |
4627 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ);
4628
4629 dev->ib_dev.query_device = mlx5_ib_query_device;
4630 dev->ib_dev.query_port = mlx5_ib_query_port;
4631 dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer;
4023 if (ll == IB_LINK_LAYER_ETHERNET)
4024 dev->ib_dev.get_netdev = mlx5_ib_get_netdev;
4025 dev->ib_dev.query_gid = mlx5_ib_query_gid;
4026 dev->ib_dev.add_gid = mlx5_ib_add_gid;
4027 dev->ib_dev.del_gid = mlx5_ib_del_gid;
4028 dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
4029 dev->ib_dev.modify_device = mlx5_ib_modify_device;
4030 dev->ib_dev.modify_port = mlx5_ib_modify_port;
4031 dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
4032 dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;

--- 40 unchanged lines hidden (view full) ---

4073 dev->ib_dev.get_vf_config = mlx5_ib_get_vf_config;
4074 dev->ib_dev.set_vf_link_state = mlx5_ib_set_vf_link_state;
4075 dev->ib_dev.get_vf_stats = mlx5_ib_get_vf_stats;
4076 dev->ib_dev.set_vf_guid = mlx5_ib_set_vf_guid;
4077 }
4078
4079 dev->ib_dev.disassociate_ucontext = mlx5_ib_disassociate_ucontext;
4080
4632 dev->ib_dev.query_gid = mlx5_ib_query_gid;
4633 dev->ib_dev.add_gid = mlx5_ib_add_gid;
4634 dev->ib_dev.del_gid = mlx5_ib_del_gid;
4635 dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
4636 dev->ib_dev.modify_device = mlx5_ib_modify_device;
4637 dev->ib_dev.modify_port = mlx5_ib_modify_port;
4638 dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
4639 dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;

--- 40 unchanged lines hidden (view full) ---

4680 dev->ib_dev.get_vf_config = mlx5_ib_get_vf_config;
4681 dev->ib_dev.set_vf_link_state = mlx5_ib_set_vf_link_state;
4682 dev->ib_dev.get_vf_stats = mlx5_ib_get_vf_stats;
4683 dev->ib_dev.set_vf_guid = mlx5_ib_set_vf_guid;
4684 }
4685
4686 dev->ib_dev.disassociate_ucontext = mlx5_ib_disassociate_ucontext;
4687
4081 mlx5_ib_internal_fill_odp_caps(dev);
4082
4083 dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4084
4085 if (MLX5_CAP_GEN(mdev, imaicl)) {
4086 dev->ib_dev.alloc_mw = mlx5_ib_alloc_mw;
4087 dev->ib_dev.dealloc_mw = mlx5_ib_dealloc_mw;
4088 dev->ib_dev.uverbs_cmd_mask |=
4089 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
4090 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
4091 }
4092
4688 dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4689
4690 if (MLX5_CAP_GEN(mdev, imaicl)) {
4691 dev->ib_dev.alloc_mw = mlx5_ib_alloc_mw;
4692 dev->ib_dev.dealloc_mw = mlx5_ib_dealloc_mw;
4693 dev->ib_dev.uverbs_cmd_mask |=
4694 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
4695 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
4696 }
4697
4093 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
4094 dev->ib_dev.get_hw_stats = mlx5_ib_get_hw_stats;
4095 dev->ib_dev.alloc_hw_stats = mlx5_ib_alloc_hw_stats;
4096 }
4097
4098 if (MLX5_CAP_GEN(mdev, xrc)) {
4099 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
4100 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
4101 dev->ib_dev.uverbs_cmd_mask |=
4102 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
4103 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
4104 }
4105
4106 dev->ib_dev.create_flow = mlx5_ib_create_flow;
4107 dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
4108 dev->ib_dev.uverbs_ex_cmd_mask |=
4109 (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
4110 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
4111
4698 if (MLX5_CAP_GEN(mdev, xrc)) {
4699 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
4700 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
4701 dev->ib_dev.uverbs_cmd_mask |=
4702 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
4703 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
4704 }
4705
4706 dev->ib_dev.create_flow = mlx5_ib_create_flow;
4707 dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
4708 dev->ib_dev.uverbs_ex_cmd_mask |=
4709 (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
4710 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
4711
4112 if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
4113 IB_LINK_LAYER_ETHERNET) {
4712 err = init_node_data(dev);
4713 if (err)
4714 return err;
4715
4716 if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
4717 MLX5_CAP_GEN(dev->mdev, disable_local_lb))
4718 mutex_init(&dev->lb_mutex);
4719
4720 return 0;
4721}
4722
4723static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)
4724{
4725 struct mlx5_core_dev *mdev = dev->mdev;
4726 enum rdma_link_layer ll;
4727 int port_type_cap;
4728 u8 port_num;
4729 int err;
4730 int i;
4731
4732 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4733 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4734 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4735
4736 if (ll == IB_LINK_LAYER_ETHERNET) {
4737 for (i = 0; i < dev->num_ports; i++) {
4738 dev->roce[i].dev = dev;
4739 dev->roce[i].native_port_num = i + 1;
4740 dev->roce[i].last_port_state = IB_PORT_DOWN;
4741 }
4742
4743 dev->ib_dev.get_netdev = mlx5_ib_get_netdev;
4114 dev->ib_dev.create_wq = mlx5_ib_create_wq;
4115 dev->ib_dev.modify_wq = mlx5_ib_modify_wq;
4116 dev->ib_dev.destroy_wq = mlx5_ib_destroy_wq;
4117 dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table;
4118 dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table;
4119 dev->ib_dev.uverbs_ex_cmd_mask |=
4120 (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
4121 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
4122 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
4123 (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
4124 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
4744 dev->ib_dev.create_wq = mlx5_ib_create_wq;
4745 dev->ib_dev.modify_wq = mlx5_ib_modify_wq;
4746 dev->ib_dev.destroy_wq = mlx5_ib_destroy_wq;
4747 dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table;
4748 dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table;
4749 dev->ib_dev.uverbs_ex_cmd_mask |=
4750 (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
4751 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
4752 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
4753 (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
4754 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
4755 err = mlx5_enable_eth(dev, port_num);
4756 if (err)
4757 return err;
4125 }
4758 }
4126 err = init_node_data(dev);
4127 if (err)
4128 goto err_free_port;
4129
4759
4130 mutex_init(&dev->flow_db.lock);
4131 mutex_init(&dev->cap_mask_mutex);
4132 INIT_LIST_HEAD(&dev->qp_list);
4133 spin_lock_init(&dev->reset_flow_resource_lock);
4760 return 0;
4761}
4134
4762
4763static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
4764{
4765 struct mlx5_core_dev *mdev = dev->mdev;
4766 enum rdma_link_layer ll;
4767 int port_type_cap;
4768 u8 port_num;
4769
4770 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4771 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4772 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4773
4135 if (ll == IB_LINK_LAYER_ETHERNET) {
4774 if (ll == IB_LINK_LAYER_ETHERNET) {
4136 err = mlx5_enable_eth(dev);
4137 if (err)
4138 goto err_free_port;
4139 dev->roce.last_port_state = IB_PORT_DOWN;
4775 mlx5_disable_eth(dev);
4776 mlx5_remove_netdev_notifier(dev, port_num);
4140 }
4777 }
4778}
4141
4779
4142 err = create_dev_resources(&dev->devr);
4143 if (err)
4144 goto err_disable_eth;
4780static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
4781{
4782 return create_dev_resources(&dev->devr);
4783}
4145
4784
4146 err = mlx5_ib_odp_init_one(dev);
4147 if (err)
4148 goto err_rsrc;
4785static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
4786{
4787 destroy_dev_resources(&dev->devr);
4788}
4149
4789
4790static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
4791{
4792 mlx5_ib_internal_fill_odp_caps(dev);
4793
4794 return mlx5_ib_odp_init_one(dev);
4795}
4796
4797static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
4798{
4150 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
4799 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
4151 err = mlx5_ib_alloc_counters(dev);
4152 if (err)
4153 goto err_odp;
4800 dev->ib_dev.get_hw_stats = mlx5_ib_get_hw_stats;
4801 dev->ib_dev.alloc_hw_stats = mlx5_ib_alloc_hw_stats;
4802
4803 return mlx5_ib_alloc_counters(dev);
4154 }
4155
4804 }
4805
4156 err = mlx5_ib_init_cong_debugfs(dev);
4157 if (err)
4158 goto err_cnt;
4806 return 0;
4807}
4159
4808
4809static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
4810{
4811 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
4812 mlx5_ib_dealloc_counters(dev);
4813}
4814
4815static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
4816{
4817 return mlx5_ib_init_cong_debugfs(dev,
4818 mlx5_core_native_port_num(dev->mdev) - 1);
4819}
4820
4821static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
4822{
4823 mlx5_ib_cleanup_cong_debugfs(dev,
4824 mlx5_core_native_port_num(dev->mdev) - 1);
4825}
4826
4827static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
4828{
4160 dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
4161 if (!dev->mdev->priv.uar)
4829 dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
4830 if (!dev->mdev->priv.uar)
4162 goto err_cong;
4831 return -ENOMEM;
4832 return 0;
4833}
4163
4834
4835static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
4836{
4837 mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
4838}
4839
4840static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
4841{
4842 int err;
4843
4164 err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4165 if (err)
4844 err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4845 if (err)
4166 goto err_uar_page;
4846 return err;
4167
4168 err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4169 if (err)
4847
4848 err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4849 if (err)
4170 goto err_bfreg;
4850 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4171
4851
4172 err = ib_register_device(&dev->ib_dev, NULL);
4173 if (err)
4174 goto err_fp_bfreg;
4852 return err;
4853}
4175
4854
4176 err = create_umr_res(dev);
4177 if (err)
4178 goto err_dev;
4855static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
4856{
4857 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4858 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4859}
4179
4860
4861static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
4862{
4863 return ib_register_device(&dev->ib_dev, NULL);
4864}
4865
4866static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
4867{
4868 ib_unregister_device(&dev->ib_dev);
4869}
4870
4871static int mlx5_ib_stage_umr_res_init(struct mlx5_ib_dev *dev)
4872{
4873 return create_umr_res(dev);
4874}
4875
4876static void mlx5_ib_stage_umr_res_cleanup(struct mlx5_ib_dev *dev)
4877{
4878 destroy_umrc_res(dev);
4879}
4880
4881static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4882{
4180 init_delay_drop(dev);
4181
4883 init_delay_drop(dev);
4884
4885 return 0;
4886}
4887
4888static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4889{
4890 cancel_delay_drop(dev);
4891}
4892
4893static int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev)
4894{
4895 int err;
4896 int i;
4897
4182 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
4183 err = device_create_file(&dev->ib_dev.dev,
4184 mlx5_class_attributes[i]);
4185 if (err)
4898 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
4899 err = device_create_file(&dev->ib_dev.dev,
4900 mlx5_class_attributes[i]);
4901 if (err)
4186 goto err_delay_drop;
4902 return err;
4187 }
4188
4903 }
4904
4189 if ((MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
4190 MLX5_CAP_GEN(mdev, disable_local_lb))
4191 mutex_init(&dev->lb_mutex);
4905 return 0;
4906}
4192
4907
4908static void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4909 const struct mlx5_ib_profile *profile,
4910 int stage)
4911{
4912 /* Number of stages to cleanup */
4913 while (stage) {
4914 stage--;
4915 if (profile->stage[stage].cleanup)
4916 profile->stage[stage].cleanup(dev);
4917 }
4918
4919 ib_dealloc_device((struct ib_device *)dev);
4920}
4921
4922static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num);
4923
4924static void *__mlx5_ib_add(struct mlx5_core_dev *mdev,
4925 const struct mlx5_ib_profile *profile)
4926{
4927 struct mlx5_ib_dev *dev;
4928 int err;
4929 int i;
4930
4931 printk_once(KERN_INFO "%s", mlx5_version);
4932
4933 dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
4934 if (!dev)
4935 return NULL;
4936
4937 dev->mdev = mdev;
4938 dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4939 MLX5_CAP_GEN(mdev, num_vhca_ports));
4940
4941 for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4942 if (profile->stage[i].init) {
4943 err = profile->stage[i].init(dev);
4944 if (err)
4945 goto err_out;
4946 }
4947 }
4948
4949 dev->profile = profile;
4193 dev->ib_active = true;
4194
4195 return dev;
4196
4950 dev->ib_active = true;
4951
4952 return dev;
4953
4197err_delay_drop:
4198 cancel_delay_drop(dev);
4199 destroy_umrc_res(dev);
4954err_out:
4955 __mlx5_ib_remove(dev, profile, i);
4200
4956
4201err_dev:
4202 ib_unregister_device(&dev->ib_dev);
4957 return NULL;
4958}
4203
4959
4204err_fp_bfreg:
4205 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4960static const struct mlx5_ib_profile pf_profile = {
4961 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4962 mlx5_ib_stage_init_init,
4963 mlx5_ib_stage_init_cleanup),
4964 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4965 mlx5_ib_stage_caps_init,
4966 NULL),
4967 STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4968 mlx5_ib_stage_roce_init,
4969 mlx5_ib_stage_roce_cleanup),
4970 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4971 mlx5_ib_stage_dev_res_init,
4972 mlx5_ib_stage_dev_res_cleanup),
4973 STAGE_CREATE(MLX5_IB_STAGE_ODP,
4974 mlx5_ib_stage_odp_init,
4975 NULL),
4976 STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4977 mlx5_ib_stage_counters_init,
4978 mlx5_ib_stage_counters_cleanup),
4979 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4980 mlx5_ib_stage_cong_debugfs_init,
4981 mlx5_ib_stage_cong_debugfs_cleanup),
4982 STAGE_CREATE(MLX5_IB_STAGE_UAR,
4983 mlx5_ib_stage_uar_init,
4984 mlx5_ib_stage_uar_cleanup),
4985 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4986 mlx5_ib_stage_bfrag_init,
4987 mlx5_ib_stage_bfrag_cleanup),
4988 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4989 mlx5_ib_stage_ib_reg_init,
4990 mlx5_ib_stage_ib_reg_cleanup),
4991 STAGE_CREATE(MLX5_IB_STAGE_UMR_RESOURCES,
4992 mlx5_ib_stage_umr_res_init,
4993 mlx5_ib_stage_umr_res_cleanup),
4994 STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4995 mlx5_ib_stage_delay_drop_init,
4996 mlx5_ib_stage_delay_drop_cleanup),
4997 STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR,
4998 mlx5_ib_stage_class_attr_init,
4999 NULL),
5000};
4206
5001
4207err_bfreg:
4208 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
5002static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num)
5003{
5004 struct mlx5_ib_multiport_info *mpi;
5005 struct mlx5_ib_dev *dev;
5006 bool bound = false;
5007 int err;
4209
5008
4210err_uar_page:
4211 mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
5009 mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
5010 if (!mpi)
5011 return NULL;
4212
5012
4213err_cong:
4214 mlx5_ib_cleanup_cong_debugfs(dev);
4215err_cnt:
4216 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
4217 mlx5_ib_dealloc_counters(dev);
5013 mpi->mdev = mdev;
4218
5014
4219err_odp:
4220 mlx5_ib_odp_remove_one(dev);
5015 err = mlx5_query_nic_vport_system_image_guid(mdev,
5016 &mpi->sys_image_guid);
5017 if (err) {
5018 kfree(mpi);
5019 return NULL;
5020 }
4221
5021
4222err_rsrc:
4223 destroy_dev_resources(&dev->devr);
5022 mutex_lock(&mlx5_ib_multiport_mutex);
5023 list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
5024 if (dev->sys_image_guid == mpi->sys_image_guid)
5025 bound = mlx5_ib_bind_slave_port(dev, mpi);
4224
5026
4225err_disable_eth:
4226 if (ll == IB_LINK_LAYER_ETHERNET) {
4227 mlx5_disable_eth(dev);
4228 mlx5_remove_netdev_notifier(dev);
5027 if (bound) {
5028 rdma_roce_rescan_device(&dev->ib_dev);
5029 break;
5030 }
4229 }
4230
5031 }
5032
4231err_free_port:
4232 kfree(dev->port);
5033 if (!bound) {
5034 list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
5035 dev_dbg(&mdev->pdev->dev, "no suitable IB device found to bind to, added to unaffiliated list.\n");
5036 } else {
5037 mlx5_ib_dbg(dev, "bound port %u\n", port_num + 1);
5038 }
5039 mutex_unlock(&mlx5_ib_multiport_mutex);
4233
5040
4234err_dealloc:
4235 ib_dealloc_device((struct ib_device *)dev);
5041 return mpi;
5042}
4236
5043
4237 return NULL;
5044static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
5045{
5046 enum rdma_link_layer ll;
5047 int port_type_cap;
5048
5049 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
5050 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
5051
5052 if (mlx5_core_is_mp_slave(mdev) && ll == IB_LINK_LAYER_ETHERNET) {
5053 u8 port_num = mlx5_core_native_port_num(mdev) - 1;
5054
5055 return mlx5_ib_add_slave_port(mdev, port_num);
5056 }
5057
5058 return __mlx5_ib_add(mdev, &pf_profile);
4238}
4239
4240static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
4241{
5059}
5060
5061static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
5062{
4242 struct mlx5_ib_dev *dev = context;
4243 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
5063 struct mlx5_ib_multiport_info *mpi;
5064 struct mlx5_ib_dev *dev;
4244
5065
4245 cancel_delay_drop(dev);
4246 mlx5_remove_netdev_notifier(dev);
4247 ib_unregister_device(&dev->ib_dev);
4248 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4249 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4250 mlx5_put_uars_page(dev->mdev, mdev->priv.uar);
4251 mlx5_ib_cleanup_cong_debugfs(dev);
4252 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
4253 mlx5_ib_dealloc_counters(dev);
4254 destroy_umrc_res(dev);
4255 mlx5_ib_odp_remove_one(dev);
4256 destroy_dev_resources(&dev->devr);
4257 if (ll == IB_LINK_LAYER_ETHERNET)
4258 mlx5_disable_eth(dev);
4259 kfree(dev->port);
4260 ib_dealloc_device(&dev->ib_dev);
5066 if (mlx5_core_is_mp_slave(mdev)) {
5067 mpi = context;
5068 mutex_lock(&mlx5_ib_multiport_mutex);
5069 if (mpi->ibdev)
5070 mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
5071 list_del(&mpi->list);
5072 mutex_unlock(&mlx5_ib_multiport_mutex);
5073 return;
5074 }
5075
5076 dev = context;
5077 __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
4261}
4262
4263static struct mlx5_interface mlx5_ib_interface = {
4264 .add = mlx5_ib_add,
4265 .remove = mlx5_ib_remove,
4266 .event = mlx5_ib_event,
4267#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4268 .pfault = mlx5_ib_pfault,
4269#endif
4270 .protocol = MLX5_INTERFACE_PROTOCOL_IB,
4271};
4272
4273static int __init mlx5_ib_init(void)
4274{
4275 int err;
4276
5078}
5079
5080static struct mlx5_interface mlx5_ib_interface = {
5081 .add = mlx5_ib_add,
5082 .remove = mlx5_ib_remove,
5083 .event = mlx5_ib_event,
5084#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
5085 .pfault = mlx5_ib_pfault,
5086#endif
5087 .protocol = MLX5_INTERFACE_PROTOCOL_IB,
5088};
5089
5090static int __init mlx5_ib_init(void)
5091{
5092 int err;
5093
5094 mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
5095 if (!mlx5_ib_event_wq)
5096 return -ENOMEM;
5097
4277 mlx5_ib_odp_init();
4278
4279 err = mlx5_register_interface(&mlx5_ib_interface);
4280
4281 return err;
4282}
4283
4284static void __exit mlx5_ib_cleanup(void)
4285{
4286 mlx5_unregister_interface(&mlx5_ib_interface);
5098 mlx5_ib_odp_init();
5099
5100 err = mlx5_register_interface(&mlx5_ib_interface);
5101
5102 return err;
5103}
5104
5105static void __exit mlx5_ib_cleanup(void)
5106{
5107 mlx5_unregister_interface(&mlx5_ib_interface);
5108 destroy_workqueue(mlx5_ib_event_wq);
4287}
4288
4289module_init(mlx5_ib_init);
4290module_exit(mlx5_ib_cleanup);
5109}
5110
5111module_init(mlx5_ib_init);
5112module_exit(mlx5_ib_cleanup);