xref: /openbmc/linux/drivers/infiniband/hw/mlx5/main.c (revision 47ec38666210485de860ab24675acb3d2e7d4954)
1 /*
2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/errno.h>
38 #include <linux/pci.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/slab.h>
41 #include <linux/bitmap.h>
42 #if defined(CONFIG_X86)
43 #include <asm/pat.h>
44 #endif
45 #include <linux/sched.h>
46 #include <linux/sched/mm.h>
47 #include <linux/sched/task.h>
48 #include <linux/delay.h>
49 #include <rdma/ib_user_verbs.h>
50 #include <rdma/ib_addr.h>
51 #include <rdma/ib_cache.h>
52 #include <linux/mlx5/port.h>
53 #include <linux/mlx5/vport.h>
54 #include <linux/mlx5/fs.h>
55 #include <linux/list.h>
56 #include <rdma/ib_smi.h>
57 #include <rdma/ib_umem.h>
58 #include <linux/in.h>
59 #include <linux/etherdevice.h>
60 #include "mlx5_ib.h"
61 #include "ib_rep.h"
62 #include "cmd.h"
63 #include <linux/mlx5/fs_helpers.h>
64 #include <linux/mlx5/accel.h>
65 #include <rdma/uverbs_std_types.h>
66 #include <rdma/mlx5_user_ioctl_verbs.h>
67 #include <rdma/mlx5_user_ioctl_cmds.h>
68 
69 #define UVERBS_MODULE_NAME mlx5_ib
70 #include <rdma/uverbs_named_ioctl.h>
71 
72 #define DRIVER_NAME "mlx5_ib"
73 #define DRIVER_VERSION "5.0-0"
74 
75 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
76 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
77 MODULE_LICENSE("Dual BSD/GPL");
78 
79 static char mlx5_version[] =
80 	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
81 	DRIVER_VERSION "\n";
82 
83 struct mlx5_ib_event_work {
84 	struct work_struct	work;
85 	struct mlx5_core_dev	*dev;
86 	void			*context;
87 	enum mlx5_dev_event	event;
88 	unsigned long		param;
89 };
90 
91 enum {
92 	MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
93 };
94 
95 static struct workqueue_struct *mlx5_ib_event_wq;
96 static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
97 static LIST_HEAD(mlx5_ib_dev_list);
98 /*
99  * This mutex should be held when accessing either of the above lists
100  */
101 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
102 
103 /* We can't use an array for xlt_emergency_page because dma_map_single
104  * doesn't work on kernel modules memory
105  */
106 static unsigned long xlt_emergency_page;
107 static struct mutex xlt_emergency_page_mutex;
108 
109 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
110 {
111 	struct mlx5_ib_dev *dev;
112 
113 	mutex_lock(&mlx5_ib_multiport_mutex);
114 	dev = mpi->ibdev;
115 	mutex_unlock(&mlx5_ib_multiport_mutex);
116 	return dev;
117 }
118 
119 static enum rdma_link_layer
120 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
121 {
122 	switch (port_type_cap) {
123 	case MLX5_CAP_PORT_TYPE_IB:
124 		return IB_LINK_LAYER_INFINIBAND;
125 	case MLX5_CAP_PORT_TYPE_ETH:
126 		return IB_LINK_LAYER_ETHERNET;
127 	default:
128 		return IB_LINK_LAYER_UNSPECIFIED;
129 	}
130 }
131 
132 static enum rdma_link_layer
133 mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
134 {
135 	struct mlx5_ib_dev *dev = to_mdev(device);
136 	int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
137 
138 	return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
139 }
140 
141 static int get_port_state(struct ib_device *ibdev,
142 			  u8 port_num,
143 			  enum ib_port_state *state)
144 {
145 	struct ib_port_attr attr;
146 	int ret;
147 
148 	memset(&attr, 0, sizeof(attr));
149 	ret = ibdev->query_port(ibdev, port_num, &attr);
150 	if (!ret)
151 		*state = attr.state;
152 	return ret;
153 }
154 
155 static int mlx5_netdev_event(struct notifier_block *this,
156 			     unsigned long event, void *ptr)
157 {
158 	struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
159 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
160 	u8 port_num = roce->native_port_num;
161 	struct mlx5_core_dev *mdev;
162 	struct mlx5_ib_dev *ibdev;
163 
164 	ibdev = roce->dev;
165 	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
166 	if (!mdev)
167 		return NOTIFY_DONE;
168 
169 	switch (event) {
170 	case NETDEV_REGISTER:
171 	case NETDEV_UNREGISTER:
172 		write_lock(&roce->netdev_lock);
173 		if (ibdev->rep) {
174 			struct mlx5_eswitch *esw = ibdev->mdev->priv.eswitch;
175 			struct net_device *rep_ndev;
176 
177 			rep_ndev = mlx5_ib_get_rep_netdev(esw,
178 							  ibdev->rep->vport);
179 			if (rep_ndev == ndev)
180 				roce->netdev = (event == NETDEV_UNREGISTER) ?
181 					NULL : ndev;
182 		} else if (ndev->dev.parent == &mdev->pdev->dev) {
183 			roce->netdev = (event == NETDEV_UNREGISTER) ?
184 				NULL : ndev;
185 		}
186 		write_unlock(&roce->netdev_lock);
187 		break;
188 
189 	case NETDEV_CHANGE:
190 	case NETDEV_UP:
191 	case NETDEV_DOWN: {
192 		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);
193 		struct net_device *upper = NULL;
194 
195 		if (lag_ndev) {
196 			upper = netdev_master_upper_dev_get(lag_ndev);
197 			dev_put(lag_ndev);
198 		}
199 
200 		if ((upper == ndev || (!upper && ndev == roce->netdev))
201 		    && ibdev->ib_active) {
202 			struct ib_event ibev = { };
203 			enum ib_port_state port_state;
204 
205 			if (get_port_state(&ibdev->ib_dev, port_num,
206 					   &port_state))
207 				goto done;
208 
209 			if (roce->last_port_state == port_state)
210 				goto done;
211 
212 			roce->last_port_state = port_state;
213 			ibev.device = &ibdev->ib_dev;
214 			if (port_state == IB_PORT_DOWN)
215 				ibev.event = IB_EVENT_PORT_ERR;
216 			else if (port_state == IB_PORT_ACTIVE)
217 				ibev.event = IB_EVENT_PORT_ACTIVE;
218 			else
219 				goto done;
220 
221 			ibev.element.port_num = port_num;
222 			ib_dispatch_event(&ibev);
223 		}
224 		break;
225 	}
226 
227 	default:
228 		break;
229 	}
230 done:
231 	mlx5_ib_put_native_port_mdev(ibdev, port_num);
232 	return NOTIFY_DONE;
233 }
234 
235 static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
236 					     u8 port_num)
237 {
238 	struct mlx5_ib_dev *ibdev = to_mdev(device);
239 	struct net_device *ndev;
240 	struct mlx5_core_dev *mdev;
241 
242 	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
243 	if (!mdev)
244 		return NULL;
245 
246 	ndev = mlx5_lag_get_roce_netdev(mdev);
247 	if (ndev)
248 		goto out;
249 
250 	/* Ensure ndev does not disappear before we invoke dev_hold()
251 	 */
252 	read_lock(&ibdev->roce[port_num - 1].netdev_lock);
253 	ndev = ibdev->roce[port_num - 1].netdev;
254 	if (ndev)
255 		dev_hold(ndev);
256 	read_unlock(&ibdev->roce[port_num - 1].netdev_lock);
257 
258 out:
259 	mlx5_ib_put_native_port_mdev(ibdev, port_num);
260 	return ndev;
261 }
262 
263 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
264 						   u8 ib_port_num,
265 						   u8 *native_port_num)
266 {
267 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
268 							  ib_port_num);
269 	struct mlx5_core_dev *mdev = NULL;
270 	struct mlx5_ib_multiport_info *mpi;
271 	struct mlx5_ib_port *port;
272 
273 	if (!mlx5_core_mp_enabled(ibdev->mdev) ||
274 	    ll != IB_LINK_LAYER_ETHERNET) {
275 		if (native_port_num)
276 			*native_port_num = ib_port_num;
277 		return ibdev->mdev;
278 	}
279 
280 	if (native_port_num)
281 		*native_port_num = 1;
282 
283 	port = &ibdev->port[ib_port_num - 1];
284 	if (!port)
285 		return NULL;
286 
287 	spin_lock(&port->mp.mpi_lock);
288 	mpi = ibdev->port[ib_port_num - 1].mp.mpi;
289 	if (mpi && !mpi->unaffiliate) {
290 		mdev = mpi->mdev;
291 		/* If it's the master no need to refcount, it'll exist
292 		 * as long as the ib_dev exists.
293 		 */
294 		if (!mpi->is_master)
295 			mpi->mdev_refcnt++;
296 	}
297 	spin_unlock(&port->mp.mpi_lock);
298 
299 	return mdev;
300 }
301 
302 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u8 port_num)
303 {
304 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
305 							  port_num);
306 	struct mlx5_ib_multiport_info *mpi;
307 	struct mlx5_ib_port *port;
308 
309 	if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
310 		return;
311 
312 	port = &ibdev->port[port_num - 1];
313 
314 	spin_lock(&port->mp.mpi_lock);
315 	mpi = ibdev->port[port_num - 1].mp.mpi;
316 	if (mpi->is_master)
317 		goto out;
318 
319 	mpi->mdev_refcnt--;
320 	if (mpi->unaffiliate)
321 		complete(&mpi->unref_comp);
322 out:
323 	spin_unlock(&port->mp.mpi_lock);
324 }
325 
326 static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
327 				    u8 *active_width)
328 {
329 	switch (eth_proto_oper) {
330 	case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
331 	case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
332 	case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
333 	case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
334 		*active_width = IB_WIDTH_1X;
335 		*active_speed = IB_SPEED_SDR;
336 		break;
337 	case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
338 	case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
339 	case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
340 	case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
341 	case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
342 	case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
343 	case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
344 		*active_width = IB_WIDTH_1X;
345 		*active_speed = IB_SPEED_QDR;
346 		break;
347 	case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
348 	case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
349 	case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
350 		*active_width = IB_WIDTH_1X;
351 		*active_speed = IB_SPEED_EDR;
352 		break;
353 	case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
354 	case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
355 	case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
356 	case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
357 		*active_width = IB_WIDTH_4X;
358 		*active_speed = IB_SPEED_QDR;
359 		break;
360 	case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
361 	case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
362 	case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
363 		*active_width = IB_WIDTH_1X;
364 		*active_speed = IB_SPEED_HDR;
365 		break;
366 	case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
367 		*active_width = IB_WIDTH_4X;
368 		*active_speed = IB_SPEED_FDR;
369 		break;
370 	case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
371 	case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
372 	case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
373 	case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
374 		*active_width = IB_WIDTH_4X;
375 		*active_speed = IB_SPEED_EDR;
376 		break;
377 	default:
378 		return -EINVAL;
379 	}
380 
381 	return 0;
382 }
383 
384 static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
385 				struct ib_port_attr *props)
386 {
387 	struct mlx5_ib_dev *dev = to_mdev(device);
388 	struct mlx5_core_dev *mdev;
389 	struct net_device *ndev, *upper;
390 	enum ib_mtu ndev_ib_mtu;
391 	bool put_mdev = true;
392 	u16 qkey_viol_cntr;
393 	u32 eth_prot_oper;
394 	u8 mdev_port_num;
395 	int err;
396 
397 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
398 	if (!mdev) {
399 		/* This means the port isn't affiliated yet. Get the
400 		 * info for the master port instead.
401 		 */
402 		put_mdev = false;
403 		mdev = dev->mdev;
404 		mdev_port_num = 1;
405 		port_num = 1;
406 	}
407 
408 	/* Possible bad flows are checked before filling out props so in case
409 	 * of an error it will still be zeroed out.
410 	 */
411 	err = mlx5_query_port_eth_proto_oper(mdev, &eth_prot_oper,
412 					     mdev_port_num);
413 	if (err)
414 		goto out;
415 
416 	props->active_width     = IB_WIDTH_4X;
417 	props->active_speed     = IB_SPEED_QDR;
418 
419 	translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
420 				 &props->active_width);
421 
422 	props->port_cap_flags  |= IB_PORT_CM_SUP;
423 	props->port_cap_flags  |= IB_PORT_IP_BASED_GIDS;
424 
425 	props->gid_tbl_len      = MLX5_CAP_ROCE(dev->mdev,
426 						roce_address_table_size);
427 	props->max_mtu          = IB_MTU_4096;
428 	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
429 	props->pkey_tbl_len     = 1;
430 	props->state            = IB_PORT_DOWN;
431 	props->phys_state       = 3;
432 
433 	mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
434 	props->qkey_viol_cntr = qkey_viol_cntr;
435 
436 	/* If this is a stub query for an unaffiliated port stop here */
437 	if (!put_mdev)
438 		goto out;
439 
440 	ndev = mlx5_ib_get_netdev(device, port_num);
441 	if (!ndev)
442 		goto out;
443 
444 	if (mlx5_lag_is_active(dev->mdev)) {
445 		rcu_read_lock();
446 		upper = netdev_master_upper_dev_get_rcu(ndev);
447 		if (upper) {
448 			dev_put(ndev);
449 			ndev = upper;
450 			dev_hold(ndev);
451 		}
452 		rcu_read_unlock();
453 	}
454 
455 	if (netif_running(ndev) && netif_carrier_ok(ndev)) {
456 		props->state      = IB_PORT_ACTIVE;
457 		props->phys_state = 5;
458 	}
459 
460 	ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
461 
462 	dev_put(ndev);
463 
464 	props->active_mtu	= min(props->max_mtu, ndev_ib_mtu);
465 out:
466 	if (put_mdev)
467 		mlx5_ib_put_native_port_mdev(dev, port_num);
468 	return err;
469 }
470 
471 static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
472 			 unsigned int index, const union ib_gid *gid,
473 			 const struct ib_gid_attr *attr)
474 {
475 	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
476 	u8 roce_version = 0;
477 	u8 roce_l3_type = 0;
478 	bool vlan = false;
479 	u8 mac[ETH_ALEN];
480 	u16 vlan_id = 0;
481 
482 	if (gid) {
483 		gid_type = attr->gid_type;
484 		ether_addr_copy(mac, attr->ndev->dev_addr);
485 
486 		if (is_vlan_dev(attr->ndev)) {
487 			vlan = true;
488 			vlan_id = vlan_dev_vlan_id(attr->ndev);
489 		}
490 	}
491 
492 	switch (gid_type) {
493 	case IB_GID_TYPE_IB:
494 		roce_version = MLX5_ROCE_VERSION_1;
495 		break;
496 	case IB_GID_TYPE_ROCE_UDP_ENCAP:
497 		roce_version = MLX5_ROCE_VERSION_2;
498 		if (ipv6_addr_v4mapped((void *)gid))
499 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
500 		else
501 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
502 		break;
503 
504 	default:
505 		mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
506 	}
507 
508 	return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
509 				      roce_l3_type, gid->raw, mac, vlan,
510 				      vlan_id, port_num);
511 }
512 
513 static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
514 			   __always_unused void **context)
515 {
516 	return set_roce_addr(to_mdev(attr->device), attr->port_num,
517 			     attr->index, &attr->gid, attr);
518 }
519 
520 static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
521 			   __always_unused void **context)
522 {
523 	return set_roce_addr(to_mdev(attr->device), attr->port_num,
524 			     attr->index, NULL, NULL);
525 }
526 
527 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
528 			       const struct ib_gid_attr *attr)
529 {
530 	if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
531 		return 0;
532 
533 	return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
534 }
535 
536 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
537 {
538 	if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
539 		return !MLX5_CAP_GEN(dev->mdev, ib_virt);
540 	return 0;
541 }
542 
543 enum {
544 	MLX5_VPORT_ACCESS_METHOD_MAD,
545 	MLX5_VPORT_ACCESS_METHOD_HCA,
546 	MLX5_VPORT_ACCESS_METHOD_NIC,
547 };
548 
549 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
550 {
551 	if (mlx5_use_mad_ifc(to_mdev(ibdev)))
552 		return MLX5_VPORT_ACCESS_METHOD_MAD;
553 
554 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
555 	    IB_LINK_LAYER_ETHERNET)
556 		return MLX5_VPORT_ACCESS_METHOD_NIC;
557 
558 	return MLX5_VPORT_ACCESS_METHOD_HCA;
559 }
560 
561 static void get_atomic_caps(struct mlx5_ib_dev *dev,
562 			    u8 atomic_size_qp,
563 			    struct ib_device_attr *props)
564 {
565 	u8 tmp;
566 	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
567 	u8 atomic_req_8B_endianness_mode =
568 		MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
569 
570 	/* Check if HW supports 8 bytes standard atomic operations and capable
571 	 * of host endianness respond
572 	 */
573 	tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
574 	if (((atomic_operations & tmp) == tmp) &&
575 	    (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
576 	    (atomic_req_8B_endianness_mode)) {
577 		props->atomic_cap = IB_ATOMIC_HCA;
578 	} else {
579 		props->atomic_cap = IB_ATOMIC_NONE;
580 	}
581 }
582 
583 static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
584 			       struct ib_device_attr *props)
585 {
586 	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
587 
588 	get_atomic_caps(dev, atomic_size_qp, props);
589 }
590 
591 static void get_atomic_caps_dc(struct mlx5_ib_dev *dev,
592 			       struct ib_device_attr *props)
593 {
594 	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
595 
596 	get_atomic_caps(dev, atomic_size_qp, props);
597 }
598 
599 bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev)
600 {
601 	struct ib_device_attr props = {};
602 
603 	get_atomic_caps_dc(dev, &props);
604 	return (props.atomic_cap == IB_ATOMIC_HCA) ? true : false;
605 }
606 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
607 					__be64 *sys_image_guid)
608 {
609 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
610 	struct mlx5_core_dev *mdev = dev->mdev;
611 	u64 tmp;
612 	int err;
613 
614 	switch (mlx5_get_vport_access_method(ibdev)) {
615 	case MLX5_VPORT_ACCESS_METHOD_MAD:
616 		return mlx5_query_mad_ifc_system_image_guid(ibdev,
617 							    sys_image_guid);
618 
619 	case MLX5_VPORT_ACCESS_METHOD_HCA:
620 		err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
621 		break;
622 
623 	case MLX5_VPORT_ACCESS_METHOD_NIC:
624 		err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
625 		break;
626 
627 	default:
628 		return -EINVAL;
629 	}
630 
631 	if (!err)
632 		*sys_image_guid = cpu_to_be64(tmp);
633 
634 	return err;
635 
636 }
637 
638 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
639 				u16 *max_pkeys)
640 {
641 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
642 	struct mlx5_core_dev *mdev = dev->mdev;
643 
644 	switch (mlx5_get_vport_access_method(ibdev)) {
645 	case MLX5_VPORT_ACCESS_METHOD_MAD:
646 		return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
647 
648 	case MLX5_VPORT_ACCESS_METHOD_HCA:
649 	case MLX5_VPORT_ACCESS_METHOD_NIC:
650 		*max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
651 						pkey_table_size));
652 		return 0;
653 
654 	default:
655 		return -EINVAL;
656 	}
657 }
658 
659 static int mlx5_query_vendor_id(struct ib_device *ibdev,
660 				u32 *vendor_id)
661 {
662 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
663 
664 	switch (mlx5_get_vport_access_method(ibdev)) {
665 	case MLX5_VPORT_ACCESS_METHOD_MAD:
666 		return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
667 
668 	case MLX5_VPORT_ACCESS_METHOD_HCA:
669 	case MLX5_VPORT_ACCESS_METHOD_NIC:
670 		return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
671 
672 	default:
673 		return -EINVAL;
674 	}
675 }
676 
677 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
678 				__be64 *node_guid)
679 {
680 	u64 tmp;
681 	int err;
682 
683 	switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
684 	case MLX5_VPORT_ACCESS_METHOD_MAD:
685 		return mlx5_query_mad_ifc_node_guid(dev, node_guid);
686 
687 	case MLX5_VPORT_ACCESS_METHOD_HCA:
688 		err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
689 		break;
690 
691 	case MLX5_VPORT_ACCESS_METHOD_NIC:
692 		err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
693 		break;
694 
695 	default:
696 		return -EINVAL;
697 	}
698 
699 	if (!err)
700 		*node_guid = cpu_to_be64(tmp);
701 
702 	return err;
703 }
704 
705 struct mlx5_reg_node_desc {
706 	u8	desc[IB_DEVICE_NODE_DESC_MAX];
707 };
708 
709 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
710 {
711 	struct mlx5_reg_node_desc in;
712 
713 	if (mlx5_use_mad_ifc(dev))
714 		return mlx5_query_mad_ifc_node_desc(dev, node_desc);
715 
716 	memset(&in, 0, sizeof(in));
717 
718 	return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
719 				    sizeof(struct mlx5_reg_node_desc),
720 				    MLX5_REG_NODE_DESC, 0, 0);
721 }
722 
723 static int mlx5_ib_query_device(struct ib_device *ibdev,
724 				struct ib_device_attr *props,
725 				struct ib_udata *uhw)
726 {
727 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
728 	struct mlx5_core_dev *mdev = dev->mdev;
729 	int err = -ENOMEM;
730 	int max_sq_desc;
731 	int max_rq_sg;
732 	int max_sq_sg;
733 	u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
734 	bool raw_support = !mlx5_core_mp_enabled(mdev);
735 	struct mlx5_ib_query_device_resp resp = {};
736 	size_t resp_len;
737 	u64 max_tso;
738 
739 	resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
740 	if (uhw->outlen && uhw->outlen < resp_len)
741 		return -EINVAL;
742 	else
743 		resp.response_length = resp_len;
744 
745 	if (uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
746 		return -EINVAL;
747 
748 	memset(props, 0, sizeof(*props));
749 	err = mlx5_query_system_image_guid(ibdev,
750 					   &props->sys_image_guid);
751 	if (err)
752 		return err;
753 
754 	err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
755 	if (err)
756 		return err;
757 
758 	err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
759 	if (err)
760 		return err;
761 
762 	props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
763 		(fw_rev_min(dev->mdev) << 16) |
764 		fw_rev_sub(dev->mdev);
765 	props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
766 		IB_DEVICE_PORT_ACTIVE_EVENT		|
767 		IB_DEVICE_SYS_IMAGE_GUID		|
768 		IB_DEVICE_RC_RNR_NAK_GEN;
769 
770 	if (MLX5_CAP_GEN(mdev, pkv))
771 		props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
772 	if (MLX5_CAP_GEN(mdev, qkv))
773 		props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
774 	if (MLX5_CAP_GEN(mdev, apm))
775 		props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
776 	if (MLX5_CAP_GEN(mdev, xrc))
777 		props->device_cap_flags |= IB_DEVICE_XRC;
778 	if (MLX5_CAP_GEN(mdev, imaicl)) {
779 		props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
780 					   IB_DEVICE_MEM_WINDOW_TYPE_2B;
781 		props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
782 		/* We support 'Gappy' memory registration too */
783 		props->device_cap_flags |= IB_DEVICE_SG_GAPS_REG;
784 	}
785 	props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
786 	if (MLX5_CAP_GEN(mdev, sho)) {
787 		props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
788 		/* At this stage no support for signature handover */
789 		props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
790 				      IB_PROT_T10DIF_TYPE_2 |
791 				      IB_PROT_T10DIF_TYPE_3;
792 		props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
793 				       IB_GUARD_T10DIF_CSUM;
794 	}
795 	if (MLX5_CAP_GEN(mdev, block_lb_mc))
796 		props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
797 
798 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
799 		if (MLX5_CAP_ETH(mdev, csum_cap)) {
800 			/* Legacy bit to support old userspace libraries */
801 			props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
802 			props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
803 		}
804 
805 		if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
806 			props->raw_packet_caps |=
807 				IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
808 
809 		if (field_avail(typeof(resp), tso_caps, uhw->outlen)) {
810 			max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
811 			if (max_tso) {
812 				resp.tso_caps.max_tso = 1 << max_tso;
813 				resp.tso_caps.supported_qpts |=
814 					1 << IB_QPT_RAW_PACKET;
815 				resp.response_length += sizeof(resp.tso_caps);
816 			}
817 		}
818 
819 		if (field_avail(typeof(resp), rss_caps, uhw->outlen)) {
820 			resp.rss_caps.rx_hash_function =
821 						MLX5_RX_HASH_FUNC_TOEPLITZ;
822 			resp.rss_caps.rx_hash_fields_mask =
823 						MLX5_RX_HASH_SRC_IPV4 |
824 						MLX5_RX_HASH_DST_IPV4 |
825 						MLX5_RX_HASH_SRC_IPV6 |
826 						MLX5_RX_HASH_DST_IPV6 |
827 						MLX5_RX_HASH_SRC_PORT_TCP |
828 						MLX5_RX_HASH_DST_PORT_TCP |
829 						MLX5_RX_HASH_SRC_PORT_UDP |
830 						MLX5_RX_HASH_DST_PORT_UDP |
831 						MLX5_RX_HASH_INNER;
832 			if (mlx5_accel_ipsec_device_caps(dev->mdev) &
833 			    MLX5_ACCEL_IPSEC_CAP_DEVICE)
834 				resp.rss_caps.rx_hash_fields_mask |=
835 					MLX5_RX_HASH_IPSEC_SPI;
836 			resp.response_length += sizeof(resp.rss_caps);
837 		}
838 	} else {
839 		if (field_avail(typeof(resp), tso_caps, uhw->outlen))
840 			resp.response_length += sizeof(resp.tso_caps);
841 		if (field_avail(typeof(resp), rss_caps, uhw->outlen))
842 			resp.response_length += sizeof(resp.rss_caps);
843 	}
844 
845 	if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
846 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
847 		props->device_cap_flags |= IB_DEVICE_UD_TSO;
848 	}
849 
850 	if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
851 	    MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
852 	    raw_support)
853 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
854 
855 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
856 	    MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
857 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
858 
859 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
860 	    MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
861 	    raw_support) {
862 		/* Legacy bit to support old userspace libraries */
863 		props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
864 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
865 	}
866 
867 	if (MLX5_CAP_DEV_MEM(mdev, memic)) {
868 		props->max_dm_size =
869 			MLX5_CAP_DEV_MEM(mdev, max_memic_size);
870 	}
871 
872 	if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
873 		props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
874 
875 	if (MLX5_CAP_GEN(mdev, end_pad))
876 		props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
877 
878 	props->vendor_part_id	   = mdev->pdev->device;
879 	props->hw_ver		   = mdev->pdev->revision;
880 
881 	props->max_mr_size	   = ~0ull;
882 	props->page_size_cap	   = ~(min_page_size - 1);
883 	props->max_qp		   = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
884 	props->max_qp_wr	   = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
885 	max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
886 		     sizeof(struct mlx5_wqe_data_seg);
887 	max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
888 	max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
889 		     sizeof(struct mlx5_wqe_raddr_seg)) /
890 		sizeof(struct mlx5_wqe_data_seg);
891 	props->max_sge = min(max_rq_sg, max_sq_sg);
892 	props->max_sge_rd	   = MLX5_MAX_SGE_RD;
893 	props->max_cq		   = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
894 	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
895 	props->max_mr		   = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
896 	props->max_pd		   = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
897 	props->max_qp_rd_atom	   = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
898 	props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
899 	props->max_srq		   = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
900 	props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
901 	props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
902 	props->max_res_rd_atom	   = props->max_qp_rd_atom * props->max_qp;
903 	props->max_srq_sge	   = max_rq_sg - 1;
904 	props->max_fast_reg_page_list_len =
905 		1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
906 	get_atomic_caps_qp(dev, props);
907 	props->masked_atomic_cap   = IB_ATOMIC_NONE;
908 	props->max_mcast_grp	   = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
909 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
910 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
911 					   props->max_mcast_grp;
912 	props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
913 	props->max_ah = INT_MAX;
914 	props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
915 	props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
916 
917 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
918 	if (MLX5_CAP_GEN(mdev, pg))
919 		props->device_cap_flags |= IB_DEVICE_ON_DEMAND_PAGING;
920 	props->odp_caps = dev->odp_caps;
921 #endif
922 
923 	if (MLX5_CAP_GEN(mdev, cd))
924 		props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
925 
926 	if (!mlx5_core_is_pf(mdev))
927 		props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
928 
929 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
930 	    IB_LINK_LAYER_ETHERNET && raw_support) {
931 		props->rss_caps.max_rwq_indirection_tables =
932 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
933 		props->rss_caps.max_rwq_indirection_table_size =
934 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
935 		props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
936 		props->max_wq_type_rq =
937 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
938 	}
939 
940 	if (MLX5_CAP_GEN(mdev, tag_matching)) {
941 		props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
942 		props->tm_caps.max_num_tags =
943 			(1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
944 		props->tm_caps.flags = IB_TM_CAP_RC;
945 		props->tm_caps.max_ops =
946 			1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
947 		props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
948 	}
949 
950 	if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
951 		props->cq_caps.max_cq_moderation_count =
952 						MLX5_MAX_CQ_COUNT;
953 		props->cq_caps.max_cq_moderation_period =
954 						MLX5_MAX_CQ_PERIOD;
955 	}
956 
957 	if (field_avail(typeof(resp), cqe_comp_caps, uhw->outlen)) {
958 		resp.response_length += sizeof(resp.cqe_comp_caps);
959 
960 		if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
961 			resp.cqe_comp_caps.max_num =
962 				MLX5_CAP_GEN(dev->mdev,
963 					     cqe_compression_max_num);
964 
965 			resp.cqe_comp_caps.supported_format =
966 				MLX5_IB_CQE_RES_FORMAT_HASH |
967 				MLX5_IB_CQE_RES_FORMAT_CSUM;
968 
969 			if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
970 				resp.cqe_comp_caps.supported_format |=
971 					MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
972 		}
973 	}
974 
975 	if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen) &&
976 	    raw_support) {
977 		if (MLX5_CAP_QOS(mdev, packet_pacing) &&
978 		    MLX5_CAP_GEN(mdev, qos)) {
979 			resp.packet_pacing_caps.qp_rate_limit_max =
980 				MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
981 			resp.packet_pacing_caps.qp_rate_limit_min =
982 				MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
983 			resp.packet_pacing_caps.supported_qpts |=
984 				1 << IB_QPT_RAW_PACKET;
985 			if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
986 			    MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
987 				resp.packet_pacing_caps.cap_flags |=
988 					MLX5_IB_PP_SUPPORT_BURST;
989 		}
990 		resp.response_length += sizeof(resp.packet_pacing_caps);
991 	}
992 
993 	if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,
994 			uhw->outlen)) {
995 		if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
996 			resp.mlx5_ib_support_multi_pkt_send_wqes =
997 				MLX5_IB_ALLOW_MPW;
998 
999 		if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1000 			resp.mlx5_ib_support_multi_pkt_send_wqes |=
1001 				MLX5_IB_SUPPORT_EMPW;
1002 
1003 		resp.response_length +=
1004 			sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1005 	}
1006 
1007 	if (field_avail(typeof(resp), flags, uhw->outlen)) {
1008 		resp.response_length += sizeof(resp.flags);
1009 
1010 		if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1011 			resp.flags |=
1012 				MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1013 
1014 		if (MLX5_CAP_GEN(mdev, cqe_128_always))
1015 			resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1016 	}
1017 
1018 	if (field_avail(typeof(resp), sw_parsing_caps,
1019 			uhw->outlen)) {
1020 		resp.response_length += sizeof(resp.sw_parsing_caps);
1021 		if (MLX5_CAP_ETH(mdev, swp)) {
1022 			resp.sw_parsing_caps.sw_parsing_offloads |=
1023 				MLX5_IB_SW_PARSING;
1024 
1025 			if (MLX5_CAP_ETH(mdev, swp_csum))
1026 				resp.sw_parsing_caps.sw_parsing_offloads |=
1027 					MLX5_IB_SW_PARSING_CSUM;
1028 
1029 			if (MLX5_CAP_ETH(mdev, swp_lso))
1030 				resp.sw_parsing_caps.sw_parsing_offloads |=
1031 					MLX5_IB_SW_PARSING_LSO;
1032 
1033 			if (resp.sw_parsing_caps.sw_parsing_offloads)
1034 				resp.sw_parsing_caps.supported_qpts =
1035 					BIT(IB_QPT_RAW_PACKET);
1036 		}
1037 	}
1038 
1039 	if (field_avail(typeof(resp), striding_rq_caps, uhw->outlen) &&
1040 	    raw_support) {
1041 		resp.response_length += sizeof(resp.striding_rq_caps);
1042 		if (MLX5_CAP_GEN(mdev, striding_rq)) {
1043 			resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1044 				MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1045 			resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1046 				MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1047 			resp.striding_rq_caps.min_single_wqe_log_num_of_strides =
1048 				MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1049 			resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1050 				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1051 			resp.striding_rq_caps.supported_qpts =
1052 				BIT(IB_QPT_RAW_PACKET);
1053 		}
1054 	}
1055 
1056 	if (field_avail(typeof(resp), tunnel_offloads_caps,
1057 			uhw->outlen)) {
1058 		resp.response_length += sizeof(resp.tunnel_offloads_caps);
1059 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1060 			resp.tunnel_offloads_caps |=
1061 				MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1062 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1063 			resp.tunnel_offloads_caps |=
1064 				MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1065 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1066 			resp.tunnel_offloads_caps |=
1067 				MLX5_IB_TUNNELED_OFFLOADS_GRE;
1068 		if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
1069 		    MLX5_FLEX_PROTO_CW_MPLS_GRE)
1070 			resp.tunnel_offloads_caps |=
1071 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1072 		if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
1073 		    MLX5_FLEX_PROTO_CW_MPLS_UDP)
1074 			resp.tunnel_offloads_caps |=
1075 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1076 	}
1077 
1078 	if (uhw->outlen) {
1079 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1080 
1081 		if (err)
1082 			return err;
1083 	}
1084 
1085 	return 0;
1086 }
1087 
1088 enum mlx5_ib_width {
1089 	MLX5_IB_WIDTH_1X	= 1 << 0,
1090 	MLX5_IB_WIDTH_2X	= 1 << 1,
1091 	MLX5_IB_WIDTH_4X	= 1 << 2,
1092 	MLX5_IB_WIDTH_8X	= 1 << 3,
1093 	MLX5_IB_WIDTH_12X	= 1 << 4
1094 };
1095 
1096 static int translate_active_width(struct ib_device *ibdev, u8 active_width,
1097 				  u8 *ib_width)
1098 {
1099 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1100 	int err = 0;
1101 
1102 	if (active_width & MLX5_IB_WIDTH_1X) {
1103 		*ib_width = IB_WIDTH_1X;
1104 	} else if (active_width & MLX5_IB_WIDTH_2X) {
1105 		mlx5_ib_dbg(dev, "active_width %d is not supported by IB spec\n",
1106 			    (int)active_width);
1107 		err = -EINVAL;
1108 	} else if (active_width & MLX5_IB_WIDTH_4X) {
1109 		*ib_width = IB_WIDTH_4X;
1110 	} else if (active_width & MLX5_IB_WIDTH_8X) {
1111 		*ib_width = IB_WIDTH_8X;
1112 	} else if (active_width & MLX5_IB_WIDTH_12X) {
1113 		*ib_width = IB_WIDTH_12X;
1114 	} else {
1115 		mlx5_ib_dbg(dev, "Invalid active_width %d\n",
1116 			    (int)active_width);
1117 		err = -EINVAL;
1118 	}
1119 
1120 	return err;
1121 }
1122 
1123 static int mlx5_mtu_to_ib_mtu(int mtu)
1124 {
1125 	switch (mtu) {
1126 	case 256: return 1;
1127 	case 512: return 2;
1128 	case 1024: return 3;
1129 	case 2048: return 4;
1130 	case 4096: return 5;
1131 	default:
1132 		pr_warn("invalid mtu\n");
1133 		return -1;
1134 	}
1135 }
1136 
1137 enum ib_max_vl_num {
1138 	__IB_MAX_VL_0		= 1,
1139 	__IB_MAX_VL_0_1		= 2,
1140 	__IB_MAX_VL_0_3		= 3,
1141 	__IB_MAX_VL_0_7		= 4,
1142 	__IB_MAX_VL_0_14	= 5,
1143 };
1144 
1145 enum mlx5_vl_hw_cap {
1146 	MLX5_VL_HW_0	= 1,
1147 	MLX5_VL_HW_0_1	= 2,
1148 	MLX5_VL_HW_0_2	= 3,
1149 	MLX5_VL_HW_0_3	= 4,
1150 	MLX5_VL_HW_0_4	= 5,
1151 	MLX5_VL_HW_0_5	= 6,
1152 	MLX5_VL_HW_0_6	= 7,
1153 	MLX5_VL_HW_0_7	= 8,
1154 	MLX5_VL_HW_0_14	= 15
1155 };
1156 
1157 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1158 				u8 *max_vl_num)
1159 {
1160 	switch (vl_hw_cap) {
1161 	case MLX5_VL_HW_0:
1162 		*max_vl_num = __IB_MAX_VL_0;
1163 		break;
1164 	case MLX5_VL_HW_0_1:
1165 		*max_vl_num = __IB_MAX_VL_0_1;
1166 		break;
1167 	case MLX5_VL_HW_0_3:
1168 		*max_vl_num = __IB_MAX_VL_0_3;
1169 		break;
1170 	case MLX5_VL_HW_0_7:
1171 		*max_vl_num = __IB_MAX_VL_0_7;
1172 		break;
1173 	case MLX5_VL_HW_0_14:
1174 		*max_vl_num = __IB_MAX_VL_0_14;
1175 		break;
1176 
1177 	default:
1178 		return -EINVAL;
1179 	}
1180 
1181 	return 0;
1182 }
1183 
1184 static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
1185 			       struct ib_port_attr *props)
1186 {
1187 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1188 	struct mlx5_core_dev *mdev = dev->mdev;
1189 	struct mlx5_hca_vport_context *rep;
1190 	u16 max_mtu;
1191 	u16 oper_mtu;
1192 	int err;
1193 	u8 ib_link_width_oper;
1194 	u8 vl_hw_cap;
1195 
1196 	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1197 	if (!rep) {
1198 		err = -ENOMEM;
1199 		goto out;
1200 	}
1201 
1202 	/* props being zeroed by the caller, avoid zeroing it here */
1203 
1204 	err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1205 	if (err)
1206 		goto out;
1207 
1208 	props->lid		= rep->lid;
1209 	props->lmc		= rep->lmc;
1210 	props->sm_lid		= rep->sm_lid;
1211 	props->sm_sl		= rep->sm_sl;
1212 	props->state		= rep->vport_state;
1213 	props->phys_state	= rep->port_physical_state;
1214 	props->port_cap_flags	= rep->cap_mask1;
1215 	props->gid_tbl_len	= mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1216 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1217 	props->pkey_tbl_len	= mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1218 	props->bad_pkey_cntr	= rep->pkey_violation_counter;
1219 	props->qkey_viol_cntr	= rep->qkey_violation_counter;
1220 	props->subnet_timeout	= rep->subnet_timeout;
1221 	props->init_type_reply	= rep->init_type_reply;
1222 	props->grh_required	= rep->grh_required;
1223 
1224 	err = mlx5_query_port_link_width_oper(mdev, &ib_link_width_oper, port);
1225 	if (err)
1226 		goto out;
1227 
1228 	err = translate_active_width(ibdev, ib_link_width_oper,
1229 				     &props->active_width);
1230 	if (err)
1231 		goto out;
1232 	err = mlx5_query_port_ib_proto_oper(mdev, &props->active_speed, port);
1233 	if (err)
1234 		goto out;
1235 
1236 	mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1237 
1238 	props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1239 
1240 	mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1241 
1242 	props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1243 
1244 	err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1245 	if (err)
1246 		goto out;
1247 
1248 	err = translate_max_vl_num(ibdev, vl_hw_cap,
1249 				   &props->max_vl_num);
1250 out:
1251 	kfree(rep);
1252 	return err;
1253 }
1254 
1255 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
1256 		       struct ib_port_attr *props)
1257 {
1258 	unsigned int count;
1259 	int ret;
1260 
1261 	switch (mlx5_get_vport_access_method(ibdev)) {
1262 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1263 		ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1264 		break;
1265 
1266 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1267 		ret = mlx5_query_hca_port(ibdev, port, props);
1268 		break;
1269 
1270 	case MLX5_VPORT_ACCESS_METHOD_NIC:
1271 		ret = mlx5_query_port_roce(ibdev, port, props);
1272 		break;
1273 
1274 	default:
1275 		ret = -EINVAL;
1276 	}
1277 
1278 	if (!ret && props) {
1279 		struct mlx5_ib_dev *dev = to_mdev(ibdev);
1280 		struct mlx5_core_dev *mdev;
1281 		bool put_mdev = true;
1282 
1283 		mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1284 		if (!mdev) {
1285 			/* If the port isn't affiliated yet query the master.
1286 			 * The master and slave will have the same values.
1287 			 */
1288 			mdev = dev->mdev;
1289 			port = 1;
1290 			put_mdev = false;
1291 		}
1292 		count = mlx5_core_reserved_gids_count(mdev);
1293 		if (put_mdev)
1294 			mlx5_ib_put_native_port_mdev(dev, port);
1295 		props->gid_tbl_len -= count;
1296 	}
1297 	return ret;
1298 }
1299 
1300 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,
1301 				  struct ib_port_attr *props)
1302 {
1303 	int ret;
1304 
1305 	/* Only link layer == ethernet is valid for representors */
1306 	ret = mlx5_query_port_roce(ibdev, port, props);
1307 	if (ret || !props)
1308 		return ret;
1309 
1310 	/* We don't support GIDS */
1311 	props->gid_tbl_len = 0;
1312 
1313 	return ret;
1314 }
1315 
1316 static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
1317 			     union ib_gid *gid)
1318 {
1319 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1320 	struct mlx5_core_dev *mdev = dev->mdev;
1321 
1322 	switch (mlx5_get_vport_access_method(ibdev)) {
1323 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1324 		return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1325 
1326 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1327 		return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1328 
1329 	default:
1330 		return -EINVAL;
1331 	}
1332 
1333 }
1334 
1335 static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port,
1336 				   u16 index, u16 *pkey)
1337 {
1338 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1339 	struct mlx5_core_dev *mdev;
1340 	bool put_mdev = true;
1341 	u8 mdev_port_num;
1342 	int err;
1343 
1344 	mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1345 	if (!mdev) {
1346 		/* The port isn't affiliated yet, get the PKey from the master
1347 		 * port. For RoCE the PKey tables will be the same.
1348 		 */
1349 		put_mdev = false;
1350 		mdev = dev->mdev;
1351 		mdev_port_num = 1;
1352 	}
1353 
1354 	err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1355 					index, pkey);
1356 	if (put_mdev)
1357 		mlx5_ib_put_native_port_mdev(dev, port);
1358 
1359 	return err;
1360 }
1361 
1362 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1363 			      u16 *pkey)
1364 {
1365 	switch (mlx5_get_vport_access_method(ibdev)) {
1366 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1367 		return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1368 
1369 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1370 	case MLX5_VPORT_ACCESS_METHOD_NIC:
1371 		return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1372 	default:
1373 		return -EINVAL;
1374 	}
1375 }
1376 
1377 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1378 				 struct ib_device_modify *props)
1379 {
1380 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1381 	struct mlx5_reg_node_desc in;
1382 	struct mlx5_reg_node_desc out;
1383 	int err;
1384 
1385 	if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1386 		return -EOPNOTSUPP;
1387 
1388 	if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1389 		return 0;
1390 
1391 	/*
1392 	 * If possible, pass node desc to FW, so it can generate
1393 	 * a 144 trap.  If cmd fails, just ignore.
1394 	 */
1395 	memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1396 	err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1397 				   sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1398 	if (err)
1399 		return err;
1400 
1401 	memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1402 
1403 	return err;
1404 }
1405 
1406 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,
1407 				u32 value)
1408 {
1409 	struct mlx5_hca_vport_context ctx = {};
1410 	struct mlx5_core_dev *mdev;
1411 	u8 mdev_port_num;
1412 	int err;
1413 
1414 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1415 	if (!mdev)
1416 		return -ENODEV;
1417 
1418 	err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1419 	if (err)
1420 		goto out;
1421 
1422 	if (~ctx.cap_mask1_perm & mask) {
1423 		mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1424 			     mask, ctx.cap_mask1_perm);
1425 		err = -EINVAL;
1426 		goto out;
1427 	}
1428 
1429 	ctx.cap_mask1 = value;
1430 	ctx.cap_mask1_perm = mask;
1431 	err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1432 						 0, &ctx);
1433 
1434 out:
1435 	mlx5_ib_put_native_port_mdev(dev, port_num);
1436 
1437 	return err;
1438 }
1439 
1440 static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
1441 			       struct ib_port_modify *props)
1442 {
1443 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1444 	struct ib_port_attr attr;
1445 	u32 tmp;
1446 	int err;
1447 	u32 change_mask;
1448 	u32 value;
1449 	bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1450 		      IB_LINK_LAYER_INFINIBAND);
1451 
1452 	/* CM layer calls ib_modify_port() regardless of the link layer. For
1453 	 * Ethernet ports, qkey violation and Port capabilities are meaningless.
1454 	 */
1455 	if (!is_ib)
1456 		return 0;
1457 
1458 	if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1459 		change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1460 		value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1461 		return set_port_caps_atomic(dev, port, change_mask, value);
1462 	}
1463 
1464 	mutex_lock(&dev->cap_mask_mutex);
1465 
1466 	err = ib_query_port(ibdev, port, &attr);
1467 	if (err)
1468 		goto out;
1469 
1470 	tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1471 		~props->clr_port_cap_mask;
1472 
1473 	err = mlx5_set_port_caps(dev->mdev, port, tmp);
1474 
1475 out:
1476 	mutex_unlock(&dev->cap_mask_mutex);
1477 	return err;
1478 }
1479 
1480 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1481 {
1482 	mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1483 		    caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1484 }
1485 
1486 static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1487 {
1488 	/* Large page with non 4k uar support might limit the dynamic size */
1489 	if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
1490 		return MLX5_MIN_DYN_BFREGS;
1491 
1492 	return MLX5_MAX_DYN_BFREGS;
1493 }
1494 
1495 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1496 			     struct mlx5_ib_alloc_ucontext_req_v2 *req,
1497 			     struct mlx5_bfreg_info *bfregi)
1498 {
1499 	int uars_per_sys_page;
1500 	int bfregs_per_sys_page;
1501 	int ref_bfregs = req->total_num_bfregs;
1502 
1503 	if (req->total_num_bfregs == 0)
1504 		return -EINVAL;
1505 
1506 	BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1507 	BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1508 
1509 	if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1510 		return -ENOMEM;
1511 
1512 	uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1513 	bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1514 	/* This holds the required static allocation asked by the user */
1515 	req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1516 	if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1517 		return -EINVAL;
1518 
1519 	bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1520 	bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1521 	bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1522 	bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1523 
1524 	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1525 		    MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1526 		    lib_uar_4k ? "yes" : "no", ref_bfregs,
1527 		    req->total_num_bfregs, bfregi->total_num_bfregs,
1528 		    bfregi->num_sys_pages);
1529 
1530 	return 0;
1531 }
1532 
1533 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1534 {
1535 	struct mlx5_bfreg_info *bfregi;
1536 	int err;
1537 	int i;
1538 
1539 	bfregi = &context->bfregi;
1540 	for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1541 		err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1542 		if (err)
1543 			goto error;
1544 
1545 		mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1546 	}
1547 
1548 	for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1549 		bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1550 
1551 	return 0;
1552 
1553 error:
1554 	for (--i; i >= 0; i--)
1555 		if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1556 			mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1557 
1558 	return err;
1559 }
1560 
1561 static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1562 {
1563 	struct mlx5_bfreg_info *bfregi;
1564 	int err;
1565 	int i;
1566 
1567 	bfregi = &context->bfregi;
1568 	for (i = 0; i < bfregi->num_sys_pages; i++) {
1569 		if (i < bfregi->num_static_sys_pages ||
1570 		    bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX) {
1571 			err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1572 			if (err) {
1573 				mlx5_ib_warn(dev, "failed to free uar %d, err=%d\n", i, err);
1574 				return err;
1575 			}
1576 		}
1577 	}
1578 
1579 	return 0;
1580 }
1581 
1582 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn)
1583 {
1584 	int err;
1585 
1586 	err = mlx5_core_alloc_transport_domain(dev->mdev, tdn);
1587 	if (err)
1588 		return err;
1589 
1590 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1591 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1592 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1593 		return err;
1594 
1595 	mutex_lock(&dev->lb_mutex);
1596 	dev->user_td++;
1597 
1598 	if (dev->user_td == 2)
1599 		err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1600 
1601 	mutex_unlock(&dev->lb_mutex);
1602 	return err;
1603 }
1604 
1605 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn)
1606 {
1607 	mlx5_core_dealloc_transport_domain(dev->mdev, tdn);
1608 
1609 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1610 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1611 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1612 		return;
1613 
1614 	mutex_lock(&dev->lb_mutex);
1615 	dev->user_td--;
1616 
1617 	if (dev->user_td < 2)
1618 		mlx5_nic_vport_update_local_lb(dev->mdev, false);
1619 
1620 	mutex_unlock(&dev->lb_mutex);
1621 }
1622 
1623 static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1624 						  struct ib_udata *udata)
1625 {
1626 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1627 	struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1628 	struct mlx5_ib_alloc_ucontext_resp resp = {};
1629 	struct mlx5_core_dev *mdev = dev->mdev;
1630 	struct mlx5_ib_ucontext *context;
1631 	struct mlx5_bfreg_info *bfregi;
1632 	int ver;
1633 	int err;
1634 	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1635 				     max_cqe_version);
1636 	bool lib_uar_4k;
1637 
1638 	if (!dev->ib_active)
1639 		return ERR_PTR(-EAGAIN);
1640 
1641 	if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
1642 		ver = 0;
1643 	else if (udata->inlen >= min_req_v2)
1644 		ver = 2;
1645 	else
1646 		return ERR_PTR(-EINVAL);
1647 
1648 	err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1649 	if (err)
1650 		return ERR_PTR(err);
1651 
1652 	if (req.flags)
1653 		return ERR_PTR(-EINVAL);
1654 
1655 	if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1656 		return ERR_PTR(-EOPNOTSUPP);
1657 
1658 	req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1659 				    MLX5_NON_FP_BFREGS_PER_UAR);
1660 	if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1661 		return ERR_PTR(-EINVAL);
1662 
1663 	resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1664 	if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
1665 		resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
1666 	resp.cache_line_size = cache_line_size();
1667 	resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1668 	resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1669 	resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1670 	resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1671 	resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1672 	resp.cqe_version = min_t(__u8,
1673 				 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1674 				 req.max_cqe_version);
1675 	resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1676 				MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1677 	resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1678 					MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
1679 	resp.response_length = min(offsetof(typeof(resp), response_length) +
1680 				   sizeof(resp.response_length), udata->outlen);
1681 
1682 	if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) {
1683 		if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_EGRESS))
1684 			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;
1685 		if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)
1686 			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;
1687 		if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
1688 			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;
1689 		if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)
1690 			resp.flow_action_flags |= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;
1691 		/* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */
1692 	}
1693 
1694 	context = kzalloc(sizeof(*context), GFP_KERNEL);
1695 	if (!context)
1696 		return ERR_PTR(-ENOMEM);
1697 
1698 	lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1699 	bfregi = &context->bfregi;
1700 
1701 	/* updates req->total_num_bfregs */
1702 	err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1703 	if (err)
1704 		goto out_ctx;
1705 
1706 	mutex_init(&bfregi->lock);
1707 	bfregi->lib_uar_4k = lib_uar_4k;
1708 	bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
1709 				GFP_KERNEL);
1710 	if (!bfregi->count) {
1711 		err = -ENOMEM;
1712 		goto out_ctx;
1713 	}
1714 
1715 	bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1716 				    sizeof(*bfregi->sys_pages),
1717 				    GFP_KERNEL);
1718 	if (!bfregi->sys_pages) {
1719 		err = -ENOMEM;
1720 		goto out_count;
1721 	}
1722 
1723 	err = allocate_uars(dev, context);
1724 	if (err)
1725 		goto out_sys_pages;
1726 
1727 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1728 	context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
1729 #endif
1730 
1731 	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
1732 		err = mlx5_ib_alloc_transport_domain(dev, &context->tdn);
1733 		if (err)
1734 			goto out_uars;
1735 	}
1736 
1737 	INIT_LIST_HEAD(&context->vma_private_list);
1738 	mutex_init(&context->vma_private_list_mutex);
1739 	INIT_LIST_HEAD(&context->db_page_list);
1740 	mutex_init(&context->db_page_mutex);
1741 
1742 	resp.tot_bfregs = req.total_num_bfregs;
1743 	resp.num_ports = dev->num_ports;
1744 
1745 	if (field_avail(typeof(resp), cqe_version, udata->outlen))
1746 		resp.response_length += sizeof(resp.cqe_version);
1747 
1748 	if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
1749 		resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1750 				      MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1751 		resp.response_length += sizeof(resp.cmds_supp_uhw);
1752 	}
1753 
1754 	if (field_avail(typeof(resp), eth_min_inline, udata->outlen)) {
1755 		if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1756 			mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
1757 			resp.eth_min_inline++;
1758 		}
1759 		resp.response_length += sizeof(resp.eth_min_inline);
1760 	}
1761 
1762 	if (field_avail(typeof(resp), clock_info_versions, udata->outlen)) {
1763 		if (mdev->clock_info)
1764 			resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1765 		resp.response_length += sizeof(resp.clock_info_versions);
1766 	}
1767 
1768 	/*
1769 	 * We don't want to expose information from the PCI bar that is located
1770 	 * after 4096 bytes, so if the arch only supports larger pages, let's
1771 	 * pretend we don't support reading the HCA's core clock. This is also
1772 	 * forced by mmap function.
1773 	 */
1774 	if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
1775 		if (PAGE_SIZE <= 4096) {
1776 			resp.comp_mask |=
1777 				MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1778 			resp.hca_core_clock_offset =
1779 				offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1780 		}
1781 		resp.response_length += sizeof(resp.hca_core_clock_offset);
1782 	}
1783 
1784 	if (field_avail(typeof(resp), log_uar_size, udata->outlen))
1785 		resp.response_length += sizeof(resp.log_uar_size);
1786 
1787 	if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
1788 		resp.response_length += sizeof(resp.num_uars_per_page);
1789 
1790 	if (field_avail(typeof(resp), num_dyn_bfregs, udata->outlen)) {
1791 		resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
1792 		resp.response_length += sizeof(resp.num_dyn_bfregs);
1793 	}
1794 
1795 	err = ib_copy_to_udata(udata, &resp, resp.response_length);
1796 	if (err)
1797 		goto out_td;
1798 
1799 	bfregi->ver = ver;
1800 	bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1801 	context->cqe_version = resp.cqe_version;
1802 	context->lib_caps = req.lib_caps;
1803 	print_lib_caps(dev, context->lib_caps);
1804 
1805 	return &context->ibucontext;
1806 
1807 out_td:
1808 	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1809 		mlx5_ib_dealloc_transport_domain(dev, context->tdn);
1810 
1811 out_uars:
1812 	deallocate_uars(dev, context);
1813 
1814 out_sys_pages:
1815 	kfree(bfregi->sys_pages);
1816 
1817 out_count:
1818 	kfree(bfregi->count);
1819 
1820 out_ctx:
1821 	kfree(context);
1822 
1823 	return ERR_PTR(err);
1824 }
1825 
1826 static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1827 {
1828 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1829 	struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1830 	struct mlx5_bfreg_info *bfregi;
1831 
1832 	bfregi = &context->bfregi;
1833 	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1834 		mlx5_ib_dealloc_transport_domain(dev, context->tdn);
1835 
1836 	deallocate_uars(dev, context);
1837 	kfree(bfregi->sys_pages);
1838 	kfree(bfregi->count);
1839 	kfree(context);
1840 
1841 	return 0;
1842 }
1843 
1844 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1845 				 int uar_idx)
1846 {
1847 	int fw_uars_per_page;
1848 
1849 	fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1850 
1851 	return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
1852 }
1853 
1854 static int get_command(unsigned long offset)
1855 {
1856 	return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1857 }
1858 
1859 static int get_arg(unsigned long offset)
1860 {
1861 	return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
1862 }
1863 
1864 static int get_index(unsigned long offset)
1865 {
1866 	return get_arg(offset);
1867 }
1868 
1869 /* Index resides in an extra byte to enable larger values than 255 */
1870 static int get_extended_index(unsigned long offset)
1871 {
1872 	return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
1873 }
1874 
1875 static void  mlx5_ib_vma_open(struct vm_area_struct *area)
1876 {
1877 	/* vma_open is called when a new VMA is created on top of our VMA.  This
1878 	 * is done through either mremap flow or split_vma (usually due to
1879 	 * mlock, madvise, munmap, etc.) We do not support a clone of the VMA,
1880 	 * as this VMA is strongly hardware related.  Therefore we set the
1881 	 * vm_ops of the newly created/cloned VMA to NULL, to prevent it from
1882 	 * calling us again and trying to do incorrect actions.  We assume that
1883 	 * the original VMA size is exactly a single page, and therefore all
1884 	 * "splitting" operation will not happen to it.
1885 	 */
1886 	area->vm_ops = NULL;
1887 }
1888 
1889 static void  mlx5_ib_vma_close(struct vm_area_struct *area)
1890 {
1891 	struct mlx5_ib_vma_private_data *mlx5_ib_vma_priv_data;
1892 
1893 	/* It's guaranteed that all VMAs opened on a FD are closed before the
1894 	 * file itself is closed, therefore no sync is needed with the regular
1895 	 * closing flow. (e.g. mlx5 ib_dealloc_ucontext)
1896 	 * However need a sync with accessing the vma as part of
1897 	 * mlx5_ib_disassociate_ucontext.
1898 	 * The close operation is usually called under mm->mmap_sem except when
1899 	 * process is exiting.
1900 	 * The exiting case is handled explicitly as part of
1901 	 * mlx5_ib_disassociate_ucontext.
1902 	 */
1903 	mlx5_ib_vma_priv_data = (struct mlx5_ib_vma_private_data *)area->vm_private_data;
1904 
1905 	/* setting the vma context pointer to null in the mlx5_ib driver's
1906 	 * private data, to protect a race condition in
1907 	 * mlx5_ib_disassociate_ucontext().
1908 	 */
1909 	mlx5_ib_vma_priv_data->vma = NULL;
1910 	mutex_lock(mlx5_ib_vma_priv_data->vma_private_list_mutex);
1911 	list_del(&mlx5_ib_vma_priv_data->list);
1912 	mutex_unlock(mlx5_ib_vma_priv_data->vma_private_list_mutex);
1913 	kfree(mlx5_ib_vma_priv_data);
1914 }
1915 
1916 static const struct vm_operations_struct mlx5_ib_vm_ops = {
1917 	.open = mlx5_ib_vma_open,
1918 	.close = mlx5_ib_vma_close
1919 };
1920 
1921 static int mlx5_ib_set_vma_data(struct vm_area_struct *vma,
1922 				struct mlx5_ib_ucontext *ctx)
1923 {
1924 	struct mlx5_ib_vma_private_data *vma_prv;
1925 	struct list_head *vma_head = &ctx->vma_private_list;
1926 
1927 	vma_prv = kzalloc(sizeof(*vma_prv), GFP_KERNEL);
1928 	if (!vma_prv)
1929 		return -ENOMEM;
1930 
1931 	vma_prv->vma = vma;
1932 	vma_prv->vma_private_list_mutex = &ctx->vma_private_list_mutex;
1933 	vma->vm_private_data = vma_prv;
1934 	vma->vm_ops =  &mlx5_ib_vm_ops;
1935 
1936 	mutex_lock(&ctx->vma_private_list_mutex);
1937 	list_add(&vma_prv->list, vma_head);
1938 	mutex_unlock(&ctx->vma_private_list_mutex);
1939 
1940 	return 0;
1941 }
1942 
1943 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
1944 {
1945 	struct vm_area_struct *vma;
1946 	struct mlx5_ib_vma_private_data *vma_private, *n;
1947 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1948 
1949 	mutex_lock(&context->vma_private_list_mutex);
1950 	list_for_each_entry_safe(vma_private, n, &context->vma_private_list,
1951 				 list) {
1952 		vma = vma_private->vma;
1953 		zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE);
1954 		/* context going to be destroyed, should
1955 		 * not access ops any more.
1956 		 */
1957 		vma->vm_flags &= ~(VM_SHARED | VM_MAYSHARE);
1958 		vma->vm_ops = NULL;
1959 		list_del(&vma_private->list);
1960 		kfree(vma_private);
1961 	}
1962 	mutex_unlock(&context->vma_private_list_mutex);
1963 }
1964 
1965 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
1966 {
1967 	switch (cmd) {
1968 	case MLX5_IB_MMAP_WC_PAGE:
1969 		return "WC";
1970 	case MLX5_IB_MMAP_REGULAR_PAGE:
1971 		return "best effort WC";
1972 	case MLX5_IB_MMAP_NC_PAGE:
1973 		return "NC";
1974 	case MLX5_IB_MMAP_DEVICE_MEM:
1975 		return "Device Memory";
1976 	default:
1977 		return NULL;
1978 	}
1979 }
1980 
1981 static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
1982 					struct vm_area_struct *vma,
1983 					struct mlx5_ib_ucontext *context)
1984 {
1985 	phys_addr_t pfn;
1986 	int err;
1987 
1988 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1989 		return -EINVAL;
1990 
1991 	if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
1992 		return -EOPNOTSUPP;
1993 
1994 	if (vma->vm_flags & VM_WRITE)
1995 		return -EPERM;
1996 
1997 	if (!dev->mdev->clock_info_page)
1998 		return -EOPNOTSUPP;
1999 
2000 	pfn = page_to_pfn(dev->mdev->clock_info_page);
2001 	err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
2002 			      vma->vm_page_prot);
2003 	if (err)
2004 		return err;
2005 
2006 	return mlx5_ib_set_vma_data(vma, context);
2007 }
2008 
2009 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2010 		    struct vm_area_struct *vma,
2011 		    struct mlx5_ib_ucontext *context)
2012 {
2013 	struct mlx5_bfreg_info *bfregi = &context->bfregi;
2014 	int err;
2015 	unsigned long idx;
2016 	phys_addr_t pfn, pa;
2017 	pgprot_t prot;
2018 	u32 bfreg_dyn_idx = 0;
2019 	u32 uar_index;
2020 	int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2021 	int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2022 				bfregi->num_static_sys_pages;
2023 
2024 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2025 		return -EINVAL;
2026 
2027 	if (dyn_uar)
2028 		idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2029 	else
2030 		idx = get_index(vma->vm_pgoff);
2031 
2032 	if (idx >= max_valid_idx) {
2033 		mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2034 			     idx, max_valid_idx);
2035 		return -EINVAL;
2036 	}
2037 
2038 	switch (cmd) {
2039 	case MLX5_IB_MMAP_WC_PAGE:
2040 	case MLX5_IB_MMAP_ALLOC_WC:
2041 /* Some architectures don't support WC memory */
2042 #if defined(CONFIG_X86)
2043 		if (!pat_enabled())
2044 			return -EPERM;
2045 #elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
2046 			return -EPERM;
2047 #endif
2048 	/* fall through */
2049 	case MLX5_IB_MMAP_REGULAR_PAGE:
2050 		/* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2051 		prot = pgprot_writecombine(vma->vm_page_prot);
2052 		break;
2053 	case MLX5_IB_MMAP_NC_PAGE:
2054 		prot = pgprot_noncached(vma->vm_page_prot);
2055 		break;
2056 	default:
2057 		return -EINVAL;
2058 	}
2059 
2060 	if (dyn_uar) {
2061 		int uars_per_page;
2062 
2063 		uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2064 		bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2065 		if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2066 			mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2067 				     bfreg_dyn_idx, bfregi->total_num_bfregs);
2068 			return -EINVAL;
2069 		}
2070 
2071 		mutex_lock(&bfregi->lock);
2072 		/* Fail if uar already allocated, first bfreg index of each
2073 		 * page holds its count.
2074 		 */
2075 		if (bfregi->count[bfreg_dyn_idx]) {
2076 			mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2077 			mutex_unlock(&bfregi->lock);
2078 			return -EINVAL;
2079 		}
2080 
2081 		bfregi->count[bfreg_dyn_idx]++;
2082 		mutex_unlock(&bfregi->lock);
2083 
2084 		err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index);
2085 		if (err) {
2086 			mlx5_ib_warn(dev, "UAR alloc failed\n");
2087 			goto free_bfreg;
2088 		}
2089 	} else {
2090 		uar_index = bfregi->sys_pages[idx];
2091 	}
2092 
2093 	pfn = uar_index2pfn(dev, uar_index);
2094 	mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2095 
2096 	vma->vm_page_prot = prot;
2097 	err = io_remap_pfn_range(vma, vma->vm_start, pfn,
2098 				 PAGE_SIZE, vma->vm_page_prot);
2099 	if (err) {
2100 		mlx5_ib_err(dev,
2101 			    "io_remap_pfn_range failed with error=%d, mmap_cmd=%s\n",
2102 			    err, mmap_cmd2str(cmd));
2103 		err = -EAGAIN;
2104 		goto err;
2105 	}
2106 
2107 	pa = pfn << PAGE_SHIFT;
2108 
2109 	err = mlx5_ib_set_vma_data(vma, context);
2110 	if (err)
2111 		goto err;
2112 
2113 	if (dyn_uar)
2114 		bfregi->sys_pages[idx] = uar_index;
2115 	return 0;
2116 
2117 err:
2118 	if (!dyn_uar)
2119 		return err;
2120 
2121 	mlx5_cmd_free_uar(dev->mdev, idx);
2122 
2123 free_bfreg:
2124 	mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2125 
2126 	return err;
2127 }
2128 
2129 static int dm_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
2130 {
2131 	struct mlx5_ib_ucontext *mctx = to_mucontext(context);
2132 	struct mlx5_ib_dev *dev = to_mdev(context->device);
2133 	u16 page_idx = get_extended_index(vma->vm_pgoff);
2134 	size_t map_size = vma->vm_end - vma->vm_start;
2135 	u32 npages = map_size >> PAGE_SHIFT;
2136 	phys_addr_t pfn;
2137 	pgprot_t prot;
2138 
2139 	if (find_next_zero_bit(mctx->dm_pages, page_idx + npages, page_idx) !=
2140 	    page_idx + npages)
2141 		return -EINVAL;
2142 
2143 	pfn = ((pci_resource_start(dev->mdev->pdev, 0) +
2144 	      MLX5_CAP64_DEV_MEM(dev->mdev, memic_bar_start_addr)) >>
2145 	      PAGE_SHIFT) +
2146 	      page_idx;
2147 	prot = pgprot_writecombine(vma->vm_page_prot);
2148 	vma->vm_page_prot = prot;
2149 
2150 	if (io_remap_pfn_range(vma, vma->vm_start, pfn, map_size,
2151 			       vma->vm_page_prot))
2152 		return -EAGAIN;
2153 
2154 	return mlx5_ib_set_vma_data(vma, mctx);
2155 }
2156 
2157 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2158 {
2159 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2160 	struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2161 	unsigned long command;
2162 	phys_addr_t pfn;
2163 
2164 	command = get_command(vma->vm_pgoff);
2165 	switch (command) {
2166 	case MLX5_IB_MMAP_WC_PAGE:
2167 	case MLX5_IB_MMAP_NC_PAGE:
2168 	case MLX5_IB_MMAP_REGULAR_PAGE:
2169 	case MLX5_IB_MMAP_ALLOC_WC:
2170 		return uar_mmap(dev, command, vma, context);
2171 
2172 	case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2173 		return -ENOSYS;
2174 
2175 	case MLX5_IB_MMAP_CORE_CLOCK:
2176 		if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2177 			return -EINVAL;
2178 
2179 		if (vma->vm_flags & VM_WRITE)
2180 			return -EPERM;
2181 
2182 		/* Don't expose to user-space information it shouldn't have */
2183 		if (PAGE_SIZE > 4096)
2184 			return -EOPNOTSUPP;
2185 
2186 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2187 		pfn = (dev->mdev->iseg_base +
2188 		       offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2189 			PAGE_SHIFT;
2190 		if (io_remap_pfn_range(vma, vma->vm_start, pfn,
2191 				       PAGE_SIZE, vma->vm_page_prot))
2192 			return -EAGAIN;
2193 		break;
2194 	case MLX5_IB_MMAP_CLOCK_INFO:
2195 		return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2196 
2197 	case MLX5_IB_MMAP_DEVICE_MEM:
2198 		return dm_mmap(ibcontext, vma);
2199 
2200 	default:
2201 		return -EINVAL;
2202 	}
2203 
2204 	return 0;
2205 }
2206 
2207 struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
2208 			       struct ib_ucontext *context,
2209 			       struct ib_dm_alloc_attr *attr,
2210 			       struct uverbs_attr_bundle *attrs)
2211 {
2212 	u64 act_size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
2213 	struct mlx5_memic *memic = &to_mdev(ibdev)->memic;
2214 	phys_addr_t memic_addr;
2215 	struct mlx5_ib_dm *dm;
2216 	u64 start_offset;
2217 	u32 page_idx;
2218 	int err;
2219 
2220 	dm = kzalloc(sizeof(*dm), GFP_KERNEL);
2221 	if (!dm)
2222 		return ERR_PTR(-ENOMEM);
2223 
2224 	mlx5_ib_dbg(to_mdev(ibdev), "alloc_memic req: user_length=0x%llx act_length=0x%llx log_alignment=%d\n",
2225 		    attr->length, act_size, attr->alignment);
2226 
2227 	err = mlx5_cmd_alloc_memic(memic, &memic_addr,
2228 				   act_size, attr->alignment);
2229 	if (err)
2230 		goto err_free;
2231 
2232 	start_offset = memic_addr & ~PAGE_MASK;
2233 	page_idx = (memic_addr - pci_resource_start(memic->dev->pdev, 0) -
2234 		    MLX5_CAP64_DEV_MEM(memic->dev, memic_bar_start_addr)) >>
2235 		    PAGE_SHIFT;
2236 
2237 	err = uverbs_copy_to(attrs,
2238 			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
2239 			     &start_offset, sizeof(start_offset));
2240 	if (err)
2241 		goto err_dealloc;
2242 
2243 	err = uverbs_copy_to(attrs,
2244 			     MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
2245 			     &page_idx, sizeof(page_idx));
2246 	if (err)
2247 		goto err_dealloc;
2248 
2249 	bitmap_set(to_mucontext(context)->dm_pages, page_idx,
2250 		   DIV_ROUND_UP(act_size, PAGE_SIZE));
2251 
2252 	dm->dev_addr = memic_addr;
2253 
2254 	return &dm->ibdm;
2255 
2256 err_dealloc:
2257 	mlx5_cmd_dealloc_memic(memic, memic_addr,
2258 			       act_size);
2259 err_free:
2260 	kfree(dm);
2261 	return ERR_PTR(err);
2262 }
2263 
2264 int mlx5_ib_dealloc_dm(struct ib_dm *ibdm)
2265 {
2266 	struct mlx5_memic *memic = &to_mdev(ibdm->device)->memic;
2267 	struct mlx5_ib_dm *dm = to_mdm(ibdm);
2268 	u64 act_size = roundup(dm->ibdm.length, MLX5_MEMIC_BASE_SIZE);
2269 	u32 page_idx;
2270 	int ret;
2271 
2272 	ret = mlx5_cmd_dealloc_memic(memic, dm->dev_addr, act_size);
2273 	if (ret)
2274 		return ret;
2275 
2276 	page_idx = (dm->dev_addr - pci_resource_start(memic->dev->pdev, 0) -
2277 		    MLX5_CAP64_DEV_MEM(memic->dev, memic_bar_start_addr)) >>
2278 		    PAGE_SHIFT;
2279 	bitmap_clear(to_mucontext(ibdm->uobject->context)->dm_pages,
2280 		     page_idx,
2281 		     DIV_ROUND_UP(act_size, PAGE_SIZE));
2282 
2283 	kfree(dm);
2284 
2285 	return 0;
2286 }
2287 
2288 static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
2289 				      struct ib_ucontext *context,
2290 				      struct ib_udata *udata)
2291 {
2292 	struct mlx5_ib_alloc_pd_resp resp;
2293 	struct mlx5_ib_pd *pd;
2294 	int err;
2295 
2296 	pd = kmalloc(sizeof(*pd), GFP_KERNEL);
2297 	if (!pd)
2298 		return ERR_PTR(-ENOMEM);
2299 
2300 	err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
2301 	if (err) {
2302 		kfree(pd);
2303 		return ERR_PTR(err);
2304 	}
2305 
2306 	if (context) {
2307 		resp.pdn = pd->pdn;
2308 		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2309 			mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
2310 			kfree(pd);
2311 			return ERR_PTR(-EFAULT);
2312 		}
2313 	}
2314 
2315 	return &pd->ibpd;
2316 }
2317 
2318 static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
2319 {
2320 	struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2321 	struct mlx5_ib_pd *mpd = to_mpd(pd);
2322 
2323 	mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
2324 	kfree(mpd);
2325 
2326 	return 0;
2327 }
2328 
2329 enum {
2330 	MATCH_CRITERIA_ENABLE_OUTER_BIT,
2331 	MATCH_CRITERIA_ENABLE_MISC_BIT,
2332 	MATCH_CRITERIA_ENABLE_INNER_BIT,
2333 	MATCH_CRITERIA_ENABLE_MISC2_BIT
2334 };
2335 
2336 #define HEADER_IS_ZERO(match_criteria, headers)			           \
2337 	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
2338 		    0, MLX5_FLD_SZ_BYTES(fte_match_param, headers)))       \
2339 
2340 static u8 get_match_criteria_enable(u32 *match_criteria)
2341 {
2342 	u8 match_criteria_enable;
2343 
2344 	match_criteria_enable =
2345 		(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
2346 		MATCH_CRITERIA_ENABLE_OUTER_BIT;
2347 	match_criteria_enable |=
2348 		(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
2349 		MATCH_CRITERIA_ENABLE_MISC_BIT;
2350 	match_criteria_enable |=
2351 		(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
2352 		MATCH_CRITERIA_ENABLE_INNER_BIT;
2353 	match_criteria_enable |=
2354 		(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
2355 		MATCH_CRITERIA_ENABLE_MISC2_BIT;
2356 
2357 	return match_criteria_enable;
2358 }
2359 
2360 static void set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
2361 {
2362 	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
2363 	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
2364 }
2365 
2366 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
2367 			   bool inner)
2368 {
2369 	if (inner) {
2370 		MLX5_SET(fte_match_set_misc,
2371 			 misc_c, inner_ipv6_flow_label, mask);
2372 		MLX5_SET(fte_match_set_misc,
2373 			 misc_v, inner_ipv6_flow_label, val);
2374 	} else {
2375 		MLX5_SET(fte_match_set_misc,
2376 			 misc_c, outer_ipv6_flow_label, mask);
2377 		MLX5_SET(fte_match_set_misc,
2378 			 misc_v, outer_ipv6_flow_label, val);
2379 	}
2380 }
2381 
2382 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
2383 {
2384 	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
2385 	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
2386 	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
2387 	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
2388 }
2389 
2390 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
2391 {
2392 	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
2393 	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
2394 		return -EOPNOTSUPP;
2395 
2396 	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
2397 	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
2398 		return -EOPNOTSUPP;
2399 
2400 	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
2401 	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
2402 		return -EOPNOTSUPP;
2403 
2404 	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
2405 	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
2406 		return -EOPNOTSUPP;
2407 
2408 	return 0;
2409 }
2410 
2411 #define LAST_ETH_FIELD vlan_tag
2412 #define LAST_IB_FIELD sl
2413 #define LAST_IPV4_FIELD tos
2414 #define LAST_IPV6_FIELD traffic_class
2415 #define LAST_TCP_UDP_FIELD src_port
2416 #define LAST_TUNNEL_FIELD tunnel_id
2417 #define LAST_FLOW_TAG_FIELD tag_id
2418 #define LAST_DROP_FIELD size
2419 #define LAST_COUNTERS_FIELD counters
2420 
2421 /* Field is the last supported field */
2422 #define FIELDS_NOT_SUPPORTED(filter, field)\
2423 	memchr_inv((void *)&filter.field  +\
2424 		   sizeof(filter.field), 0,\
2425 		   sizeof(filter) -\
2426 		   offsetof(typeof(filter), field) -\
2427 		   sizeof(filter.field))
2428 
2429 static int parse_flow_flow_action(const union ib_flow_spec *ib_spec,
2430 				  const struct ib_flow_attr *flow_attr,
2431 				  struct mlx5_flow_act *action)
2432 {
2433 	struct mlx5_ib_flow_action *maction = to_mflow_act(ib_spec->action.act);
2434 
2435 	switch (maction->ib_action.type) {
2436 	case IB_FLOW_ACTION_ESP:
2437 		/* Currently only AES_GCM keymat is supported by the driver */
2438 		action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
2439 		action->action |= flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS ?
2440 			MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
2441 			MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
2442 		return 0;
2443 	default:
2444 		return -EOPNOTSUPP;
2445 	}
2446 }
2447 
2448 static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c,
2449 			   u32 *match_v, const union ib_flow_spec *ib_spec,
2450 			   const struct ib_flow_attr *flow_attr,
2451 			   struct mlx5_flow_act *action, u32 prev_type)
2452 {
2453 	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
2454 					   misc_parameters);
2455 	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
2456 					   misc_parameters);
2457 	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
2458 					    misc_parameters_2);
2459 	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
2460 					    misc_parameters_2);
2461 	void *headers_c;
2462 	void *headers_v;
2463 	int match_ipv;
2464 	int ret;
2465 
2466 	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
2467 		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
2468 					 inner_headers);
2469 		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
2470 					 inner_headers);
2471 		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2472 					ft_field_support.inner_ip_version);
2473 	} else {
2474 		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
2475 					 outer_headers);
2476 		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
2477 					 outer_headers);
2478 		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2479 					ft_field_support.outer_ip_version);
2480 	}
2481 
2482 	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2483 	case IB_FLOW_SPEC_ETH:
2484 		if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
2485 			return -EOPNOTSUPP;
2486 
2487 		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2488 					     dmac_47_16),
2489 				ib_spec->eth.mask.dst_mac);
2490 		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2491 					     dmac_47_16),
2492 				ib_spec->eth.val.dst_mac);
2493 
2494 		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2495 					     smac_47_16),
2496 				ib_spec->eth.mask.src_mac);
2497 		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2498 					     smac_47_16),
2499 				ib_spec->eth.val.src_mac);
2500 
2501 		if (ib_spec->eth.mask.vlan_tag) {
2502 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2503 				 cvlan_tag, 1);
2504 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2505 				 cvlan_tag, 1);
2506 
2507 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2508 				 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
2509 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2510 				 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
2511 
2512 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2513 				 first_cfi,
2514 				 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
2515 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2516 				 first_cfi,
2517 				 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
2518 
2519 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2520 				 first_prio,
2521 				 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
2522 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2523 				 first_prio,
2524 				 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
2525 		}
2526 		MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2527 			 ethertype, ntohs(ib_spec->eth.mask.ether_type));
2528 		MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2529 			 ethertype, ntohs(ib_spec->eth.val.ether_type));
2530 		break;
2531 	case IB_FLOW_SPEC_IPV4:
2532 		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
2533 			return -EOPNOTSUPP;
2534 
2535 		if (match_ipv) {
2536 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2537 				 ip_version, 0xf);
2538 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2539 				 ip_version, MLX5_FS_IPV4_VERSION);
2540 		} else {
2541 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2542 				 ethertype, 0xffff);
2543 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2544 				 ethertype, ETH_P_IP);
2545 		}
2546 
2547 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2548 				    src_ipv4_src_ipv6.ipv4_layout.ipv4),
2549 		       &ib_spec->ipv4.mask.src_ip,
2550 		       sizeof(ib_spec->ipv4.mask.src_ip));
2551 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2552 				    src_ipv4_src_ipv6.ipv4_layout.ipv4),
2553 		       &ib_spec->ipv4.val.src_ip,
2554 		       sizeof(ib_spec->ipv4.val.src_ip));
2555 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2556 				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2557 		       &ib_spec->ipv4.mask.dst_ip,
2558 		       sizeof(ib_spec->ipv4.mask.dst_ip));
2559 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2560 				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2561 		       &ib_spec->ipv4.val.dst_ip,
2562 		       sizeof(ib_spec->ipv4.val.dst_ip));
2563 
2564 		set_tos(headers_c, headers_v,
2565 			ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
2566 
2567 		set_proto(headers_c, headers_v,
2568 			  ib_spec->ipv4.mask.proto, ib_spec->ipv4.val.proto);
2569 		break;
2570 	case IB_FLOW_SPEC_IPV6:
2571 		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
2572 			return -EOPNOTSUPP;
2573 
2574 		if (match_ipv) {
2575 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2576 				 ip_version, 0xf);
2577 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2578 				 ip_version, MLX5_FS_IPV6_VERSION);
2579 		} else {
2580 			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2581 				 ethertype, 0xffff);
2582 			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2583 				 ethertype, ETH_P_IPV6);
2584 		}
2585 
2586 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2587 				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
2588 		       &ib_spec->ipv6.mask.src_ip,
2589 		       sizeof(ib_spec->ipv6.mask.src_ip));
2590 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2591 				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
2592 		       &ib_spec->ipv6.val.src_ip,
2593 		       sizeof(ib_spec->ipv6.val.src_ip));
2594 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2595 				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2596 		       &ib_spec->ipv6.mask.dst_ip,
2597 		       sizeof(ib_spec->ipv6.mask.dst_ip));
2598 		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2599 				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2600 		       &ib_spec->ipv6.val.dst_ip,
2601 		       sizeof(ib_spec->ipv6.val.dst_ip));
2602 
2603 		set_tos(headers_c, headers_v,
2604 			ib_spec->ipv6.mask.traffic_class,
2605 			ib_spec->ipv6.val.traffic_class);
2606 
2607 		set_proto(headers_c, headers_v,
2608 			  ib_spec->ipv6.mask.next_hdr,
2609 			  ib_spec->ipv6.val.next_hdr);
2610 
2611 		set_flow_label(misc_params_c, misc_params_v,
2612 			       ntohl(ib_spec->ipv6.mask.flow_label),
2613 			       ntohl(ib_spec->ipv6.val.flow_label),
2614 			       ib_spec->type & IB_FLOW_SPEC_INNER);
2615 		break;
2616 	case IB_FLOW_SPEC_ESP:
2617 		if (ib_spec->esp.mask.seq)
2618 			return -EOPNOTSUPP;
2619 
2620 		MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,
2621 			 ntohl(ib_spec->esp.mask.spi));
2622 		MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
2623 			 ntohl(ib_spec->esp.val.spi));
2624 		break;
2625 	case IB_FLOW_SPEC_TCP:
2626 		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2627 					 LAST_TCP_UDP_FIELD))
2628 			return -EOPNOTSUPP;
2629 
2630 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2631 			 0xff);
2632 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2633 			 IPPROTO_TCP);
2634 
2635 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
2636 			 ntohs(ib_spec->tcp_udp.mask.src_port));
2637 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
2638 			 ntohs(ib_spec->tcp_udp.val.src_port));
2639 
2640 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
2641 			 ntohs(ib_spec->tcp_udp.mask.dst_port));
2642 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
2643 			 ntohs(ib_spec->tcp_udp.val.dst_port));
2644 		break;
2645 	case IB_FLOW_SPEC_UDP:
2646 		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2647 					 LAST_TCP_UDP_FIELD))
2648 			return -EOPNOTSUPP;
2649 
2650 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2651 			 0xff);
2652 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2653 			 IPPROTO_UDP);
2654 
2655 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
2656 			 ntohs(ib_spec->tcp_udp.mask.src_port));
2657 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
2658 			 ntohs(ib_spec->tcp_udp.val.src_port));
2659 
2660 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
2661 			 ntohs(ib_spec->tcp_udp.mask.dst_port));
2662 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2663 			 ntohs(ib_spec->tcp_udp.val.dst_port));
2664 		break;
2665 	case IB_FLOW_SPEC_GRE:
2666 		if (ib_spec->gre.mask.c_ks_res0_ver)
2667 			return -EOPNOTSUPP;
2668 
2669 		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2670 			 0xff);
2671 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2672 			 IPPROTO_GRE);
2673 
2674 		MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
2675 			 0xffff);
2676 		MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
2677 			 ntohs(ib_spec->gre.val.protocol));
2678 
2679 		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
2680 				    gre_key_h),
2681 		       &ib_spec->gre.mask.key,
2682 		       sizeof(ib_spec->gre.mask.key));
2683 		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
2684 				    gre_key_h),
2685 		       &ib_spec->gre.val.key,
2686 		       sizeof(ib_spec->gre.val.key));
2687 		break;
2688 	case IB_FLOW_SPEC_MPLS:
2689 		switch (prev_type) {
2690 		case IB_FLOW_SPEC_UDP:
2691 			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2692 						   ft_field_support.outer_first_mpls_over_udp),
2693 						   &ib_spec->mpls.mask.tag))
2694 				return -EOPNOTSUPP;
2695 
2696 			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2697 					    outer_first_mpls_over_udp),
2698 			       &ib_spec->mpls.val.tag,
2699 			       sizeof(ib_spec->mpls.val.tag));
2700 			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2701 					    outer_first_mpls_over_udp),
2702 			       &ib_spec->mpls.mask.tag,
2703 			       sizeof(ib_spec->mpls.mask.tag));
2704 			break;
2705 		case IB_FLOW_SPEC_GRE:
2706 			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2707 						   ft_field_support.outer_first_mpls_over_gre),
2708 						   &ib_spec->mpls.mask.tag))
2709 				return -EOPNOTSUPP;
2710 
2711 			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2712 					    outer_first_mpls_over_gre),
2713 			       &ib_spec->mpls.val.tag,
2714 			       sizeof(ib_spec->mpls.val.tag));
2715 			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2716 					    outer_first_mpls_over_gre),
2717 			       &ib_spec->mpls.mask.tag,
2718 			       sizeof(ib_spec->mpls.mask.tag));
2719 			break;
2720 		default:
2721 			if (ib_spec->type & IB_FLOW_SPEC_INNER) {
2722 				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2723 							   ft_field_support.inner_first_mpls),
2724 							   &ib_spec->mpls.mask.tag))
2725 					return -EOPNOTSUPP;
2726 
2727 				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2728 						    inner_first_mpls),
2729 				       &ib_spec->mpls.val.tag,
2730 				       sizeof(ib_spec->mpls.val.tag));
2731 				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2732 						    inner_first_mpls),
2733 				       &ib_spec->mpls.mask.tag,
2734 				       sizeof(ib_spec->mpls.mask.tag));
2735 			} else {
2736 				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2737 							   ft_field_support.outer_first_mpls),
2738 							   &ib_spec->mpls.mask.tag))
2739 					return -EOPNOTSUPP;
2740 
2741 				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
2742 						    outer_first_mpls),
2743 				       &ib_spec->mpls.val.tag,
2744 				       sizeof(ib_spec->mpls.val.tag));
2745 				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
2746 						    outer_first_mpls),
2747 				       &ib_spec->mpls.mask.tag,
2748 				       sizeof(ib_spec->mpls.mask.tag));
2749 			}
2750 		}
2751 		break;
2752 	case IB_FLOW_SPEC_VXLAN_TUNNEL:
2753 		if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
2754 					 LAST_TUNNEL_FIELD))
2755 			return -EOPNOTSUPP;
2756 
2757 		MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
2758 			 ntohl(ib_spec->tunnel.mask.tunnel_id));
2759 		MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
2760 			 ntohl(ib_spec->tunnel.val.tunnel_id));
2761 		break;
2762 	case IB_FLOW_SPEC_ACTION_TAG:
2763 		if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
2764 					 LAST_FLOW_TAG_FIELD))
2765 			return -EOPNOTSUPP;
2766 		if (ib_spec->flow_tag.tag_id >= BIT(24))
2767 			return -EINVAL;
2768 
2769 		action->flow_tag = ib_spec->flow_tag.tag_id;
2770 		action->has_flow_tag = true;
2771 		break;
2772 	case IB_FLOW_SPEC_ACTION_DROP:
2773 		if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
2774 					 LAST_DROP_FIELD))
2775 			return -EOPNOTSUPP;
2776 		action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
2777 		break;
2778 	case IB_FLOW_SPEC_ACTION_HANDLE:
2779 		ret = parse_flow_flow_action(ib_spec, flow_attr, action);
2780 		if (ret)
2781 			return ret;
2782 		break;
2783 	case IB_FLOW_SPEC_ACTION_COUNT:
2784 		if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
2785 					 LAST_COUNTERS_FIELD))
2786 			return -EOPNOTSUPP;
2787 
2788 		/* for now support only one counters spec per flow */
2789 		if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
2790 			return -EINVAL;
2791 
2792 		action->counters = ib_spec->flow_count.counters;
2793 		action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2794 		break;
2795 	default:
2796 		return -EINVAL;
2797 	}
2798 
2799 	return 0;
2800 }
2801 
2802 /* If a flow could catch both multicast and unicast packets,
2803  * it won't fall into the multicast flow steering table and this rule
2804  * could steal other multicast packets.
2805  */
2806 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
2807 {
2808 	union ib_flow_spec *flow_spec;
2809 
2810 	if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
2811 	    ib_attr->num_of_specs < 1)
2812 		return false;
2813 
2814 	flow_spec = (union ib_flow_spec *)(ib_attr + 1);
2815 	if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
2816 		struct ib_flow_spec_ipv4 *ipv4_spec;
2817 
2818 		ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
2819 		if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
2820 			return true;
2821 
2822 		return false;
2823 	}
2824 
2825 	if (flow_spec->type == IB_FLOW_SPEC_ETH) {
2826 		struct ib_flow_spec_eth *eth_spec;
2827 
2828 		eth_spec = (struct ib_flow_spec_eth *)flow_spec;
2829 		return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
2830 		       is_multicast_ether_addr(eth_spec->val.dst_mac);
2831 	}
2832 
2833 	return false;
2834 }
2835 
2836 enum valid_spec {
2837 	VALID_SPEC_INVALID,
2838 	VALID_SPEC_VALID,
2839 	VALID_SPEC_NA,
2840 };
2841 
2842 static enum valid_spec
2843 is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
2844 		     const struct mlx5_flow_spec *spec,
2845 		     const struct mlx5_flow_act *flow_act,
2846 		     bool egress)
2847 {
2848 	const u32 *match_c = spec->match_criteria;
2849 	bool is_crypto =
2850 		(flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
2851 				     MLX5_FLOW_CONTEXT_ACTION_DECRYPT));
2852 	bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
2853 	bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
2854 
2855 	/*
2856 	 * Currently only crypto is supported in egress, when regular egress
2857 	 * rules would be supported, always return VALID_SPEC_NA.
2858 	 */
2859 	if (!is_crypto)
2860 		return egress ? VALID_SPEC_INVALID : VALID_SPEC_NA;
2861 
2862 	return is_crypto && is_ipsec &&
2863 		(!egress || (!is_drop && !flow_act->has_flow_tag)) ?
2864 		VALID_SPEC_VALID : VALID_SPEC_INVALID;
2865 }
2866 
2867 static bool is_valid_spec(struct mlx5_core_dev *mdev,
2868 			  const struct mlx5_flow_spec *spec,
2869 			  const struct mlx5_flow_act *flow_act,
2870 			  bool egress)
2871 {
2872 	/* We curretly only support ipsec egress flow */
2873 	return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
2874 }
2875 
2876 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
2877 			       const struct ib_flow_attr *flow_attr,
2878 			       bool check_inner)
2879 {
2880 	union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
2881 	int match_ipv = check_inner ?
2882 			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2883 					ft_field_support.inner_ip_version) :
2884 			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2885 					ft_field_support.outer_ip_version);
2886 	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
2887 	bool ipv4_spec_valid, ipv6_spec_valid;
2888 	unsigned int ip_spec_type = 0;
2889 	bool has_ethertype = false;
2890 	unsigned int spec_index;
2891 	bool mask_valid = true;
2892 	u16 eth_type = 0;
2893 	bool type_valid;
2894 
2895 	/* Validate that ethertype is correct */
2896 	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
2897 		if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
2898 		    ib_spec->eth.mask.ether_type) {
2899 			mask_valid = (ib_spec->eth.mask.ether_type ==
2900 				      htons(0xffff));
2901 			has_ethertype = true;
2902 			eth_type = ntohs(ib_spec->eth.val.ether_type);
2903 		} else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
2904 			   (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
2905 			ip_spec_type = ib_spec->type;
2906 		}
2907 		ib_spec = (void *)ib_spec + ib_spec->size;
2908 	}
2909 
2910 	type_valid = (!has_ethertype) || (!ip_spec_type);
2911 	if (!type_valid && mask_valid) {
2912 		ipv4_spec_valid = (eth_type == ETH_P_IP) &&
2913 			(ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
2914 		ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
2915 			(ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
2916 
2917 		type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
2918 			     (((eth_type == ETH_P_MPLS_UC) ||
2919 			       (eth_type == ETH_P_MPLS_MC)) && match_ipv);
2920 	}
2921 
2922 	return type_valid;
2923 }
2924 
2925 static bool is_valid_attr(struct mlx5_core_dev *mdev,
2926 			  const struct ib_flow_attr *flow_attr)
2927 {
2928 	return is_valid_ethertype(mdev, flow_attr, false) &&
2929 	       is_valid_ethertype(mdev, flow_attr, true);
2930 }
2931 
2932 static void put_flow_table(struct mlx5_ib_dev *dev,
2933 			   struct mlx5_ib_flow_prio *prio, bool ft_added)
2934 {
2935 	prio->refcount -= !!ft_added;
2936 	if (!prio->refcount) {
2937 		mlx5_destroy_flow_table(prio->flow_table);
2938 		prio->flow_table = NULL;
2939 	}
2940 }
2941 
2942 static void counters_clear_description(struct ib_counters *counters)
2943 {
2944 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
2945 
2946 	mutex_lock(&mcounters->mcntrs_mutex);
2947 	kfree(mcounters->counters_data);
2948 	mcounters->counters_data = NULL;
2949 	mcounters->cntrs_max_index = 0;
2950 	mutex_unlock(&mcounters->mcntrs_mutex);
2951 }
2952 
2953 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
2954 {
2955 	struct mlx5_ib_dev *dev = to_mdev(flow_id->qp->device);
2956 	struct mlx5_ib_flow_handler *handler = container_of(flow_id,
2957 							  struct mlx5_ib_flow_handler,
2958 							  ibflow);
2959 	struct mlx5_ib_flow_handler *iter, *tmp;
2960 
2961 	mutex_lock(&dev->flow_db->lock);
2962 
2963 	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
2964 		mlx5_del_flow_rules(iter->rule);
2965 		put_flow_table(dev, iter->prio, true);
2966 		list_del(&iter->list);
2967 		kfree(iter);
2968 	}
2969 
2970 	mlx5_del_flow_rules(handler->rule);
2971 	put_flow_table(dev, handler->prio, true);
2972 	if (handler->ibcounters &&
2973 	    atomic_read(&handler->ibcounters->usecnt) == 1)
2974 		counters_clear_description(handler->ibcounters);
2975 
2976 	mutex_unlock(&dev->flow_db->lock);
2977 	kfree(handler);
2978 
2979 	return 0;
2980 }
2981 
2982 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
2983 {
2984 	priority *= 2;
2985 	if (!dont_trap)
2986 		priority++;
2987 	return priority;
2988 }
2989 
2990 enum flow_table_type {
2991 	MLX5_IB_FT_RX,
2992 	MLX5_IB_FT_TX
2993 };
2994 
2995 #define MLX5_FS_MAX_TYPES	 6
2996 #define MLX5_FS_MAX_ENTRIES	 BIT(16)
2997 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
2998 						struct ib_flow_attr *flow_attr,
2999 						enum flow_table_type ft_type)
3000 {
3001 	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
3002 	struct mlx5_flow_namespace *ns = NULL;
3003 	struct mlx5_ib_flow_prio *prio;
3004 	struct mlx5_flow_table *ft;
3005 	int max_table_size;
3006 	int num_entries;
3007 	int num_groups;
3008 	int priority;
3009 	int err = 0;
3010 
3011 	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3012 						       log_max_ft_size));
3013 	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
3014 		if (ft_type == MLX5_IB_FT_TX)
3015 			priority = 0;
3016 		else if (flow_is_multicast_only(flow_attr) &&
3017 			 !dont_trap)
3018 			priority = MLX5_IB_FLOW_MCAST_PRIO;
3019 		else
3020 			priority = ib_prio_to_core_prio(flow_attr->priority,
3021 							dont_trap);
3022 		ns = mlx5_get_flow_namespace(dev->mdev,
3023 					     ft_type == MLX5_IB_FT_TX ?
3024 					     MLX5_FLOW_NAMESPACE_EGRESS :
3025 					     MLX5_FLOW_NAMESPACE_BYPASS);
3026 		num_entries = MLX5_FS_MAX_ENTRIES;
3027 		num_groups = MLX5_FS_MAX_TYPES;
3028 		prio = &dev->flow_db->prios[priority];
3029 	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3030 		   flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
3031 		ns = mlx5_get_flow_namespace(dev->mdev,
3032 					     MLX5_FLOW_NAMESPACE_LEFTOVERS);
3033 		build_leftovers_ft_param(&priority,
3034 					 &num_entries,
3035 					 &num_groups);
3036 		prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
3037 	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3038 		if (!MLX5_CAP_FLOWTABLE(dev->mdev,
3039 					allow_sniffer_and_nic_rx_shared_tir))
3040 			return ERR_PTR(-ENOTSUPP);
3041 
3042 		ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
3043 					     MLX5_FLOW_NAMESPACE_SNIFFER_RX :
3044 					     MLX5_FLOW_NAMESPACE_SNIFFER_TX);
3045 
3046 		prio = &dev->flow_db->sniffer[ft_type];
3047 		priority = 0;
3048 		num_entries = 1;
3049 		num_groups = 1;
3050 	}
3051 
3052 	if (!ns)
3053 		return ERR_PTR(-ENOTSUPP);
3054 
3055 	if (num_entries > max_table_size)
3056 		return ERR_PTR(-ENOMEM);
3057 
3058 	ft = prio->flow_table;
3059 	if (!ft) {
3060 		ft = mlx5_create_auto_grouped_flow_table(ns, priority,
3061 							 num_entries,
3062 							 num_groups,
3063 							 0, 0);
3064 
3065 		if (!IS_ERR(ft)) {
3066 			prio->refcount = 0;
3067 			prio->flow_table = ft;
3068 		} else {
3069 			err = PTR_ERR(ft);
3070 		}
3071 	}
3072 
3073 	return err ? ERR_PTR(err) : prio;
3074 }
3075 
3076 static void set_underlay_qp(struct mlx5_ib_dev *dev,
3077 			    struct mlx5_flow_spec *spec,
3078 			    u32 underlay_qpn)
3079 {
3080 	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
3081 					   spec->match_criteria,
3082 					   misc_parameters);
3083 	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3084 					   misc_parameters);
3085 
3086 	if (underlay_qpn &&
3087 	    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3088 				      ft_field_support.bth_dst_qp)) {
3089 		MLX5_SET(fte_match_set_misc,
3090 			 misc_params_v, bth_dst_qp, underlay_qpn);
3091 		MLX5_SET(fte_match_set_misc,
3092 			 misc_params_c, bth_dst_qp, 0xffffff);
3093 	}
3094 }
3095 
3096 static int read_flow_counters(struct ib_device *ibdev,
3097 			      struct mlx5_read_counters_attr *read_attr)
3098 {
3099 	struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
3100 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
3101 
3102 	return mlx5_fc_query(dev->mdev, fc,
3103 			     &read_attr->out[IB_COUNTER_PACKETS],
3104 			     &read_attr->out[IB_COUNTER_BYTES]);
3105 }
3106 
3107 /* flow counters currently expose two counters packets and bytes */
3108 #define FLOW_COUNTERS_NUM 2
3109 static int counters_set_description(struct ib_counters *counters,
3110 				    enum mlx5_ib_counters_type counters_type,
3111 				    struct mlx5_ib_flow_counters_desc *desc_data,
3112 				    u32 ncounters)
3113 {
3114 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
3115 	u32 cntrs_max_index = 0;
3116 	int i;
3117 
3118 	if (counters_type != MLX5_IB_COUNTERS_FLOW)
3119 		return -EINVAL;
3120 
3121 	/* init the fields for the object */
3122 	mcounters->type = counters_type;
3123 	mcounters->read_counters = read_flow_counters;
3124 	mcounters->counters_num = FLOW_COUNTERS_NUM;
3125 	mcounters->ncounters = ncounters;
3126 	/* each counter entry have both description and index pair */
3127 	for (i = 0; i < ncounters; i++) {
3128 		if (desc_data[i].description > IB_COUNTER_BYTES)
3129 			return -EINVAL;
3130 
3131 		if (cntrs_max_index <= desc_data[i].index)
3132 			cntrs_max_index = desc_data[i].index + 1;
3133 	}
3134 
3135 	mutex_lock(&mcounters->mcntrs_mutex);
3136 	mcounters->counters_data = desc_data;
3137 	mcounters->cntrs_max_index = cntrs_max_index;
3138 	mutex_unlock(&mcounters->mcntrs_mutex);
3139 
3140 	return 0;
3141 }
3142 
3143 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
3144 static int flow_counters_set_data(struct ib_counters *ibcounters,
3145 				  struct mlx5_ib_create_flow *ucmd)
3146 {
3147 	struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
3148 	struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
3149 	struct mlx5_ib_flow_counters_desc *desc_data = NULL;
3150 	bool hw_hndl = false;
3151 	int ret = 0;
3152 
3153 	if (ucmd && ucmd->ncounters_data != 0) {
3154 		cntrs_data = ucmd->data;
3155 		if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
3156 			return -EINVAL;
3157 
3158 		desc_data = kcalloc(cntrs_data->ncounters,
3159 				    sizeof(*desc_data),
3160 				    GFP_KERNEL);
3161 		if (!desc_data)
3162 			return  -ENOMEM;
3163 
3164 		if (copy_from_user(desc_data,
3165 				   u64_to_user_ptr(cntrs_data->counters_data),
3166 				   sizeof(*desc_data) * cntrs_data->ncounters)) {
3167 			ret = -EFAULT;
3168 			goto free;
3169 		}
3170 	}
3171 
3172 	if (!mcounters->hw_cntrs_hndl) {
3173 		mcounters->hw_cntrs_hndl = mlx5_fc_create(
3174 			to_mdev(ibcounters->device)->mdev, false);
3175 		if (!mcounters->hw_cntrs_hndl) {
3176 			ret = -ENOMEM;
3177 			goto free;
3178 		}
3179 		hw_hndl = true;
3180 	}
3181 
3182 	if (desc_data) {
3183 		/* counters already bound to at least one flow */
3184 		if (mcounters->cntrs_max_index) {
3185 			ret = -EINVAL;
3186 			goto free_hndl;
3187 		}
3188 
3189 		ret = counters_set_description(ibcounters,
3190 					       MLX5_IB_COUNTERS_FLOW,
3191 					       desc_data,
3192 					       cntrs_data->ncounters);
3193 		if (ret)
3194 			goto free_hndl;
3195 
3196 	} else if (!mcounters->cntrs_max_index) {
3197 		/* counters not bound yet, must have udata passed */
3198 		ret = -EINVAL;
3199 		goto free_hndl;
3200 	}
3201 
3202 	return 0;
3203 
3204 free_hndl:
3205 	if (hw_hndl) {
3206 		mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
3207 				mcounters->hw_cntrs_hndl);
3208 		mcounters->hw_cntrs_hndl = NULL;
3209 	}
3210 free:
3211 	kfree(desc_data);
3212 	return ret;
3213 }
3214 
3215 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
3216 						      struct mlx5_ib_flow_prio *ft_prio,
3217 						      const struct ib_flow_attr *flow_attr,
3218 						      struct mlx5_flow_destination *dst,
3219 						      u32 underlay_qpn,
3220 						      struct mlx5_ib_create_flow *ucmd)
3221 {
3222 	struct mlx5_flow_table	*ft = ft_prio->flow_table;
3223 	struct mlx5_ib_flow_handler *handler;
3224 	struct mlx5_flow_act flow_act = {.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG};
3225 	struct mlx5_flow_spec *spec;
3226 	struct mlx5_flow_destination dest_arr[2] = {};
3227 	struct mlx5_flow_destination *rule_dst = dest_arr;
3228 	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
3229 	unsigned int spec_index;
3230 	u32 prev_type = 0;
3231 	int err = 0;
3232 	int dest_num = 0;
3233 	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
3234 
3235 	if (!is_valid_attr(dev->mdev, flow_attr))
3236 		return ERR_PTR(-EINVAL);
3237 
3238 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
3239 	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
3240 	if (!handler || !spec) {
3241 		err = -ENOMEM;
3242 		goto free;
3243 	}
3244 
3245 	INIT_LIST_HEAD(&handler->list);
3246 	if (dst) {
3247 		memcpy(&dest_arr[0], dst, sizeof(*dst));
3248 		dest_num++;
3249 	}
3250 
3251 	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
3252 		err = parse_flow_attr(dev->mdev, spec->match_criteria,
3253 				      spec->match_value,
3254 				      ib_flow, flow_attr, &flow_act,
3255 				      prev_type);
3256 		if (err < 0)
3257 			goto free;
3258 
3259 		prev_type = ((union ib_flow_spec *)ib_flow)->type;
3260 		ib_flow += ((union ib_flow_spec *)ib_flow)->size;
3261 	}
3262 
3263 	if (!flow_is_multicast_only(flow_attr))
3264 		set_underlay_qp(dev, spec, underlay_qpn);
3265 
3266 	if (dev->rep) {
3267 		void *misc;
3268 
3269 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3270 				    misc_parameters);
3271 		MLX5_SET(fte_match_set_misc, misc, source_port,
3272 			 dev->rep->vport);
3273 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
3274 				    misc_parameters);
3275 		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
3276 	}
3277 
3278 	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
3279 
3280 	if (is_egress &&
3281 	    !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
3282 		err = -EINVAL;
3283 		goto free;
3284 	}
3285 
3286 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
3287 		err = flow_counters_set_data(flow_act.counters, ucmd);
3288 		if (err)
3289 			goto free;
3290 
3291 		handler->ibcounters = flow_act.counters;
3292 		dest_arr[dest_num].type =
3293 			MLX5_FLOW_DESTINATION_TYPE_COUNTER;
3294 		dest_arr[dest_num].counter =
3295 			to_mcounters(flow_act.counters)->hw_cntrs_hndl;
3296 		dest_num++;
3297 	}
3298 
3299 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
3300 		if (!(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT)) {
3301 			rule_dst = NULL;
3302 			dest_num = 0;
3303 		}
3304 	} else {
3305 		if (is_egress)
3306 			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
3307 		else
3308 			flow_act.action |=
3309 				dest_num ?  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST :
3310 					MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
3311 	}
3312 
3313 	if (flow_act.has_flow_tag &&
3314 	    (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3315 	     flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
3316 		mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
3317 			     flow_act.flow_tag, flow_attr->type);
3318 		err = -EINVAL;
3319 		goto free;
3320 	}
3321 	handler->rule = mlx5_add_flow_rules(ft, spec,
3322 					    &flow_act,
3323 					    rule_dst, dest_num);
3324 
3325 	if (IS_ERR(handler->rule)) {
3326 		err = PTR_ERR(handler->rule);
3327 		goto free;
3328 	}
3329 
3330 	ft_prio->refcount++;
3331 	handler->prio = ft_prio;
3332 
3333 	ft_prio->flow_table = ft;
3334 free:
3335 	if (err && handler) {
3336 		if (handler->ibcounters &&
3337 		    atomic_read(&handler->ibcounters->usecnt) == 1)
3338 			counters_clear_description(handler->ibcounters);
3339 		kfree(handler);
3340 	}
3341 	kvfree(spec);
3342 	return err ? ERR_PTR(err) : handler;
3343 }
3344 
3345 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
3346 						     struct mlx5_ib_flow_prio *ft_prio,
3347 						     const struct ib_flow_attr *flow_attr,
3348 						     struct mlx5_flow_destination *dst)
3349 {
3350 	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
3351 }
3352 
3353 static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,
3354 							  struct mlx5_ib_flow_prio *ft_prio,
3355 							  struct ib_flow_attr *flow_attr,
3356 							  struct mlx5_flow_destination *dst)
3357 {
3358 	struct mlx5_ib_flow_handler *handler_dst = NULL;
3359 	struct mlx5_ib_flow_handler *handler = NULL;
3360 
3361 	handler = create_flow_rule(dev, ft_prio, flow_attr, NULL);
3362 	if (!IS_ERR(handler)) {
3363 		handler_dst = create_flow_rule(dev, ft_prio,
3364 					       flow_attr, dst);
3365 		if (IS_ERR(handler_dst)) {
3366 			mlx5_del_flow_rules(handler->rule);
3367 			ft_prio->refcount--;
3368 			kfree(handler);
3369 			handler = handler_dst;
3370 		} else {
3371 			list_add(&handler_dst->list, &handler->list);
3372 		}
3373 	}
3374 
3375 	return handler;
3376 }
3377 enum {
3378 	LEFTOVERS_MC,
3379 	LEFTOVERS_UC,
3380 };
3381 
3382 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
3383 							  struct mlx5_ib_flow_prio *ft_prio,
3384 							  struct ib_flow_attr *flow_attr,
3385 							  struct mlx5_flow_destination *dst)
3386 {
3387 	struct mlx5_ib_flow_handler *handler_ucast = NULL;
3388 	struct mlx5_ib_flow_handler *handler = NULL;
3389 
3390 	static struct {
3391 		struct ib_flow_attr	flow_attr;
3392 		struct ib_flow_spec_eth eth_flow;
3393 	} leftovers_specs[] = {
3394 		[LEFTOVERS_MC] = {
3395 			.flow_attr = {
3396 				.num_of_specs = 1,
3397 				.size = sizeof(leftovers_specs[0])
3398 			},
3399 			.eth_flow = {
3400 				.type = IB_FLOW_SPEC_ETH,
3401 				.size = sizeof(struct ib_flow_spec_eth),
3402 				.mask = {.dst_mac = {0x1} },
3403 				.val =  {.dst_mac = {0x1} }
3404 			}
3405 		},
3406 		[LEFTOVERS_UC] = {
3407 			.flow_attr = {
3408 				.num_of_specs = 1,
3409 				.size = sizeof(leftovers_specs[0])
3410 			},
3411 			.eth_flow = {
3412 				.type = IB_FLOW_SPEC_ETH,
3413 				.size = sizeof(struct ib_flow_spec_eth),
3414 				.mask = {.dst_mac = {0x1} },
3415 				.val = {.dst_mac = {} }
3416 			}
3417 		}
3418 	};
3419 
3420 	handler = create_flow_rule(dev, ft_prio,
3421 				   &leftovers_specs[LEFTOVERS_MC].flow_attr,
3422 				   dst);
3423 	if (!IS_ERR(handler) &&
3424 	    flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
3425 		handler_ucast = create_flow_rule(dev, ft_prio,
3426 						 &leftovers_specs[LEFTOVERS_UC].flow_attr,
3427 						 dst);
3428 		if (IS_ERR(handler_ucast)) {
3429 			mlx5_del_flow_rules(handler->rule);
3430 			ft_prio->refcount--;
3431 			kfree(handler);
3432 			handler = handler_ucast;
3433 		} else {
3434 			list_add(&handler_ucast->list, &handler->list);
3435 		}
3436 	}
3437 
3438 	return handler;
3439 }
3440 
3441 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
3442 							struct mlx5_ib_flow_prio *ft_rx,
3443 							struct mlx5_ib_flow_prio *ft_tx,
3444 							struct mlx5_flow_destination *dst)
3445 {
3446 	struct mlx5_ib_flow_handler *handler_rx;
3447 	struct mlx5_ib_flow_handler *handler_tx;
3448 	int err;
3449 	static const struct ib_flow_attr flow_attr  = {
3450 		.num_of_specs = 0,
3451 		.size = sizeof(flow_attr)
3452 	};
3453 
3454 	handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
3455 	if (IS_ERR(handler_rx)) {
3456 		err = PTR_ERR(handler_rx);
3457 		goto err;
3458 	}
3459 
3460 	handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
3461 	if (IS_ERR(handler_tx)) {
3462 		err = PTR_ERR(handler_tx);
3463 		goto err_tx;
3464 	}
3465 
3466 	list_add(&handler_tx->list, &handler_rx->list);
3467 
3468 	return handler_rx;
3469 
3470 err_tx:
3471 	mlx5_del_flow_rules(handler_rx->rule);
3472 	ft_rx->refcount--;
3473 	kfree(handler_rx);
3474 err:
3475 	return ERR_PTR(err);
3476 }
3477 
3478 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
3479 					   struct ib_flow_attr *flow_attr,
3480 					   int domain,
3481 					   struct ib_udata *udata)
3482 {
3483 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
3484 	struct mlx5_ib_qp *mqp = to_mqp(qp);
3485 	struct mlx5_ib_flow_handler *handler = NULL;
3486 	struct mlx5_flow_destination *dst = NULL;
3487 	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
3488 	struct mlx5_ib_flow_prio *ft_prio;
3489 	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
3490 	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
3491 	size_t min_ucmd_sz, required_ucmd_sz;
3492 	int err;
3493 	int underlay_qpn;
3494 
3495 	if (udata && udata->inlen) {
3496 		min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +
3497 				sizeof(ucmd_hdr.reserved);
3498 		if (udata->inlen < min_ucmd_sz)
3499 			return ERR_PTR(-EOPNOTSUPP);
3500 
3501 		err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
3502 		if (err)
3503 			return ERR_PTR(err);
3504 
3505 		/* currently supports only one counters data */
3506 		if (ucmd_hdr.ncounters_data > 1)
3507 			return ERR_PTR(-EINVAL);
3508 
3509 		required_ucmd_sz = min_ucmd_sz +
3510 			sizeof(struct mlx5_ib_flow_counters_data) *
3511 			ucmd_hdr.ncounters_data;
3512 		if (udata->inlen > required_ucmd_sz &&
3513 		    !ib_is_udata_cleared(udata, required_ucmd_sz,
3514 					 udata->inlen - required_ucmd_sz))
3515 			return ERR_PTR(-EOPNOTSUPP);
3516 
3517 		ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
3518 		if (!ucmd)
3519 			return ERR_PTR(-ENOMEM);
3520 
3521 		err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
3522 		if (err) {
3523 			kfree(ucmd);
3524 			return ERR_PTR(err);
3525 		}
3526 	}
3527 
3528 	if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO)
3529 		return ERR_PTR(-ENOMEM);
3530 
3531 	if (domain != IB_FLOW_DOMAIN_USER ||
3532 	    flow_attr->port > dev->num_ports ||
3533 	    (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP |
3534 				  IB_FLOW_ATTR_FLAGS_EGRESS)))
3535 		return ERR_PTR(-EINVAL);
3536 
3537 	if (is_egress &&
3538 	    (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3539 	     flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT))
3540 		return ERR_PTR(-EINVAL);
3541 
3542 	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
3543 	if (!dst)
3544 		return ERR_PTR(-ENOMEM);
3545 
3546 	mutex_lock(&dev->flow_db->lock);
3547 
3548 	ft_prio = get_flow_table(dev, flow_attr,
3549 				 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
3550 	if (IS_ERR(ft_prio)) {
3551 		err = PTR_ERR(ft_prio);
3552 		goto unlock;
3553 	}
3554 	if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3555 		ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
3556 		if (IS_ERR(ft_prio_tx)) {
3557 			err = PTR_ERR(ft_prio_tx);
3558 			ft_prio_tx = NULL;
3559 			goto destroy_ft;
3560 		}
3561 	}
3562 
3563 	if (is_egress) {
3564 		dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
3565 	} else {
3566 		dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
3567 		if (mqp->flags & MLX5_IB_QP_RSS)
3568 			dst->tir_num = mqp->rss_qp.tirn;
3569 		else
3570 			dst->tir_num = mqp->raw_packet_qp.rq.tirn;
3571 	}
3572 
3573 	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
3574 		if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)  {
3575 			handler = create_dont_trap_rule(dev, ft_prio,
3576 							flow_attr, dst);
3577 		} else {
3578 			underlay_qpn = (mqp->flags & MLX5_IB_QP_UNDERLAY) ?
3579 					mqp->underlay_qpn : 0;
3580 			handler = _create_flow_rule(dev, ft_prio, flow_attr,
3581 						    dst, underlay_qpn, ucmd);
3582 		}
3583 	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
3584 		   flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
3585 		handler = create_leftovers_rule(dev, ft_prio, flow_attr,
3586 						dst);
3587 	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3588 		handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
3589 	} else {
3590 		err = -EINVAL;
3591 		goto destroy_ft;
3592 	}
3593 
3594 	if (IS_ERR(handler)) {
3595 		err = PTR_ERR(handler);
3596 		handler = NULL;
3597 		goto destroy_ft;
3598 	}
3599 
3600 	mutex_unlock(&dev->flow_db->lock);
3601 	kfree(dst);
3602 	kfree(ucmd);
3603 
3604 	return &handler->ibflow;
3605 
3606 destroy_ft:
3607 	put_flow_table(dev, ft_prio, false);
3608 	if (ft_prio_tx)
3609 		put_flow_table(dev, ft_prio_tx, false);
3610 unlock:
3611 	mutex_unlock(&dev->flow_db->lock);
3612 	kfree(dst);
3613 	kfree(ucmd);
3614 	kfree(handler);
3615 	return ERR_PTR(err);
3616 }
3617 
3618 static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
3619 {
3620 	u32 flags = 0;
3621 
3622 	if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
3623 		flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
3624 
3625 	return flags;
3626 }
3627 
3628 #define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED	MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
3629 static struct ib_flow_action *
3630 mlx5_ib_create_flow_action_esp(struct ib_device *device,
3631 			       const struct ib_flow_action_attrs_esp *attr,
3632 			       struct uverbs_attr_bundle *attrs)
3633 {
3634 	struct mlx5_ib_dev *mdev = to_mdev(device);
3635 	struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
3636 	struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
3637 	struct mlx5_ib_flow_action *action;
3638 	u64 action_flags;
3639 	u64 flags;
3640 	int err = 0;
3641 
3642 	if (IS_UVERBS_COPY_ERR(uverbs_copy_from(&action_flags, attrs,
3643 						MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS)))
3644 		return ERR_PTR(-EFAULT);
3645 
3646 	if (action_flags >= (MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1))
3647 		return ERR_PTR(-EOPNOTSUPP);
3648 
3649 	flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
3650 
3651 	/* We current only support a subset of the standard features. Only a
3652 	 * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn
3653 	 * (with overlap). Full offload mode isn't supported.
3654 	 */
3655 	if (!attr->keymat || attr->replay || attr->encap ||
3656 	    attr->spi || attr->seq || attr->tfc_pad ||
3657 	    attr->hard_limit_pkts ||
3658 	    (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
3659 			     IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
3660 		return ERR_PTR(-EOPNOTSUPP);
3661 
3662 	if (attr->keymat->protocol !=
3663 	    IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
3664 		return ERR_PTR(-EOPNOTSUPP);
3665 
3666 	aes_gcm = &attr->keymat->keymat.aes_gcm;
3667 
3668 	if (aes_gcm->icv_len != 16 ||
3669 	    aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
3670 		return ERR_PTR(-EOPNOTSUPP);
3671 
3672 	action = kmalloc(sizeof(*action), GFP_KERNEL);
3673 	if (!action)
3674 		return ERR_PTR(-ENOMEM);
3675 
3676 	action->esp_aes_gcm.ib_flags = attr->flags;
3677 	memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
3678 	       sizeof(accel_attrs.keymat.aes_gcm.aes_key));
3679 	accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
3680 	memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
3681 	       sizeof(accel_attrs.keymat.aes_gcm.salt));
3682 	memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
3683 	       sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
3684 	accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
3685 	accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
3686 	accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
3687 
3688 	accel_attrs.esn = attr->esn;
3689 	if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
3690 		accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
3691 	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
3692 		accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
3693 
3694 	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
3695 		accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
3696 
3697 	action->esp_aes_gcm.ctx =
3698 		mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
3699 	if (IS_ERR(action->esp_aes_gcm.ctx)) {
3700 		err = PTR_ERR(action->esp_aes_gcm.ctx);
3701 		goto err_parse;
3702 	}
3703 
3704 	action->esp_aes_gcm.ib_flags = attr->flags;
3705 
3706 	return &action->ib_action;
3707 
3708 err_parse:
3709 	kfree(action);
3710 	return ERR_PTR(err);
3711 }
3712 
3713 static int
3714 mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
3715 			       const struct ib_flow_action_attrs_esp *attr,
3716 			       struct uverbs_attr_bundle *attrs)
3717 {
3718 	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
3719 	struct mlx5_accel_esp_xfrm_attrs accel_attrs;
3720 	int err = 0;
3721 
3722 	if (attr->keymat || attr->replay || attr->encap ||
3723 	    attr->spi || attr->seq || attr->tfc_pad ||
3724 	    attr->hard_limit_pkts ||
3725 	    (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
3726 			     IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS |
3727 			     IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
3728 		return -EOPNOTSUPP;
3729 
3730 	/* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can
3731 	 * be modified.
3732 	 */
3733 	if (!(maction->esp_aes_gcm.ib_flags &
3734 	      IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
3735 	    attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
3736 			   IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
3737 		return -EINVAL;
3738 
3739 	memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
3740 	       sizeof(accel_attrs));
3741 
3742 	accel_attrs.esn = attr->esn;
3743 	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
3744 		accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
3745 	else
3746 		accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
3747 
3748 	err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
3749 					 &accel_attrs);
3750 	if (err)
3751 		return err;
3752 
3753 	maction->esp_aes_gcm.ib_flags &=
3754 		~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
3755 	maction->esp_aes_gcm.ib_flags |=
3756 		attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
3757 
3758 	return 0;
3759 }
3760 
3761 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
3762 {
3763 	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
3764 
3765 	switch (action->type) {
3766 	case IB_FLOW_ACTION_ESP:
3767 		/*
3768 		 * We only support aes_gcm by now, so we implicitly know this is
3769 		 * the underline crypto.
3770 		 */
3771 		mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
3772 		break;
3773 	default:
3774 		WARN_ON(true);
3775 		break;
3776 	}
3777 
3778 	kfree(maction);
3779 	return 0;
3780 }
3781 
3782 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
3783 {
3784 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3785 	struct mlx5_ib_qp *mqp = to_mqp(ibqp);
3786 	int err;
3787 
3788 	if (mqp->flags & MLX5_IB_QP_UNDERLAY) {
3789 		mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
3790 		return -EOPNOTSUPP;
3791 	}
3792 
3793 	err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
3794 	if (err)
3795 		mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
3796 			     ibqp->qp_num, gid->raw);
3797 
3798 	return err;
3799 }
3800 
3801 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
3802 {
3803 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3804 	int err;
3805 
3806 	err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
3807 	if (err)
3808 		mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
3809 			     ibqp->qp_num, gid->raw);
3810 
3811 	return err;
3812 }
3813 
3814 static int init_node_data(struct mlx5_ib_dev *dev)
3815 {
3816 	int err;
3817 
3818 	err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
3819 	if (err)
3820 		return err;
3821 
3822 	dev->mdev->rev_id = dev->mdev->pdev->revision;
3823 
3824 	return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
3825 }
3826 
3827 static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
3828 			     char *buf)
3829 {
3830 	struct mlx5_ib_dev *dev =
3831 		container_of(device, struct mlx5_ib_dev, ib_dev.dev);
3832 
3833 	return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
3834 }
3835 
3836 static ssize_t show_reg_pages(struct device *device,
3837 			      struct device_attribute *attr, char *buf)
3838 {
3839 	struct mlx5_ib_dev *dev =
3840 		container_of(device, struct mlx5_ib_dev, ib_dev.dev);
3841 
3842 	return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
3843 }
3844 
3845 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
3846 			char *buf)
3847 {
3848 	struct mlx5_ib_dev *dev =
3849 		container_of(device, struct mlx5_ib_dev, ib_dev.dev);
3850 	return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
3851 }
3852 
3853 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
3854 			char *buf)
3855 {
3856 	struct mlx5_ib_dev *dev =
3857 		container_of(device, struct mlx5_ib_dev, ib_dev.dev);
3858 	return sprintf(buf, "%x\n", dev->mdev->rev_id);
3859 }
3860 
3861 static ssize_t show_board(struct device *device, struct device_attribute *attr,
3862 			  char *buf)
3863 {
3864 	struct mlx5_ib_dev *dev =
3865 		container_of(device, struct mlx5_ib_dev, ib_dev.dev);
3866 	return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
3867 		       dev->mdev->board_id);
3868 }
3869 
3870 static DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,    NULL);
3871 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca,    NULL);
3872 static DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);
3873 static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
3874 static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
3875 
3876 static struct device_attribute *mlx5_class_attributes[] = {
3877 	&dev_attr_hw_rev,
3878 	&dev_attr_hca_type,
3879 	&dev_attr_board_id,
3880 	&dev_attr_fw_pages,
3881 	&dev_attr_reg_pages,
3882 };
3883 
3884 static void pkey_change_handler(struct work_struct *work)
3885 {
3886 	struct mlx5_ib_port_resources *ports =
3887 		container_of(work, struct mlx5_ib_port_resources,
3888 			     pkey_change_work);
3889 
3890 	mutex_lock(&ports->devr->mutex);
3891 	mlx5_ib_gsi_pkey_change(ports->gsi);
3892 	mutex_unlock(&ports->devr->mutex);
3893 }
3894 
3895 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
3896 {
3897 	struct mlx5_ib_qp *mqp;
3898 	struct mlx5_ib_cq *send_mcq, *recv_mcq;
3899 	struct mlx5_core_cq *mcq;
3900 	struct list_head cq_armed_list;
3901 	unsigned long flags_qp;
3902 	unsigned long flags_cq;
3903 	unsigned long flags;
3904 
3905 	INIT_LIST_HEAD(&cq_armed_list);
3906 
3907 	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
3908 	spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
3909 	list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
3910 		spin_lock_irqsave(&mqp->sq.lock, flags_qp);
3911 		if (mqp->sq.tail != mqp->sq.head) {
3912 			send_mcq = to_mcq(mqp->ibqp.send_cq);
3913 			spin_lock_irqsave(&send_mcq->lock, flags_cq);
3914 			if (send_mcq->mcq.comp &&
3915 			    mqp->ibqp.send_cq->comp_handler) {
3916 				if (!send_mcq->mcq.reset_notify_added) {
3917 					send_mcq->mcq.reset_notify_added = 1;
3918 					list_add_tail(&send_mcq->mcq.reset_notify,
3919 						      &cq_armed_list);
3920 				}
3921 			}
3922 			spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
3923 		}
3924 		spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
3925 		spin_lock_irqsave(&mqp->rq.lock, flags_qp);
3926 		/* no handling is needed for SRQ */
3927 		if (!mqp->ibqp.srq) {
3928 			if (mqp->rq.tail != mqp->rq.head) {
3929 				recv_mcq = to_mcq(mqp->ibqp.recv_cq);
3930 				spin_lock_irqsave(&recv_mcq->lock, flags_cq);
3931 				if (recv_mcq->mcq.comp &&
3932 				    mqp->ibqp.recv_cq->comp_handler) {
3933 					if (!recv_mcq->mcq.reset_notify_added) {
3934 						recv_mcq->mcq.reset_notify_added = 1;
3935 						list_add_tail(&recv_mcq->mcq.reset_notify,
3936 							      &cq_armed_list);
3937 					}
3938 				}
3939 				spin_unlock_irqrestore(&recv_mcq->lock,
3940 						       flags_cq);
3941 			}
3942 		}
3943 		spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
3944 	}
3945 	/*At that point all inflight post send were put to be executed as of we
3946 	 * lock/unlock above locks Now need to arm all involved CQs.
3947 	 */
3948 	list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
3949 		mcq->comp(mcq);
3950 	}
3951 	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
3952 }
3953 
3954 static void delay_drop_handler(struct work_struct *work)
3955 {
3956 	int err;
3957 	struct mlx5_ib_delay_drop *delay_drop =
3958 		container_of(work, struct mlx5_ib_delay_drop,
3959 			     delay_drop_work);
3960 
3961 	atomic_inc(&delay_drop->events_cnt);
3962 
3963 	mutex_lock(&delay_drop->lock);
3964 	err = mlx5_core_set_delay_drop(delay_drop->dev->mdev,
3965 				       delay_drop->timeout);
3966 	if (err) {
3967 		mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
3968 			     delay_drop->timeout);
3969 		delay_drop->activate = false;
3970 	}
3971 	mutex_unlock(&delay_drop->lock);
3972 }
3973 
3974 static void mlx5_ib_handle_event(struct work_struct *_work)
3975 {
3976 	struct mlx5_ib_event_work *work =
3977 		container_of(_work, struct mlx5_ib_event_work, work);
3978 	struct mlx5_ib_dev *ibdev;
3979 	struct ib_event ibev;
3980 	bool fatal = false;
3981 	u8 port = (u8)work->param;
3982 
3983 	if (mlx5_core_is_mp_slave(work->dev)) {
3984 		ibdev = mlx5_ib_get_ibdev_from_mpi(work->context);
3985 		if (!ibdev)
3986 			goto out;
3987 	} else {
3988 		ibdev = work->context;
3989 	}
3990 
3991 	switch (work->event) {
3992 	case MLX5_DEV_EVENT_SYS_ERROR:
3993 		ibev.event = IB_EVENT_DEVICE_FATAL;
3994 		mlx5_ib_handle_internal_error(ibdev);
3995 		fatal = true;
3996 		break;
3997 
3998 	case MLX5_DEV_EVENT_PORT_UP:
3999 	case MLX5_DEV_EVENT_PORT_DOWN:
4000 	case MLX5_DEV_EVENT_PORT_INITIALIZED:
4001 		/* In RoCE, port up/down events are handled in
4002 		 * mlx5_netdev_event().
4003 		 */
4004 		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
4005 			IB_LINK_LAYER_ETHERNET)
4006 			goto out;
4007 
4008 		ibev.event = (work->event == MLX5_DEV_EVENT_PORT_UP) ?
4009 			     IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
4010 		break;
4011 
4012 	case MLX5_DEV_EVENT_LID_CHANGE:
4013 		ibev.event = IB_EVENT_LID_CHANGE;
4014 		break;
4015 
4016 	case MLX5_DEV_EVENT_PKEY_CHANGE:
4017 		ibev.event = IB_EVENT_PKEY_CHANGE;
4018 		schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
4019 		break;
4020 
4021 	case MLX5_DEV_EVENT_GUID_CHANGE:
4022 		ibev.event = IB_EVENT_GID_CHANGE;
4023 		break;
4024 
4025 	case MLX5_DEV_EVENT_CLIENT_REREG:
4026 		ibev.event = IB_EVENT_CLIENT_REREGISTER;
4027 		break;
4028 	case MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT:
4029 		schedule_work(&ibdev->delay_drop.delay_drop_work);
4030 		goto out;
4031 	default:
4032 		goto out;
4033 	}
4034 
4035 	ibev.device	      = &ibdev->ib_dev;
4036 	ibev.element.port_num = port;
4037 
4038 	if (!rdma_is_port_valid(&ibdev->ib_dev, port)) {
4039 		mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
4040 		goto out;
4041 	}
4042 
4043 	if (ibdev->ib_active)
4044 		ib_dispatch_event(&ibev);
4045 
4046 	if (fatal)
4047 		ibdev->ib_active = false;
4048 out:
4049 	kfree(work);
4050 }
4051 
4052 static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
4053 			  enum mlx5_dev_event event, unsigned long param)
4054 {
4055 	struct mlx5_ib_event_work *work;
4056 
4057 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
4058 	if (!work)
4059 		return;
4060 
4061 	INIT_WORK(&work->work, mlx5_ib_handle_event);
4062 	work->dev = dev;
4063 	work->param = param;
4064 	work->context = context;
4065 	work->event = event;
4066 
4067 	queue_work(mlx5_ib_event_wq, &work->work);
4068 }
4069 
4070 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
4071 {
4072 	struct mlx5_hca_vport_context vport_ctx;
4073 	int err;
4074 	int port;
4075 
4076 	for (port = 1; port <= dev->num_ports; port++) {
4077 		dev->mdev->port_caps[port - 1].has_smi = false;
4078 		if (MLX5_CAP_GEN(dev->mdev, port_type) ==
4079 		    MLX5_CAP_PORT_TYPE_IB) {
4080 			if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
4081 				err = mlx5_query_hca_vport_context(dev->mdev, 0,
4082 								   port, 0,
4083 								   &vport_ctx);
4084 				if (err) {
4085 					mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
4086 						    port, err);
4087 					return err;
4088 				}
4089 				dev->mdev->port_caps[port - 1].has_smi =
4090 					vport_ctx.has_smi;
4091 			} else {
4092 				dev->mdev->port_caps[port - 1].has_smi = true;
4093 			}
4094 		}
4095 	}
4096 	return 0;
4097 }
4098 
4099 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
4100 {
4101 	int port;
4102 
4103 	for (port = 1; port <= dev->num_ports; port++)
4104 		mlx5_query_ext_port_caps(dev, port);
4105 }
4106 
4107 static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
4108 {
4109 	struct ib_device_attr *dprops = NULL;
4110 	struct ib_port_attr *pprops = NULL;
4111 	int err = -ENOMEM;
4112 	struct ib_udata uhw = {.inlen = 0, .outlen = 0};
4113 
4114 	pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
4115 	if (!pprops)
4116 		goto out;
4117 
4118 	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
4119 	if (!dprops)
4120 		goto out;
4121 
4122 	err = set_has_smi_cap(dev);
4123 	if (err)
4124 		goto out;
4125 
4126 	err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
4127 	if (err) {
4128 		mlx5_ib_warn(dev, "query_device failed %d\n", err);
4129 		goto out;
4130 	}
4131 
4132 	memset(pprops, 0, sizeof(*pprops));
4133 	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
4134 	if (err) {
4135 		mlx5_ib_warn(dev, "query_port %d failed %d\n",
4136 			     port, err);
4137 		goto out;
4138 	}
4139 
4140 	dev->mdev->port_caps[port - 1].pkey_table_len =
4141 					dprops->max_pkeys;
4142 	dev->mdev->port_caps[port - 1].gid_table_len =
4143 					pprops->gid_tbl_len;
4144 	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
4145 		    port, dprops->max_pkeys, pprops->gid_tbl_len);
4146 
4147 out:
4148 	kfree(pprops);
4149 	kfree(dprops);
4150 
4151 	return err;
4152 }
4153 
4154 static void destroy_umrc_res(struct mlx5_ib_dev *dev)
4155 {
4156 	int err;
4157 
4158 	err = mlx5_mr_cache_cleanup(dev);
4159 	if (err)
4160 		mlx5_ib_warn(dev, "mr cache cleanup failed\n");
4161 
4162 	if (dev->umrc.qp)
4163 		mlx5_ib_destroy_qp(dev->umrc.qp);
4164 	if (dev->umrc.cq)
4165 		ib_free_cq(dev->umrc.cq);
4166 	if (dev->umrc.pd)
4167 		ib_dealloc_pd(dev->umrc.pd);
4168 }
4169 
4170 enum {
4171 	MAX_UMR_WR = 128,
4172 };
4173 
4174 static int create_umr_res(struct mlx5_ib_dev *dev)
4175 {
4176 	struct ib_qp_init_attr *init_attr = NULL;
4177 	struct ib_qp_attr *attr = NULL;
4178 	struct ib_pd *pd;
4179 	struct ib_cq *cq;
4180 	struct ib_qp *qp;
4181 	int ret;
4182 
4183 	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
4184 	init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
4185 	if (!attr || !init_attr) {
4186 		ret = -ENOMEM;
4187 		goto error_0;
4188 	}
4189 
4190 	pd = ib_alloc_pd(&dev->ib_dev, 0);
4191 	if (IS_ERR(pd)) {
4192 		mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
4193 		ret = PTR_ERR(pd);
4194 		goto error_0;
4195 	}
4196 
4197 	cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
4198 	if (IS_ERR(cq)) {
4199 		mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
4200 		ret = PTR_ERR(cq);
4201 		goto error_2;
4202 	}
4203 
4204 	init_attr->send_cq = cq;
4205 	init_attr->recv_cq = cq;
4206 	init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
4207 	init_attr->cap.max_send_wr = MAX_UMR_WR;
4208 	init_attr->cap.max_send_sge = 1;
4209 	init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
4210 	init_attr->port_num = 1;
4211 	qp = mlx5_ib_create_qp(pd, init_attr, NULL);
4212 	if (IS_ERR(qp)) {
4213 		mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
4214 		ret = PTR_ERR(qp);
4215 		goto error_3;
4216 	}
4217 	qp->device     = &dev->ib_dev;
4218 	qp->real_qp    = qp;
4219 	qp->uobject    = NULL;
4220 	qp->qp_type    = MLX5_IB_QPT_REG_UMR;
4221 	qp->send_cq    = init_attr->send_cq;
4222 	qp->recv_cq    = init_attr->recv_cq;
4223 
4224 	attr->qp_state = IB_QPS_INIT;
4225 	attr->port_num = 1;
4226 	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
4227 				IB_QP_PORT, NULL);
4228 	if (ret) {
4229 		mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
4230 		goto error_4;
4231 	}
4232 
4233 	memset(attr, 0, sizeof(*attr));
4234 	attr->qp_state = IB_QPS_RTR;
4235 	attr->path_mtu = IB_MTU_256;
4236 
4237 	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
4238 	if (ret) {
4239 		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
4240 		goto error_4;
4241 	}
4242 
4243 	memset(attr, 0, sizeof(*attr));
4244 	attr->qp_state = IB_QPS_RTS;
4245 	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
4246 	if (ret) {
4247 		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
4248 		goto error_4;
4249 	}
4250 
4251 	dev->umrc.qp = qp;
4252 	dev->umrc.cq = cq;
4253 	dev->umrc.pd = pd;
4254 
4255 	sema_init(&dev->umrc.sem, MAX_UMR_WR);
4256 	ret = mlx5_mr_cache_init(dev);
4257 	if (ret) {
4258 		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4259 		goto error_4;
4260 	}
4261 
4262 	kfree(attr);
4263 	kfree(init_attr);
4264 
4265 	return 0;
4266 
4267 error_4:
4268 	mlx5_ib_destroy_qp(qp);
4269 	dev->umrc.qp = NULL;
4270 
4271 error_3:
4272 	ib_free_cq(cq);
4273 	dev->umrc.cq = NULL;
4274 
4275 error_2:
4276 	ib_dealloc_pd(pd);
4277 	dev->umrc.pd = NULL;
4278 
4279 error_0:
4280 	kfree(attr);
4281 	kfree(init_attr);
4282 	return ret;
4283 }
4284 
4285 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
4286 {
4287 	switch (umr_fence_cap) {
4288 	case MLX5_CAP_UMR_FENCE_NONE:
4289 		return MLX5_FENCE_MODE_NONE;
4290 	case MLX5_CAP_UMR_FENCE_SMALL:
4291 		return MLX5_FENCE_MODE_INITIATOR_SMALL;
4292 	default:
4293 		return MLX5_FENCE_MODE_STRONG_ORDERING;
4294 	}
4295 }
4296 
4297 static int create_dev_resources(struct mlx5_ib_resources *devr)
4298 {
4299 	struct ib_srq_init_attr attr;
4300 	struct mlx5_ib_dev *dev;
4301 	struct ib_cq_init_attr cq_attr = {.cqe = 1};
4302 	int port;
4303 	int ret = 0;
4304 
4305 	dev = container_of(devr, struct mlx5_ib_dev, devr);
4306 
4307 	mutex_init(&devr->mutex);
4308 
4309 	devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
4310 	if (IS_ERR(devr->p0)) {
4311 		ret = PTR_ERR(devr->p0);
4312 		goto error0;
4313 	}
4314 	devr->p0->device  = &dev->ib_dev;
4315 	devr->p0->uobject = NULL;
4316 	atomic_set(&devr->p0->usecnt, 0);
4317 
4318 	devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL, NULL);
4319 	if (IS_ERR(devr->c0)) {
4320 		ret = PTR_ERR(devr->c0);
4321 		goto error1;
4322 	}
4323 	devr->c0->device        = &dev->ib_dev;
4324 	devr->c0->uobject       = NULL;
4325 	devr->c0->comp_handler  = NULL;
4326 	devr->c0->event_handler = NULL;
4327 	devr->c0->cq_context    = NULL;
4328 	atomic_set(&devr->c0->usecnt, 0);
4329 
4330 	devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
4331 	if (IS_ERR(devr->x0)) {
4332 		ret = PTR_ERR(devr->x0);
4333 		goto error2;
4334 	}
4335 	devr->x0->device = &dev->ib_dev;
4336 	devr->x0->inode = NULL;
4337 	atomic_set(&devr->x0->usecnt, 0);
4338 	mutex_init(&devr->x0->tgt_qp_mutex);
4339 	INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
4340 
4341 	devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
4342 	if (IS_ERR(devr->x1)) {
4343 		ret = PTR_ERR(devr->x1);
4344 		goto error3;
4345 	}
4346 	devr->x1->device = &dev->ib_dev;
4347 	devr->x1->inode = NULL;
4348 	atomic_set(&devr->x1->usecnt, 0);
4349 	mutex_init(&devr->x1->tgt_qp_mutex);
4350 	INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
4351 
4352 	memset(&attr, 0, sizeof(attr));
4353 	attr.attr.max_sge = 1;
4354 	attr.attr.max_wr = 1;
4355 	attr.srq_type = IB_SRQT_XRC;
4356 	attr.ext.cq = devr->c0;
4357 	attr.ext.xrc.xrcd = devr->x0;
4358 
4359 	devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
4360 	if (IS_ERR(devr->s0)) {
4361 		ret = PTR_ERR(devr->s0);
4362 		goto error4;
4363 	}
4364 	devr->s0->device	= &dev->ib_dev;
4365 	devr->s0->pd		= devr->p0;
4366 	devr->s0->uobject       = NULL;
4367 	devr->s0->event_handler = NULL;
4368 	devr->s0->srq_context   = NULL;
4369 	devr->s0->srq_type      = IB_SRQT_XRC;
4370 	devr->s0->ext.xrc.xrcd	= devr->x0;
4371 	devr->s0->ext.cq	= devr->c0;
4372 	atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
4373 	atomic_inc(&devr->s0->ext.cq->usecnt);
4374 	atomic_inc(&devr->p0->usecnt);
4375 	atomic_set(&devr->s0->usecnt, 0);
4376 
4377 	memset(&attr, 0, sizeof(attr));
4378 	attr.attr.max_sge = 1;
4379 	attr.attr.max_wr = 1;
4380 	attr.srq_type = IB_SRQT_BASIC;
4381 	devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
4382 	if (IS_ERR(devr->s1)) {
4383 		ret = PTR_ERR(devr->s1);
4384 		goto error5;
4385 	}
4386 	devr->s1->device	= &dev->ib_dev;
4387 	devr->s1->pd		= devr->p0;
4388 	devr->s1->uobject       = NULL;
4389 	devr->s1->event_handler = NULL;
4390 	devr->s1->srq_context   = NULL;
4391 	devr->s1->srq_type      = IB_SRQT_BASIC;
4392 	devr->s1->ext.cq	= devr->c0;
4393 	atomic_inc(&devr->p0->usecnt);
4394 	atomic_set(&devr->s1->usecnt, 0);
4395 
4396 	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
4397 		INIT_WORK(&devr->ports[port].pkey_change_work,
4398 			  pkey_change_handler);
4399 		devr->ports[port].devr = devr;
4400 	}
4401 
4402 	return 0;
4403 
4404 error5:
4405 	mlx5_ib_destroy_srq(devr->s0);
4406 error4:
4407 	mlx5_ib_dealloc_xrcd(devr->x1);
4408 error3:
4409 	mlx5_ib_dealloc_xrcd(devr->x0);
4410 error2:
4411 	mlx5_ib_destroy_cq(devr->c0);
4412 error1:
4413 	mlx5_ib_dealloc_pd(devr->p0);
4414 error0:
4415 	return ret;
4416 }
4417 
4418 static void destroy_dev_resources(struct mlx5_ib_resources *devr)
4419 {
4420 	struct mlx5_ib_dev *dev =
4421 		container_of(devr, struct mlx5_ib_dev, devr);
4422 	int port;
4423 
4424 	mlx5_ib_destroy_srq(devr->s1);
4425 	mlx5_ib_destroy_srq(devr->s0);
4426 	mlx5_ib_dealloc_xrcd(devr->x0);
4427 	mlx5_ib_dealloc_xrcd(devr->x1);
4428 	mlx5_ib_destroy_cq(devr->c0);
4429 	mlx5_ib_dealloc_pd(devr->p0);
4430 
4431 	/* Make sure no change P_Key work items are still executing */
4432 	for (port = 0; port < dev->num_ports; ++port)
4433 		cancel_work_sync(&devr->ports[port].pkey_change_work);
4434 }
4435 
4436 static u32 get_core_cap_flags(struct ib_device *ibdev)
4437 {
4438 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
4439 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
4440 	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
4441 	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
4442 	bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
4443 	u32 ret = 0;
4444 
4445 	if (ll == IB_LINK_LAYER_INFINIBAND)
4446 		return RDMA_CORE_PORT_IBA_IB;
4447 
4448 	if (raw_support)
4449 		ret = RDMA_CORE_PORT_RAW_PACKET;
4450 
4451 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
4452 		return ret;
4453 
4454 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
4455 		return ret;
4456 
4457 	if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
4458 		ret |= RDMA_CORE_PORT_IBA_ROCE;
4459 
4460 	if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
4461 		ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
4462 
4463 	return ret;
4464 }
4465 
4466 static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
4467 			       struct ib_port_immutable *immutable)
4468 {
4469 	struct ib_port_attr attr;
4470 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
4471 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
4472 	int err;
4473 
4474 	immutable->core_cap_flags = get_core_cap_flags(ibdev);
4475 
4476 	err = ib_query_port(ibdev, port_num, &attr);
4477 	if (err)
4478 		return err;
4479 
4480 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
4481 	immutable->gid_tbl_len = attr.gid_tbl_len;
4482 	immutable->core_cap_flags = get_core_cap_flags(ibdev);
4483 	if ((ll == IB_LINK_LAYER_INFINIBAND) || MLX5_CAP_GEN(dev->mdev, roce))
4484 		immutable->max_mad_size = IB_MGMT_MAD_SIZE;
4485 
4486 	return 0;
4487 }
4488 
4489 static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num,
4490 				   struct ib_port_immutable *immutable)
4491 {
4492 	struct ib_port_attr attr;
4493 	int err;
4494 
4495 	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
4496 
4497 	err = ib_query_port(ibdev, port_num, &attr);
4498 	if (err)
4499 		return err;
4500 
4501 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
4502 	immutable->gid_tbl_len = attr.gid_tbl_len;
4503 	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
4504 
4505 	return 0;
4506 }
4507 
4508 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
4509 {
4510 	struct mlx5_ib_dev *dev =
4511 		container_of(ibdev, struct mlx5_ib_dev, ib_dev);
4512 	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
4513 		 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
4514 		 fw_rev_sub(dev->mdev));
4515 }
4516 
4517 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
4518 {
4519 	struct mlx5_core_dev *mdev = dev->mdev;
4520 	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
4521 								 MLX5_FLOW_NAMESPACE_LAG);
4522 	struct mlx5_flow_table *ft;
4523 	int err;
4524 
4525 	if (!ns || !mlx5_lag_is_active(mdev))
4526 		return 0;
4527 
4528 	err = mlx5_cmd_create_vport_lag(mdev);
4529 	if (err)
4530 		return err;
4531 
4532 	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
4533 	if (IS_ERR(ft)) {
4534 		err = PTR_ERR(ft);
4535 		goto err_destroy_vport_lag;
4536 	}
4537 
4538 	dev->flow_db->lag_demux_ft = ft;
4539 	return 0;
4540 
4541 err_destroy_vport_lag:
4542 	mlx5_cmd_destroy_vport_lag(mdev);
4543 	return err;
4544 }
4545 
4546 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
4547 {
4548 	struct mlx5_core_dev *mdev = dev->mdev;
4549 
4550 	if (dev->flow_db->lag_demux_ft) {
4551 		mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
4552 		dev->flow_db->lag_demux_ft = NULL;
4553 
4554 		mlx5_cmd_destroy_vport_lag(mdev);
4555 	}
4556 }
4557 
4558 static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
4559 {
4560 	int err;
4561 
4562 	dev->roce[port_num].nb.notifier_call = mlx5_netdev_event;
4563 	err = register_netdevice_notifier(&dev->roce[port_num].nb);
4564 	if (err) {
4565 		dev->roce[port_num].nb.notifier_call = NULL;
4566 		return err;
4567 	}
4568 
4569 	return 0;
4570 }
4571 
4572 static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
4573 {
4574 	if (dev->roce[port_num].nb.notifier_call) {
4575 		unregister_netdevice_notifier(&dev->roce[port_num].nb);
4576 		dev->roce[port_num].nb.notifier_call = NULL;
4577 	}
4578 }
4579 
4580 static int mlx5_enable_eth(struct mlx5_ib_dev *dev, u8 port_num)
4581 {
4582 	int err;
4583 
4584 	if (MLX5_CAP_GEN(dev->mdev, roce)) {
4585 		err = mlx5_nic_vport_enable_roce(dev->mdev);
4586 		if (err)
4587 			return err;
4588 	}
4589 
4590 	err = mlx5_eth_lag_init(dev);
4591 	if (err)
4592 		goto err_disable_roce;
4593 
4594 	return 0;
4595 
4596 err_disable_roce:
4597 	if (MLX5_CAP_GEN(dev->mdev, roce))
4598 		mlx5_nic_vport_disable_roce(dev->mdev);
4599 
4600 	return err;
4601 }
4602 
4603 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
4604 {
4605 	mlx5_eth_lag_cleanup(dev);
4606 	if (MLX5_CAP_GEN(dev->mdev, roce))
4607 		mlx5_nic_vport_disable_roce(dev->mdev);
4608 }
4609 
4610 struct mlx5_ib_counter {
4611 	const char *name;
4612 	size_t offset;
4613 };
4614 
4615 #define INIT_Q_COUNTER(_name)		\
4616 	{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
4617 
4618 static const struct mlx5_ib_counter basic_q_cnts[] = {
4619 	INIT_Q_COUNTER(rx_write_requests),
4620 	INIT_Q_COUNTER(rx_read_requests),
4621 	INIT_Q_COUNTER(rx_atomic_requests),
4622 	INIT_Q_COUNTER(out_of_buffer),
4623 };
4624 
4625 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
4626 	INIT_Q_COUNTER(out_of_sequence),
4627 };
4628 
4629 static const struct mlx5_ib_counter retrans_q_cnts[] = {
4630 	INIT_Q_COUNTER(duplicate_request),
4631 	INIT_Q_COUNTER(rnr_nak_retry_err),
4632 	INIT_Q_COUNTER(packet_seq_err),
4633 	INIT_Q_COUNTER(implied_nak_seq_err),
4634 	INIT_Q_COUNTER(local_ack_timeout_err),
4635 };
4636 
4637 #define INIT_CONG_COUNTER(_name)		\
4638 	{ .name = #_name, .offset =	\
4639 		MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
4640 
4641 static const struct mlx5_ib_counter cong_cnts[] = {
4642 	INIT_CONG_COUNTER(rp_cnp_ignored),
4643 	INIT_CONG_COUNTER(rp_cnp_handled),
4644 	INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
4645 	INIT_CONG_COUNTER(np_cnp_sent),
4646 };
4647 
4648 static const struct mlx5_ib_counter extended_err_cnts[] = {
4649 	INIT_Q_COUNTER(resp_local_length_error),
4650 	INIT_Q_COUNTER(resp_cqe_error),
4651 	INIT_Q_COUNTER(req_cqe_error),
4652 	INIT_Q_COUNTER(req_remote_invalid_request),
4653 	INIT_Q_COUNTER(req_remote_access_errors),
4654 	INIT_Q_COUNTER(resp_remote_access_errors),
4655 	INIT_Q_COUNTER(resp_cqe_flush_error),
4656 	INIT_Q_COUNTER(req_cqe_flush_error),
4657 };
4658 
4659 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
4660 {
4661 	int i;
4662 
4663 	for (i = 0; i < dev->num_ports; i++) {
4664 		if (dev->port[i].cnts.set_id)
4665 			mlx5_core_dealloc_q_counter(dev->mdev,
4666 						    dev->port[i].cnts.set_id);
4667 		kfree(dev->port[i].cnts.names);
4668 		kfree(dev->port[i].cnts.offsets);
4669 	}
4670 }
4671 
4672 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
4673 				    struct mlx5_ib_counters *cnts)
4674 {
4675 	u32 num_counters;
4676 
4677 	num_counters = ARRAY_SIZE(basic_q_cnts);
4678 
4679 	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
4680 		num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
4681 
4682 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
4683 		num_counters += ARRAY_SIZE(retrans_q_cnts);
4684 
4685 	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
4686 		num_counters += ARRAY_SIZE(extended_err_cnts);
4687 
4688 	cnts->num_q_counters = num_counters;
4689 
4690 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
4691 		cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
4692 		num_counters += ARRAY_SIZE(cong_cnts);
4693 	}
4694 
4695 	cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
4696 	if (!cnts->names)
4697 		return -ENOMEM;
4698 
4699 	cnts->offsets = kcalloc(num_counters,
4700 				sizeof(cnts->offsets), GFP_KERNEL);
4701 	if (!cnts->offsets)
4702 		goto err_names;
4703 
4704 	return 0;
4705 
4706 err_names:
4707 	kfree(cnts->names);
4708 	cnts->names = NULL;
4709 	return -ENOMEM;
4710 }
4711 
4712 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
4713 				  const char **names,
4714 				  size_t *offsets)
4715 {
4716 	int i;
4717 	int j = 0;
4718 
4719 	for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
4720 		names[j] = basic_q_cnts[i].name;
4721 		offsets[j] = basic_q_cnts[i].offset;
4722 	}
4723 
4724 	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
4725 		for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
4726 			names[j] = out_of_seq_q_cnts[i].name;
4727 			offsets[j] = out_of_seq_q_cnts[i].offset;
4728 		}
4729 	}
4730 
4731 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
4732 		for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
4733 			names[j] = retrans_q_cnts[i].name;
4734 			offsets[j] = retrans_q_cnts[i].offset;
4735 		}
4736 	}
4737 
4738 	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
4739 		for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
4740 			names[j] = extended_err_cnts[i].name;
4741 			offsets[j] = extended_err_cnts[i].offset;
4742 		}
4743 	}
4744 
4745 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
4746 		for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
4747 			names[j] = cong_cnts[i].name;
4748 			offsets[j] = cong_cnts[i].offset;
4749 		}
4750 	}
4751 }
4752 
4753 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
4754 {
4755 	int err = 0;
4756 	int i;
4757 
4758 	for (i = 0; i < dev->num_ports; i++) {
4759 		err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
4760 		if (err)
4761 			goto err_alloc;
4762 
4763 		mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
4764 				      dev->port[i].cnts.offsets);
4765 
4766 		err = mlx5_core_alloc_q_counter(dev->mdev,
4767 						&dev->port[i].cnts.set_id);
4768 		if (err) {
4769 			mlx5_ib_warn(dev,
4770 				     "couldn't allocate queue counter for port %d, err %d\n",
4771 				     i + 1, err);
4772 			goto err_alloc;
4773 		}
4774 		dev->port[i].cnts.set_id_valid = true;
4775 	}
4776 
4777 	return 0;
4778 
4779 err_alloc:
4780 	mlx5_ib_dealloc_counters(dev);
4781 	return err;
4782 }
4783 
4784 static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
4785 						    u8 port_num)
4786 {
4787 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
4788 	struct mlx5_ib_port *port = &dev->port[port_num - 1];
4789 
4790 	/* We support only per port stats */
4791 	if (port_num == 0)
4792 		return NULL;
4793 
4794 	return rdma_alloc_hw_stats_struct(port->cnts.names,
4795 					  port->cnts.num_q_counters +
4796 					  port->cnts.num_cong_counters,
4797 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
4798 }
4799 
4800 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
4801 				    struct mlx5_ib_port *port,
4802 				    struct rdma_hw_stats *stats)
4803 {
4804 	int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
4805 	void *out;
4806 	__be32 val;
4807 	int ret, i;
4808 
4809 	out = kvzalloc(outlen, GFP_KERNEL);
4810 	if (!out)
4811 		return -ENOMEM;
4812 
4813 	ret = mlx5_core_query_q_counter(mdev,
4814 					port->cnts.set_id, 0,
4815 					out, outlen);
4816 	if (ret)
4817 		goto free;
4818 
4819 	for (i = 0; i < port->cnts.num_q_counters; i++) {
4820 		val = *(__be32 *)(out + port->cnts.offsets[i]);
4821 		stats->value[i] = (u64)be32_to_cpu(val);
4822 	}
4823 
4824 free:
4825 	kvfree(out);
4826 	return ret;
4827 }
4828 
4829 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
4830 				struct rdma_hw_stats *stats,
4831 				u8 port_num, int index)
4832 {
4833 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
4834 	struct mlx5_ib_port *port = &dev->port[port_num - 1];
4835 	struct mlx5_core_dev *mdev;
4836 	int ret, num_counters;
4837 	u8 mdev_port_num;
4838 
4839 	if (!stats)
4840 		return -EINVAL;
4841 
4842 	num_counters = port->cnts.num_q_counters + port->cnts.num_cong_counters;
4843 
4844 	/* q_counters are per IB device, query the master mdev */
4845 	ret = mlx5_ib_query_q_counters(dev->mdev, port, stats);
4846 	if (ret)
4847 		return ret;
4848 
4849 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
4850 		mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
4851 						    &mdev_port_num);
4852 		if (!mdev) {
4853 			/* If port is not affiliated yet, its in down state
4854 			 * which doesn't have any counters yet, so it would be
4855 			 * zero. So no need to read from the HCA.
4856 			 */
4857 			goto done;
4858 		}
4859 		ret = mlx5_lag_query_cong_counters(dev->mdev,
4860 						   stats->value +
4861 						   port->cnts.num_q_counters,
4862 						   port->cnts.num_cong_counters,
4863 						   port->cnts.offsets +
4864 						   port->cnts.num_q_counters);
4865 
4866 		mlx5_ib_put_native_port_mdev(dev, port_num);
4867 		if (ret)
4868 			return ret;
4869 	}
4870 
4871 done:
4872 	return num_counters;
4873 }
4874 
4875 static void mlx5_ib_free_rdma_netdev(struct net_device *netdev)
4876 {
4877 	return mlx5_rdma_netdev_free(netdev);
4878 }
4879 
4880 static struct net_device*
4881 mlx5_ib_alloc_rdma_netdev(struct ib_device *hca,
4882 			  u8 port_num,
4883 			  enum rdma_netdev_t type,
4884 			  const char *name,
4885 			  unsigned char name_assign_type,
4886 			  void (*setup)(struct net_device *))
4887 {
4888 	struct net_device *netdev;
4889 	struct rdma_netdev *rn;
4890 
4891 	if (type != RDMA_NETDEV_IPOIB)
4892 		return ERR_PTR(-EOPNOTSUPP);
4893 
4894 	netdev = mlx5_rdma_netdev_alloc(to_mdev(hca)->mdev, hca,
4895 					name, setup);
4896 	if (likely(!IS_ERR_OR_NULL(netdev))) {
4897 		rn = netdev_priv(netdev);
4898 		rn->free_rdma_netdev = mlx5_ib_free_rdma_netdev;
4899 	}
4900 	return netdev;
4901 }
4902 
4903 static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)
4904 {
4905 	if (!dev->delay_drop.dbg)
4906 		return;
4907 	debugfs_remove_recursive(dev->delay_drop.dbg->dir_debugfs);
4908 	kfree(dev->delay_drop.dbg);
4909 	dev->delay_drop.dbg = NULL;
4910 }
4911 
4912 static void cancel_delay_drop(struct mlx5_ib_dev *dev)
4913 {
4914 	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4915 		return;
4916 
4917 	cancel_work_sync(&dev->delay_drop.delay_drop_work);
4918 	delay_drop_debugfs_cleanup(dev);
4919 }
4920 
4921 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
4922 				       size_t count, loff_t *pos)
4923 {
4924 	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
4925 	char lbuf[20];
4926 	int len;
4927 
4928 	len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
4929 	return simple_read_from_buffer(buf, count, pos, lbuf, len);
4930 }
4931 
4932 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
4933 					size_t count, loff_t *pos)
4934 {
4935 	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
4936 	u32 timeout;
4937 	u32 var;
4938 
4939 	if (kstrtouint_from_user(buf, count, 0, &var))
4940 		return -EFAULT;
4941 
4942 	timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
4943 			1000);
4944 	if (timeout != var)
4945 		mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
4946 			    timeout);
4947 
4948 	delay_drop->timeout = timeout;
4949 
4950 	return count;
4951 }
4952 
4953 static const struct file_operations fops_delay_drop_timeout = {
4954 	.owner	= THIS_MODULE,
4955 	.open	= simple_open,
4956 	.write	= delay_drop_timeout_write,
4957 	.read	= delay_drop_timeout_read,
4958 };
4959 
4960 static int delay_drop_debugfs_init(struct mlx5_ib_dev *dev)
4961 {
4962 	struct mlx5_ib_dbg_delay_drop *dbg;
4963 
4964 	if (!mlx5_debugfs_root)
4965 		return 0;
4966 
4967 	dbg = kzalloc(sizeof(*dbg), GFP_KERNEL);
4968 	if (!dbg)
4969 		return -ENOMEM;
4970 
4971 	dev->delay_drop.dbg = dbg;
4972 
4973 	dbg->dir_debugfs =
4974 		debugfs_create_dir("delay_drop",
4975 				   dev->mdev->priv.dbg_root);
4976 	if (!dbg->dir_debugfs)
4977 		goto out_debugfs;
4978 
4979 	dbg->events_cnt_debugfs =
4980 		debugfs_create_atomic_t("num_timeout_events", 0400,
4981 					dbg->dir_debugfs,
4982 					&dev->delay_drop.events_cnt);
4983 	if (!dbg->events_cnt_debugfs)
4984 		goto out_debugfs;
4985 
4986 	dbg->rqs_cnt_debugfs =
4987 		debugfs_create_atomic_t("num_rqs", 0400,
4988 					dbg->dir_debugfs,
4989 					&dev->delay_drop.rqs_cnt);
4990 	if (!dbg->rqs_cnt_debugfs)
4991 		goto out_debugfs;
4992 
4993 	dbg->timeout_debugfs =
4994 		debugfs_create_file("timeout", 0600,
4995 				    dbg->dir_debugfs,
4996 				    &dev->delay_drop,
4997 				    &fops_delay_drop_timeout);
4998 	if (!dbg->timeout_debugfs)
4999 		goto out_debugfs;
5000 
5001 	return 0;
5002 
5003 out_debugfs:
5004 	delay_drop_debugfs_cleanup(dev);
5005 	return -ENOMEM;
5006 }
5007 
5008 static void init_delay_drop(struct mlx5_ib_dev *dev)
5009 {
5010 	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
5011 		return;
5012 
5013 	mutex_init(&dev->delay_drop.lock);
5014 	dev->delay_drop.dev = dev;
5015 	dev->delay_drop.activate = false;
5016 	dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
5017 	INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
5018 	atomic_set(&dev->delay_drop.rqs_cnt, 0);
5019 	atomic_set(&dev->delay_drop.events_cnt, 0);
5020 
5021 	if (delay_drop_debugfs_init(dev))
5022 		mlx5_ib_warn(dev, "Failed to init delay drop debugfs\n");
5023 }
5024 
5025 static const struct cpumask *
5026 mlx5_ib_get_vector_affinity(struct ib_device *ibdev, int comp_vector)
5027 {
5028 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
5029 
5030 	return mlx5_get_vector_affinity_hint(dev->mdev, comp_vector);
5031 }
5032 
5033 /* The mlx5_ib_multiport_mutex should be held when calling this function */
5034 static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
5035 				      struct mlx5_ib_multiport_info *mpi)
5036 {
5037 	u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
5038 	struct mlx5_ib_port *port = &ibdev->port[port_num];
5039 	int comps;
5040 	int err;
5041 	int i;
5042 
5043 	mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
5044 
5045 	spin_lock(&port->mp.mpi_lock);
5046 	if (!mpi->ibdev) {
5047 		spin_unlock(&port->mp.mpi_lock);
5048 		return;
5049 	}
5050 	mpi->ibdev = NULL;
5051 
5052 	spin_unlock(&port->mp.mpi_lock);
5053 	mlx5_remove_netdev_notifier(ibdev, port_num);
5054 	spin_lock(&port->mp.mpi_lock);
5055 
5056 	comps = mpi->mdev_refcnt;
5057 	if (comps) {
5058 		mpi->unaffiliate = true;
5059 		init_completion(&mpi->unref_comp);
5060 		spin_unlock(&port->mp.mpi_lock);
5061 
5062 		for (i = 0; i < comps; i++)
5063 			wait_for_completion(&mpi->unref_comp);
5064 
5065 		spin_lock(&port->mp.mpi_lock);
5066 		mpi->unaffiliate = false;
5067 	}
5068 
5069 	port->mp.mpi = NULL;
5070 
5071 	list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
5072 
5073 	spin_unlock(&port->mp.mpi_lock);
5074 
5075 	err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
5076 
5077 	mlx5_ib_dbg(ibdev, "unaffiliated port %d\n", port_num + 1);
5078 	/* Log an error, still needed to cleanup the pointers and add
5079 	 * it back to the list.
5080 	 */
5081 	if (err)
5082 		mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
5083 			    port_num + 1);
5084 
5085 	ibdev->roce[port_num].last_port_state = IB_PORT_DOWN;
5086 }
5087 
5088 /* The mlx5_ib_multiport_mutex should be held when calling this function */
5089 static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
5090 				    struct mlx5_ib_multiport_info *mpi)
5091 {
5092 	u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
5093 	int err;
5094 
5095 	spin_lock(&ibdev->port[port_num].mp.mpi_lock);
5096 	if (ibdev->port[port_num].mp.mpi) {
5097 		mlx5_ib_warn(ibdev, "port %d already affiliated.\n",
5098 			     port_num + 1);
5099 		spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
5100 		return false;
5101 	}
5102 
5103 	ibdev->port[port_num].mp.mpi = mpi;
5104 	mpi->ibdev = ibdev;
5105 	spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
5106 
5107 	err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
5108 	if (err)
5109 		goto unbind;
5110 
5111 	err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi->mdev));
5112 	if (err)
5113 		goto unbind;
5114 
5115 	err = mlx5_add_netdev_notifier(ibdev, port_num);
5116 	if (err) {
5117 		mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
5118 			    port_num + 1);
5119 		goto unbind;
5120 	}
5121 
5122 	err = mlx5_ib_init_cong_debugfs(ibdev, port_num);
5123 	if (err)
5124 		goto unbind;
5125 
5126 	return true;
5127 
5128 unbind:
5129 	mlx5_ib_unbind_slave_port(ibdev, mpi);
5130 	return false;
5131 }
5132 
5133 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
5134 {
5135 	int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5136 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
5137 							  port_num + 1);
5138 	struct mlx5_ib_multiport_info *mpi;
5139 	int err;
5140 	int i;
5141 
5142 	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
5143 		return 0;
5144 
5145 	err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
5146 						     &dev->sys_image_guid);
5147 	if (err)
5148 		return err;
5149 
5150 	err = mlx5_nic_vport_enable_roce(dev->mdev);
5151 	if (err)
5152 		return err;
5153 
5154 	mutex_lock(&mlx5_ib_multiport_mutex);
5155 	for (i = 0; i < dev->num_ports; i++) {
5156 		bool bound = false;
5157 
5158 		/* build a stub multiport info struct for the native port. */
5159 		if (i == port_num) {
5160 			mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
5161 			if (!mpi) {
5162 				mutex_unlock(&mlx5_ib_multiport_mutex);
5163 				mlx5_nic_vport_disable_roce(dev->mdev);
5164 				return -ENOMEM;
5165 			}
5166 
5167 			mpi->is_master = true;
5168 			mpi->mdev = dev->mdev;
5169 			mpi->sys_image_guid = dev->sys_image_guid;
5170 			dev->port[i].mp.mpi = mpi;
5171 			mpi->ibdev = dev;
5172 			mpi = NULL;
5173 			continue;
5174 		}
5175 
5176 		list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
5177 				    list) {
5178 			if (dev->sys_image_guid == mpi->sys_image_guid &&
5179 			    (mlx5_core_native_port_num(mpi->mdev) - 1) == i) {
5180 				bound = mlx5_ib_bind_slave_port(dev, mpi);
5181 			}
5182 
5183 			if (bound) {
5184 				dev_dbg(&mpi->mdev->pdev->dev, "removing port from unaffiliated list.\n");
5185 				mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
5186 				list_del(&mpi->list);
5187 				break;
5188 			}
5189 		}
5190 		if (!bound) {
5191 			get_port_caps(dev, i + 1);
5192 			mlx5_ib_dbg(dev, "no free port found for port %d\n",
5193 				    i + 1);
5194 		}
5195 	}
5196 
5197 	list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
5198 	mutex_unlock(&mlx5_ib_multiport_mutex);
5199 	return err;
5200 }
5201 
5202 static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
5203 {
5204 	int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5205 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
5206 							  port_num + 1);
5207 	int i;
5208 
5209 	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
5210 		return;
5211 
5212 	mutex_lock(&mlx5_ib_multiport_mutex);
5213 	for (i = 0; i < dev->num_ports; i++) {
5214 		if (dev->port[i].mp.mpi) {
5215 			/* Destroy the native port stub */
5216 			if (i == port_num) {
5217 				kfree(dev->port[i].mp.mpi);
5218 				dev->port[i].mp.mpi = NULL;
5219 			} else {
5220 				mlx5_ib_dbg(dev, "unbinding port_num: %d\n", i + 1);
5221 				mlx5_ib_unbind_slave_port(dev, dev->port[i].mp.mpi);
5222 			}
5223 		}
5224 	}
5225 
5226 	mlx5_ib_dbg(dev, "removing from devlist\n");
5227 	list_del(&dev->ib_dev_list);
5228 	mutex_unlock(&mlx5_ib_multiport_mutex);
5229 
5230 	mlx5_nic_vport_disable_roce(dev->mdev);
5231 }
5232 
5233 ADD_UVERBS_ATTRIBUTES_SIMPLE(mlx5_ib_dm, UVERBS_OBJECT_DM,
5234 			     UVERBS_METHOD_DM_ALLOC,
5235 			     &UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
5236 						  UVERBS_ATTR_TYPE(u64),
5237 						  UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
5238 			     &UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
5239 						  UVERBS_ATTR_TYPE(u16),
5240 						  UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)));
5241 
5242 ADD_UVERBS_ATTRIBUTES_SIMPLE(mlx5_ib_flow_action, UVERBS_OBJECT_FLOW_ACTION,
5243 			     UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,
5244 			     &UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
5245 						 UVERBS_ATTR_TYPE(u64),
5246 						 UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)));
5247 
5248 #define NUM_TREES	2
5249 static int populate_specs_root(struct mlx5_ib_dev *dev)
5250 {
5251 	const struct uverbs_object_tree_def *default_root[NUM_TREES + 1] = {
5252 		uverbs_default_get_objects()};
5253 	size_t num_trees = 1;
5254 
5255 	if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE &&
5256 	    !WARN_ON(num_trees >= ARRAY_SIZE(default_root)))
5257 		default_root[num_trees++] = &mlx5_ib_flow_action;
5258 
5259 	if (MLX5_CAP_DEV_MEM(dev->mdev, memic) &&
5260 	    !WARN_ON(num_trees >= ARRAY_SIZE(default_root)))
5261 		default_root[num_trees++] = &mlx5_ib_dm;
5262 
5263 	dev->ib_dev.specs_root =
5264 		uverbs_alloc_spec_tree(num_trees, default_root);
5265 
5266 	return PTR_ERR_OR_ZERO(dev->ib_dev.specs_root);
5267 }
5268 
5269 static void depopulate_specs_root(struct mlx5_ib_dev *dev)
5270 {
5271 	uverbs_free_spec_tree(dev->ib_dev.specs_root);
5272 }
5273 
5274 static int mlx5_ib_read_counters(struct ib_counters *counters,
5275 				 struct ib_counters_read_attr *read_attr,
5276 				 struct uverbs_attr_bundle *attrs)
5277 {
5278 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
5279 	struct mlx5_read_counters_attr mread_attr = {};
5280 	struct mlx5_ib_flow_counters_desc *desc;
5281 	int ret, i;
5282 
5283 	mutex_lock(&mcounters->mcntrs_mutex);
5284 	if (mcounters->cntrs_max_index > read_attr->ncounters) {
5285 		ret = -EINVAL;
5286 		goto err_bound;
5287 	}
5288 
5289 	mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
5290 				 GFP_KERNEL);
5291 	if (!mread_attr.out) {
5292 		ret = -ENOMEM;
5293 		goto err_bound;
5294 	}
5295 
5296 	mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
5297 	mread_attr.flags = read_attr->flags;
5298 	ret = mcounters->read_counters(counters->device, &mread_attr);
5299 	if (ret)
5300 		goto err_read;
5301 
5302 	/* do the pass over the counters data array to assign according to the
5303 	 * descriptions and indexing pairs
5304 	 */
5305 	desc = mcounters->counters_data;
5306 	for (i = 0; i < mcounters->ncounters; i++)
5307 		read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
5308 
5309 err_read:
5310 	kfree(mread_attr.out);
5311 err_bound:
5312 	mutex_unlock(&mcounters->mcntrs_mutex);
5313 	return ret;
5314 }
5315 
5316 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
5317 {
5318 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
5319 
5320 	counters_clear_description(counters);
5321 	if (mcounters->hw_cntrs_hndl)
5322 		mlx5_fc_destroy(to_mdev(counters->device)->mdev,
5323 				mcounters->hw_cntrs_hndl);
5324 
5325 	kfree(mcounters);
5326 
5327 	return 0;
5328 }
5329 
5330 static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device,
5331 						   struct uverbs_attr_bundle *attrs)
5332 {
5333 	struct mlx5_ib_mcounters *mcounters;
5334 
5335 	mcounters = kzalloc(sizeof(*mcounters), GFP_KERNEL);
5336 	if (!mcounters)
5337 		return ERR_PTR(-ENOMEM);
5338 
5339 	mutex_init(&mcounters->mcntrs_mutex);
5340 
5341 	return &mcounters->ibcntrs;
5342 }
5343 
5344 void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
5345 {
5346 	mlx5_ib_cleanup_multiport_master(dev);
5347 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
5348 	cleanup_srcu_struct(&dev->mr_srcu);
5349 #endif
5350 	kfree(dev->port);
5351 }
5352 
5353 int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
5354 {
5355 	struct mlx5_core_dev *mdev = dev->mdev;
5356 	const char *name;
5357 	int err;
5358 	int i;
5359 
5360 	dev->port = kcalloc(dev->num_ports, sizeof(*dev->port),
5361 			    GFP_KERNEL);
5362 	if (!dev->port)
5363 		return -ENOMEM;
5364 
5365 	for (i = 0; i < dev->num_ports; i++) {
5366 		spin_lock_init(&dev->port[i].mp.mpi_lock);
5367 		rwlock_init(&dev->roce[i].netdev_lock);
5368 	}
5369 
5370 	err = mlx5_ib_init_multiport_master(dev);
5371 	if (err)
5372 		goto err_free_port;
5373 
5374 	if (!mlx5_core_mp_enabled(mdev)) {
5375 		for (i = 1; i <= dev->num_ports; i++) {
5376 			err = get_port_caps(dev, i);
5377 			if (err)
5378 				break;
5379 		}
5380 	} else {
5381 		err = get_port_caps(dev, mlx5_core_native_port_num(mdev));
5382 	}
5383 	if (err)
5384 		goto err_mp;
5385 
5386 	if (mlx5_use_mad_ifc(dev))
5387 		get_ext_port_caps(dev);
5388 
5389 	if (!mlx5_lag_is_active(mdev))
5390 		name = "mlx5_%d";
5391 	else
5392 		name = "mlx5_bond_%d";
5393 
5394 	strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
5395 	dev->ib_dev.owner		= THIS_MODULE;
5396 	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
5397 	dev->ib_dev.local_dma_lkey	= 0 /* not supported for now */;
5398 	dev->ib_dev.phys_port_cnt	= dev->num_ports;
5399 	dev->ib_dev.num_comp_vectors    =
5400 		dev->mdev->priv.eq_table.num_comp_vectors;
5401 	dev->ib_dev.dev.parent		= &mdev->pdev->dev;
5402 
5403 	mutex_init(&dev->cap_mask_mutex);
5404 	INIT_LIST_HEAD(&dev->qp_list);
5405 	spin_lock_init(&dev->reset_flow_resource_lock);
5406 
5407 	spin_lock_init(&dev->memic.memic_lock);
5408 	dev->memic.dev = mdev;
5409 
5410 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
5411 	err = init_srcu_struct(&dev->mr_srcu);
5412 	if (err)
5413 		goto err_free_port;
5414 #endif
5415 
5416 	return 0;
5417 err_mp:
5418 	mlx5_ib_cleanup_multiport_master(dev);
5419 
5420 err_free_port:
5421 	kfree(dev->port);
5422 
5423 	return -ENOMEM;
5424 }
5425 
5426 static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)
5427 {
5428 	dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
5429 
5430 	if (!dev->flow_db)
5431 		return -ENOMEM;
5432 
5433 	mutex_init(&dev->flow_db->lock);
5434 
5435 	return 0;
5436 }
5437 
5438 int mlx5_ib_stage_rep_flow_db_init(struct mlx5_ib_dev *dev)
5439 {
5440 	struct mlx5_ib_dev *nic_dev;
5441 
5442 	nic_dev = mlx5_ib_get_uplink_ibdev(dev->mdev->priv.eswitch);
5443 
5444 	if (!nic_dev)
5445 		return -EINVAL;
5446 
5447 	dev->flow_db = nic_dev->flow_db;
5448 
5449 	return 0;
5450 }
5451 
5452 static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
5453 {
5454 	kfree(dev->flow_db);
5455 }
5456 
5457 int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
5458 {
5459 	struct mlx5_core_dev *mdev = dev->mdev;
5460 	int err;
5461 
5462 	dev->ib_dev.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION;
5463 	dev->ib_dev.uverbs_cmd_mask	=
5464 		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
5465 		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
5466 		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
5467 		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
5468 		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
5469 		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
5470 		(1ull << IB_USER_VERBS_CMD_DESTROY_AH)		|
5471 		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
5472 		(1ull << IB_USER_VERBS_CMD_REREG_MR)		|
5473 		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
5474 		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|
5475 		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
5476 		(1ull << IB_USER_VERBS_CMD_RESIZE_CQ)		|
5477 		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
5478 		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
5479 		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
5480 		(1ull << IB_USER_VERBS_CMD_QUERY_QP)		|
5481 		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
5482 		(1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)	|
5483 		(1ull << IB_USER_VERBS_CMD_DETACH_MCAST)	|
5484 		(1ull << IB_USER_VERBS_CMD_CREATE_SRQ)		|
5485 		(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)		|
5486 		(1ull << IB_USER_VERBS_CMD_QUERY_SRQ)		|
5487 		(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)		|
5488 		(1ull << IB_USER_VERBS_CMD_CREATE_XSRQ)		|
5489 		(1ull << IB_USER_VERBS_CMD_OPEN_QP);
5490 	dev->ib_dev.uverbs_ex_cmd_mask =
5491 		(1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE)	|
5492 		(1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ)	|
5493 		(1ull << IB_USER_VERBS_EX_CMD_CREATE_QP)	|
5494 		(1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP)	|
5495 		(1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ);
5496 
5497 	dev->ib_dev.query_device	= mlx5_ib_query_device;
5498 	dev->ib_dev.get_link_layer	= mlx5_ib_port_link_layer;
5499 	dev->ib_dev.query_gid		= mlx5_ib_query_gid;
5500 	dev->ib_dev.add_gid		= mlx5_ib_add_gid;
5501 	dev->ib_dev.del_gid		= mlx5_ib_del_gid;
5502 	dev->ib_dev.query_pkey		= mlx5_ib_query_pkey;
5503 	dev->ib_dev.modify_device	= mlx5_ib_modify_device;
5504 	dev->ib_dev.modify_port		= mlx5_ib_modify_port;
5505 	dev->ib_dev.alloc_ucontext	= mlx5_ib_alloc_ucontext;
5506 	dev->ib_dev.dealloc_ucontext	= mlx5_ib_dealloc_ucontext;
5507 	dev->ib_dev.mmap		= mlx5_ib_mmap;
5508 	dev->ib_dev.alloc_pd		= mlx5_ib_alloc_pd;
5509 	dev->ib_dev.dealloc_pd		= mlx5_ib_dealloc_pd;
5510 	dev->ib_dev.create_ah		= mlx5_ib_create_ah;
5511 	dev->ib_dev.query_ah		= mlx5_ib_query_ah;
5512 	dev->ib_dev.destroy_ah		= mlx5_ib_destroy_ah;
5513 	dev->ib_dev.create_srq		= mlx5_ib_create_srq;
5514 	dev->ib_dev.modify_srq		= mlx5_ib_modify_srq;
5515 	dev->ib_dev.query_srq		= mlx5_ib_query_srq;
5516 	dev->ib_dev.destroy_srq		= mlx5_ib_destroy_srq;
5517 	dev->ib_dev.post_srq_recv	= mlx5_ib_post_srq_recv;
5518 	dev->ib_dev.create_qp		= mlx5_ib_create_qp;
5519 	dev->ib_dev.modify_qp		= mlx5_ib_modify_qp;
5520 	dev->ib_dev.query_qp		= mlx5_ib_query_qp;
5521 	dev->ib_dev.destroy_qp		= mlx5_ib_destroy_qp;
5522 	dev->ib_dev.post_send		= mlx5_ib_post_send;
5523 	dev->ib_dev.post_recv		= mlx5_ib_post_recv;
5524 	dev->ib_dev.create_cq		= mlx5_ib_create_cq;
5525 	dev->ib_dev.modify_cq		= mlx5_ib_modify_cq;
5526 	dev->ib_dev.resize_cq		= mlx5_ib_resize_cq;
5527 	dev->ib_dev.destroy_cq		= mlx5_ib_destroy_cq;
5528 	dev->ib_dev.poll_cq		= mlx5_ib_poll_cq;
5529 	dev->ib_dev.req_notify_cq	= mlx5_ib_arm_cq;
5530 	dev->ib_dev.get_dma_mr		= mlx5_ib_get_dma_mr;
5531 	dev->ib_dev.reg_user_mr		= mlx5_ib_reg_user_mr;
5532 	dev->ib_dev.rereg_user_mr	= mlx5_ib_rereg_user_mr;
5533 	dev->ib_dev.dereg_mr		= mlx5_ib_dereg_mr;
5534 	dev->ib_dev.attach_mcast	= mlx5_ib_mcg_attach;
5535 	dev->ib_dev.detach_mcast	= mlx5_ib_mcg_detach;
5536 	dev->ib_dev.process_mad		= mlx5_ib_process_mad;
5537 	dev->ib_dev.alloc_mr		= mlx5_ib_alloc_mr;
5538 	dev->ib_dev.map_mr_sg		= mlx5_ib_map_mr_sg;
5539 	dev->ib_dev.check_mr_status	= mlx5_ib_check_mr_status;
5540 	dev->ib_dev.get_dev_fw_str      = get_dev_fw_str;
5541 	dev->ib_dev.get_vector_affinity	= mlx5_ib_get_vector_affinity;
5542 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads))
5543 		dev->ib_dev.alloc_rdma_netdev	= mlx5_ib_alloc_rdma_netdev;
5544 
5545 	if (mlx5_core_is_pf(mdev)) {
5546 		dev->ib_dev.get_vf_config	= mlx5_ib_get_vf_config;
5547 		dev->ib_dev.set_vf_link_state	= mlx5_ib_set_vf_link_state;
5548 		dev->ib_dev.get_vf_stats	= mlx5_ib_get_vf_stats;
5549 		dev->ib_dev.set_vf_guid		= mlx5_ib_set_vf_guid;
5550 	}
5551 
5552 	dev->ib_dev.disassociate_ucontext = mlx5_ib_disassociate_ucontext;
5553 
5554 	dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
5555 
5556 	if (MLX5_CAP_GEN(mdev, imaicl)) {
5557 		dev->ib_dev.alloc_mw		= mlx5_ib_alloc_mw;
5558 		dev->ib_dev.dealloc_mw		= mlx5_ib_dealloc_mw;
5559 		dev->ib_dev.uverbs_cmd_mask |=
5560 			(1ull << IB_USER_VERBS_CMD_ALLOC_MW)	|
5561 			(1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
5562 	}
5563 
5564 	if (MLX5_CAP_GEN(mdev, xrc)) {
5565 		dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
5566 		dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
5567 		dev->ib_dev.uverbs_cmd_mask |=
5568 			(1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
5569 			(1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
5570 	}
5571 
5572 	if (MLX5_CAP_DEV_MEM(mdev, memic)) {
5573 		dev->ib_dev.alloc_dm = mlx5_ib_alloc_dm;
5574 		dev->ib_dev.dealloc_dm = mlx5_ib_dealloc_dm;
5575 		dev->ib_dev.reg_dm_mr = mlx5_ib_reg_dm_mr;
5576 	}
5577 
5578 	dev->ib_dev.create_flow	= mlx5_ib_create_flow;
5579 	dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
5580 	dev->ib_dev.uverbs_ex_cmd_mask |=
5581 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
5582 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
5583 	dev->ib_dev.create_flow_action_esp = mlx5_ib_create_flow_action_esp;
5584 	dev->ib_dev.destroy_flow_action = mlx5_ib_destroy_flow_action;
5585 	dev->ib_dev.modify_flow_action_esp = mlx5_ib_modify_flow_action_esp;
5586 	dev->ib_dev.driver_id = RDMA_DRIVER_MLX5;
5587 	dev->ib_dev.create_counters = mlx5_ib_create_counters;
5588 	dev->ib_dev.destroy_counters = mlx5_ib_destroy_counters;
5589 	dev->ib_dev.read_counters = mlx5_ib_read_counters;
5590 
5591 	err = init_node_data(dev);
5592 	if (err)
5593 		return err;
5594 
5595 	if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
5596 	    (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
5597 	     MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
5598 		mutex_init(&dev->lb_mutex);
5599 
5600 	return 0;
5601 }
5602 
5603 static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
5604 {
5605 	dev->ib_dev.get_port_immutable  = mlx5_port_immutable;
5606 	dev->ib_dev.query_port		= mlx5_ib_query_port;
5607 
5608 	return 0;
5609 }
5610 
5611 int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev)
5612 {
5613 	dev->ib_dev.get_port_immutable  = mlx5_port_rep_immutable;
5614 	dev->ib_dev.query_port		= mlx5_ib_rep_query_port;
5615 
5616 	return 0;
5617 }
5618 
5619 static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev,
5620 					  u8 port_num)
5621 {
5622 	int i;
5623 
5624 	for (i = 0; i < dev->num_ports; i++) {
5625 		dev->roce[i].dev = dev;
5626 		dev->roce[i].native_port_num = i + 1;
5627 		dev->roce[i].last_port_state = IB_PORT_DOWN;
5628 	}
5629 
5630 	dev->ib_dev.get_netdev	= mlx5_ib_get_netdev;
5631 	dev->ib_dev.create_wq	 = mlx5_ib_create_wq;
5632 	dev->ib_dev.modify_wq	 = mlx5_ib_modify_wq;
5633 	dev->ib_dev.destroy_wq	 = mlx5_ib_destroy_wq;
5634 	dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table;
5635 	dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table;
5636 
5637 	dev->ib_dev.uverbs_ex_cmd_mask |=
5638 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
5639 			(1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
5640 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
5641 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
5642 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
5643 
5644 	return mlx5_add_netdev_notifier(dev, port_num);
5645 }
5646 
5647 static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev)
5648 {
5649 	u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5650 
5651 	mlx5_remove_netdev_notifier(dev, port_num);
5652 }
5653 
5654 int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev)
5655 {
5656 	struct mlx5_core_dev *mdev = dev->mdev;
5657 	enum rdma_link_layer ll;
5658 	int port_type_cap;
5659 	int err = 0;
5660 	u8 port_num;
5661 
5662 	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5663 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
5664 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
5665 
5666 	if (ll == IB_LINK_LAYER_ETHERNET)
5667 		err = mlx5_ib_stage_common_roce_init(dev, port_num);
5668 
5669 	return err;
5670 }
5671 
5672 void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev)
5673 {
5674 	mlx5_ib_stage_common_roce_cleanup(dev);
5675 }
5676 
5677 static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)
5678 {
5679 	struct mlx5_core_dev *mdev = dev->mdev;
5680 	enum rdma_link_layer ll;
5681 	int port_type_cap;
5682 	u8 port_num;
5683 	int err;
5684 
5685 	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5686 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
5687 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
5688 
5689 	if (ll == IB_LINK_LAYER_ETHERNET) {
5690 		err = mlx5_ib_stage_common_roce_init(dev, port_num);
5691 		if (err)
5692 			return err;
5693 
5694 		err = mlx5_enable_eth(dev, port_num);
5695 		if (err)
5696 			goto cleanup;
5697 	}
5698 
5699 	return 0;
5700 cleanup:
5701 	mlx5_ib_stage_common_roce_cleanup(dev);
5702 
5703 	return err;
5704 }
5705 
5706 static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
5707 {
5708 	struct mlx5_core_dev *mdev = dev->mdev;
5709 	enum rdma_link_layer ll;
5710 	int port_type_cap;
5711 	u8 port_num;
5712 
5713 	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
5714 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
5715 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
5716 
5717 	if (ll == IB_LINK_LAYER_ETHERNET) {
5718 		mlx5_disable_eth(dev);
5719 		mlx5_ib_stage_common_roce_cleanup(dev);
5720 	}
5721 }
5722 
5723 int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
5724 {
5725 	return create_dev_resources(&dev->devr);
5726 }
5727 
5728 void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
5729 {
5730 	destroy_dev_resources(&dev->devr);
5731 }
5732 
5733 static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
5734 {
5735 	mlx5_ib_internal_fill_odp_caps(dev);
5736 
5737 	return mlx5_ib_odp_init_one(dev);
5738 }
5739 
5740 int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
5741 {
5742 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
5743 		dev->ib_dev.get_hw_stats	= mlx5_ib_get_hw_stats;
5744 		dev->ib_dev.alloc_hw_stats	= mlx5_ib_alloc_hw_stats;
5745 
5746 		return mlx5_ib_alloc_counters(dev);
5747 	}
5748 
5749 	return 0;
5750 }
5751 
5752 void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
5753 {
5754 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
5755 		mlx5_ib_dealloc_counters(dev);
5756 }
5757 
5758 static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
5759 {
5760 	return mlx5_ib_init_cong_debugfs(dev,
5761 					 mlx5_core_native_port_num(dev->mdev) - 1);
5762 }
5763 
5764 static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
5765 {
5766 	mlx5_ib_cleanup_cong_debugfs(dev,
5767 				     mlx5_core_native_port_num(dev->mdev) - 1);
5768 }
5769 
5770 static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
5771 {
5772 	dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
5773 	return PTR_ERR_OR_ZERO(dev->mdev->priv.uar);
5774 }
5775 
5776 static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
5777 {
5778 	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
5779 }
5780 
5781 int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
5782 {
5783 	int err;
5784 
5785 	err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
5786 	if (err)
5787 		return err;
5788 
5789 	err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
5790 	if (err)
5791 		mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
5792 
5793 	return err;
5794 }
5795 
5796 void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
5797 {
5798 	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
5799 	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
5800 }
5801 
5802 static int mlx5_ib_stage_populate_specs(struct mlx5_ib_dev *dev)
5803 {
5804 	return populate_specs_root(dev);
5805 }
5806 
5807 int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
5808 {
5809 	return ib_register_device(&dev->ib_dev, NULL);
5810 }
5811 
5812 static void mlx5_ib_stage_depopulate_specs(struct mlx5_ib_dev *dev)
5813 {
5814 	depopulate_specs_root(dev);
5815 }
5816 
5817 void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
5818 {
5819 	destroy_umrc_res(dev);
5820 }
5821 
5822 void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
5823 {
5824 	ib_unregister_device(&dev->ib_dev);
5825 }
5826 
5827 int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
5828 {
5829 	return create_umr_res(dev);
5830 }
5831 
5832 static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
5833 {
5834 	init_delay_drop(dev);
5835 
5836 	return 0;
5837 }
5838 
5839 static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
5840 {
5841 	cancel_delay_drop(dev);
5842 }
5843 
5844 int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev)
5845 {
5846 	int err;
5847 	int i;
5848 
5849 	for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
5850 		err = device_create_file(&dev->ib_dev.dev,
5851 					 mlx5_class_attributes[i]);
5852 		if (err)
5853 			return err;
5854 	}
5855 
5856 	return 0;
5857 }
5858 
5859 static int mlx5_ib_stage_rep_reg_init(struct mlx5_ib_dev *dev)
5860 {
5861 	mlx5_ib_register_vport_reps(dev);
5862 
5863 	return 0;
5864 }
5865 
5866 static void mlx5_ib_stage_rep_reg_cleanup(struct mlx5_ib_dev *dev)
5867 {
5868 	mlx5_ib_unregister_vport_reps(dev);
5869 }
5870 
5871 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
5872 		      const struct mlx5_ib_profile *profile,
5873 		      int stage)
5874 {
5875 	/* Number of stages to cleanup */
5876 	while (stage) {
5877 		stage--;
5878 		if (profile->stage[stage].cleanup)
5879 			profile->stage[stage].cleanup(dev);
5880 	}
5881 
5882 	ib_dealloc_device((struct ib_device *)dev);
5883 }
5884 
5885 static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num);
5886 
5887 void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
5888 		    const struct mlx5_ib_profile *profile)
5889 {
5890 	int err;
5891 	int i;
5892 
5893 	printk_once(KERN_INFO "%s", mlx5_version);
5894 
5895 	for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
5896 		if (profile->stage[i].init) {
5897 			err = profile->stage[i].init(dev);
5898 			if (err)
5899 				goto err_out;
5900 		}
5901 	}
5902 
5903 	dev->profile = profile;
5904 	dev->ib_active = true;
5905 
5906 	return dev;
5907 
5908 err_out:
5909 	__mlx5_ib_remove(dev, profile, i);
5910 
5911 	return NULL;
5912 }
5913 
5914 static const struct mlx5_ib_profile pf_profile = {
5915 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
5916 		     mlx5_ib_stage_init_init,
5917 		     mlx5_ib_stage_init_cleanup),
5918 	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
5919 		     mlx5_ib_stage_flow_db_init,
5920 		     mlx5_ib_stage_flow_db_cleanup),
5921 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
5922 		     mlx5_ib_stage_caps_init,
5923 		     NULL),
5924 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
5925 		     mlx5_ib_stage_non_default_cb,
5926 		     NULL),
5927 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
5928 		     mlx5_ib_stage_roce_init,
5929 		     mlx5_ib_stage_roce_cleanup),
5930 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
5931 		     mlx5_ib_stage_dev_res_init,
5932 		     mlx5_ib_stage_dev_res_cleanup),
5933 	STAGE_CREATE(MLX5_IB_STAGE_ODP,
5934 		     mlx5_ib_stage_odp_init,
5935 		     NULL),
5936 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
5937 		     mlx5_ib_stage_counters_init,
5938 		     mlx5_ib_stage_counters_cleanup),
5939 	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
5940 		     mlx5_ib_stage_cong_debugfs_init,
5941 		     mlx5_ib_stage_cong_debugfs_cleanup),
5942 	STAGE_CREATE(MLX5_IB_STAGE_UAR,
5943 		     mlx5_ib_stage_uar_init,
5944 		     mlx5_ib_stage_uar_cleanup),
5945 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
5946 		     mlx5_ib_stage_bfrag_init,
5947 		     mlx5_ib_stage_bfrag_cleanup),
5948 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
5949 		     NULL,
5950 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
5951 	STAGE_CREATE(MLX5_IB_STAGE_SPECS,
5952 		     mlx5_ib_stage_populate_specs,
5953 		     mlx5_ib_stage_depopulate_specs),
5954 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
5955 		     mlx5_ib_stage_ib_reg_init,
5956 		     mlx5_ib_stage_ib_reg_cleanup),
5957 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
5958 		     mlx5_ib_stage_post_ib_reg_umr_init,
5959 		     NULL),
5960 	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
5961 		     mlx5_ib_stage_delay_drop_init,
5962 		     mlx5_ib_stage_delay_drop_cleanup),
5963 	STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR,
5964 		     mlx5_ib_stage_class_attr_init,
5965 		     NULL),
5966 };
5967 
5968 static const struct mlx5_ib_profile nic_rep_profile = {
5969 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
5970 		     mlx5_ib_stage_init_init,
5971 		     mlx5_ib_stage_init_cleanup),
5972 	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
5973 		     mlx5_ib_stage_flow_db_init,
5974 		     mlx5_ib_stage_flow_db_cleanup),
5975 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
5976 		     mlx5_ib_stage_caps_init,
5977 		     NULL),
5978 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
5979 		     mlx5_ib_stage_rep_non_default_cb,
5980 		     NULL),
5981 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
5982 		     mlx5_ib_stage_rep_roce_init,
5983 		     mlx5_ib_stage_rep_roce_cleanup),
5984 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
5985 		     mlx5_ib_stage_dev_res_init,
5986 		     mlx5_ib_stage_dev_res_cleanup),
5987 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
5988 		     mlx5_ib_stage_counters_init,
5989 		     mlx5_ib_stage_counters_cleanup),
5990 	STAGE_CREATE(MLX5_IB_STAGE_UAR,
5991 		     mlx5_ib_stage_uar_init,
5992 		     mlx5_ib_stage_uar_cleanup),
5993 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
5994 		     mlx5_ib_stage_bfrag_init,
5995 		     mlx5_ib_stage_bfrag_cleanup),
5996 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
5997 		     NULL,
5998 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
5999 	STAGE_CREATE(MLX5_IB_STAGE_SPECS,
6000 		     mlx5_ib_stage_populate_specs,
6001 		     mlx5_ib_stage_depopulate_specs),
6002 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
6003 		     mlx5_ib_stage_ib_reg_init,
6004 		     mlx5_ib_stage_ib_reg_cleanup),
6005 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
6006 		     mlx5_ib_stage_post_ib_reg_umr_init,
6007 		     NULL),
6008 	STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR,
6009 		     mlx5_ib_stage_class_attr_init,
6010 		     NULL),
6011 	STAGE_CREATE(MLX5_IB_STAGE_REP_REG,
6012 		     mlx5_ib_stage_rep_reg_init,
6013 		     mlx5_ib_stage_rep_reg_cleanup),
6014 };
6015 
6016 static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num)
6017 {
6018 	struct mlx5_ib_multiport_info *mpi;
6019 	struct mlx5_ib_dev *dev;
6020 	bool bound = false;
6021 	int err;
6022 
6023 	mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
6024 	if (!mpi)
6025 		return NULL;
6026 
6027 	mpi->mdev = mdev;
6028 
6029 	err = mlx5_query_nic_vport_system_image_guid(mdev,
6030 						     &mpi->sys_image_guid);
6031 	if (err) {
6032 		kfree(mpi);
6033 		return NULL;
6034 	}
6035 
6036 	mutex_lock(&mlx5_ib_multiport_mutex);
6037 	list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
6038 		if (dev->sys_image_guid == mpi->sys_image_guid)
6039 			bound = mlx5_ib_bind_slave_port(dev, mpi);
6040 
6041 		if (bound) {
6042 			rdma_roce_rescan_device(&dev->ib_dev);
6043 			break;
6044 		}
6045 	}
6046 
6047 	if (!bound) {
6048 		list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
6049 		dev_dbg(&mdev->pdev->dev, "no suitable IB device found to bind to, added to unaffiliated list.\n");
6050 	} else {
6051 		mlx5_ib_dbg(dev, "bound port %u\n", port_num + 1);
6052 	}
6053 	mutex_unlock(&mlx5_ib_multiport_mutex);
6054 
6055 	return mpi;
6056 }
6057 
6058 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
6059 {
6060 	enum rdma_link_layer ll;
6061 	struct mlx5_ib_dev *dev;
6062 	int port_type_cap;
6063 
6064 	printk_once(KERN_INFO "%s", mlx5_version);
6065 
6066 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6067 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6068 
6069 	if (mlx5_core_is_mp_slave(mdev) && ll == IB_LINK_LAYER_ETHERNET) {
6070 		u8 port_num = mlx5_core_native_port_num(mdev) - 1;
6071 
6072 		return mlx5_ib_add_slave_port(mdev, port_num);
6073 	}
6074 
6075 	dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
6076 	if (!dev)
6077 		return NULL;
6078 
6079 	dev->mdev = mdev;
6080 	dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
6081 			     MLX5_CAP_GEN(mdev, num_vhca_ports));
6082 
6083 	if (MLX5_VPORT_MANAGER(mdev) &&
6084 	    mlx5_ib_eswitch_mode(mdev->priv.eswitch) == SRIOV_OFFLOADS) {
6085 		dev->rep = mlx5_ib_vport_rep(mdev->priv.eswitch, 0);
6086 
6087 		return __mlx5_ib_add(dev, &nic_rep_profile);
6088 	}
6089 
6090 	return __mlx5_ib_add(dev, &pf_profile);
6091 }
6092 
6093 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
6094 {
6095 	struct mlx5_ib_multiport_info *mpi;
6096 	struct mlx5_ib_dev *dev;
6097 
6098 	if (mlx5_core_is_mp_slave(mdev)) {
6099 		mpi = context;
6100 		mutex_lock(&mlx5_ib_multiport_mutex);
6101 		if (mpi->ibdev)
6102 			mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
6103 		list_del(&mpi->list);
6104 		mutex_unlock(&mlx5_ib_multiport_mutex);
6105 		return;
6106 	}
6107 
6108 	dev = context;
6109 	__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
6110 }
6111 
6112 static struct mlx5_interface mlx5_ib_interface = {
6113 	.add            = mlx5_ib_add,
6114 	.remove         = mlx5_ib_remove,
6115 	.event          = mlx5_ib_event,
6116 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
6117 	.pfault		= mlx5_ib_pfault,
6118 #endif
6119 	.protocol	= MLX5_INTERFACE_PROTOCOL_IB,
6120 };
6121 
6122 unsigned long mlx5_ib_get_xlt_emergency_page(void)
6123 {
6124 	mutex_lock(&xlt_emergency_page_mutex);
6125 	return xlt_emergency_page;
6126 }
6127 
6128 void mlx5_ib_put_xlt_emergency_page(void)
6129 {
6130 	mutex_unlock(&xlt_emergency_page_mutex);
6131 }
6132 
6133 static int __init mlx5_ib_init(void)
6134 {
6135 	int err;
6136 
6137 	xlt_emergency_page = __get_free_page(GFP_KERNEL);
6138 	if (!xlt_emergency_page)
6139 		return -ENOMEM;
6140 
6141 	mutex_init(&xlt_emergency_page_mutex);
6142 
6143 	mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
6144 	if (!mlx5_ib_event_wq) {
6145 		free_page(xlt_emergency_page);
6146 		return -ENOMEM;
6147 	}
6148 
6149 	mlx5_ib_odp_init();
6150 
6151 	err = mlx5_register_interface(&mlx5_ib_interface);
6152 
6153 	return err;
6154 }
6155 
6156 static void __exit mlx5_ib_cleanup(void)
6157 {
6158 	mlx5_unregister_interface(&mlx5_ib_interface);
6159 	destroy_workqueue(mlx5_ib_event_wq);
6160 	mutex_destroy(&xlt_emergency_page_mutex);
6161 	free_page(xlt_emergency_page);
6162 }
6163 
6164 module_init(mlx5_ib_init);
6165 module_exit(mlx5_ib_cleanup);
6166