1 /*
2  * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/string.h>
35 #include <linux/etherdevice.h>
36 
37 #include <linux/mlx4/cmd.h>
38 #include <linux/mlx4/qp.h>
39 #include <linux/export.h>
40 
41 #include "mlx4.h"
42 
43 int mlx4_get_mgm_entry_size(struct mlx4_dev *dev)
44 {
45 	return 1 << dev->oper_log_mgm_entry_size;
46 }
47 
48 int mlx4_get_qp_per_mgm(struct mlx4_dev *dev)
49 {
50 	return 4 * (mlx4_get_mgm_entry_size(dev) / 16 - 2);
51 }
52 
53 static int mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev *dev,
54 					struct mlx4_cmd_mailbox *mailbox,
55 					u32 size,
56 					u64 *reg_id)
57 {
58 	u64 imm;
59 	int err = 0;
60 
61 	err = mlx4_cmd_imm(dev, mailbox->dma, &imm, size, 0,
62 			   MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A,
63 			   MLX4_CMD_NATIVE);
64 	if (err)
65 		return err;
66 	*reg_id = imm;
67 
68 	return err;
69 }
70 
71 static int mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev *dev, u64 regid)
72 {
73 	int err = 0;
74 
75 	err = mlx4_cmd(dev, regid, 0, 0,
76 		       MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A,
77 		       MLX4_CMD_NATIVE);
78 
79 	return err;
80 }
81 
82 static int mlx4_READ_ENTRY(struct mlx4_dev *dev, int index,
83 			   struct mlx4_cmd_mailbox *mailbox)
84 {
85 	return mlx4_cmd_box(dev, 0, mailbox->dma, index, 0, MLX4_CMD_READ_MCG,
86 			    MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
87 }
88 
89 static int mlx4_WRITE_ENTRY(struct mlx4_dev *dev, int index,
90 			    struct mlx4_cmd_mailbox *mailbox)
91 {
92 	return mlx4_cmd(dev, mailbox->dma, index, 0, MLX4_CMD_WRITE_MCG,
93 			MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
94 }
95 
96 static int mlx4_WRITE_PROMISC(struct mlx4_dev *dev, u8 port, u8 steer,
97 			      struct mlx4_cmd_mailbox *mailbox)
98 {
99 	u32 in_mod;
100 
101 	in_mod = (u32) port << 16 | steer << 1;
102 	return mlx4_cmd(dev, mailbox->dma, in_mod, 0x1,
103 			MLX4_CMD_WRITE_MCG, MLX4_CMD_TIME_CLASS_A,
104 			MLX4_CMD_NATIVE);
105 }
106 
107 static int mlx4_GID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
108 			 u16 *hash, u8 op_mod)
109 {
110 	u64 imm;
111 	int err;
112 
113 	err = mlx4_cmd_imm(dev, mailbox->dma, &imm, 0, op_mod,
114 			   MLX4_CMD_MGID_HASH, MLX4_CMD_TIME_CLASS_A,
115 			   MLX4_CMD_NATIVE);
116 
117 	if (!err)
118 		*hash = imm;
119 
120 	return err;
121 }
122 
123 static struct mlx4_promisc_qp *get_promisc_qp(struct mlx4_dev *dev, u8 port,
124 					      enum mlx4_steer_type steer,
125 					      u32 qpn)
126 {
127 	struct mlx4_steer *s_steer;
128 	struct mlx4_promisc_qp *pqp;
129 
130 	if (port < 1 || port > dev->caps.num_ports)
131 		return NULL;
132 
133 	s_steer = &mlx4_priv(dev)->steer[port - 1];
134 
135 	list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) {
136 		if (pqp->qpn == qpn)
137 			return pqp;
138 	}
139 	/* not found */
140 	return NULL;
141 }
142 
143 /*
144  * Add new entry to steering data structure.
145  * All promisc QPs should be added as well
146  */
147 static int new_steering_entry(struct mlx4_dev *dev, u8 port,
148 			      enum mlx4_steer_type steer,
149 			      unsigned int index, u32 qpn)
150 {
151 	struct mlx4_steer *s_steer;
152 	struct mlx4_cmd_mailbox *mailbox;
153 	struct mlx4_mgm *mgm;
154 	u32 members_count;
155 	struct mlx4_steer_index *new_entry;
156 	struct mlx4_promisc_qp *pqp;
157 	struct mlx4_promisc_qp *dqp = NULL;
158 	u32 prot;
159 	int err;
160 
161 	if (port < 1 || port > dev->caps.num_ports)
162 		return -EINVAL;
163 
164 	s_steer = &mlx4_priv(dev)->steer[port - 1];
165 	new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
166 	if (!new_entry)
167 		return -ENOMEM;
168 
169 	INIT_LIST_HEAD(&new_entry->duplicates);
170 	new_entry->index = index;
171 	list_add_tail(&new_entry->list, &s_steer->steer_entries[steer]);
172 
173 	/* If the given qpn is also a promisc qp,
174 	 * it should be inserted to duplicates list
175 	 */
176 	pqp = get_promisc_qp(dev, port, steer, qpn);
177 	if (pqp) {
178 		dqp = kmalloc(sizeof(*dqp), GFP_KERNEL);
179 		if (!dqp) {
180 			err = -ENOMEM;
181 			goto out_alloc;
182 		}
183 		dqp->qpn = qpn;
184 		list_add_tail(&dqp->list, &new_entry->duplicates);
185 	}
186 
187 	/* if no promisc qps for this vep, we are done */
188 	if (list_empty(&s_steer->promisc_qps[steer]))
189 		return 0;
190 
191 	/* now need to add all the promisc qps to the new
192 	 * steering entry, as they should also receive the packets
193 	 * destined to this address */
194 	mailbox = mlx4_alloc_cmd_mailbox(dev);
195 	if (IS_ERR(mailbox)) {
196 		err = -ENOMEM;
197 		goto out_alloc;
198 	}
199 	mgm = mailbox->buf;
200 
201 	err = mlx4_READ_ENTRY(dev, index, mailbox);
202 	if (err)
203 		goto out_mailbox;
204 
205 	members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
206 	prot = be32_to_cpu(mgm->members_count) >> 30;
207 	list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) {
208 		/* don't add already existing qpn */
209 		if (pqp->qpn == qpn)
210 			continue;
211 		if (members_count == dev->caps.num_qp_per_mgm) {
212 			/* out of space */
213 			err = -ENOMEM;
214 			goto out_mailbox;
215 		}
216 
217 		/* add the qpn */
218 		mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK);
219 	}
220 	/* update the qps count and update the entry with all the promisc qps*/
221 	mgm->members_count = cpu_to_be32(members_count | (prot << 30));
222 	err = mlx4_WRITE_ENTRY(dev, index, mailbox);
223 
224 out_mailbox:
225 	mlx4_free_cmd_mailbox(dev, mailbox);
226 	if (!err)
227 		return 0;
228 out_alloc:
229 	if (dqp) {
230 		list_del(&dqp->list);
231 		kfree(dqp);
232 	}
233 	list_del(&new_entry->list);
234 	kfree(new_entry);
235 	return err;
236 }
237 
238 /* update the data structures with existing steering entry */
239 static int existing_steering_entry(struct mlx4_dev *dev, u8 port,
240 				   enum mlx4_steer_type steer,
241 				   unsigned int index, u32 qpn)
242 {
243 	struct mlx4_steer *s_steer;
244 	struct mlx4_steer_index *tmp_entry, *entry = NULL;
245 	struct mlx4_promisc_qp *pqp;
246 	struct mlx4_promisc_qp *dqp;
247 
248 	if (port < 1 || port > dev->caps.num_ports)
249 		return -EINVAL;
250 
251 	s_steer = &mlx4_priv(dev)->steer[port - 1];
252 
253 	pqp = get_promisc_qp(dev, port, steer, qpn);
254 	if (!pqp)
255 		return 0; /* nothing to do */
256 
257 	list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) {
258 		if (tmp_entry->index == index) {
259 			entry = tmp_entry;
260 			break;
261 		}
262 	}
263 	if (unlikely(!entry)) {
264 		mlx4_warn(dev, "Steering entry at index %x is not registered\n", index);
265 		return -EINVAL;
266 	}
267 
268 	/* the given qpn is listed as a promisc qpn
269 	 * we need to add it as a duplicate to this entry
270 	 * for future references */
271 	list_for_each_entry(dqp, &entry->duplicates, list) {
272 		if (qpn == dqp->qpn)
273 			return 0; /* qp is already duplicated */
274 	}
275 
276 	/* add the qp as a duplicate on this index */
277 	dqp = kmalloc(sizeof(*dqp), GFP_KERNEL);
278 	if (!dqp)
279 		return -ENOMEM;
280 	dqp->qpn = qpn;
281 	list_add_tail(&dqp->list, &entry->duplicates);
282 
283 	return 0;
284 }
285 
286 /* Check whether a qpn is a duplicate on steering entry
287  * If so, it should not be removed from mgm */
288 static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port,
289 				  enum mlx4_steer_type steer,
290 				  unsigned int index, u32 qpn)
291 {
292 	struct mlx4_steer *s_steer;
293 	struct mlx4_steer_index *tmp_entry, *entry = NULL;
294 	struct mlx4_promisc_qp *dqp, *tmp_dqp;
295 
296 	if (port < 1 || port > dev->caps.num_ports)
297 		return false;
298 
299 	s_steer = &mlx4_priv(dev)->steer[port - 1];
300 
301 	/* if qp is not promisc, it cannot be duplicated */
302 	if (!get_promisc_qp(dev, port, steer, qpn))
303 		return false;
304 
305 	/* The qp is promisc qp so it is a duplicate on this index
306 	 * Find the index entry, and remove the duplicate */
307 	list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) {
308 		if (tmp_entry->index == index) {
309 			entry = tmp_entry;
310 			break;
311 		}
312 	}
313 	if (unlikely(!entry)) {
314 		mlx4_warn(dev, "Steering entry for index %x is not registered\n", index);
315 		return false;
316 	}
317 	list_for_each_entry_safe(dqp, tmp_dqp, &entry->duplicates, list) {
318 		if (dqp->qpn == qpn) {
319 			list_del(&dqp->list);
320 			kfree(dqp);
321 		}
322 	}
323 	return true;
324 }
325 
326 /* Returns true if all the QPs != tqpn contained in this entry
327  * are Promisc QPs. Returns false otherwise.
328  */
329 static bool promisc_steering_entry(struct mlx4_dev *dev, u8 port,
330 				   enum mlx4_steer_type steer,
331 				   unsigned int index, u32 tqpn,
332 				   u32 *members_count)
333 {
334 	struct mlx4_cmd_mailbox *mailbox;
335 	struct mlx4_mgm *mgm;
336 	u32 m_count;
337 	bool ret = false;
338 	int i;
339 
340 	if (port < 1 || port > dev->caps.num_ports)
341 		return false;
342 
343 	mailbox = mlx4_alloc_cmd_mailbox(dev);
344 	if (IS_ERR(mailbox))
345 		return false;
346 	mgm = mailbox->buf;
347 
348 	if (mlx4_READ_ENTRY(dev, index, mailbox))
349 		goto out;
350 	m_count = be32_to_cpu(mgm->members_count) & 0xffffff;
351 	if (members_count)
352 		*members_count = m_count;
353 
354 	for (i = 0;  i < m_count; i++) {
355 		u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK;
356 		if (!get_promisc_qp(dev, port, steer, qpn) && qpn != tqpn) {
357 			/* the qp is not promisc, the entry can't be removed */
358 			goto out;
359 		}
360 	}
361 	ret = true;
362 out:
363 	mlx4_free_cmd_mailbox(dev, mailbox);
364 	return ret;
365 }
366 
367 /* IF a steering entry contains only promisc QPs, it can be removed. */
368 static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port,
369 				      enum mlx4_steer_type steer,
370 				      unsigned int index, u32 tqpn)
371 {
372 	struct mlx4_steer *s_steer;
373 	struct mlx4_steer_index *entry = NULL, *tmp_entry;
374 	u32 members_count;
375 	bool ret = false;
376 
377 	if (port < 1 || port > dev->caps.num_ports)
378 		return false;
379 
380 	s_steer = &mlx4_priv(dev)->steer[port - 1];
381 
382 	if (!promisc_steering_entry(dev, port, steer, index,
383 				    tqpn, &members_count))
384 		goto out;
385 
386 	/* All the qps currently registered for this entry are promiscuous,
387 	  * Checking for duplicates */
388 	ret = true;
389 	list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) {
390 		if (entry->index == index) {
391 			if (list_empty(&entry->duplicates) ||
392 			    members_count == 1) {
393 				struct mlx4_promisc_qp *pqp, *tmp_pqp;
394 				/* If there is only 1 entry in duplicates then
395 				 * this is the QP we want to delete, going over
396 				 * the list and deleting the entry.
397 				 */
398 				list_del(&entry->list);
399 				list_for_each_entry_safe(pqp, tmp_pqp,
400 							 &entry->duplicates,
401 							 list) {
402 					list_del(&pqp->list);
403 					kfree(pqp);
404 				}
405 				kfree(entry);
406 			} else {
407 				/* This entry contains duplicates so it shouldn't be removed */
408 				ret = false;
409 				goto out;
410 			}
411 		}
412 	}
413 
414 out:
415 	return ret;
416 }
417 
418 static int add_promisc_qp(struct mlx4_dev *dev, u8 port,
419 			  enum mlx4_steer_type steer, u32 qpn)
420 {
421 	struct mlx4_steer *s_steer;
422 	struct mlx4_cmd_mailbox *mailbox;
423 	struct mlx4_mgm *mgm;
424 	struct mlx4_steer_index *entry;
425 	struct mlx4_promisc_qp *pqp;
426 	struct mlx4_promisc_qp *dqp;
427 	u32 members_count;
428 	u32 prot;
429 	int i;
430 	bool found;
431 	int err;
432 	struct mlx4_priv *priv = mlx4_priv(dev);
433 
434 	if (port < 1 || port > dev->caps.num_ports)
435 		return -EINVAL;
436 
437 	s_steer = &mlx4_priv(dev)->steer[port - 1];
438 
439 	mutex_lock(&priv->mcg_table.mutex);
440 
441 	if (get_promisc_qp(dev, port, steer, qpn)) {
442 		err = 0;  /* Noting to do, already exists */
443 		goto out_mutex;
444 	}
445 
446 	pqp = kmalloc(sizeof(*pqp), GFP_KERNEL);
447 	if (!pqp) {
448 		err = -ENOMEM;
449 		goto out_mutex;
450 	}
451 	pqp->qpn = qpn;
452 
453 	mailbox = mlx4_alloc_cmd_mailbox(dev);
454 	if (IS_ERR(mailbox)) {
455 		err = -ENOMEM;
456 		goto out_alloc;
457 	}
458 	mgm = mailbox->buf;
459 
460 	if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) {
461 		/* The promisc QP needs to be added for each one of the steering
462 		 * entries. If it already exists, needs to be added as
463 		 * a duplicate for this entry.
464 		 */
465 		list_for_each_entry(entry,
466 				    &s_steer->steer_entries[steer],
467 				    list) {
468 			err = mlx4_READ_ENTRY(dev, entry->index, mailbox);
469 			if (err)
470 				goto out_mailbox;
471 
472 			members_count = be32_to_cpu(mgm->members_count) &
473 					0xffffff;
474 			prot = be32_to_cpu(mgm->members_count) >> 30;
475 			found = false;
476 			for (i = 0; i < members_count; i++) {
477 				if ((be32_to_cpu(mgm->qp[i]) &
478 				     MGM_QPN_MASK) == qpn) {
479 					/* Entry already exists.
480 					 * Add to duplicates.
481 					 */
482 					dqp = kmalloc(sizeof(*dqp), GFP_KERNEL);
483 					if (!dqp) {
484 						err = -ENOMEM;
485 						goto out_mailbox;
486 					}
487 					dqp->qpn = qpn;
488 					list_add_tail(&dqp->list,
489 						      &entry->duplicates);
490 					found = true;
491 				}
492 			}
493 			if (!found) {
494 				/* Need to add the qpn to mgm */
495 				if (members_count ==
496 				    dev->caps.num_qp_per_mgm) {
497 					/* entry is full */
498 					err = -ENOMEM;
499 					goto out_mailbox;
500 				}
501 				mgm->qp[members_count++] =
502 					cpu_to_be32(qpn & MGM_QPN_MASK);
503 				mgm->members_count =
504 					cpu_to_be32(members_count |
505 						    (prot << 30));
506 				err = mlx4_WRITE_ENTRY(dev, entry->index,
507 						       mailbox);
508 				if (err)
509 					goto out_mailbox;
510 			}
511 		}
512 	}
513 
514 	/* add the new qpn to list of promisc qps */
515 	list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]);
516 	/* now need to add all the promisc qps to default entry */
517 	memset(mgm, 0, sizeof(*mgm));
518 	members_count = 0;
519 	list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) {
520 		if (members_count == dev->caps.num_qp_per_mgm) {
521 			/* entry is full */
522 			err = -ENOMEM;
523 			goto out_list;
524 		}
525 		mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK);
526 	}
527 	mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30);
528 
529 	err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox);
530 	if (err)
531 		goto out_list;
532 
533 	mlx4_free_cmd_mailbox(dev, mailbox);
534 	mutex_unlock(&priv->mcg_table.mutex);
535 	return 0;
536 
537 out_list:
538 	list_del(&pqp->list);
539 out_mailbox:
540 	mlx4_free_cmd_mailbox(dev, mailbox);
541 out_alloc:
542 	kfree(pqp);
543 out_mutex:
544 	mutex_unlock(&priv->mcg_table.mutex);
545 	return err;
546 }
547 
548 static int remove_promisc_qp(struct mlx4_dev *dev, u8 port,
549 			     enum mlx4_steer_type steer, u32 qpn)
550 {
551 	struct mlx4_priv *priv = mlx4_priv(dev);
552 	struct mlx4_steer *s_steer;
553 	struct mlx4_cmd_mailbox *mailbox;
554 	struct mlx4_mgm *mgm;
555 	struct mlx4_steer_index *entry, *tmp_entry;
556 	struct mlx4_promisc_qp *pqp;
557 	struct mlx4_promisc_qp *dqp;
558 	u32 members_count;
559 	bool found;
560 	bool back_to_list = false;
561 	int i;
562 	int err;
563 
564 	if (port < 1 || port > dev->caps.num_ports)
565 		return -EINVAL;
566 
567 	s_steer = &mlx4_priv(dev)->steer[port - 1];
568 	mutex_lock(&priv->mcg_table.mutex);
569 
570 	pqp = get_promisc_qp(dev, port, steer, qpn);
571 	if (unlikely(!pqp)) {
572 		mlx4_warn(dev, "QP %x is not promiscuous QP\n", qpn);
573 		/* nothing to do */
574 		err = 0;
575 		goto out_mutex;
576 	}
577 
578 	/*remove from list of promisc qps */
579 	list_del(&pqp->list);
580 
581 	/* set the default entry not to include the removed one */
582 	mailbox = mlx4_alloc_cmd_mailbox(dev);
583 	if (IS_ERR(mailbox)) {
584 		err = -ENOMEM;
585 		back_to_list = true;
586 		goto out_list;
587 	}
588 	mgm = mailbox->buf;
589 	members_count = 0;
590 	list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list)
591 		mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK);
592 	mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30);
593 
594 	err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox);
595 	if (err)
596 		goto out_mailbox;
597 
598 	if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) {
599 		/* Remove the QP from all the steering entries */
600 		list_for_each_entry_safe(entry, tmp_entry,
601 					 &s_steer->steer_entries[steer],
602 					 list) {
603 			found = false;
604 			list_for_each_entry(dqp, &entry->duplicates, list) {
605 				if (dqp->qpn == qpn) {
606 					found = true;
607 					break;
608 				}
609 			}
610 			if (found) {
611 				/* A duplicate, no need to change the MGM,
612 				 * only update the duplicates list
613 				 */
614 				list_del(&dqp->list);
615 				kfree(dqp);
616 			} else {
617 				int loc = -1;
618 
619 				err = mlx4_READ_ENTRY(dev,
620 						      entry->index,
621 						      mailbox);
622 				if (err)
623 					goto out_mailbox;
624 				members_count =
625 					be32_to_cpu(mgm->members_count) &
626 					0xffffff;
627 				if (!members_count) {
628 					mlx4_warn(dev, "QP %06x wasn't found in entry %x mcount=0. deleting entry...\n",
629 						  qpn, entry->index);
630 					list_del(&entry->list);
631 					kfree(entry);
632 					continue;
633 				}
634 
635 				for (i = 0; i < members_count; ++i)
636 					if ((be32_to_cpu(mgm->qp[i]) &
637 					     MGM_QPN_MASK) == qpn) {
638 						loc = i;
639 						break;
640 					}
641 
642 				if (loc < 0) {
643 					mlx4_err(dev, "QP %06x wasn't found in entry %d\n",
644 						 qpn, entry->index);
645 					err = -EINVAL;
646 					goto out_mailbox;
647 				}
648 
649 				/* Copy the last QP in this MGM
650 				 * over removed QP
651 				 */
652 				mgm->qp[loc] = mgm->qp[members_count - 1];
653 				mgm->qp[members_count - 1] = 0;
654 				mgm->members_count =
655 					cpu_to_be32(--members_count |
656 						    (MLX4_PROT_ETH << 30));
657 
658 				err = mlx4_WRITE_ENTRY(dev,
659 						       entry->index,
660 						       mailbox);
661 				if (err)
662 					goto out_mailbox;
663 			}
664 		}
665 	}
666 
667 out_mailbox:
668 	mlx4_free_cmd_mailbox(dev, mailbox);
669 out_list:
670 	if (back_to_list)
671 		list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]);
672 	else
673 		kfree(pqp);
674 out_mutex:
675 	mutex_unlock(&priv->mcg_table.mutex);
676 	return err;
677 }
678 
679 /*
680  * Caller must hold MCG table semaphore.  gid and mgm parameters must
681  * be properly aligned for command interface.
682  *
683  *  Returns 0 unless a firmware command error occurs.
684  *
685  * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1
686  * and *mgm holds MGM entry.
687  *
688  * if GID is found in AMGM, *index = index in AMGM, *prev = index of
689  * previous entry in hash chain and *mgm holds AMGM entry.
690  *
691  * If no AMGM exists for given gid, *index = -1, *prev = index of last
692  * entry in hash chain and *mgm holds end of hash chain.
693  */
694 static int find_entry(struct mlx4_dev *dev, u8 port,
695 		      u8 *gid, enum mlx4_protocol prot,
696 		      struct mlx4_cmd_mailbox *mgm_mailbox,
697 		      int *prev, int *index)
698 {
699 	struct mlx4_cmd_mailbox *mailbox;
700 	struct mlx4_mgm *mgm = mgm_mailbox->buf;
701 	u8 *mgid;
702 	int err;
703 	u16 hash;
704 	u8 op_mod = (prot == MLX4_PROT_ETH) ?
705 		!!(dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) : 0;
706 
707 	mailbox = mlx4_alloc_cmd_mailbox(dev);
708 	if (IS_ERR(mailbox))
709 		return -ENOMEM;
710 	mgid = mailbox->buf;
711 
712 	memcpy(mgid, gid, 16);
713 
714 	err = mlx4_GID_HASH(dev, mailbox, &hash, op_mod);
715 	mlx4_free_cmd_mailbox(dev, mailbox);
716 	if (err)
717 		return err;
718 
719 	if (0)
720 		mlx4_dbg(dev, "Hash for %pI6 is %04x\n", gid, hash);
721 
722 	*index = hash;
723 	*prev  = -1;
724 
725 	do {
726 		err = mlx4_READ_ENTRY(dev, *index, mgm_mailbox);
727 		if (err)
728 			return err;
729 
730 		if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) {
731 			if (*index != hash) {
732 				mlx4_err(dev, "Found zero MGID in AMGM\n");
733 				err = -EINVAL;
734 			}
735 			return err;
736 		}
737 
738 		if (!memcmp(mgm->gid, gid, 16) &&
739 		    be32_to_cpu(mgm->members_count) >> 30 == prot)
740 			return err;
741 
742 		*prev = *index;
743 		*index = be32_to_cpu(mgm->next_gid_index) >> 6;
744 	} while (*index);
745 
746 	*index = -1;
747 	return err;
748 }
749 
750 static const u8 __promisc_mode[] = {
751 	[MLX4_FS_REGULAR]   = 0x0,
752 	[MLX4_FS_ALL_DEFAULT] = 0x1,
753 	[MLX4_FS_MC_DEFAULT] = 0x3,
754 	[MLX4_FS_MIRROR_RX_PORT] = 0x4,
755 	[MLX4_FS_MIRROR_SX_PORT] = 0x5,
756 	[MLX4_FS_UC_SNIFFER] = 0x6,
757 	[MLX4_FS_MC_SNIFFER] = 0x7,
758 };
759 
760 int mlx4_map_sw_to_hw_steering_mode(struct mlx4_dev *dev,
761 				    enum mlx4_net_trans_promisc_mode flow_type)
762 {
763 	if (flow_type >= MLX4_FS_MODE_NUM) {
764 		mlx4_err(dev, "Invalid flow type. type = %d\n", flow_type);
765 		return -EINVAL;
766 	}
767 	return __promisc_mode[flow_type];
768 }
769 EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_mode);
770 
771 static void trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule *ctrl,
772 				  struct mlx4_net_trans_rule_hw_ctrl *hw)
773 {
774 	u8 flags = 0;
775 
776 	flags = ctrl->queue_mode == MLX4_NET_TRANS_Q_LIFO ? 1 : 0;
777 	flags |= ctrl->exclusive ? (1 << 2) : 0;
778 	flags |= ctrl->allow_loopback ? (1 << 3) : 0;
779 
780 	hw->flags = flags;
781 	hw->type = __promisc_mode[ctrl->promisc_mode];
782 	hw->prio = cpu_to_be16(ctrl->priority);
783 	hw->port = ctrl->port;
784 	hw->qpn = cpu_to_be32(ctrl->qpn);
785 }
786 
787 const u16 __sw_id_hw[] = {
788 	[MLX4_NET_TRANS_RULE_ID_ETH]     = 0xE001,
789 	[MLX4_NET_TRANS_RULE_ID_IB]      = 0xE005,
790 	[MLX4_NET_TRANS_RULE_ID_IPV6]    = 0xE003,
791 	[MLX4_NET_TRANS_RULE_ID_IPV4]    = 0xE002,
792 	[MLX4_NET_TRANS_RULE_ID_TCP]     = 0xE004,
793 	[MLX4_NET_TRANS_RULE_ID_UDP]     = 0xE006,
794 	[MLX4_NET_TRANS_RULE_ID_VXLAN]	 = 0xE008
795 };
796 
797 int mlx4_map_sw_to_hw_steering_id(struct mlx4_dev *dev,
798 				  enum mlx4_net_trans_rule_id id)
799 {
800 	if (id >= MLX4_NET_TRANS_RULE_NUM) {
801 		mlx4_err(dev, "Invalid network rule id. id = %d\n", id);
802 		return -EINVAL;
803 	}
804 	return __sw_id_hw[id];
805 }
806 EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_id);
807 
808 static const int __rule_hw_sz[] = {
809 	[MLX4_NET_TRANS_RULE_ID_ETH] =
810 		sizeof(struct mlx4_net_trans_rule_hw_eth),
811 	[MLX4_NET_TRANS_RULE_ID_IB] =
812 		sizeof(struct mlx4_net_trans_rule_hw_ib),
813 	[MLX4_NET_TRANS_RULE_ID_IPV6] = 0,
814 	[MLX4_NET_TRANS_RULE_ID_IPV4] =
815 		sizeof(struct mlx4_net_trans_rule_hw_ipv4),
816 	[MLX4_NET_TRANS_RULE_ID_TCP] =
817 		sizeof(struct mlx4_net_trans_rule_hw_tcp_udp),
818 	[MLX4_NET_TRANS_RULE_ID_UDP] =
819 		sizeof(struct mlx4_net_trans_rule_hw_tcp_udp),
820 	[MLX4_NET_TRANS_RULE_ID_VXLAN] =
821 		sizeof(struct mlx4_net_trans_rule_hw_vxlan)
822 };
823 
824 int mlx4_hw_rule_sz(struct mlx4_dev *dev,
825 	       enum mlx4_net_trans_rule_id id)
826 {
827 	if (id >= MLX4_NET_TRANS_RULE_NUM) {
828 		mlx4_err(dev, "Invalid network rule id. id = %d\n", id);
829 		return -EINVAL;
830 	}
831 
832 	return __rule_hw_sz[id];
833 }
834 EXPORT_SYMBOL_GPL(mlx4_hw_rule_sz);
835 
836 static int parse_trans_rule(struct mlx4_dev *dev, struct mlx4_spec_list *spec,
837 			    struct _rule_hw *rule_hw)
838 {
839 	if (mlx4_hw_rule_sz(dev, spec->id) < 0)
840 		return -EINVAL;
841 	memset(rule_hw, 0, mlx4_hw_rule_sz(dev, spec->id));
842 	rule_hw->id = cpu_to_be16(__sw_id_hw[spec->id]);
843 	rule_hw->size = mlx4_hw_rule_sz(dev, spec->id) >> 2;
844 
845 	switch (spec->id) {
846 	case MLX4_NET_TRANS_RULE_ID_ETH:
847 		memcpy(rule_hw->eth.dst_mac, spec->eth.dst_mac, ETH_ALEN);
848 		memcpy(rule_hw->eth.dst_mac_msk, spec->eth.dst_mac_msk,
849 		       ETH_ALEN);
850 		memcpy(rule_hw->eth.src_mac, spec->eth.src_mac, ETH_ALEN);
851 		memcpy(rule_hw->eth.src_mac_msk, spec->eth.src_mac_msk,
852 		       ETH_ALEN);
853 		if (spec->eth.ether_type_enable) {
854 			rule_hw->eth.ether_type_enable = 1;
855 			rule_hw->eth.ether_type = spec->eth.ether_type;
856 		}
857 		rule_hw->eth.vlan_tag = spec->eth.vlan_id;
858 		rule_hw->eth.vlan_tag_msk = spec->eth.vlan_id_msk;
859 		break;
860 
861 	case MLX4_NET_TRANS_RULE_ID_IB:
862 		rule_hw->ib.l3_qpn = spec->ib.l3_qpn;
863 		rule_hw->ib.qpn_mask = spec->ib.qpn_msk;
864 		memcpy(&rule_hw->ib.dst_gid, &spec->ib.dst_gid, 16);
865 		memcpy(&rule_hw->ib.dst_gid_msk, &spec->ib.dst_gid_msk, 16);
866 		break;
867 
868 	case MLX4_NET_TRANS_RULE_ID_IPV6:
869 		return -EOPNOTSUPP;
870 
871 	case MLX4_NET_TRANS_RULE_ID_IPV4:
872 		rule_hw->ipv4.src_ip = spec->ipv4.src_ip;
873 		rule_hw->ipv4.src_ip_msk = spec->ipv4.src_ip_msk;
874 		rule_hw->ipv4.dst_ip = spec->ipv4.dst_ip;
875 		rule_hw->ipv4.dst_ip_msk = spec->ipv4.dst_ip_msk;
876 		break;
877 
878 	case MLX4_NET_TRANS_RULE_ID_TCP:
879 	case MLX4_NET_TRANS_RULE_ID_UDP:
880 		rule_hw->tcp_udp.dst_port = spec->tcp_udp.dst_port;
881 		rule_hw->tcp_udp.dst_port_msk = spec->tcp_udp.dst_port_msk;
882 		rule_hw->tcp_udp.src_port = spec->tcp_udp.src_port;
883 		rule_hw->tcp_udp.src_port_msk = spec->tcp_udp.src_port_msk;
884 		break;
885 
886 	case MLX4_NET_TRANS_RULE_ID_VXLAN:
887 		rule_hw->vxlan.vni =
888 			cpu_to_be32(be32_to_cpu(spec->vxlan.vni) << 8);
889 		rule_hw->vxlan.vni_mask =
890 			cpu_to_be32(be32_to_cpu(spec->vxlan.vni_mask) << 8);
891 		break;
892 
893 	default:
894 		return -EINVAL;
895 	}
896 
897 	return __rule_hw_sz[spec->id];
898 }
899 
900 static void mlx4_err_rule(struct mlx4_dev *dev, char *str,
901 			  struct mlx4_net_trans_rule *rule)
902 {
903 #define BUF_SIZE 256
904 	struct mlx4_spec_list *cur;
905 	char buf[BUF_SIZE];
906 	int len = 0;
907 
908 	mlx4_err(dev, "%s", str);
909 	len += scnprintf(buf + len, BUF_SIZE - len,
910 			 "port = %d prio = 0x%x qp = 0x%x ",
911 			 rule->port, rule->priority, rule->qpn);
912 
913 	list_for_each_entry(cur, &rule->list, list) {
914 		switch (cur->id) {
915 		case MLX4_NET_TRANS_RULE_ID_ETH:
916 			len += scnprintf(buf + len, BUF_SIZE - len,
917 					 "dmac = %pM ", &cur->eth.dst_mac);
918 			if (cur->eth.ether_type)
919 				len += scnprintf(buf + len, BUF_SIZE - len,
920 						 "ethertype = 0x%x ",
921 						 be16_to_cpu(cur->eth.ether_type));
922 			if (cur->eth.vlan_id)
923 				len += scnprintf(buf + len, BUF_SIZE - len,
924 						 "vlan-id = %d ",
925 						 be16_to_cpu(cur->eth.vlan_id));
926 			break;
927 
928 		case MLX4_NET_TRANS_RULE_ID_IPV4:
929 			if (cur->ipv4.src_ip)
930 				len += scnprintf(buf + len, BUF_SIZE - len,
931 						 "src-ip = %pI4 ",
932 						 &cur->ipv4.src_ip);
933 			if (cur->ipv4.dst_ip)
934 				len += scnprintf(buf + len, BUF_SIZE - len,
935 						 "dst-ip = %pI4 ",
936 						 &cur->ipv4.dst_ip);
937 			break;
938 
939 		case MLX4_NET_TRANS_RULE_ID_TCP:
940 		case MLX4_NET_TRANS_RULE_ID_UDP:
941 			if (cur->tcp_udp.src_port)
942 				len += scnprintf(buf + len, BUF_SIZE - len,
943 						 "src-port = %d ",
944 						 be16_to_cpu(cur->tcp_udp.src_port));
945 			if (cur->tcp_udp.dst_port)
946 				len += scnprintf(buf + len, BUF_SIZE - len,
947 						 "dst-port = %d ",
948 						 be16_to_cpu(cur->tcp_udp.dst_port));
949 			break;
950 
951 		case MLX4_NET_TRANS_RULE_ID_IB:
952 			len += scnprintf(buf + len, BUF_SIZE - len,
953 					 "dst-gid = %pI6\n", cur->ib.dst_gid);
954 			len += scnprintf(buf + len, BUF_SIZE - len,
955 					 "dst-gid-mask = %pI6\n",
956 					 cur->ib.dst_gid_msk);
957 			break;
958 
959 		case MLX4_NET_TRANS_RULE_ID_VXLAN:
960 			len += scnprintf(buf + len, BUF_SIZE - len,
961 					 "VNID = %d ", be32_to_cpu(cur->vxlan.vni));
962 			break;
963 		case MLX4_NET_TRANS_RULE_ID_IPV6:
964 			break;
965 
966 		default:
967 			break;
968 		}
969 	}
970 	len += scnprintf(buf + len, BUF_SIZE - len, "\n");
971 	mlx4_err(dev, "%s", buf);
972 
973 	if (len >= BUF_SIZE)
974 		mlx4_err(dev, "Network rule error message was truncated, print buffer is too small\n");
975 }
976 
977 int mlx4_flow_attach(struct mlx4_dev *dev,
978 		     struct mlx4_net_trans_rule *rule, u64 *reg_id)
979 {
980 	struct mlx4_cmd_mailbox *mailbox;
981 	struct mlx4_spec_list *cur;
982 	u32 size = 0;
983 	int ret;
984 
985 	mailbox = mlx4_alloc_cmd_mailbox(dev);
986 	if (IS_ERR(mailbox))
987 		return PTR_ERR(mailbox);
988 
989 	if (!mlx4_qp_lookup(dev, rule->qpn)) {
990 		mlx4_err_rule(dev, "QP doesn't exist\n", rule);
991 		ret = -EINVAL;
992 		goto out;
993 	}
994 
995 	trans_rule_ctrl_to_hw(rule, mailbox->buf);
996 
997 	size += sizeof(struct mlx4_net_trans_rule_hw_ctrl);
998 
999 	list_for_each_entry(cur, &rule->list, list) {
1000 		ret = parse_trans_rule(dev, cur, mailbox->buf + size);
1001 		if (ret < 0)
1002 			goto out;
1003 
1004 		size += ret;
1005 	}
1006 
1007 	ret = mlx4_QP_FLOW_STEERING_ATTACH(dev, mailbox, size >> 2, reg_id);
1008 	if (ret == -ENOMEM) {
1009 		mlx4_err_rule(dev,
1010 			      "mcg table is full. Fail to register network rule\n",
1011 			      rule);
1012 	} else if (ret) {
1013 		if (ret == -ENXIO) {
1014 			if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED)
1015 				mlx4_err_rule(dev,
1016 					      "DMFS is not enabled, "
1017 					      "failed to register network rule.\n",
1018 					      rule);
1019 			else
1020 				mlx4_err_rule(dev,
1021 					      "Rule exceeds the dmfs_high_rate_mode limitations, "
1022 					      "failed to register network rule.\n",
1023 					      rule);
1024 
1025 		} else {
1026 			mlx4_err_rule(dev, "Fail to register network rule.\n", rule);
1027 		}
1028 	}
1029 
1030 out:
1031 	mlx4_free_cmd_mailbox(dev, mailbox);
1032 
1033 	return ret;
1034 }
1035 EXPORT_SYMBOL_GPL(mlx4_flow_attach);
1036 
1037 int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id)
1038 {
1039 	int err;
1040 
1041 	err = mlx4_QP_FLOW_STEERING_DETACH(dev, reg_id);
1042 	if (err)
1043 		mlx4_err(dev, "Fail to detach network rule. registration id = 0x%llx\n",
1044 			 reg_id);
1045 	return err;
1046 }
1047 EXPORT_SYMBOL_GPL(mlx4_flow_detach);
1048 
1049 int mlx4_tunnel_steer_add(struct mlx4_dev *dev, const unsigned char *addr,
1050 			  int port, int qpn, u16 prio, u64 *reg_id)
1051 {
1052 	int err;
1053 	struct mlx4_spec_list spec_eth_outer = { {NULL} };
1054 	struct mlx4_spec_list spec_vxlan     = { {NULL} };
1055 	struct mlx4_spec_list spec_eth_inner = { {NULL} };
1056 
1057 	struct mlx4_net_trans_rule rule = {
1058 		.queue_mode = MLX4_NET_TRANS_Q_FIFO,
1059 		.exclusive = 0,
1060 		.allow_loopback = 1,
1061 		.promisc_mode = MLX4_FS_REGULAR,
1062 	};
1063 
1064 	__be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
1065 
1066 	rule.port = port;
1067 	rule.qpn = qpn;
1068 	rule.priority = prio;
1069 	INIT_LIST_HEAD(&rule.list);
1070 
1071 	spec_eth_outer.id = MLX4_NET_TRANS_RULE_ID_ETH;
1072 	memcpy(spec_eth_outer.eth.dst_mac, addr, ETH_ALEN);
1073 	memcpy(spec_eth_outer.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
1074 
1075 	spec_vxlan.id = MLX4_NET_TRANS_RULE_ID_VXLAN;    /* any vxlan header */
1076 	spec_eth_inner.id = MLX4_NET_TRANS_RULE_ID_ETH;	 /* any inner eth header */
1077 
1078 	list_add_tail(&spec_eth_outer.list, &rule.list);
1079 	list_add_tail(&spec_vxlan.list,     &rule.list);
1080 	list_add_tail(&spec_eth_inner.list, &rule.list);
1081 
1082 	err = mlx4_flow_attach(dev, &rule, reg_id);
1083 	return err;
1084 }
1085 EXPORT_SYMBOL(mlx4_tunnel_steer_add);
1086 
1087 int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn,
1088 				      u32 max_range_qpn)
1089 {
1090 	int err;
1091 	u64 in_param;
1092 
1093 	in_param = ((u64) min_range_qpn) << 32;
1094 	in_param |= ((u64) max_range_qpn) & 0xFFFFFFFF;
1095 
1096 	err = mlx4_cmd(dev, in_param, 0, 0,
1097 			MLX4_FLOW_STEERING_IB_UC_QP_RANGE,
1098 			MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
1099 
1100 	return err;
1101 }
1102 EXPORT_SYMBOL_GPL(mlx4_FLOW_STEERING_IB_UC_QP_RANGE);
1103 
1104 int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1105 			  int block_mcast_loopback, enum mlx4_protocol prot,
1106 			  enum mlx4_steer_type steer)
1107 {
1108 	struct mlx4_priv *priv = mlx4_priv(dev);
1109 	struct mlx4_cmd_mailbox *mailbox;
1110 	struct mlx4_mgm *mgm;
1111 	u32 members_count;
1112 	int index = -1, prev;
1113 	int link = 0;
1114 	int i;
1115 	int err;
1116 	u8 port = gid[5];
1117 	u8 new_entry = 0;
1118 
1119 	mailbox = mlx4_alloc_cmd_mailbox(dev);
1120 	if (IS_ERR(mailbox))
1121 		return PTR_ERR(mailbox);
1122 	mgm = mailbox->buf;
1123 
1124 	mutex_lock(&priv->mcg_table.mutex);
1125 	err = find_entry(dev, port, gid, prot,
1126 			 mailbox, &prev, &index);
1127 	if (err)
1128 		goto out;
1129 
1130 	if (index != -1) {
1131 		if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) {
1132 			new_entry = 1;
1133 			memcpy(mgm->gid, gid, 16);
1134 		}
1135 	} else {
1136 		link = 1;
1137 
1138 		index = mlx4_bitmap_alloc(&priv->mcg_table.bitmap);
1139 		if (index == -1) {
1140 			mlx4_err(dev, "No AMGM entries left\n");
1141 			err = -ENOMEM;
1142 			goto out;
1143 		}
1144 		index += dev->caps.num_mgms;
1145 
1146 		new_entry = 1;
1147 		memset(mgm, 0, sizeof(*mgm));
1148 		memcpy(mgm->gid, gid, 16);
1149 	}
1150 
1151 	members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
1152 	if (members_count == dev->caps.num_qp_per_mgm) {
1153 		mlx4_err(dev, "MGM at index %x is full\n", index);
1154 		err = -ENOMEM;
1155 		goto out;
1156 	}
1157 
1158 	for (i = 0; i < members_count; ++i)
1159 		if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) {
1160 			mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn);
1161 			err = 0;
1162 			goto out;
1163 		}
1164 
1165 	if (block_mcast_loopback)
1166 		mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) |
1167 						       (1U << MGM_BLCK_LB_BIT));
1168 	else
1169 		mgm->qp[members_count++] = cpu_to_be32(qp->qpn & MGM_QPN_MASK);
1170 
1171 	mgm->members_count = cpu_to_be32(members_count | (u32) prot << 30);
1172 
1173 	err = mlx4_WRITE_ENTRY(dev, index, mailbox);
1174 	if (err)
1175 		goto out;
1176 
1177 	if (!link)
1178 		goto out;
1179 
1180 	err = mlx4_READ_ENTRY(dev, prev, mailbox);
1181 	if (err)
1182 		goto out;
1183 
1184 	mgm->next_gid_index = cpu_to_be32(index << 6);
1185 
1186 	err = mlx4_WRITE_ENTRY(dev, prev, mailbox);
1187 	if (err)
1188 		goto out;
1189 
1190 out:
1191 	if (prot == MLX4_PROT_ETH && index != -1) {
1192 		/* manage the steering entry for promisc mode */
1193 		if (new_entry)
1194 			err = new_steering_entry(dev, port, steer,
1195 						 index, qp->qpn);
1196 		else
1197 			err = existing_steering_entry(dev, port, steer,
1198 						      index, qp->qpn);
1199 	}
1200 	if (err && link && index != -1) {
1201 		if (index < dev->caps.num_mgms)
1202 			mlx4_warn(dev, "Got AMGM index %d < %d\n",
1203 				  index, dev->caps.num_mgms);
1204 		else
1205 			mlx4_bitmap_free(&priv->mcg_table.bitmap,
1206 					 index - dev->caps.num_mgms, MLX4_USE_RR);
1207 	}
1208 	mutex_unlock(&priv->mcg_table.mutex);
1209 
1210 	mlx4_free_cmd_mailbox(dev, mailbox);
1211 	return err;
1212 }
1213 
1214 int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1215 			  enum mlx4_protocol prot, enum mlx4_steer_type steer)
1216 {
1217 	struct mlx4_priv *priv = mlx4_priv(dev);
1218 	struct mlx4_cmd_mailbox *mailbox;
1219 	struct mlx4_mgm *mgm;
1220 	u32 members_count;
1221 	int prev, index;
1222 	int i, loc = -1;
1223 	int err;
1224 	u8 port = gid[5];
1225 	bool removed_entry = false;
1226 
1227 	mailbox = mlx4_alloc_cmd_mailbox(dev);
1228 	if (IS_ERR(mailbox))
1229 		return PTR_ERR(mailbox);
1230 	mgm = mailbox->buf;
1231 
1232 	mutex_lock(&priv->mcg_table.mutex);
1233 
1234 	err = find_entry(dev, port, gid, prot,
1235 			 mailbox, &prev, &index);
1236 	if (err)
1237 		goto out;
1238 
1239 	if (index == -1) {
1240 		mlx4_err(dev, "MGID %pI6 not found\n", gid);
1241 		err = -EINVAL;
1242 		goto out;
1243 	}
1244 
1245 	/* If this QP is also a promisc QP, it shouldn't be removed only if
1246 	 * at least one none promisc QP is also attached to this MCG
1247 	 */
1248 	if (prot == MLX4_PROT_ETH &&
1249 	    check_duplicate_entry(dev, port, steer, index, qp->qpn) &&
1250 	    !promisc_steering_entry(dev, port, steer, index, qp->qpn, NULL))
1251 			goto out;
1252 
1253 	members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
1254 	for (i = 0; i < members_count; ++i)
1255 		if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) {
1256 			loc = i;
1257 			break;
1258 		}
1259 
1260 	if (loc == -1) {
1261 		mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn);
1262 		err = -EINVAL;
1263 		goto out;
1264 	}
1265 
1266 	/* copy the last QP in this MGM over removed QP */
1267 	mgm->qp[loc] = mgm->qp[members_count - 1];
1268 	mgm->qp[members_count - 1] = 0;
1269 	mgm->members_count = cpu_to_be32(--members_count | (u32) prot << 30);
1270 
1271 	if (prot == MLX4_PROT_ETH)
1272 		removed_entry = can_remove_steering_entry(dev, port, steer,
1273 								index, qp->qpn);
1274 	if (members_count && (prot != MLX4_PROT_ETH || !removed_entry)) {
1275 		err = mlx4_WRITE_ENTRY(dev, index, mailbox);
1276 		goto out;
1277 	}
1278 
1279 	/* We are going to delete the entry, members count should be 0 */
1280 	mgm->members_count = cpu_to_be32((u32) prot << 30);
1281 
1282 	if (prev == -1) {
1283 		/* Remove entry from MGM */
1284 		int amgm_index = be32_to_cpu(mgm->next_gid_index) >> 6;
1285 		if (amgm_index) {
1286 			err = mlx4_READ_ENTRY(dev, amgm_index, mailbox);
1287 			if (err)
1288 				goto out;
1289 		} else
1290 			memset(mgm->gid, 0, 16);
1291 
1292 		err = mlx4_WRITE_ENTRY(dev, index, mailbox);
1293 		if (err)
1294 			goto out;
1295 
1296 		if (amgm_index) {
1297 			if (amgm_index < dev->caps.num_mgms)
1298 				mlx4_warn(dev, "MGM entry %d had AMGM index %d < %d\n",
1299 					  index, amgm_index, dev->caps.num_mgms);
1300 			else
1301 				mlx4_bitmap_free(&priv->mcg_table.bitmap,
1302 						 amgm_index - dev->caps.num_mgms, MLX4_USE_RR);
1303 		}
1304 	} else {
1305 		/* Remove entry from AMGM */
1306 		int cur_next_index = be32_to_cpu(mgm->next_gid_index) >> 6;
1307 		err = mlx4_READ_ENTRY(dev, prev, mailbox);
1308 		if (err)
1309 			goto out;
1310 
1311 		mgm->next_gid_index = cpu_to_be32(cur_next_index << 6);
1312 
1313 		err = mlx4_WRITE_ENTRY(dev, prev, mailbox);
1314 		if (err)
1315 			goto out;
1316 
1317 		if (index < dev->caps.num_mgms)
1318 			mlx4_warn(dev, "entry %d had next AMGM index %d < %d\n",
1319 				  prev, index, dev->caps.num_mgms);
1320 		else
1321 			mlx4_bitmap_free(&priv->mcg_table.bitmap,
1322 					 index - dev->caps.num_mgms, MLX4_USE_RR);
1323 	}
1324 
1325 out:
1326 	mutex_unlock(&priv->mcg_table.mutex);
1327 
1328 	mlx4_free_cmd_mailbox(dev, mailbox);
1329 	if (err && dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)
1330 		/* In case device is under an error, return success as a closing command */
1331 		err = 0;
1332 	return err;
1333 }
1334 
1335 static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp,
1336 			  u8 gid[16], u8 attach, u8 block_loopback,
1337 			  enum mlx4_protocol prot)
1338 {
1339 	struct mlx4_cmd_mailbox *mailbox;
1340 	int err = 0;
1341 	int qpn;
1342 
1343 	if (!mlx4_is_mfunc(dev))
1344 		return -EBADF;
1345 
1346 	mailbox = mlx4_alloc_cmd_mailbox(dev);
1347 	if (IS_ERR(mailbox))
1348 		return PTR_ERR(mailbox);
1349 
1350 	memcpy(mailbox->buf, gid, 16);
1351 	qpn = qp->qpn;
1352 	qpn |= (prot << 28);
1353 	if (attach && block_loopback)
1354 		qpn |= (1 << 31);
1355 
1356 	err = mlx4_cmd(dev, mailbox->dma, qpn, attach,
1357 		       MLX4_CMD_QP_ATTACH, MLX4_CMD_TIME_CLASS_A,
1358 		       MLX4_CMD_WRAPPED);
1359 
1360 	mlx4_free_cmd_mailbox(dev, mailbox);
1361 	if (err && !attach &&
1362 	    dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)
1363 		err = 0;
1364 	return err;
1365 }
1366 
1367 int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp,
1368 			      u8 gid[16], u8 port,
1369 			      int block_mcast_loopback,
1370 			      enum mlx4_protocol prot, u64 *reg_id)
1371 {
1372 		struct mlx4_spec_list spec = { {NULL} };
1373 		__be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
1374 
1375 		struct mlx4_net_trans_rule rule = {
1376 			.queue_mode = MLX4_NET_TRANS_Q_FIFO,
1377 			.exclusive = 0,
1378 			.promisc_mode = MLX4_FS_REGULAR,
1379 			.priority = MLX4_DOMAIN_NIC,
1380 		};
1381 
1382 		rule.allow_loopback = !block_mcast_loopback;
1383 		rule.port = port;
1384 		rule.qpn = qp->qpn;
1385 		INIT_LIST_HEAD(&rule.list);
1386 
1387 		switch (prot) {
1388 		case MLX4_PROT_ETH:
1389 			spec.id = MLX4_NET_TRANS_RULE_ID_ETH;
1390 			memcpy(spec.eth.dst_mac, &gid[10], ETH_ALEN);
1391 			memcpy(spec.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
1392 			break;
1393 
1394 		case MLX4_PROT_IB_IPV6:
1395 			spec.id = MLX4_NET_TRANS_RULE_ID_IB;
1396 			memcpy(spec.ib.dst_gid, gid, 16);
1397 			memset(&spec.ib.dst_gid_msk, 0xff, 16);
1398 			break;
1399 		default:
1400 			return -EINVAL;
1401 		}
1402 		list_add_tail(&spec.list, &rule.list);
1403 
1404 		return mlx4_flow_attach(dev, &rule, reg_id);
1405 }
1406 
1407 int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1408 			  u8 port, int block_mcast_loopback,
1409 			  enum mlx4_protocol prot, u64 *reg_id)
1410 {
1411 	switch (dev->caps.steering_mode) {
1412 	case MLX4_STEERING_MODE_A0:
1413 		if (prot == MLX4_PROT_ETH)
1414 			return 0;
1415 		fallthrough;
1416 
1417 	case MLX4_STEERING_MODE_B0:
1418 		if (prot == MLX4_PROT_ETH)
1419 			gid[7] |= (MLX4_MC_STEER << 1);
1420 
1421 		if (mlx4_is_mfunc(dev))
1422 			return mlx4_QP_ATTACH(dev, qp, gid, 1,
1423 					      block_mcast_loopback, prot);
1424 		return mlx4_qp_attach_common(dev, qp, gid,
1425 					     block_mcast_loopback, prot,
1426 					     MLX4_MC_STEER);
1427 
1428 	case MLX4_STEERING_MODE_DEVICE_MANAGED:
1429 		return mlx4_trans_to_dmfs_attach(dev, qp, gid, port,
1430 						 block_mcast_loopback,
1431 						 prot, reg_id);
1432 	default:
1433 		return -EINVAL;
1434 	}
1435 }
1436 EXPORT_SYMBOL_GPL(mlx4_multicast_attach);
1437 
1438 int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1439 			  enum mlx4_protocol prot, u64 reg_id)
1440 {
1441 	switch (dev->caps.steering_mode) {
1442 	case MLX4_STEERING_MODE_A0:
1443 		if (prot == MLX4_PROT_ETH)
1444 			return 0;
1445 		fallthrough;
1446 
1447 	case MLX4_STEERING_MODE_B0:
1448 		if (prot == MLX4_PROT_ETH)
1449 			gid[7] |= (MLX4_MC_STEER << 1);
1450 
1451 		if (mlx4_is_mfunc(dev))
1452 			return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot);
1453 
1454 		return mlx4_qp_detach_common(dev, qp, gid, prot,
1455 					     MLX4_MC_STEER);
1456 
1457 	case MLX4_STEERING_MODE_DEVICE_MANAGED:
1458 		return mlx4_flow_detach(dev, reg_id);
1459 
1460 	default:
1461 		return -EINVAL;
1462 	}
1463 }
1464 EXPORT_SYMBOL_GPL(mlx4_multicast_detach);
1465 
1466 int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port,
1467 				u32 qpn, enum mlx4_net_trans_promisc_mode mode)
1468 {
1469 	struct mlx4_net_trans_rule rule = {
1470 		.queue_mode = MLX4_NET_TRANS_Q_FIFO,
1471 		.exclusive = 0,
1472 		.allow_loopback = 1,
1473 	};
1474 
1475 	u64 *regid_p;
1476 
1477 	switch (mode) {
1478 	case MLX4_FS_ALL_DEFAULT:
1479 		regid_p = &dev->regid_promisc_array[port];
1480 		break;
1481 	case MLX4_FS_MC_DEFAULT:
1482 		regid_p = &dev->regid_allmulti_array[port];
1483 		break;
1484 	default:
1485 		return -1;
1486 	}
1487 
1488 	if (*regid_p != 0)
1489 		return -1;
1490 
1491 	rule.promisc_mode = mode;
1492 	rule.port = port;
1493 	rule.qpn = qpn;
1494 	INIT_LIST_HEAD(&rule.list);
1495 	mlx4_info(dev, "going promisc on %x\n", port);
1496 
1497 	return  mlx4_flow_attach(dev, &rule, regid_p);
1498 }
1499 EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_add);
1500 
1501 int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port,
1502 				   enum mlx4_net_trans_promisc_mode mode)
1503 {
1504 	int ret;
1505 	u64 *regid_p;
1506 
1507 	switch (mode) {
1508 	case MLX4_FS_ALL_DEFAULT:
1509 		regid_p = &dev->regid_promisc_array[port];
1510 		break;
1511 	case MLX4_FS_MC_DEFAULT:
1512 		regid_p = &dev->regid_allmulti_array[port];
1513 		break;
1514 	default:
1515 		return -1;
1516 	}
1517 
1518 	if (*regid_p == 0)
1519 		return -1;
1520 
1521 	ret =  mlx4_flow_detach(dev, *regid_p);
1522 	if (ret == 0)
1523 		*regid_p = 0;
1524 
1525 	return ret;
1526 }
1527 EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_remove);
1528 
1529 int mlx4_unicast_attach(struct mlx4_dev *dev,
1530 			struct mlx4_qp *qp, u8 gid[16],
1531 			int block_mcast_loopback, enum mlx4_protocol prot)
1532 {
1533 	if (prot == MLX4_PROT_ETH)
1534 		gid[7] |= (MLX4_UC_STEER << 1);
1535 
1536 	if (mlx4_is_mfunc(dev))
1537 		return mlx4_QP_ATTACH(dev, qp, gid, 1,
1538 					block_mcast_loopback, prot);
1539 
1540 	return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback,
1541 					prot, MLX4_UC_STEER);
1542 }
1543 EXPORT_SYMBOL_GPL(mlx4_unicast_attach);
1544 
1545 int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp,
1546 			       u8 gid[16], enum mlx4_protocol prot)
1547 {
1548 	if (prot == MLX4_PROT_ETH)
1549 		gid[7] |= (MLX4_UC_STEER << 1);
1550 
1551 	if (mlx4_is_mfunc(dev))
1552 		return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot);
1553 
1554 	return mlx4_qp_detach_common(dev, qp, gid, prot, MLX4_UC_STEER);
1555 }
1556 EXPORT_SYMBOL_GPL(mlx4_unicast_detach);
1557 
1558 int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave,
1559 			 struct mlx4_vhcr *vhcr,
1560 			 struct mlx4_cmd_mailbox *inbox,
1561 			 struct mlx4_cmd_mailbox *outbox,
1562 			 struct mlx4_cmd_info *cmd)
1563 {
1564 	u32 qpn = (u32) vhcr->in_param & 0xffffffff;
1565 	int port = mlx4_slave_convert_port(dev, slave, vhcr->in_param >> 62);
1566 	enum mlx4_steer_type steer = vhcr->in_modifier;
1567 
1568 	if (port < 0)
1569 		return -EINVAL;
1570 
1571 	/* Promiscuous unicast is not allowed in mfunc */
1572 	if (mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)
1573 		return 0;
1574 
1575 	if (vhcr->op_modifier)
1576 		return add_promisc_qp(dev, port, steer, qpn);
1577 	else
1578 		return remove_promisc_qp(dev, port, steer, qpn);
1579 }
1580 
1581 static int mlx4_PROMISC(struct mlx4_dev *dev, u32 qpn,
1582 			enum mlx4_steer_type steer, u8 add, u8 port)
1583 {
1584 	return mlx4_cmd(dev, (u64) qpn | (u64) port << 62, (u32) steer, add,
1585 			MLX4_CMD_PROMISC, MLX4_CMD_TIME_CLASS_A,
1586 			MLX4_CMD_WRAPPED);
1587 }
1588 
1589 int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port)
1590 {
1591 	if (mlx4_is_mfunc(dev))
1592 		return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 1, port);
1593 
1594 	return add_promisc_qp(dev, port, MLX4_MC_STEER, qpn);
1595 }
1596 EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_add);
1597 
1598 int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port)
1599 {
1600 	if (mlx4_is_mfunc(dev))
1601 		return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 0, port);
1602 
1603 	return remove_promisc_qp(dev, port, MLX4_MC_STEER, qpn);
1604 }
1605 EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_remove);
1606 
1607 int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port)
1608 {
1609 	if (mlx4_is_mfunc(dev))
1610 		return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 1, port);
1611 
1612 	return add_promisc_qp(dev, port, MLX4_UC_STEER, qpn);
1613 }
1614 EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_add);
1615 
1616 int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port)
1617 {
1618 	if (mlx4_is_mfunc(dev))
1619 		return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 0, port);
1620 
1621 	return remove_promisc_qp(dev, port, MLX4_UC_STEER, qpn);
1622 }
1623 EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_remove);
1624 
1625 int mlx4_init_mcg_table(struct mlx4_dev *dev)
1626 {
1627 	struct mlx4_priv *priv = mlx4_priv(dev);
1628 	int err;
1629 
1630 	/* No need for mcg_table when fw managed the mcg table*/
1631 	if (dev->caps.steering_mode ==
1632 	    MLX4_STEERING_MODE_DEVICE_MANAGED)
1633 		return 0;
1634 	err = mlx4_bitmap_init(&priv->mcg_table.bitmap, dev->caps.num_amgms,
1635 			       dev->caps.num_amgms - 1, 0, 0);
1636 	if (err)
1637 		return err;
1638 
1639 	mutex_init(&priv->mcg_table.mutex);
1640 
1641 	return 0;
1642 }
1643 
1644 void mlx4_cleanup_mcg_table(struct mlx4_dev *dev)
1645 {
1646 	if (dev->caps.steering_mode !=
1647 	    MLX4_STEERING_MODE_DEVICE_MANAGED)
1648 		mlx4_bitmap_cleanup(&mlx4_priv(dev)->mcg_table.bitmap);
1649 }
1650