xref: /openbmc/linux/drivers/infiniband/hw/mlx4/qp.c (revision c819e2cf)
1 /*
2  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/log2.h>
35 #include <linux/slab.h>
36 #include <linux/netdevice.h>
37 
38 #include <rdma/ib_cache.h>
39 #include <rdma/ib_pack.h>
40 #include <rdma/ib_addr.h>
41 #include <rdma/ib_mad.h>
42 
43 #include <linux/mlx4/qp.h>
44 
45 #include "mlx4_ib.h"
46 #include "user.h"
47 
48 enum {
49 	MLX4_IB_ACK_REQ_FREQ	= 8,
50 };
51 
52 enum {
53 	MLX4_IB_DEFAULT_SCHED_QUEUE	= 0x83,
54 	MLX4_IB_DEFAULT_QP0_SCHED_QUEUE	= 0x3f,
55 	MLX4_IB_LINK_TYPE_IB		= 0,
56 	MLX4_IB_LINK_TYPE_ETH		= 1
57 };
58 
59 enum {
60 	/*
61 	 * Largest possible UD header: send with GRH and immediate
62 	 * data plus 18 bytes for an Ethernet header with VLAN/802.1Q
63 	 * tag.  (LRH would only use 8 bytes, so Ethernet is the
64 	 * biggest case)
65 	 */
66 	MLX4_IB_UD_HEADER_SIZE		= 82,
67 	MLX4_IB_LSO_HEADER_SPARE	= 128,
68 };
69 
70 enum {
71 	MLX4_IB_IBOE_ETHERTYPE		= 0x8915
72 };
73 
74 struct mlx4_ib_sqp {
75 	struct mlx4_ib_qp	qp;
76 	int			pkey_index;
77 	u32			qkey;
78 	u32			send_psn;
79 	struct ib_ud_header	ud_header;
80 	u8			header_buf[MLX4_IB_UD_HEADER_SIZE];
81 };
82 
83 enum {
84 	MLX4_IB_MIN_SQ_STRIDE	= 6,
85 	MLX4_IB_CACHE_LINE_SIZE	= 64,
86 };
87 
88 enum {
89 	MLX4_RAW_QP_MTU		= 7,
90 	MLX4_RAW_QP_MSGMAX	= 31,
91 };
92 
93 #ifndef ETH_ALEN
94 #define ETH_ALEN        6
95 #endif
96 static inline u64 mlx4_mac_to_u64(u8 *addr)
97 {
98 	u64 mac = 0;
99 	int i;
100 
101 	for (i = 0; i < ETH_ALEN; i++) {
102 		mac <<= 8;
103 		mac |= addr[i];
104 	}
105 	return mac;
106 }
107 
108 static const __be32 mlx4_ib_opcode[] = {
109 	[IB_WR_SEND]				= cpu_to_be32(MLX4_OPCODE_SEND),
110 	[IB_WR_LSO]				= cpu_to_be32(MLX4_OPCODE_LSO),
111 	[IB_WR_SEND_WITH_IMM]			= cpu_to_be32(MLX4_OPCODE_SEND_IMM),
112 	[IB_WR_RDMA_WRITE]			= cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
113 	[IB_WR_RDMA_WRITE_WITH_IMM]		= cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
114 	[IB_WR_RDMA_READ]			= cpu_to_be32(MLX4_OPCODE_RDMA_READ),
115 	[IB_WR_ATOMIC_CMP_AND_SWP]		= cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
116 	[IB_WR_ATOMIC_FETCH_AND_ADD]		= cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
117 	[IB_WR_SEND_WITH_INV]			= cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
118 	[IB_WR_LOCAL_INV]			= cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
119 	[IB_WR_FAST_REG_MR]			= cpu_to_be32(MLX4_OPCODE_FMR),
120 	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
121 	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
122 	[IB_WR_BIND_MW]				= cpu_to_be32(MLX4_OPCODE_BIND_MW),
123 };
124 
125 static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
126 {
127 	return container_of(mqp, struct mlx4_ib_sqp, qp);
128 }
129 
130 static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
131 {
132 	if (!mlx4_is_master(dev->dev))
133 		return 0;
134 
135 	return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
136 	       qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
137 		8 * MLX4_MFUNC_MAX;
138 }
139 
140 static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
141 {
142 	int proxy_sqp = 0;
143 	int real_sqp = 0;
144 	int i;
145 	/* PPF or Native -- real SQP */
146 	real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
147 		    qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
148 		    qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
149 	if (real_sqp)
150 		return 1;
151 	/* VF or PF -- proxy SQP */
152 	if (mlx4_is_mfunc(dev->dev)) {
153 		for (i = 0; i < dev->dev->caps.num_ports; i++) {
154 			if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i] ||
155 			    qp->mqp.qpn == dev->dev->caps.qp1_proxy[i]) {
156 				proxy_sqp = 1;
157 				break;
158 			}
159 		}
160 	}
161 	return proxy_sqp;
162 }
163 
164 /* used for INIT/CLOSE port logic */
165 static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
166 {
167 	int proxy_qp0 = 0;
168 	int real_qp0 = 0;
169 	int i;
170 	/* PPF or Native -- real QP0 */
171 	real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
172 		    qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
173 		    qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
174 	if (real_qp0)
175 		return 1;
176 	/* VF or PF -- proxy QP0 */
177 	if (mlx4_is_mfunc(dev->dev)) {
178 		for (i = 0; i < dev->dev->caps.num_ports; i++) {
179 			if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i]) {
180 				proxy_qp0 = 1;
181 				break;
182 			}
183 		}
184 	}
185 	return proxy_qp0;
186 }
187 
188 static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
189 {
190 	return mlx4_buf_offset(&qp->buf, offset);
191 }
192 
193 static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
194 {
195 	return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
196 }
197 
198 static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
199 {
200 	return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
201 }
202 
203 /*
204  * Stamp a SQ WQE so that it is invalid if prefetched by marking the
205  * first four bytes of every 64 byte chunk with
206  *     0x7FFFFFF | (invalid_ownership_value << 31).
207  *
208  * When the max work request size is less than or equal to the WQE
209  * basic block size, as an optimization, we can stamp all WQEs with
210  * 0xffffffff, and skip the very first chunk of each WQE.
211  */
212 static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
213 {
214 	__be32 *wqe;
215 	int i;
216 	int s;
217 	int ind;
218 	void *buf;
219 	__be32 stamp;
220 	struct mlx4_wqe_ctrl_seg *ctrl;
221 
222 	if (qp->sq_max_wqes_per_wr > 1) {
223 		s = roundup(size, 1U << qp->sq.wqe_shift);
224 		for (i = 0; i < s; i += 64) {
225 			ind = (i >> qp->sq.wqe_shift) + n;
226 			stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
227 						       cpu_to_be32(0xffffffff);
228 			buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
229 			wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
230 			*wqe = stamp;
231 		}
232 	} else {
233 		ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
234 		s = (ctrl->fence_size & 0x3f) << 4;
235 		for (i = 64; i < s; i += 64) {
236 			wqe = buf + i;
237 			*wqe = cpu_to_be32(0xffffffff);
238 		}
239 	}
240 }
241 
242 static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
243 {
244 	struct mlx4_wqe_ctrl_seg *ctrl;
245 	struct mlx4_wqe_inline_seg *inl;
246 	void *wqe;
247 	int s;
248 
249 	ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
250 	s = sizeof(struct mlx4_wqe_ctrl_seg);
251 
252 	if (qp->ibqp.qp_type == IB_QPT_UD) {
253 		struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
254 		struct mlx4_av *av = (struct mlx4_av *)dgram->av;
255 		memset(dgram, 0, sizeof *dgram);
256 		av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
257 		s += sizeof(struct mlx4_wqe_datagram_seg);
258 	}
259 
260 	/* Pad the remainder of the WQE with an inline data segment. */
261 	if (size > s) {
262 		inl = wqe + s;
263 		inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
264 	}
265 	ctrl->srcrb_flags = 0;
266 	ctrl->fence_size = size / 16;
267 	/*
268 	 * Make sure descriptor is fully written before setting ownership bit
269 	 * (because HW can start executing as soon as we do).
270 	 */
271 	wmb();
272 
273 	ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
274 		(n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
275 
276 	stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
277 }
278 
279 /* Post NOP WQE to prevent wrap-around in the middle of WR */
280 static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
281 {
282 	unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
283 	if (unlikely(s < qp->sq_max_wqes_per_wr)) {
284 		post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
285 		ind += s;
286 	}
287 	return ind;
288 }
289 
290 static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
291 {
292 	struct ib_event event;
293 	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
294 
295 	if (type == MLX4_EVENT_TYPE_PATH_MIG)
296 		to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
297 
298 	if (ibqp->event_handler) {
299 		event.device     = ibqp->device;
300 		event.element.qp = ibqp;
301 		switch (type) {
302 		case MLX4_EVENT_TYPE_PATH_MIG:
303 			event.event = IB_EVENT_PATH_MIG;
304 			break;
305 		case MLX4_EVENT_TYPE_COMM_EST:
306 			event.event = IB_EVENT_COMM_EST;
307 			break;
308 		case MLX4_EVENT_TYPE_SQ_DRAINED:
309 			event.event = IB_EVENT_SQ_DRAINED;
310 			break;
311 		case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
312 			event.event = IB_EVENT_QP_LAST_WQE_REACHED;
313 			break;
314 		case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
315 			event.event = IB_EVENT_QP_FATAL;
316 			break;
317 		case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
318 			event.event = IB_EVENT_PATH_MIG_ERR;
319 			break;
320 		case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
321 			event.event = IB_EVENT_QP_REQ_ERR;
322 			break;
323 		case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
324 			event.event = IB_EVENT_QP_ACCESS_ERR;
325 			break;
326 		default:
327 			pr_warn("Unexpected event type %d "
328 			       "on QP %06x\n", type, qp->qpn);
329 			return;
330 		}
331 
332 		ibqp->event_handler(&event, ibqp->qp_context);
333 	}
334 }
335 
336 static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
337 {
338 	/*
339 	 * UD WQEs must have a datagram segment.
340 	 * RC and UC WQEs might have a remote address segment.
341 	 * MLX WQEs need two extra inline data segments (for the UD
342 	 * header and space for the ICRC).
343 	 */
344 	switch (type) {
345 	case MLX4_IB_QPT_UD:
346 		return sizeof (struct mlx4_wqe_ctrl_seg) +
347 			sizeof (struct mlx4_wqe_datagram_seg) +
348 			((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
349 	case MLX4_IB_QPT_PROXY_SMI_OWNER:
350 	case MLX4_IB_QPT_PROXY_SMI:
351 	case MLX4_IB_QPT_PROXY_GSI:
352 		return sizeof (struct mlx4_wqe_ctrl_seg) +
353 			sizeof (struct mlx4_wqe_datagram_seg) + 64;
354 	case MLX4_IB_QPT_TUN_SMI_OWNER:
355 	case MLX4_IB_QPT_TUN_GSI:
356 		return sizeof (struct mlx4_wqe_ctrl_seg) +
357 			sizeof (struct mlx4_wqe_datagram_seg);
358 
359 	case MLX4_IB_QPT_UC:
360 		return sizeof (struct mlx4_wqe_ctrl_seg) +
361 			sizeof (struct mlx4_wqe_raddr_seg);
362 	case MLX4_IB_QPT_RC:
363 		return sizeof (struct mlx4_wqe_ctrl_seg) +
364 			sizeof (struct mlx4_wqe_atomic_seg) +
365 			sizeof (struct mlx4_wqe_raddr_seg);
366 	case MLX4_IB_QPT_SMI:
367 	case MLX4_IB_QPT_GSI:
368 		return sizeof (struct mlx4_wqe_ctrl_seg) +
369 			ALIGN(MLX4_IB_UD_HEADER_SIZE +
370 			      DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
371 					   MLX4_INLINE_ALIGN) *
372 			      sizeof (struct mlx4_wqe_inline_seg),
373 			      sizeof (struct mlx4_wqe_data_seg)) +
374 			ALIGN(4 +
375 			      sizeof (struct mlx4_wqe_inline_seg),
376 			      sizeof (struct mlx4_wqe_data_seg));
377 	default:
378 		return sizeof (struct mlx4_wqe_ctrl_seg);
379 	}
380 }
381 
382 static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
383 		       int is_user, int has_rq, struct mlx4_ib_qp *qp)
384 {
385 	/* Sanity check RQ size before proceeding */
386 	if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
387 	    cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
388 		return -EINVAL;
389 
390 	if (!has_rq) {
391 		if (cap->max_recv_wr)
392 			return -EINVAL;
393 
394 		qp->rq.wqe_cnt = qp->rq.max_gs = 0;
395 	} else {
396 		/* HW requires >= 1 RQ entry with >= 1 gather entry */
397 		if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
398 			return -EINVAL;
399 
400 		qp->rq.wqe_cnt	 = roundup_pow_of_two(max(1U, cap->max_recv_wr));
401 		qp->rq.max_gs	 = roundup_pow_of_two(max(1U, cap->max_recv_sge));
402 		qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
403 	}
404 
405 	/* leave userspace return values as they were, so as not to break ABI */
406 	if (is_user) {
407 		cap->max_recv_wr  = qp->rq.max_post = qp->rq.wqe_cnt;
408 		cap->max_recv_sge = qp->rq.max_gs;
409 	} else {
410 		cap->max_recv_wr  = qp->rq.max_post =
411 			min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
412 		cap->max_recv_sge = min(qp->rq.max_gs,
413 					min(dev->dev->caps.max_sq_sg,
414 					    dev->dev->caps.max_rq_sg));
415 	}
416 
417 	return 0;
418 }
419 
420 static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
421 			      enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp)
422 {
423 	int s;
424 
425 	/* Sanity check SQ size before proceeding */
426 	if (cap->max_send_wr  > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
427 	    cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
428 	    cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
429 	    sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
430 		return -EINVAL;
431 
432 	/*
433 	 * For MLX transport we need 2 extra S/G entries:
434 	 * one for the header and one for the checksum at the end
435 	 */
436 	if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
437 	     type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
438 	    cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
439 		return -EINVAL;
440 
441 	s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
442 		cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
443 		send_wqe_overhead(type, qp->flags);
444 
445 	if (s > dev->dev->caps.max_sq_desc_sz)
446 		return -EINVAL;
447 
448 	/*
449 	 * Hermon supports shrinking WQEs, such that a single work
450 	 * request can include multiple units of 1 << wqe_shift.  This
451 	 * way, work requests can differ in size, and do not have to
452 	 * be a power of 2 in size, saving memory and speeding up send
453 	 * WR posting.  Unfortunately, if we do this then the
454 	 * wqe_index field in CQEs can't be used to look up the WR ID
455 	 * anymore, so we do this only if selective signaling is off.
456 	 *
457 	 * Further, on 32-bit platforms, we can't use vmap() to make
458 	 * the QP buffer virtually contiguous.  Thus we have to use
459 	 * constant-sized WRs to make sure a WR is always fully within
460 	 * a single page-sized chunk.
461 	 *
462 	 * Finally, we use NOP work requests to pad the end of the
463 	 * work queue, to avoid wrap-around in the middle of WR.  We
464 	 * set NEC bit to avoid getting completions with error for
465 	 * these NOP WRs, but since NEC is only supported starting
466 	 * with firmware 2.2.232, we use constant-sized WRs for older
467 	 * firmware.
468 	 *
469 	 * And, since MLX QPs only support SEND, we use constant-sized
470 	 * WRs in this case.
471 	 *
472 	 * We look for the smallest value of wqe_shift such that the
473 	 * resulting number of wqes does not exceed device
474 	 * capabilities.
475 	 *
476 	 * We set WQE size to at least 64 bytes, this way stamping
477 	 * invalidates each WQE.
478 	 */
479 	if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
480 	    qp->sq_signal_bits && BITS_PER_LONG == 64 &&
481 	    type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI &&
482 	    !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI |
483 		      MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER)))
484 		qp->sq.wqe_shift = ilog2(64);
485 	else
486 		qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
487 
488 	for (;;) {
489 		qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
490 
491 		/*
492 		 * We need to leave 2 KB + 1 WR of headroom in the SQ to
493 		 * allow HW to prefetch.
494 		 */
495 		qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
496 		qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
497 						    qp->sq_max_wqes_per_wr +
498 						    qp->sq_spare_wqes);
499 
500 		if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
501 			break;
502 
503 		if (qp->sq_max_wqes_per_wr <= 1)
504 			return -EINVAL;
505 
506 		++qp->sq.wqe_shift;
507 	}
508 
509 	qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
510 			     (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
511 			 send_wqe_overhead(type, qp->flags)) /
512 		sizeof (struct mlx4_wqe_data_seg);
513 
514 	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
515 		(qp->sq.wqe_cnt << qp->sq.wqe_shift);
516 	if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
517 		qp->rq.offset = 0;
518 		qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
519 	} else {
520 		qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
521 		qp->sq.offset = 0;
522 	}
523 
524 	cap->max_send_wr  = qp->sq.max_post =
525 		(qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
526 	cap->max_send_sge = min(qp->sq.max_gs,
527 				min(dev->dev->caps.max_sq_sg,
528 				    dev->dev->caps.max_rq_sg));
529 	/* We don't support inline sends for kernel QPs (yet) */
530 	cap->max_inline_data = 0;
531 
532 	return 0;
533 }
534 
535 static int set_user_sq_size(struct mlx4_ib_dev *dev,
536 			    struct mlx4_ib_qp *qp,
537 			    struct mlx4_ib_create_qp *ucmd)
538 {
539 	/* Sanity check SQ size before proceeding */
540 	if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes	 ||
541 	    ucmd->log_sq_stride >
542 		ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
543 	    ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
544 		return -EINVAL;
545 
546 	qp->sq.wqe_cnt   = 1 << ucmd->log_sq_bb_count;
547 	qp->sq.wqe_shift = ucmd->log_sq_stride;
548 
549 	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
550 		(qp->sq.wqe_cnt << qp->sq.wqe_shift);
551 
552 	return 0;
553 }
554 
555 static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
556 {
557 	int i;
558 
559 	qp->sqp_proxy_rcv =
560 		kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt,
561 			GFP_KERNEL);
562 	if (!qp->sqp_proxy_rcv)
563 		return -ENOMEM;
564 	for (i = 0; i < qp->rq.wqe_cnt; i++) {
565 		qp->sqp_proxy_rcv[i].addr =
566 			kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
567 				GFP_KERNEL);
568 		if (!qp->sqp_proxy_rcv[i].addr)
569 			goto err;
570 		qp->sqp_proxy_rcv[i].map =
571 			ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
572 					  sizeof (struct mlx4_ib_proxy_sqp_hdr),
573 					  DMA_FROM_DEVICE);
574 	}
575 	return 0;
576 
577 err:
578 	while (i > 0) {
579 		--i;
580 		ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
581 				    sizeof (struct mlx4_ib_proxy_sqp_hdr),
582 				    DMA_FROM_DEVICE);
583 		kfree(qp->sqp_proxy_rcv[i].addr);
584 	}
585 	kfree(qp->sqp_proxy_rcv);
586 	qp->sqp_proxy_rcv = NULL;
587 	return -ENOMEM;
588 }
589 
590 static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
591 {
592 	int i;
593 
594 	for (i = 0; i < qp->rq.wqe_cnt; i++) {
595 		ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
596 				    sizeof (struct mlx4_ib_proxy_sqp_hdr),
597 				    DMA_FROM_DEVICE);
598 		kfree(qp->sqp_proxy_rcv[i].addr);
599 	}
600 	kfree(qp->sqp_proxy_rcv);
601 }
602 
603 static int qp_has_rq(struct ib_qp_init_attr *attr)
604 {
605 	if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
606 		return 0;
607 
608 	return !attr->srq;
609 }
610 
611 static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
612 {
613 	int i;
614 	for (i = 0; i < dev->caps.num_ports; i++) {
615 		if (qpn == dev->caps.qp0_proxy[i])
616 			return !!dev->caps.qp0_qkey[i];
617 	}
618 	return 0;
619 }
620 
621 static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
622 			    struct ib_qp_init_attr *init_attr,
623 			    struct ib_udata *udata, int sqpn, struct mlx4_ib_qp **caller_qp,
624 			    gfp_t gfp)
625 {
626 	int qpn;
627 	int err;
628 	struct mlx4_ib_sqp *sqp;
629 	struct mlx4_ib_qp *qp;
630 	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
631 
632 	/* When tunneling special qps, we use a plain UD qp */
633 	if (sqpn) {
634 		if (mlx4_is_mfunc(dev->dev) &&
635 		    (!mlx4_is_master(dev->dev) ||
636 		     !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
637 			if (init_attr->qp_type == IB_QPT_GSI)
638 				qp_type = MLX4_IB_QPT_PROXY_GSI;
639 			else {
640 				if (mlx4_is_master(dev->dev) ||
641 				    qp0_enabled_vf(dev->dev, sqpn))
642 					qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
643 				else
644 					qp_type = MLX4_IB_QPT_PROXY_SMI;
645 			}
646 		}
647 		qpn = sqpn;
648 		/* add extra sg entry for tunneling */
649 		init_attr->cap.max_recv_sge++;
650 	} else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
651 		struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
652 			container_of(init_attr,
653 				     struct mlx4_ib_qp_tunnel_init_attr, init_attr);
654 		if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
655 		     tnl_init->proxy_qp_type != IB_QPT_GSI)   ||
656 		    !mlx4_is_master(dev->dev))
657 			return -EINVAL;
658 		if (tnl_init->proxy_qp_type == IB_QPT_GSI)
659 			qp_type = MLX4_IB_QPT_TUN_GSI;
660 		else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
661 			 mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
662 					     tnl_init->port))
663 			qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
664 		else
665 			qp_type = MLX4_IB_QPT_TUN_SMI;
666 		/* we are definitely in the PPF here, since we are creating
667 		 * tunnel QPs. base_tunnel_sqpn is therefore valid. */
668 		qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
669 			+ tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
670 		sqpn = qpn;
671 	}
672 
673 	if (!*caller_qp) {
674 		if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||
675 		    (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
676 				MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
677 			sqp = kzalloc(sizeof (struct mlx4_ib_sqp), gfp);
678 			if (!sqp)
679 				return -ENOMEM;
680 			qp = &sqp->qp;
681 			qp->pri.vid = 0xFFFF;
682 			qp->alt.vid = 0xFFFF;
683 		} else {
684 			qp = kzalloc(sizeof (struct mlx4_ib_qp), gfp);
685 			if (!qp)
686 				return -ENOMEM;
687 			qp->pri.vid = 0xFFFF;
688 			qp->alt.vid = 0xFFFF;
689 		}
690 	} else
691 		qp = *caller_qp;
692 
693 	qp->mlx4_ib_qp_type = qp_type;
694 
695 	mutex_init(&qp->mutex);
696 	spin_lock_init(&qp->sq.lock);
697 	spin_lock_init(&qp->rq.lock);
698 	INIT_LIST_HEAD(&qp->gid_list);
699 	INIT_LIST_HEAD(&qp->steering_rules);
700 
701 	qp->state	 = IB_QPS_RESET;
702 	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
703 		qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
704 
705 	err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, qp_has_rq(init_attr), qp);
706 	if (err)
707 		goto err;
708 
709 	if (pd->uobject) {
710 		struct mlx4_ib_create_qp ucmd;
711 
712 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
713 			err = -EFAULT;
714 			goto err;
715 		}
716 
717 		qp->sq_no_prefetch = ucmd.sq_no_prefetch;
718 
719 		err = set_user_sq_size(dev, qp, &ucmd);
720 		if (err)
721 			goto err;
722 
723 		qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
724 				       qp->buf_size, 0, 0);
725 		if (IS_ERR(qp->umem)) {
726 			err = PTR_ERR(qp->umem);
727 			goto err;
728 		}
729 
730 		err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),
731 				    ilog2(qp->umem->page_size), &qp->mtt);
732 		if (err)
733 			goto err_buf;
734 
735 		err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
736 		if (err)
737 			goto err_mtt;
738 
739 		if (qp_has_rq(init_attr)) {
740 			err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
741 						  ucmd.db_addr, &qp->db);
742 			if (err)
743 				goto err_mtt;
744 		}
745 	} else {
746 		qp->sq_no_prefetch = 0;
747 
748 		if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
749 			qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
750 
751 		if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
752 			qp->flags |= MLX4_IB_QP_LSO;
753 
754 		if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
755 			if (dev->steering_support ==
756 			    MLX4_STEERING_MODE_DEVICE_MANAGED)
757 				qp->flags |= MLX4_IB_QP_NETIF;
758 			else
759 				goto err;
760 		}
761 
762 		err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp);
763 		if (err)
764 			goto err;
765 
766 		if (qp_has_rq(init_attr)) {
767 			err = mlx4_db_alloc(dev->dev, &qp->db, 0, gfp);
768 			if (err)
769 				goto err;
770 
771 			*qp->db.db = 0;
772 		}
773 
774 		if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf, gfp)) {
775 			err = -ENOMEM;
776 			goto err_db;
777 		}
778 
779 		err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
780 				    &qp->mtt);
781 		if (err)
782 			goto err_buf;
783 
784 		err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf, gfp);
785 		if (err)
786 			goto err_mtt;
787 
788 		qp->sq.wrid  = kmalloc(qp->sq.wqe_cnt * sizeof (u64), gfp);
789 		qp->rq.wrid  = kmalloc(qp->rq.wqe_cnt * sizeof (u64), gfp);
790 		if (!qp->sq.wrid || !qp->rq.wrid) {
791 			err = -ENOMEM;
792 			goto err_wrid;
793 		}
794 	}
795 
796 	if (sqpn) {
797 		if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
798 		    MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
799 			if (alloc_proxy_bufs(pd->device, qp)) {
800 				err = -ENOMEM;
801 				goto err_wrid;
802 			}
803 		}
804 	} else {
805 		/* Raw packet QPNs may not have bits 6,7 set in their qp_num;
806 		 * otherwise, the WQE BlueFlame setup flow wrongly causes
807 		 * VLAN insertion. */
808 		if (init_attr->qp_type == IB_QPT_RAW_PACKET)
809 			err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
810 						    (init_attr->cap.max_send_wr ?
811 						     MLX4_RESERVE_ETH_BF_QP : 0) |
812 						    (init_attr->cap.max_recv_wr ?
813 						     MLX4_RESERVE_A0_QP : 0));
814 		else
815 			if (qp->flags & MLX4_IB_QP_NETIF)
816 				err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
817 			else
818 				err = mlx4_qp_reserve_range(dev->dev, 1, 1,
819 							    &qpn, 0);
820 		if (err)
821 			goto err_proxy;
822 	}
823 
824 	err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp, gfp);
825 	if (err)
826 		goto err_qpn;
827 
828 	if (init_attr->qp_type == IB_QPT_XRC_TGT)
829 		qp->mqp.qpn |= (1 << 23);
830 
831 	/*
832 	 * Hardware wants QPN written in big-endian order (after
833 	 * shifting) for send doorbell.  Precompute this value to save
834 	 * a little bit when posting sends.
835 	 */
836 	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
837 
838 	qp->mqp.event = mlx4_ib_qp_event;
839 	if (!*caller_qp)
840 		*caller_qp = qp;
841 	return 0;
842 
843 err_qpn:
844 	if (!sqpn) {
845 		if (qp->flags & MLX4_IB_QP_NETIF)
846 			mlx4_ib_steer_qp_free(dev, qpn, 1);
847 		else
848 			mlx4_qp_release_range(dev->dev, qpn, 1);
849 	}
850 err_proxy:
851 	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
852 		free_proxy_bufs(pd->device, qp);
853 err_wrid:
854 	if (pd->uobject) {
855 		if (qp_has_rq(init_attr))
856 			mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
857 	} else {
858 		kfree(qp->sq.wrid);
859 		kfree(qp->rq.wrid);
860 	}
861 
862 err_mtt:
863 	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
864 
865 err_buf:
866 	if (pd->uobject)
867 		ib_umem_release(qp->umem);
868 	else
869 		mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
870 
871 err_db:
872 	if (!pd->uobject && qp_has_rq(init_attr))
873 		mlx4_db_free(dev->dev, &qp->db);
874 
875 err:
876 	if (!*caller_qp)
877 		kfree(qp);
878 	return err;
879 }
880 
881 static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
882 {
883 	switch (state) {
884 	case IB_QPS_RESET:	return MLX4_QP_STATE_RST;
885 	case IB_QPS_INIT:	return MLX4_QP_STATE_INIT;
886 	case IB_QPS_RTR:	return MLX4_QP_STATE_RTR;
887 	case IB_QPS_RTS:	return MLX4_QP_STATE_RTS;
888 	case IB_QPS_SQD:	return MLX4_QP_STATE_SQD;
889 	case IB_QPS_SQE:	return MLX4_QP_STATE_SQER;
890 	case IB_QPS_ERR:	return MLX4_QP_STATE_ERR;
891 	default:		return -1;
892 	}
893 }
894 
895 static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
896 	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
897 {
898 	if (send_cq == recv_cq) {
899 		spin_lock_irq(&send_cq->lock);
900 		__acquire(&recv_cq->lock);
901 	} else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
902 		spin_lock_irq(&send_cq->lock);
903 		spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
904 	} else {
905 		spin_lock_irq(&recv_cq->lock);
906 		spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
907 	}
908 }
909 
910 static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
911 	__releases(&send_cq->lock) __releases(&recv_cq->lock)
912 {
913 	if (send_cq == recv_cq) {
914 		__release(&recv_cq->lock);
915 		spin_unlock_irq(&send_cq->lock);
916 	} else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
917 		spin_unlock(&recv_cq->lock);
918 		spin_unlock_irq(&send_cq->lock);
919 	} else {
920 		spin_unlock(&send_cq->lock);
921 		spin_unlock_irq(&recv_cq->lock);
922 	}
923 }
924 
925 static void del_gid_entries(struct mlx4_ib_qp *qp)
926 {
927 	struct mlx4_ib_gid_entry *ge, *tmp;
928 
929 	list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
930 		list_del(&ge->list);
931 		kfree(ge);
932 	}
933 }
934 
935 static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
936 {
937 	if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
938 		return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
939 	else
940 		return to_mpd(qp->ibqp.pd);
941 }
942 
943 static void get_cqs(struct mlx4_ib_qp *qp,
944 		    struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
945 {
946 	switch (qp->ibqp.qp_type) {
947 	case IB_QPT_XRC_TGT:
948 		*send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
949 		*recv_cq = *send_cq;
950 		break;
951 	case IB_QPT_XRC_INI:
952 		*send_cq = to_mcq(qp->ibqp.send_cq);
953 		*recv_cq = *send_cq;
954 		break;
955 	default:
956 		*send_cq = to_mcq(qp->ibqp.send_cq);
957 		*recv_cq = to_mcq(qp->ibqp.recv_cq);
958 		break;
959 	}
960 }
961 
962 static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
963 			      int is_user)
964 {
965 	struct mlx4_ib_cq *send_cq, *recv_cq;
966 
967 	if (qp->state != IB_QPS_RESET) {
968 		if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
969 				   MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
970 			pr_warn("modify QP %06x to RESET failed.\n",
971 			       qp->mqp.qpn);
972 		if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
973 			mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
974 			qp->pri.smac = 0;
975 			qp->pri.smac_port = 0;
976 		}
977 		if (qp->alt.smac) {
978 			mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
979 			qp->alt.smac = 0;
980 		}
981 		if (qp->pri.vid < 0x1000) {
982 			mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
983 			qp->pri.vid = 0xFFFF;
984 			qp->pri.candidate_vid = 0xFFFF;
985 			qp->pri.update_vid = 0;
986 		}
987 		if (qp->alt.vid < 0x1000) {
988 			mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
989 			qp->alt.vid = 0xFFFF;
990 			qp->alt.candidate_vid = 0xFFFF;
991 			qp->alt.update_vid = 0;
992 		}
993 	}
994 
995 	get_cqs(qp, &send_cq, &recv_cq);
996 
997 	mlx4_ib_lock_cqs(send_cq, recv_cq);
998 
999 	if (!is_user) {
1000 		__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1001 				 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1002 		if (send_cq != recv_cq)
1003 			__mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1004 	}
1005 
1006 	mlx4_qp_remove(dev->dev, &qp->mqp);
1007 
1008 	mlx4_ib_unlock_cqs(send_cq, recv_cq);
1009 
1010 	mlx4_qp_free(dev->dev, &qp->mqp);
1011 
1012 	if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
1013 		if (qp->flags & MLX4_IB_QP_NETIF)
1014 			mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1015 		else
1016 			mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1017 	}
1018 
1019 	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1020 
1021 	if (is_user) {
1022 		if (qp->rq.wqe_cnt)
1023 			mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
1024 					      &qp->db);
1025 		ib_umem_release(qp->umem);
1026 	} else {
1027 		kfree(qp->sq.wrid);
1028 		kfree(qp->rq.wrid);
1029 		if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1030 		    MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
1031 			free_proxy_bufs(&dev->ib_dev, qp);
1032 		mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1033 		if (qp->rq.wqe_cnt)
1034 			mlx4_db_free(dev->dev, &qp->db);
1035 	}
1036 
1037 	del_gid_entries(qp);
1038 }
1039 
1040 static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
1041 {
1042 	/* Native or PPF */
1043 	if (!mlx4_is_mfunc(dev->dev) ||
1044 	    (mlx4_is_master(dev->dev) &&
1045 	     attr->create_flags & MLX4_IB_SRIOV_SQP)) {
1046 		return  dev->dev->phys_caps.base_sqpn +
1047 			(attr->qp_type == IB_QPT_SMI ? 0 : 2) +
1048 			attr->port_num - 1;
1049 	}
1050 	/* PF or VF -- creating proxies */
1051 	if (attr->qp_type == IB_QPT_SMI)
1052 		return dev->dev->caps.qp0_proxy[attr->port_num - 1];
1053 	else
1054 		return dev->dev->caps.qp1_proxy[attr->port_num - 1];
1055 }
1056 
1057 struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
1058 				struct ib_qp_init_attr *init_attr,
1059 				struct ib_udata *udata)
1060 {
1061 	struct mlx4_ib_qp *qp = NULL;
1062 	int err;
1063 	u16 xrcdn = 0;
1064 	gfp_t gfp;
1065 
1066 	gfp = (init_attr->create_flags & MLX4_IB_QP_CREATE_USE_GFP_NOIO) ?
1067 		GFP_NOIO : GFP_KERNEL;
1068 	/*
1069 	 * We only support LSO, vendor flag1, and multicast loopback blocking,
1070 	 * and only for kernel UD QPs.
1071 	 */
1072 	if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
1073 					MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
1074 					MLX4_IB_SRIOV_TUNNEL_QP |
1075 					MLX4_IB_SRIOV_SQP |
1076 					MLX4_IB_QP_NETIF |
1077 					MLX4_IB_QP_CREATE_USE_GFP_NOIO))
1078 		return ERR_PTR(-EINVAL);
1079 
1080 	if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1081 		if (init_attr->qp_type != IB_QPT_UD)
1082 			return ERR_PTR(-EINVAL);
1083 	}
1084 
1085 	if (init_attr->create_flags &&
1086 	    (udata ||
1087 	     ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP | MLX4_IB_QP_CREATE_USE_GFP_NOIO)) &&
1088 	      init_attr->qp_type != IB_QPT_UD) ||
1089 	     ((init_attr->create_flags & MLX4_IB_SRIOV_SQP) &&
1090 	      init_attr->qp_type > IB_QPT_GSI)))
1091 		return ERR_PTR(-EINVAL);
1092 
1093 	switch (init_attr->qp_type) {
1094 	case IB_QPT_XRC_TGT:
1095 		pd = to_mxrcd(init_attr->xrcd)->pd;
1096 		xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1097 		init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1098 		/* fall through */
1099 	case IB_QPT_XRC_INI:
1100 		if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1101 			return ERR_PTR(-ENOSYS);
1102 		init_attr->recv_cq = init_attr->send_cq;
1103 		/* fall through */
1104 	case IB_QPT_RC:
1105 	case IB_QPT_UC:
1106 	case IB_QPT_RAW_PACKET:
1107 		qp = kzalloc(sizeof *qp, gfp);
1108 		if (!qp)
1109 			return ERR_PTR(-ENOMEM);
1110 		qp->pri.vid = 0xFFFF;
1111 		qp->alt.vid = 0xFFFF;
1112 		/* fall through */
1113 	case IB_QPT_UD:
1114 	{
1115 		err = create_qp_common(to_mdev(pd->device), pd, init_attr,
1116 				       udata, 0, &qp, gfp);
1117 		if (err)
1118 			return ERR_PTR(err);
1119 
1120 		qp->ibqp.qp_num = qp->mqp.qpn;
1121 		qp->xrcdn = xrcdn;
1122 
1123 		break;
1124 	}
1125 	case IB_QPT_SMI:
1126 	case IB_QPT_GSI:
1127 	{
1128 		/* Userspace is not allowed to create special QPs: */
1129 		if (udata)
1130 			return ERR_PTR(-EINVAL);
1131 
1132 		err = create_qp_common(to_mdev(pd->device), pd, init_attr, udata,
1133 				       get_sqp_num(to_mdev(pd->device), init_attr),
1134 				       &qp, gfp);
1135 		if (err)
1136 			return ERR_PTR(err);
1137 
1138 		qp->port	= init_attr->port_num;
1139 		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
1140 
1141 		break;
1142 	}
1143 	default:
1144 		/* Don't support raw QPs */
1145 		return ERR_PTR(-EINVAL);
1146 	}
1147 
1148 	return &qp->ibqp;
1149 }
1150 
1151 int mlx4_ib_destroy_qp(struct ib_qp *qp)
1152 {
1153 	struct mlx4_ib_dev *dev = to_mdev(qp->device);
1154 	struct mlx4_ib_qp *mqp = to_mqp(qp);
1155 	struct mlx4_ib_pd *pd;
1156 
1157 	if (is_qp0(dev, mqp))
1158 		mlx4_CLOSE_PORT(dev->dev, mqp->port);
1159 
1160 	if (dev->qp1_proxy[mqp->port - 1] == mqp) {
1161 		mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
1162 		dev->qp1_proxy[mqp->port - 1] = NULL;
1163 		mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
1164 	}
1165 
1166 	pd = get_pd(mqp);
1167 	destroy_qp_common(dev, mqp, !!pd->ibpd.uobject);
1168 
1169 	if (is_sqp(dev, mqp))
1170 		kfree(to_msqp(mqp));
1171 	else
1172 		kfree(mqp);
1173 
1174 	return 0;
1175 }
1176 
1177 static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
1178 {
1179 	switch (type) {
1180 	case MLX4_IB_QPT_RC:		return MLX4_QP_ST_RC;
1181 	case MLX4_IB_QPT_UC:		return MLX4_QP_ST_UC;
1182 	case MLX4_IB_QPT_UD:		return MLX4_QP_ST_UD;
1183 	case MLX4_IB_QPT_XRC_INI:
1184 	case MLX4_IB_QPT_XRC_TGT:	return MLX4_QP_ST_XRC;
1185 	case MLX4_IB_QPT_SMI:
1186 	case MLX4_IB_QPT_GSI:
1187 	case MLX4_IB_QPT_RAW_PACKET:	return MLX4_QP_ST_MLX;
1188 
1189 	case MLX4_IB_QPT_PROXY_SMI_OWNER:
1190 	case MLX4_IB_QPT_TUN_SMI_OWNER:	return (mlx4_is_mfunc(dev->dev) ?
1191 						MLX4_QP_ST_MLX : -1);
1192 	case MLX4_IB_QPT_PROXY_SMI:
1193 	case MLX4_IB_QPT_TUN_SMI:
1194 	case MLX4_IB_QPT_PROXY_GSI:
1195 	case MLX4_IB_QPT_TUN_GSI:	return (mlx4_is_mfunc(dev->dev) ?
1196 						MLX4_QP_ST_UD : -1);
1197 	default:			return -1;
1198 	}
1199 }
1200 
1201 static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
1202 				   int attr_mask)
1203 {
1204 	u8 dest_rd_atomic;
1205 	u32 access_flags;
1206 	u32 hw_access_flags = 0;
1207 
1208 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1209 		dest_rd_atomic = attr->max_dest_rd_atomic;
1210 	else
1211 		dest_rd_atomic = qp->resp_depth;
1212 
1213 	if (attr_mask & IB_QP_ACCESS_FLAGS)
1214 		access_flags = attr->qp_access_flags;
1215 	else
1216 		access_flags = qp->atomic_rd_en;
1217 
1218 	if (!dest_rd_atomic)
1219 		access_flags &= IB_ACCESS_REMOTE_WRITE;
1220 
1221 	if (access_flags & IB_ACCESS_REMOTE_READ)
1222 		hw_access_flags |= MLX4_QP_BIT_RRE;
1223 	if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
1224 		hw_access_flags |= MLX4_QP_BIT_RAE;
1225 	if (access_flags & IB_ACCESS_REMOTE_WRITE)
1226 		hw_access_flags |= MLX4_QP_BIT_RWE;
1227 
1228 	return cpu_to_be32(hw_access_flags);
1229 }
1230 
1231 static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
1232 			    int attr_mask)
1233 {
1234 	if (attr_mask & IB_QP_PKEY_INDEX)
1235 		sqp->pkey_index = attr->pkey_index;
1236 	if (attr_mask & IB_QP_QKEY)
1237 		sqp->qkey = attr->qkey;
1238 	if (attr_mask & IB_QP_SQ_PSN)
1239 		sqp->send_psn = attr->sq_psn;
1240 }
1241 
1242 static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
1243 {
1244 	path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
1245 }
1246 
1247 static int _mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
1248 			  u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
1249 			  struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
1250 {
1251 	int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) ==
1252 		IB_LINK_LAYER_ETHERNET;
1253 	int vidx;
1254 	int smac_index;
1255 	int err;
1256 
1257 
1258 	path->grh_mylmc     = ah->src_path_bits & 0x7f;
1259 	path->rlid	    = cpu_to_be16(ah->dlid);
1260 	if (ah->static_rate) {
1261 		path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;
1262 		while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
1263 		       !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
1264 			--path->static_rate;
1265 	} else
1266 		path->static_rate = 0;
1267 
1268 	if (ah->ah_flags & IB_AH_GRH) {
1269 		if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
1270 			pr_err("sgid_index (%u) too large. max is %d\n",
1271 			       ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
1272 			return -1;
1273 		}
1274 
1275 		path->grh_mylmc |= 1 << 7;
1276 		path->mgid_index = ah->grh.sgid_index;
1277 		path->hop_limit  = ah->grh.hop_limit;
1278 		path->tclass_flowlabel =
1279 			cpu_to_be32((ah->grh.traffic_class << 20) |
1280 				    (ah->grh.flow_label));
1281 		memcpy(path->rgid, ah->grh.dgid.raw, 16);
1282 	}
1283 
1284 	if (is_eth) {
1285 		if (!(ah->ah_flags & IB_AH_GRH))
1286 			return -1;
1287 
1288 		path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1289 			((port - 1) << 6) | ((ah->sl & 7) << 3);
1290 
1291 		path->feup |= MLX4_FEUP_FORCE_ETH_UP;
1292 		if (vlan_tag < 0x1000) {
1293 			if (smac_info->vid < 0x1000) {
1294 				/* both valid vlan ids */
1295 				if (smac_info->vid != vlan_tag) {
1296 					/* different VIDs.  unreg old and reg new */
1297 					err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1298 					if (err)
1299 						return err;
1300 					smac_info->candidate_vid = vlan_tag;
1301 					smac_info->candidate_vlan_index = vidx;
1302 					smac_info->candidate_vlan_port = port;
1303 					smac_info->update_vid = 1;
1304 					path->vlan_index = vidx;
1305 				} else {
1306 					path->vlan_index = smac_info->vlan_index;
1307 				}
1308 			} else {
1309 				/* no current vlan tag in qp */
1310 				err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1311 				if (err)
1312 					return err;
1313 				smac_info->candidate_vid = vlan_tag;
1314 				smac_info->candidate_vlan_index = vidx;
1315 				smac_info->candidate_vlan_port = port;
1316 				smac_info->update_vid = 1;
1317 				path->vlan_index = vidx;
1318 			}
1319 			path->feup |= MLX4_FVL_FORCE_ETH_VLAN;
1320 			path->fl = 1 << 6;
1321 		} else {
1322 			/* have current vlan tag. unregister it at modify-qp success */
1323 			if (smac_info->vid < 0x1000) {
1324 				smac_info->candidate_vid = 0xFFFF;
1325 				smac_info->update_vid = 1;
1326 			}
1327 		}
1328 
1329 		/* get smac_index for RoCE use.
1330 		 * If no smac was yet assigned, register one.
1331 		 * If one was already assigned, but the new mac differs,
1332 		 * unregister the old one and register the new one.
1333 		*/
1334 		if ((!smac_info->smac && !smac_info->smac_port) ||
1335 		    smac_info->smac != smac) {
1336 			/* register candidate now, unreg if needed, after success */
1337 			smac_index = mlx4_register_mac(dev->dev, port, smac);
1338 			if (smac_index >= 0) {
1339 				smac_info->candidate_smac_index = smac_index;
1340 				smac_info->candidate_smac = smac;
1341 				smac_info->candidate_smac_port = port;
1342 			} else {
1343 				return -EINVAL;
1344 			}
1345 		} else {
1346 			smac_index = smac_info->smac_index;
1347 		}
1348 
1349 		memcpy(path->dmac, ah->dmac, 6);
1350 		path->ackto = MLX4_IB_LINK_TYPE_ETH;
1351 		/* put MAC table smac index for IBoE */
1352 		path->grh_mylmc = (u8) (smac_index) | 0x80;
1353 	} else {
1354 		path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1355 			((port - 1) << 6) | ((ah->sl & 0xf) << 2);
1356 	}
1357 
1358 	return 0;
1359 }
1360 
1361 static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,
1362 			 enum ib_qp_attr_mask qp_attr_mask,
1363 			 struct mlx4_ib_qp *mqp,
1364 			 struct mlx4_qp_path *path, u8 port)
1365 {
1366 	return _mlx4_set_path(dev, &qp->ah_attr,
1367 			      mlx4_mac_to_u64((u8 *)qp->smac),
1368 			      (qp_attr_mask & IB_QP_VID) ? qp->vlan_id : 0xffff,
1369 			      path, &mqp->pri, port);
1370 }
1371 
1372 static int mlx4_set_alt_path(struct mlx4_ib_dev *dev,
1373 			     const struct ib_qp_attr *qp,
1374 			     enum ib_qp_attr_mask qp_attr_mask,
1375 			     struct mlx4_ib_qp *mqp,
1376 			     struct mlx4_qp_path *path, u8 port)
1377 {
1378 	return _mlx4_set_path(dev, &qp->alt_ah_attr,
1379 			      mlx4_mac_to_u64((u8 *)qp->alt_smac),
1380 			      (qp_attr_mask & IB_QP_ALT_VID) ?
1381 			      qp->alt_vlan_id : 0xffff,
1382 			      path, &mqp->alt, port);
1383 }
1384 
1385 static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1386 {
1387 	struct mlx4_ib_gid_entry *ge, *tmp;
1388 
1389 	list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1390 		if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
1391 			ge->added = 1;
1392 			ge->port = qp->port;
1393 		}
1394 	}
1395 }
1396 
1397 static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, u8 *smac,
1398 				    struct mlx4_qp_context *context)
1399 {
1400 	u64 u64_mac;
1401 	int smac_index;
1402 
1403 	u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]);
1404 
1405 	context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6);
1406 	if (!qp->pri.smac && !qp->pri.smac_port) {
1407 		smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac);
1408 		if (smac_index >= 0) {
1409 			qp->pri.candidate_smac_index = smac_index;
1410 			qp->pri.candidate_smac = u64_mac;
1411 			qp->pri.candidate_smac_port = qp->port;
1412 			context->pri_path.grh_mylmc = 0x80 | (u8) smac_index;
1413 		} else {
1414 			return -ENOENT;
1415 		}
1416 	}
1417 	return 0;
1418 }
1419 
1420 static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
1421 			       const struct ib_qp_attr *attr, int attr_mask,
1422 			       enum ib_qp_state cur_state, enum ib_qp_state new_state)
1423 {
1424 	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1425 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
1426 	struct mlx4_ib_pd *pd;
1427 	struct mlx4_ib_cq *send_cq, *recv_cq;
1428 	struct mlx4_qp_context *context;
1429 	enum mlx4_qp_optpar optpar = 0;
1430 	int sqd_event;
1431 	int steer_qp = 0;
1432 	int err = -EINVAL;
1433 
1434 	/* APM is not supported under RoCE */
1435 	if (attr_mask & IB_QP_ALT_PATH &&
1436 	    rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
1437 	    IB_LINK_LAYER_ETHERNET)
1438 		return -ENOTSUPP;
1439 
1440 	context = kzalloc(sizeof *context, GFP_KERNEL);
1441 	if (!context)
1442 		return -ENOMEM;
1443 
1444 	context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
1445 				     (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
1446 
1447 	if (!(attr_mask & IB_QP_PATH_MIG_STATE))
1448 		context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
1449 	else {
1450 		optpar |= MLX4_QP_OPTPAR_PM_STATE;
1451 		switch (attr->path_mig_state) {
1452 		case IB_MIG_MIGRATED:
1453 			context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
1454 			break;
1455 		case IB_MIG_REARM:
1456 			context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
1457 			break;
1458 		case IB_MIG_ARMED:
1459 			context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
1460 			break;
1461 		}
1462 	}
1463 
1464 	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
1465 		context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
1466 	else if (ibqp->qp_type == IB_QPT_RAW_PACKET)
1467 		context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
1468 	else if (ibqp->qp_type == IB_QPT_UD) {
1469 		if (qp->flags & MLX4_IB_QP_LSO)
1470 			context->mtu_msgmax = (IB_MTU_4096 << 5) |
1471 					      ilog2(dev->dev->caps.max_gso_sz);
1472 		else
1473 			context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
1474 	} else if (attr_mask & IB_QP_PATH_MTU) {
1475 		if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
1476 			pr_err("path MTU (%u) is invalid\n",
1477 			       attr->path_mtu);
1478 			goto out;
1479 		}
1480 		context->mtu_msgmax = (attr->path_mtu << 5) |
1481 			ilog2(dev->dev->caps.max_msg_sz);
1482 	}
1483 
1484 	if (qp->rq.wqe_cnt)
1485 		context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
1486 	context->rq_size_stride |= qp->rq.wqe_shift - 4;
1487 
1488 	if (qp->sq.wqe_cnt)
1489 		context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
1490 	context->sq_size_stride |= qp->sq.wqe_shift - 4;
1491 
1492 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1493 		context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
1494 		context->xrcd = cpu_to_be32((u32) qp->xrcdn);
1495 		if (ibqp->qp_type == IB_QPT_RAW_PACKET)
1496 			context->param3 |= cpu_to_be32(1 << 30);
1497 	}
1498 
1499 	if (qp->ibqp.uobject)
1500 		context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index);
1501 	else
1502 		context->usr_page = cpu_to_be32(dev->priv_uar.index);
1503 
1504 	if (attr_mask & IB_QP_DEST_QPN)
1505 		context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
1506 
1507 	if (attr_mask & IB_QP_PORT) {
1508 		if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
1509 		    !(attr_mask & IB_QP_AV)) {
1510 			mlx4_set_sched(&context->pri_path, attr->port_num);
1511 			optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
1512 		}
1513 	}
1514 
1515 	if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
1516 		if (dev->counters[qp->port - 1] != -1) {
1517 			context->pri_path.counter_index =
1518 						dev->counters[qp->port - 1];
1519 			optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
1520 		} else
1521 			context->pri_path.counter_index = 0xff;
1522 
1523 		if (qp->flags & MLX4_IB_QP_NETIF) {
1524 			mlx4_ib_steer_qp_reg(dev, qp, 1);
1525 			steer_qp = 1;
1526 		}
1527 	}
1528 
1529 	if (attr_mask & IB_QP_PKEY_INDEX) {
1530 		if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
1531 			context->pri_path.disable_pkey_check = 0x40;
1532 		context->pri_path.pkey_index = attr->pkey_index;
1533 		optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
1534 	}
1535 
1536 	if (attr_mask & IB_QP_AV) {
1537 		if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
1538 				  attr_mask & IB_QP_PORT ?
1539 				  attr->port_num : qp->port))
1540 			goto out;
1541 
1542 		optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
1543 			   MLX4_QP_OPTPAR_SCHED_QUEUE);
1544 	}
1545 
1546 	if (attr_mask & IB_QP_TIMEOUT) {
1547 		context->pri_path.ackto |= attr->timeout << 3;
1548 		optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
1549 	}
1550 
1551 	if (attr_mask & IB_QP_ALT_PATH) {
1552 		if (attr->alt_port_num == 0 ||
1553 		    attr->alt_port_num > dev->dev->caps.num_ports)
1554 			goto out;
1555 
1556 		if (attr->alt_pkey_index >=
1557 		    dev->dev->caps.pkey_table_len[attr->alt_port_num])
1558 			goto out;
1559 
1560 		if (mlx4_set_alt_path(dev, attr, attr_mask, qp,
1561 				      &context->alt_path,
1562 				      attr->alt_port_num))
1563 			goto out;
1564 
1565 		context->alt_path.pkey_index = attr->alt_pkey_index;
1566 		context->alt_path.ackto = attr->alt_timeout << 3;
1567 		optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
1568 	}
1569 
1570 	pd = get_pd(qp);
1571 	get_cqs(qp, &send_cq, &recv_cq);
1572 	context->pd       = cpu_to_be32(pd->pdn);
1573 	context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
1574 	context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
1575 	context->params1  = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
1576 
1577 	/* Set "fast registration enabled" for all kernel QPs */
1578 	if (!qp->ibqp.uobject)
1579 		context->params1 |= cpu_to_be32(1 << 11);
1580 
1581 	if (attr_mask & IB_QP_RNR_RETRY) {
1582 		context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
1583 		optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
1584 	}
1585 
1586 	if (attr_mask & IB_QP_RETRY_CNT) {
1587 		context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
1588 		optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
1589 	}
1590 
1591 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1592 		if (attr->max_rd_atomic)
1593 			context->params1 |=
1594 				cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
1595 		optpar |= MLX4_QP_OPTPAR_SRA_MAX;
1596 	}
1597 
1598 	if (attr_mask & IB_QP_SQ_PSN)
1599 		context->next_send_psn = cpu_to_be32(attr->sq_psn);
1600 
1601 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1602 		if (attr->max_dest_rd_atomic)
1603 			context->params2 |=
1604 				cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
1605 		optpar |= MLX4_QP_OPTPAR_RRA_MAX;
1606 	}
1607 
1608 	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
1609 		context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
1610 		optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
1611 	}
1612 
1613 	if (ibqp->srq)
1614 		context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
1615 
1616 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1617 		context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
1618 		optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
1619 	}
1620 	if (attr_mask & IB_QP_RQ_PSN)
1621 		context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
1622 
1623 	/* proxy and tunnel qp qkeys will be changed in modify-qp wrappers */
1624 	if (attr_mask & IB_QP_QKEY) {
1625 		if (qp->mlx4_ib_qp_type &
1626 		    (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))
1627 			context->qkey = cpu_to_be32(IB_QP_SET_QKEY);
1628 		else {
1629 			if (mlx4_is_mfunc(dev->dev) &&
1630 			    !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) &&
1631 			    (attr->qkey & MLX4_RESERVED_QKEY_MASK) ==
1632 			    MLX4_RESERVED_QKEY_BASE) {
1633 				pr_err("Cannot use reserved QKEY"
1634 				       " 0x%x (range 0xffff0000..0xffffffff"
1635 				       " is reserved)\n", attr->qkey);
1636 				err = -EINVAL;
1637 				goto out;
1638 			}
1639 			context->qkey = cpu_to_be32(attr->qkey);
1640 		}
1641 		optpar |= MLX4_QP_OPTPAR_Q_KEY;
1642 	}
1643 
1644 	if (ibqp->srq)
1645 		context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
1646 
1647 	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1648 		context->db_rec_addr = cpu_to_be64(qp->db.dma);
1649 
1650 	if (cur_state == IB_QPS_INIT &&
1651 	    new_state == IB_QPS_RTR  &&
1652 	    (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
1653 	     ibqp->qp_type == IB_QPT_UD ||
1654 	     ibqp->qp_type == IB_QPT_RAW_PACKET)) {
1655 		context->pri_path.sched_queue = (qp->port - 1) << 6;
1656 		if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
1657 		    qp->mlx4_ib_qp_type &
1658 		    (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) {
1659 			context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
1660 			if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI)
1661 				context->pri_path.fl = 0x80;
1662 		} else {
1663 			if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
1664 				context->pri_path.fl = 0x80;
1665 			context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
1666 		}
1667 		if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
1668 		    IB_LINK_LAYER_ETHERNET) {
1669 			if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI ||
1670 			    qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI)
1671 				context->pri_path.feup = 1 << 7; /* don't fsm */
1672 			/* handle smac_index */
1673 			if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
1674 			    qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI ||
1675 			    qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
1676 				err = handle_eth_ud_smac_index(dev, qp, (u8 *)attr->smac, context);
1677 				if (err)
1678 					return -EINVAL;
1679 				if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1680 					dev->qp1_proxy[qp->port - 1] = qp;
1681 			}
1682 		}
1683 	}
1684 
1685 	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
1686 		context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
1687 					MLX4_IB_LINK_TYPE_ETH;
1688 		if (dev->dev->caps.tunnel_offload_mode ==  MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
1689 			/* set QP to receive both tunneled & non-tunneled packets */
1690 			if (!(context->flags & cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET)))
1691 				context->srqn = cpu_to_be32(7 << 28);
1692 		}
1693 	}
1694 
1695 	if (ibqp->qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
1696 		int is_eth = rdma_port_get_link_layer(
1697 				&dev->ib_dev, qp->port) ==
1698 				IB_LINK_LAYER_ETHERNET;
1699 		if (is_eth) {
1700 			context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH;
1701 			optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH;
1702 		}
1703 	}
1704 
1705 
1706 	if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD	&&
1707 	    attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
1708 		sqd_event = 1;
1709 	else
1710 		sqd_event = 0;
1711 
1712 	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1713 		context->rlkey |= (1 << 4);
1714 
1715 	/*
1716 	 * Before passing a kernel QP to the HW, make sure that the
1717 	 * ownership bits of the send queue are set and the SQ
1718 	 * headroom is stamped so that the hardware doesn't start
1719 	 * processing stale work requests.
1720 	 */
1721 	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1722 		struct mlx4_wqe_ctrl_seg *ctrl;
1723 		int i;
1724 
1725 		for (i = 0; i < qp->sq.wqe_cnt; ++i) {
1726 			ctrl = get_send_wqe(qp, i);
1727 			ctrl->owner_opcode = cpu_to_be32(1 << 31);
1728 			if (qp->sq_max_wqes_per_wr == 1)
1729 				ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4);
1730 
1731 			stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
1732 		}
1733 	}
1734 
1735 	err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
1736 			     to_mlx4_state(new_state), context, optpar,
1737 			     sqd_event, &qp->mqp);
1738 	if (err)
1739 		goto out;
1740 
1741 	qp->state = new_state;
1742 
1743 	if (attr_mask & IB_QP_ACCESS_FLAGS)
1744 		qp->atomic_rd_en = attr->qp_access_flags;
1745 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1746 		qp->resp_depth = attr->max_dest_rd_atomic;
1747 	if (attr_mask & IB_QP_PORT) {
1748 		qp->port = attr->port_num;
1749 		update_mcg_macs(dev, qp);
1750 	}
1751 	if (attr_mask & IB_QP_ALT_PATH)
1752 		qp->alt_port = attr->alt_port_num;
1753 
1754 	if (is_sqp(dev, qp))
1755 		store_sqp_attrs(to_msqp(qp), attr, attr_mask);
1756 
1757 	/*
1758 	 * If we moved QP0 to RTR, bring the IB link up; if we moved
1759 	 * QP0 to RESET or ERROR, bring the link back down.
1760 	 */
1761 	if (is_qp0(dev, qp)) {
1762 		if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
1763 			if (mlx4_INIT_PORT(dev->dev, qp->port))
1764 				pr_warn("INIT_PORT failed for port %d\n",
1765 				       qp->port);
1766 
1767 		if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
1768 		    (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
1769 			mlx4_CLOSE_PORT(dev->dev, qp->port);
1770 	}
1771 
1772 	/*
1773 	 * If we moved a kernel QP to RESET, clean up all old CQ
1774 	 * entries and reinitialize the QP.
1775 	 */
1776 	if (new_state == IB_QPS_RESET) {
1777 		if (!ibqp->uobject) {
1778 			mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1779 					 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
1780 			if (send_cq != recv_cq)
1781 				mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1782 
1783 			qp->rq.head = 0;
1784 			qp->rq.tail = 0;
1785 			qp->sq.head = 0;
1786 			qp->sq.tail = 0;
1787 			qp->sq_next_wqe = 0;
1788 			if (qp->rq.wqe_cnt)
1789 				*qp->db.db  = 0;
1790 
1791 			if (qp->flags & MLX4_IB_QP_NETIF)
1792 				mlx4_ib_steer_qp_reg(dev, qp, 0);
1793 		}
1794 		if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
1795 			mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1796 			qp->pri.smac = 0;
1797 			qp->pri.smac_port = 0;
1798 		}
1799 		if (qp->alt.smac) {
1800 			mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1801 			qp->alt.smac = 0;
1802 		}
1803 		if (qp->pri.vid < 0x1000) {
1804 			mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
1805 			qp->pri.vid = 0xFFFF;
1806 			qp->pri.candidate_vid = 0xFFFF;
1807 			qp->pri.update_vid = 0;
1808 		}
1809 
1810 		if (qp->alt.vid < 0x1000) {
1811 			mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
1812 			qp->alt.vid = 0xFFFF;
1813 			qp->alt.candidate_vid = 0xFFFF;
1814 			qp->alt.update_vid = 0;
1815 		}
1816 	}
1817 out:
1818 	if (err && steer_qp)
1819 		mlx4_ib_steer_qp_reg(dev, qp, 0);
1820 	kfree(context);
1821 	if (qp->pri.candidate_smac ||
1822 	    (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) {
1823 		if (err) {
1824 			mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac);
1825 		} else {
1826 			if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port))
1827 				mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1828 			qp->pri.smac = qp->pri.candidate_smac;
1829 			qp->pri.smac_index = qp->pri.candidate_smac_index;
1830 			qp->pri.smac_port = qp->pri.candidate_smac_port;
1831 		}
1832 		qp->pri.candidate_smac = 0;
1833 		qp->pri.candidate_smac_index = 0;
1834 		qp->pri.candidate_smac_port = 0;
1835 	}
1836 	if (qp->alt.candidate_smac) {
1837 		if (err) {
1838 			mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac);
1839 		} else {
1840 			if (qp->alt.smac)
1841 				mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1842 			qp->alt.smac = qp->alt.candidate_smac;
1843 			qp->alt.smac_index = qp->alt.candidate_smac_index;
1844 			qp->alt.smac_port = qp->alt.candidate_smac_port;
1845 		}
1846 		qp->alt.candidate_smac = 0;
1847 		qp->alt.candidate_smac_index = 0;
1848 		qp->alt.candidate_smac_port = 0;
1849 	}
1850 
1851 	if (qp->pri.update_vid) {
1852 		if (err) {
1853 			if (qp->pri.candidate_vid < 0x1000)
1854 				mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port,
1855 						     qp->pri.candidate_vid);
1856 		} else {
1857 			if (qp->pri.vid < 0x1000)
1858 				mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port,
1859 						     qp->pri.vid);
1860 			qp->pri.vid = qp->pri.candidate_vid;
1861 			qp->pri.vlan_port = qp->pri.candidate_vlan_port;
1862 			qp->pri.vlan_index =  qp->pri.candidate_vlan_index;
1863 		}
1864 		qp->pri.candidate_vid = 0xFFFF;
1865 		qp->pri.update_vid = 0;
1866 	}
1867 
1868 	if (qp->alt.update_vid) {
1869 		if (err) {
1870 			if (qp->alt.candidate_vid < 0x1000)
1871 				mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port,
1872 						     qp->alt.candidate_vid);
1873 		} else {
1874 			if (qp->alt.vid < 0x1000)
1875 				mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port,
1876 						     qp->alt.vid);
1877 			qp->alt.vid = qp->alt.candidate_vid;
1878 			qp->alt.vlan_port = qp->alt.candidate_vlan_port;
1879 			qp->alt.vlan_index =  qp->alt.candidate_vlan_index;
1880 		}
1881 		qp->alt.candidate_vid = 0xFFFF;
1882 		qp->alt.update_vid = 0;
1883 	}
1884 
1885 	return err;
1886 }
1887 
1888 int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1889 		      int attr_mask, struct ib_udata *udata)
1890 {
1891 	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1892 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
1893 	enum ib_qp_state cur_state, new_state;
1894 	int err = -EINVAL;
1895 	int ll;
1896 	mutex_lock(&qp->mutex);
1897 
1898 	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
1899 	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1900 
1901 	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1902 		ll = IB_LINK_LAYER_UNSPECIFIED;
1903 	} else {
1904 		int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1905 		ll = rdma_port_get_link_layer(&dev->ib_dev, port);
1906 	}
1907 
1908 	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1909 				attr_mask, ll)) {
1910 		pr_debug("qpn 0x%x: invalid attribute mask specified "
1911 			 "for transition %d to %d. qp_type %d,"
1912 			 " attr_mask 0x%x\n",
1913 			 ibqp->qp_num, cur_state, new_state,
1914 			 ibqp->qp_type, attr_mask);
1915 		goto out;
1916 	}
1917 
1918 	if ((attr_mask & IB_QP_PORT) &&
1919 	    (attr->port_num == 0 || attr->port_num > dev->num_ports)) {
1920 		pr_debug("qpn 0x%x: invalid port number (%d) specified "
1921 			 "for transition %d to %d. qp_type %d\n",
1922 			 ibqp->qp_num, attr->port_num, cur_state,
1923 			 new_state, ibqp->qp_type);
1924 		goto out;
1925 	}
1926 
1927 	if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
1928 	    (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
1929 	     IB_LINK_LAYER_ETHERNET))
1930 		goto out;
1931 
1932 	if (attr_mask & IB_QP_PKEY_INDEX) {
1933 		int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1934 		if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) {
1935 			pr_debug("qpn 0x%x: invalid pkey index (%d) specified "
1936 				 "for transition %d to %d. qp_type %d\n",
1937 				 ibqp->qp_num, attr->pkey_index, cur_state,
1938 				 new_state, ibqp->qp_type);
1939 			goto out;
1940 		}
1941 	}
1942 
1943 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1944 	    attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
1945 		pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. "
1946 			 "Transition %d to %d. qp_type %d\n",
1947 			 ibqp->qp_num, attr->max_rd_atomic, cur_state,
1948 			 new_state, ibqp->qp_type);
1949 		goto out;
1950 	}
1951 
1952 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1953 	    attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
1954 		pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. "
1955 			 "Transition %d to %d. qp_type %d\n",
1956 			 ibqp->qp_num, attr->max_dest_rd_atomic, cur_state,
1957 			 new_state, ibqp->qp_type);
1958 		goto out;
1959 	}
1960 
1961 	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1962 		err = 0;
1963 		goto out;
1964 	}
1965 
1966 	err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
1967 
1968 out:
1969 	mutex_unlock(&qp->mutex);
1970 	return err;
1971 }
1972 
1973 static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
1974 {
1975 	int i;
1976 	for (i = 0; i < dev->caps.num_ports; i++) {
1977 		if (qpn == dev->caps.qp0_proxy[i] ||
1978 		    qpn == dev->caps.qp0_tunnel[i]) {
1979 			*qkey = dev->caps.qp0_qkey[i];
1980 			return 0;
1981 		}
1982 	}
1983 	return -EINVAL;
1984 }
1985 
1986 static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
1987 				  struct ib_send_wr *wr,
1988 				  void *wqe, unsigned *mlx_seg_len)
1989 {
1990 	struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
1991 	struct ib_device *ib_dev = &mdev->ib_dev;
1992 	struct mlx4_wqe_mlx_seg *mlx = wqe;
1993 	struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
1994 	struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah);
1995 	u16 pkey;
1996 	u32 qkey;
1997 	int send_size;
1998 	int header_size;
1999 	int spc;
2000 	int i;
2001 
2002 	if (wr->opcode != IB_WR_SEND)
2003 		return -EINVAL;
2004 
2005 	send_size = 0;
2006 
2007 	for (i = 0; i < wr->num_sge; ++i)
2008 		send_size += wr->sg_list[i].length;
2009 
2010 	/* for proxy-qp0 sends, need to add in size of tunnel header */
2011 	/* for tunnel-qp0 sends, tunnel header is already in s/g list */
2012 	if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2013 		send_size += sizeof (struct mlx4_ib_tunnel_header);
2014 
2015 	ib_ud_header_init(send_size, 1, 0, 0, 0, 0, &sqp->ud_header);
2016 
2017 	if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2018 		sqp->ud_header.lrh.service_level =
2019 			be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2020 		sqp->ud_header.lrh.destination_lid =
2021 			cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2022 		sqp->ud_header.lrh.source_lid =
2023 			cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2024 	}
2025 
2026 	mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2027 
2028 	/* force loopback */
2029 	mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR);
2030 	mlx->rlid = sqp->ud_header.lrh.destination_lid;
2031 
2032 	sqp->ud_header.lrh.virtual_lane    = 0;
2033 	sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
2034 	ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
2035 	sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2036 	if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2037 		sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
2038 	else
2039 		sqp->ud_header.bth.destination_qpn =
2040 			cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]);
2041 
2042 	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2043 	if (mlx4_is_master(mdev->dev)) {
2044 		if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2045 			return -EINVAL;
2046 	} else {
2047 		if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2048 			return -EINVAL;
2049 	}
2050 	sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
2051 	sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
2052 
2053 	sqp->ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY;
2054 	sqp->ud_header.immediate_present = 0;
2055 
2056 	header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
2057 
2058 	/*
2059 	 * Inline data segments may not cross a 64 byte boundary.  If
2060 	 * our UD header is bigger than the space available up to the
2061 	 * next 64 byte boundary in the WQE, use two inline data
2062 	 * segments to hold the UD header.
2063 	 */
2064 	spc = MLX4_INLINE_ALIGN -
2065 	      ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2066 	if (header_size <= spc) {
2067 		inl->byte_count = cpu_to_be32(1 << 31 | header_size);
2068 		memcpy(inl + 1, sqp->header_buf, header_size);
2069 		i = 1;
2070 	} else {
2071 		inl->byte_count = cpu_to_be32(1 << 31 | spc);
2072 		memcpy(inl + 1, sqp->header_buf, spc);
2073 
2074 		inl = (void *) (inl + 1) + spc;
2075 		memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
2076 		/*
2077 		 * Need a barrier here to make sure all the data is
2078 		 * visible before the byte_count field is set.
2079 		 * Otherwise the HCA prefetcher could grab the 64-byte
2080 		 * chunk with this inline segment and get a valid (!=
2081 		 * 0xffffffff) byte count but stale data, and end up
2082 		 * generating a packet with bad headers.
2083 		 *
2084 		 * The first inline segment's byte_count field doesn't
2085 		 * need a barrier, because it comes after a
2086 		 * control/MLX segment and therefore is at an offset
2087 		 * of 16 mod 64.
2088 		 */
2089 		wmb();
2090 		inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
2091 		i = 2;
2092 	}
2093 
2094 	*mlx_seg_len =
2095 	ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
2096 	return 0;
2097 }
2098 
2099 static void mlx4_u64_to_smac(u8 *dst_mac, u64 src_mac)
2100 {
2101 	int i;
2102 
2103 	for (i = ETH_ALEN; i; i--) {
2104 		dst_mac[i - 1] = src_mac & 0xff;
2105 		src_mac >>= 8;
2106 	}
2107 }
2108 
2109 static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
2110 			    void *wqe, unsigned *mlx_seg_len)
2111 {
2112 	struct ib_device *ib_dev = sqp->qp.ibqp.device;
2113 	struct mlx4_wqe_mlx_seg *mlx = wqe;
2114 	struct mlx4_wqe_ctrl_seg *ctrl = wqe;
2115 	struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2116 	struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah);
2117 	union ib_gid sgid;
2118 	u16 pkey;
2119 	int send_size;
2120 	int header_size;
2121 	int spc;
2122 	int i;
2123 	int err = 0;
2124 	u16 vlan = 0xffff;
2125 	bool is_eth;
2126 	bool is_vlan = false;
2127 	bool is_grh;
2128 
2129 	send_size = 0;
2130 	for (i = 0; i < wr->num_sge; ++i)
2131 		send_size += wr->sg_list[i].length;
2132 
2133 	is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
2134 	is_grh = mlx4_ib_ah_grh_present(ah);
2135 	if (is_eth) {
2136 		if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
2137 			/* When multi-function is enabled, the ib_core gid
2138 			 * indexes don't necessarily match the hw ones, so
2139 			 * we must use our own cache */
2140 			err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev,
2141 							   be32_to_cpu(ah->av.ib.port_pd) >> 24,
2142 							   ah->av.ib.gid_index, &sgid.raw[0]);
2143 			if (err)
2144 				return err;
2145 		} else  {
2146 			err = ib_get_cached_gid(ib_dev,
2147 						be32_to_cpu(ah->av.ib.port_pd) >> 24,
2148 						ah->av.ib.gid_index, &sgid);
2149 			if (err)
2150 				return err;
2151 		}
2152 
2153 		if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
2154 			vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
2155 			is_vlan = 1;
2156 		}
2157 	}
2158 	ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header);
2159 
2160 	if (!is_eth) {
2161 		sqp->ud_header.lrh.service_level =
2162 			be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2163 		sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
2164 		sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2165 	}
2166 
2167 	if (is_grh) {
2168 		sqp->ud_header.grh.traffic_class =
2169 			(be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
2170 		sqp->ud_header.grh.flow_label    =
2171 			ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
2172 		sqp->ud_header.grh.hop_limit     = ah->av.ib.hop_limit;
2173 		if (is_eth)
2174 			memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
2175 		else {
2176 		if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
2177 			/* When multi-function is enabled, the ib_core gid
2178 			 * indexes don't necessarily match the hw ones, so
2179 			 * we must use our own cache */
2180 			sqp->ud_header.grh.source_gid.global.subnet_prefix =
2181 				to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
2182 						       subnet_prefix;
2183 			sqp->ud_header.grh.source_gid.global.interface_id =
2184 				to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
2185 					       guid_cache[ah->av.ib.gid_index];
2186 		} else
2187 			ib_get_cached_gid(ib_dev,
2188 					  be32_to_cpu(ah->av.ib.port_pd) >> 24,
2189 					  ah->av.ib.gid_index,
2190 					  &sqp->ud_header.grh.source_gid);
2191 		}
2192 		memcpy(sqp->ud_header.grh.destination_gid.raw,
2193 		       ah->av.ib.dgid, 16);
2194 	}
2195 
2196 	mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2197 
2198 	if (!is_eth) {
2199 		mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
2200 					  (sqp->ud_header.lrh.destination_lid ==
2201 					   IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
2202 					  (sqp->ud_header.lrh.service_level << 8));
2203 		if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
2204 			mlx->flags |= cpu_to_be32(0x1); /* force loopback */
2205 		mlx->rlid = sqp->ud_header.lrh.destination_lid;
2206 	}
2207 
2208 	switch (wr->opcode) {
2209 	case IB_WR_SEND:
2210 		sqp->ud_header.bth.opcode	 = IB_OPCODE_UD_SEND_ONLY;
2211 		sqp->ud_header.immediate_present = 0;
2212 		break;
2213 	case IB_WR_SEND_WITH_IMM:
2214 		sqp->ud_header.bth.opcode	 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2215 		sqp->ud_header.immediate_present = 1;
2216 		sqp->ud_header.immediate_data    = wr->ex.imm_data;
2217 		break;
2218 	default:
2219 		return -EINVAL;
2220 	}
2221 
2222 	if (is_eth) {
2223 		struct in6_addr in6;
2224 
2225 		u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
2226 
2227 		mlx->sched_prio = cpu_to_be16(pcp);
2228 
2229 		memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
2230 		/* FIXME: cache smac value? */
2231 		memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
2232 		memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
2233 		memcpy(&in6, sgid.raw, sizeof(in6));
2234 
2235 		if (!mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
2236 			u64 mac = atomic64_read(&to_mdev(ib_dev)->iboe.mac[sqp->qp.port - 1]);
2237 			u8 smac[ETH_ALEN];
2238 
2239 			mlx4_u64_to_smac(smac, mac);
2240 			memcpy(sqp->ud_header.eth.smac_h, smac, ETH_ALEN);
2241 		} else {
2242 			/* use the src mac of the tunnel */
2243 			memcpy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac, ETH_ALEN);
2244 		}
2245 
2246 		if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
2247 			mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
2248 		if (!is_vlan) {
2249 			sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
2250 		} else {
2251 			sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
2252 			sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
2253 		}
2254 	} else {
2255 		sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 : 0;
2256 		if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
2257 			sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
2258 	}
2259 	sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
2260 	if (!sqp->qp.ibqp.qp_num)
2261 		ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
2262 	else
2263 		ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey);
2264 	sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2265 	sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
2266 	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2267 	sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ?
2268 					       sqp->qkey : wr->wr.ud.remote_qkey);
2269 	sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
2270 
2271 	header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
2272 
2273 	if (0) {
2274 		pr_err("built UD header of size %d:\n", header_size);
2275 		for (i = 0; i < header_size / 4; ++i) {
2276 			if (i % 8 == 0)
2277 				pr_err("  [%02x] ", i * 4);
2278 			pr_cont(" %08x",
2279 				be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
2280 			if ((i + 1) % 8 == 0)
2281 				pr_cont("\n");
2282 		}
2283 		pr_err("\n");
2284 	}
2285 
2286 	/*
2287 	 * Inline data segments may not cross a 64 byte boundary.  If
2288 	 * our UD header is bigger than the space available up to the
2289 	 * next 64 byte boundary in the WQE, use two inline data
2290 	 * segments to hold the UD header.
2291 	 */
2292 	spc = MLX4_INLINE_ALIGN -
2293 		((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2294 	if (header_size <= spc) {
2295 		inl->byte_count = cpu_to_be32(1 << 31 | header_size);
2296 		memcpy(inl + 1, sqp->header_buf, header_size);
2297 		i = 1;
2298 	} else {
2299 		inl->byte_count = cpu_to_be32(1 << 31 | spc);
2300 		memcpy(inl + 1, sqp->header_buf, spc);
2301 
2302 		inl = (void *) (inl + 1) + spc;
2303 		memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
2304 		/*
2305 		 * Need a barrier here to make sure all the data is
2306 		 * visible before the byte_count field is set.
2307 		 * Otherwise the HCA prefetcher could grab the 64-byte
2308 		 * chunk with this inline segment and get a valid (!=
2309 		 * 0xffffffff) byte count but stale data, and end up
2310 		 * generating a packet with bad headers.
2311 		 *
2312 		 * The first inline segment's byte_count field doesn't
2313 		 * need a barrier, because it comes after a
2314 		 * control/MLX segment and therefore is at an offset
2315 		 * of 16 mod 64.
2316 		 */
2317 		wmb();
2318 		inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
2319 		i = 2;
2320 	}
2321 
2322 	*mlx_seg_len =
2323 		ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
2324 	return 0;
2325 }
2326 
2327 static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
2328 {
2329 	unsigned cur;
2330 	struct mlx4_ib_cq *cq;
2331 
2332 	cur = wq->head - wq->tail;
2333 	if (likely(cur + nreq < wq->max_post))
2334 		return 0;
2335 
2336 	cq = to_mcq(ib_cq);
2337 	spin_lock(&cq->lock);
2338 	cur = wq->head - wq->tail;
2339 	spin_unlock(&cq->lock);
2340 
2341 	return cur + nreq >= wq->max_post;
2342 }
2343 
2344 static __be32 convert_access(int acc)
2345 {
2346 	return (acc & IB_ACCESS_REMOTE_ATOMIC ?
2347 		cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC)       : 0) |
2348 	       (acc & IB_ACCESS_REMOTE_WRITE  ?
2349 		cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) |
2350 	       (acc & IB_ACCESS_REMOTE_READ   ?
2351 		cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ)  : 0) |
2352 	       (acc & IB_ACCESS_LOCAL_WRITE   ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE)  : 0) |
2353 		cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
2354 }
2355 
2356 static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr)
2357 {
2358 	struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list);
2359 	int i;
2360 
2361 	for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i)
2362 		mfrpl->mapped_page_list[i] =
2363 			cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] |
2364 				    MLX4_MTT_FLAG_PRESENT);
2365 
2366 	fseg->flags		= convert_access(wr->wr.fast_reg.access_flags);
2367 	fseg->mem_key		= cpu_to_be32(wr->wr.fast_reg.rkey);
2368 	fseg->buf_list		= cpu_to_be64(mfrpl->map);
2369 	fseg->start_addr	= cpu_to_be64(wr->wr.fast_reg.iova_start);
2370 	fseg->reg_len		= cpu_to_be64(wr->wr.fast_reg.length);
2371 	fseg->offset		= 0; /* XXX -- is this just for ZBVA? */
2372 	fseg->page_size		= cpu_to_be32(wr->wr.fast_reg.page_shift);
2373 	fseg->reserved[0]	= 0;
2374 	fseg->reserved[1]	= 0;
2375 }
2376 
2377 static void set_bind_seg(struct mlx4_wqe_bind_seg *bseg, struct ib_send_wr *wr)
2378 {
2379 	bseg->flags1 =
2380 		convert_access(wr->wr.bind_mw.bind_info.mw_access_flags) &
2381 		cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ  |
2382 			    MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE |
2383 			    MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC);
2384 	bseg->flags2 = 0;
2385 	if (wr->wr.bind_mw.mw->type == IB_MW_TYPE_2)
2386 		bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_TYPE_2);
2387 	if (wr->wr.bind_mw.bind_info.mw_access_flags & IB_ZERO_BASED)
2388 		bseg->flags2 |= cpu_to_be32(MLX4_WQE_BIND_ZERO_BASED);
2389 	bseg->new_rkey = cpu_to_be32(wr->wr.bind_mw.rkey);
2390 	bseg->lkey = cpu_to_be32(wr->wr.bind_mw.bind_info.mr->lkey);
2391 	bseg->addr = cpu_to_be64(wr->wr.bind_mw.bind_info.addr);
2392 	bseg->length = cpu_to_be64(wr->wr.bind_mw.bind_info.length);
2393 }
2394 
2395 static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
2396 {
2397 	memset(iseg, 0, sizeof(*iseg));
2398 	iseg->mem_key = cpu_to_be32(rkey);
2399 }
2400 
2401 static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
2402 					  u64 remote_addr, u32 rkey)
2403 {
2404 	rseg->raddr    = cpu_to_be64(remote_addr);
2405 	rseg->rkey     = cpu_to_be32(rkey);
2406 	rseg->reserved = 0;
2407 }
2408 
2409 static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr)
2410 {
2411 	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
2412 		aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap);
2413 		aseg->compare  = cpu_to_be64(wr->wr.atomic.compare_add);
2414 	} else if (wr->opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
2415 		aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
2416 		aseg->compare  = cpu_to_be64(wr->wr.atomic.compare_add_mask);
2417 	} else {
2418 		aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
2419 		aseg->compare  = 0;
2420 	}
2421 
2422 }
2423 
2424 static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
2425 				  struct ib_send_wr *wr)
2426 {
2427 	aseg->swap_add		= cpu_to_be64(wr->wr.atomic.swap);
2428 	aseg->swap_add_mask	= cpu_to_be64(wr->wr.atomic.swap_mask);
2429 	aseg->compare		= cpu_to_be64(wr->wr.atomic.compare_add);
2430 	aseg->compare_mask	= cpu_to_be64(wr->wr.atomic.compare_add_mask);
2431 }
2432 
2433 static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
2434 			     struct ib_send_wr *wr)
2435 {
2436 	memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
2437 	dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn);
2438 	dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
2439 	dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan;
2440 	memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->av.eth.mac, 6);
2441 }
2442 
2443 static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
2444 				    struct mlx4_wqe_datagram_seg *dseg,
2445 				    struct ib_send_wr *wr,
2446 				    enum mlx4_ib_qp_type qpt)
2447 {
2448 	union mlx4_ext_av *av = &to_mah(wr->wr.ud.ah)->av;
2449 	struct mlx4_av sqp_av = {0};
2450 	int port = *((u8 *) &av->ib.port_pd) & 0x3;
2451 
2452 	/* force loopback */
2453 	sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000);
2454 	sqp_av.g_slid = av->ib.g_slid & 0x7f; /* no GRH */
2455 	sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel &
2456 			cpu_to_be32(0xf0000000);
2457 
2458 	memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
2459 	if (qpt == MLX4_IB_QPT_PROXY_GSI)
2460 		dseg->dqpn = cpu_to_be32(dev->dev->caps.qp1_tunnel[port - 1]);
2461 	else
2462 		dseg->dqpn = cpu_to_be32(dev->dev->caps.qp0_tunnel[port - 1]);
2463 	/* Use QKEY from the QP context, which is set by master */
2464 	dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
2465 }
2466 
2467 static void build_tunnel_header(struct ib_send_wr *wr, void *wqe, unsigned *mlx_seg_len)
2468 {
2469 	struct mlx4_wqe_inline_seg *inl = wqe;
2470 	struct mlx4_ib_tunnel_header hdr;
2471 	struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah);
2472 	int spc;
2473 	int i;
2474 
2475 	memcpy(&hdr.av, &ah->av, sizeof hdr.av);
2476 	hdr.remote_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
2477 	hdr.pkey_index = cpu_to_be16(wr->wr.ud.pkey_index);
2478 	hdr.qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
2479 	memcpy(hdr.mac, ah->av.eth.mac, 6);
2480 	hdr.vlan = ah->av.eth.vlan;
2481 
2482 	spc = MLX4_INLINE_ALIGN -
2483 		((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2484 	if (sizeof (hdr) <= spc) {
2485 		memcpy(inl + 1, &hdr, sizeof (hdr));
2486 		wmb();
2487 		inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr));
2488 		i = 1;
2489 	} else {
2490 		memcpy(inl + 1, &hdr, spc);
2491 		wmb();
2492 		inl->byte_count = cpu_to_be32(1 << 31 | spc);
2493 
2494 		inl = (void *) (inl + 1) + spc;
2495 		memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc);
2496 		wmb();
2497 		inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc));
2498 		i = 2;
2499 	}
2500 
2501 	*mlx_seg_len =
2502 		ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16);
2503 }
2504 
2505 static void set_mlx_icrc_seg(void *dseg)
2506 {
2507 	u32 *t = dseg;
2508 	struct mlx4_wqe_inline_seg *iseg = dseg;
2509 
2510 	t[1] = 0;
2511 
2512 	/*
2513 	 * Need a barrier here before writing the byte_count field to
2514 	 * make sure that all the data is visible before the
2515 	 * byte_count field is set.  Otherwise, if the segment begins
2516 	 * a new cacheline, the HCA prefetcher could grab the 64-byte
2517 	 * chunk and get a valid (!= * 0xffffffff) byte count but
2518 	 * stale data, and end up sending the wrong data.
2519 	 */
2520 	wmb();
2521 
2522 	iseg->byte_count = cpu_to_be32((1 << 31) | 4);
2523 }
2524 
2525 static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
2526 {
2527 	dseg->lkey       = cpu_to_be32(sg->lkey);
2528 	dseg->addr       = cpu_to_be64(sg->addr);
2529 
2530 	/*
2531 	 * Need a barrier here before writing the byte_count field to
2532 	 * make sure that all the data is visible before the
2533 	 * byte_count field is set.  Otherwise, if the segment begins
2534 	 * a new cacheline, the HCA prefetcher could grab the 64-byte
2535 	 * chunk and get a valid (!= * 0xffffffff) byte count but
2536 	 * stale data, and end up sending the wrong data.
2537 	 */
2538 	wmb();
2539 
2540 	dseg->byte_count = cpu_to_be32(sg->length);
2541 }
2542 
2543 static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
2544 {
2545 	dseg->byte_count = cpu_to_be32(sg->length);
2546 	dseg->lkey       = cpu_to_be32(sg->lkey);
2547 	dseg->addr       = cpu_to_be64(sg->addr);
2548 }
2549 
2550 static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr,
2551 			 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
2552 			 __be32 *lso_hdr_sz, __be32 *blh)
2553 {
2554 	unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16);
2555 
2556 	if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
2557 		*blh = cpu_to_be32(1 << 6);
2558 
2559 	if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
2560 		     wr->num_sge > qp->sq.max_gs - (halign >> 4)))
2561 		return -EINVAL;
2562 
2563 	memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen);
2564 
2565 	*lso_hdr_sz  = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 |
2566 				   wr->wr.ud.hlen);
2567 	*lso_seg_len = halign;
2568 	return 0;
2569 }
2570 
2571 static __be32 send_ieth(struct ib_send_wr *wr)
2572 {
2573 	switch (wr->opcode) {
2574 	case IB_WR_SEND_WITH_IMM:
2575 	case IB_WR_RDMA_WRITE_WITH_IMM:
2576 		return wr->ex.imm_data;
2577 
2578 	case IB_WR_SEND_WITH_INV:
2579 		return cpu_to_be32(wr->ex.invalidate_rkey);
2580 
2581 	default:
2582 		return 0;
2583 	}
2584 }
2585 
2586 static void add_zero_len_inline(void *wqe)
2587 {
2588 	struct mlx4_wqe_inline_seg *inl = wqe;
2589 	memset(wqe, 0, 16);
2590 	inl->byte_count = cpu_to_be32(1 << 31);
2591 }
2592 
2593 int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2594 		      struct ib_send_wr **bad_wr)
2595 {
2596 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
2597 	void *wqe;
2598 	struct mlx4_wqe_ctrl_seg *ctrl;
2599 	struct mlx4_wqe_data_seg *dseg;
2600 	unsigned long flags;
2601 	int nreq;
2602 	int err = 0;
2603 	unsigned ind;
2604 	int uninitialized_var(stamp);
2605 	int uninitialized_var(size);
2606 	unsigned uninitialized_var(seglen);
2607 	__be32 dummy;
2608 	__be32 *lso_wqe;
2609 	__be32 uninitialized_var(lso_hdr_sz);
2610 	__be32 blh;
2611 	int i;
2612 
2613 	spin_lock_irqsave(&qp->sq.lock, flags);
2614 
2615 	ind = qp->sq_next_wqe;
2616 
2617 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
2618 		lso_wqe = &dummy;
2619 		blh = 0;
2620 
2621 		if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
2622 			err = -ENOMEM;
2623 			*bad_wr = wr;
2624 			goto out;
2625 		}
2626 
2627 		if (unlikely(wr->num_sge > qp->sq.max_gs)) {
2628 			err = -EINVAL;
2629 			*bad_wr = wr;
2630 			goto out;
2631 		}
2632 
2633 		ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
2634 		qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
2635 
2636 		ctrl->srcrb_flags =
2637 			(wr->send_flags & IB_SEND_SIGNALED ?
2638 			 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
2639 			(wr->send_flags & IB_SEND_SOLICITED ?
2640 			 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
2641 			((wr->send_flags & IB_SEND_IP_CSUM) ?
2642 			 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
2643 				     MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
2644 			qp->sq_signal_bits;
2645 
2646 		ctrl->imm = send_ieth(wr);
2647 
2648 		wqe += sizeof *ctrl;
2649 		size = sizeof *ctrl / 16;
2650 
2651 		switch (qp->mlx4_ib_qp_type) {
2652 		case MLX4_IB_QPT_RC:
2653 		case MLX4_IB_QPT_UC:
2654 			switch (wr->opcode) {
2655 			case IB_WR_ATOMIC_CMP_AND_SWP:
2656 			case IB_WR_ATOMIC_FETCH_AND_ADD:
2657 			case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
2658 				set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
2659 					      wr->wr.atomic.rkey);
2660 				wqe  += sizeof (struct mlx4_wqe_raddr_seg);
2661 
2662 				set_atomic_seg(wqe, wr);
2663 				wqe  += sizeof (struct mlx4_wqe_atomic_seg);
2664 
2665 				size += (sizeof (struct mlx4_wqe_raddr_seg) +
2666 					 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
2667 
2668 				break;
2669 
2670 			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
2671 				set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
2672 					      wr->wr.atomic.rkey);
2673 				wqe  += sizeof (struct mlx4_wqe_raddr_seg);
2674 
2675 				set_masked_atomic_seg(wqe, wr);
2676 				wqe  += sizeof (struct mlx4_wqe_masked_atomic_seg);
2677 
2678 				size += (sizeof (struct mlx4_wqe_raddr_seg) +
2679 					 sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
2680 
2681 				break;
2682 
2683 			case IB_WR_RDMA_READ:
2684 			case IB_WR_RDMA_WRITE:
2685 			case IB_WR_RDMA_WRITE_WITH_IMM:
2686 				set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
2687 					      wr->wr.rdma.rkey);
2688 				wqe  += sizeof (struct mlx4_wqe_raddr_seg);
2689 				size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
2690 				break;
2691 
2692 			case IB_WR_LOCAL_INV:
2693 				ctrl->srcrb_flags |=
2694 					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
2695 				set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
2696 				wqe  += sizeof (struct mlx4_wqe_local_inval_seg);
2697 				size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
2698 				break;
2699 
2700 			case IB_WR_FAST_REG_MR:
2701 				ctrl->srcrb_flags |=
2702 					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
2703 				set_fmr_seg(wqe, wr);
2704 				wqe  += sizeof (struct mlx4_wqe_fmr_seg);
2705 				size += sizeof (struct mlx4_wqe_fmr_seg) / 16;
2706 				break;
2707 
2708 			case IB_WR_BIND_MW:
2709 				ctrl->srcrb_flags |=
2710 					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
2711 				set_bind_seg(wqe, wr);
2712 				wqe  += sizeof(struct mlx4_wqe_bind_seg);
2713 				size += sizeof(struct mlx4_wqe_bind_seg) / 16;
2714 				break;
2715 			default:
2716 				/* No extra segments required for sends */
2717 				break;
2718 			}
2719 			break;
2720 
2721 		case MLX4_IB_QPT_TUN_SMI_OWNER:
2722 			err =  build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen);
2723 			if (unlikely(err)) {
2724 				*bad_wr = wr;
2725 				goto out;
2726 			}
2727 			wqe  += seglen;
2728 			size += seglen / 16;
2729 			break;
2730 		case MLX4_IB_QPT_TUN_SMI:
2731 		case MLX4_IB_QPT_TUN_GSI:
2732 			/* this is a UD qp used in MAD responses to slaves. */
2733 			set_datagram_seg(wqe, wr);
2734 			/* set the forced-loopback bit in the data seg av */
2735 			*(__be32 *) wqe |= cpu_to_be32(0x80000000);
2736 			wqe  += sizeof (struct mlx4_wqe_datagram_seg);
2737 			size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
2738 			break;
2739 		case MLX4_IB_QPT_UD:
2740 			set_datagram_seg(wqe, wr);
2741 			wqe  += sizeof (struct mlx4_wqe_datagram_seg);
2742 			size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
2743 
2744 			if (wr->opcode == IB_WR_LSO) {
2745 				err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz, &blh);
2746 				if (unlikely(err)) {
2747 					*bad_wr = wr;
2748 					goto out;
2749 				}
2750 				lso_wqe = (__be32 *) wqe;
2751 				wqe  += seglen;
2752 				size += seglen / 16;
2753 			}
2754 			break;
2755 
2756 		case MLX4_IB_QPT_PROXY_SMI_OWNER:
2757 			err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen);
2758 			if (unlikely(err)) {
2759 				*bad_wr = wr;
2760 				goto out;
2761 			}
2762 			wqe  += seglen;
2763 			size += seglen / 16;
2764 			/* to start tunnel header on a cache-line boundary */
2765 			add_zero_len_inline(wqe);
2766 			wqe += 16;
2767 			size++;
2768 			build_tunnel_header(wr, wqe, &seglen);
2769 			wqe  += seglen;
2770 			size += seglen / 16;
2771 			break;
2772 		case MLX4_IB_QPT_PROXY_SMI:
2773 		case MLX4_IB_QPT_PROXY_GSI:
2774 			/* If we are tunneling special qps, this is a UD qp.
2775 			 * In this case we first add a UD segment targeting
2776 			 * the tunnel qp, and then add a header with address
2777 			 * information */
2778 			set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe, wr,
2779 						qp->mlx4_ib_qp_type);
2780 			wqe  += sizeof (struct mlx4_wqe_datagram_seg);
2781 			size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
2782 			build_tunnel_header(wr, wqe, &seglen);
2783 			wqe  += seglen;
2784 			size += seglen / 16;
2785 			break;
2786 
2787 		case MLX4_IB_QPT_SMI:
2788 		case MLX4_IB_QPT_GSI:
2789 			err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen);
2790 			if (unlikely(err)) {
2791 				*bad_wr = wr;
2792 				goto out;
2793 			}
2794 			wqe  += seglen;
2795 			size += seglen / 16;
2796 			break;
2797 
2798 		default:
2799 			break;
2800 		}
2801 
2802 		/*
2803 		 * Write data segments in reverse order, so as to
2804 		 * overwrite cacheline stamp last within each
2805 		 * cacheline.  This avoids issues with WQE
2806 		 * prefetching.
2807 		 */
2808 
2809 		dseg = wqe;
2810 		dseg += wr->num_sge - 1;
2811 		size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
2812 
2813 		/* Add one more inline data segment for ICRC for MLX sends */
2814 		if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
2815 			     qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI ||
2816 			     qp->mlx4_ib_qp_type &
2817 			     (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) {
2818 			set_mlx_icrc_seg(dseg + 1);
2819 			size += sizeof (struct mlx4_wqe_data_seg) / 16;
2820 		}
2821 
2822 		for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
2823 			set_data_seg(dseg, wr->sg_list + i);
2824 
2825 		/*
2826 		 * Possibly overwrite stamping in cacheline with LSO
2827 		 * segment only after making sure all data segments
2828 		 * are written.
2829 		 */
2830 		wmb();
2831 		*lso_wqe = lso_hdr_sz;
2832 
2833 		ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ?
2834 				    MLX4_WQE_CTRL_FENCE : 0) | size;
2835 
2836 		/*
2837 		 * Make sure descriptor is fully written before
2838 		 * setting ownership bit (because HW can start
2839 		 * executing as soon as we do).
2840 		 */
2841 		wmb();
2842 
2843 		if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
2844 			*bad_wr = wr;
2845 			err = -EINVAL;
2846 			goto out;
2847 		}
2848 
2849 		ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
2850 			(ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
2851 
2852 		stamp = ind + qp->sq_spare_wqes;
2853 		ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
2854 
2855 		/*
2856 		 * We can improve latency by not stamping the last
2857 		 * send queue WQE until after ringing the doorbell, so
2858 		 * only stamp here if there are still more WQEs to post.
2859 		 *
2860 		 * Same optimization applies to padding with NOP wqe
2861 		 * in case of WQE shrinking (used to prevent wrap-around
2862 		 * in the middle of WR).
2863 		 */
2864 		if (wr->next) {
2865 			stamp_send_wqe(qp, stamp, size * 16);
2866 			ind = pad_wraparound(qp, ind);
2867 		}
2868 	}
2869 
2870 out:
2871 	if (likely(nreq)) {
2872 		qp->sq.head += nreq;
2873 
2874 		/*
2875 		 * Make sure that descriptors are written before
2876 		 * doorbell record.
2877 		 */
2878 		wmb();
2879 
2880 		writel(qp->doorbell_qpn,
2881 		       to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
2882 
2883 		/*
2884 		 * Make sure doorbells don't leak out of SQ spinlock
2885 		 * and reach the HCA out of order.
2886 		 */
2887 		mmiowb();
2888 
2889 		stamp_send_wqe(qp, stamp, size * 16);
2890 
2891 		ind = pad_wraparound(qp, ind);
2892 		qp->sq_next_wqe = ind;
2893 	}
2894 
2895 	spin_unlock_irqrestore(&qp->sq.lock, flags);
2896 
2897 	return err;
2898 }
2899 
2900 int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
2901 		      struct ib_recv_wr **bad_wr)
2902 {
2903 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
2904 	struct mlx4_wqe_data_seg *scat;
2905 	unsigned long flags;
2906 	int err = 0;
2907 	int nreq;
2908 	int ind;
2909 	int max_gs;
2910 	int i;
2911 
2912 	max_gs = qp->rq.max_gs;
2913 	spin_lock_irqsave(&qp->rq.lock, flags);
2914 
2915 	ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
2916 
2917 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
2918 		if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
2919 			err = -ENOMEM;
2920 			*bad_wr = wr;
2921 			goto out;
2922 		}
2923 
2924 		if (unlikely(wr->num_sge > qp->rq.max_gs)) {
2925 			err = -EINVAL;
2926 			*bad_wr = wr;
2927 			goto out;
2928 		}
2929 
2930 		scat = get_recv_wqe(qp, ind);
2931 
2932 		if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
2933 		    MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
2934 			ib_dma_sync_single_for_device(ibqp->device,
2935 						      qp->sqp_proxy_rcv[ind].map,
2936 						      sizeof (struct mlx4_ib_proxy_sqp_hdr),
2937 						      DMA_FROM_DEVICE);
2938 			scat->byte_count =
2939 				cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr));
2940 			/* use dma lkey from upper layer entry */
2941 			scat->lkey = cpu_to_be32(wr->sg_list->lkey);
2942 			scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map);
2943 			scat++;
2944 			max_gs--;
2945 		}
2946 
2947 		for (i = 0; i < wr->num_sge; ++i)
2948 			__set_data_seg(scat + i, wr->sg_list + i);
2949 
2950 		if (i < max_gs) {
2951 			scat[i].byte_count = 0;
2952 			scat[i].lkey       = cpu_to_be32(MLX4_INVALID_LKEY);
2953 			scat[i].addr       = 0;
2954 		}
2955 
2956 		qp->rq.wrid[ind] = wr->wr_id;
2957 
2958 		ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
2959 	}
2960 
2961 out:
2962 	if (likely(nreq)) {
2963 		qp->rq.head += nreq;
2964 
2965 		/*
2966 		 * Make sure that descriptors are written before
2967 		 * doorbell record.
2968 		 */
2969 		wmb();
2970 
2971 		*qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
2972 	}
2973 
2974 	spin_unlock_irqrestore(&qp->rq.lock, flags);
2975 
2976 	return err;
2977 }
2978 
2979 static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
2980 {
2981 	switch (mlx4_state) {
2982 	case MLX4_QP_STATE_RST:      return IB_QPS_RESET;
2983 	case MLX4_QP_STATE_INIT:     return IB_QPS_INIT;
2984 	case MLX4_QP_STATE_RTR:      return IB_QPS_RTR;
2985 	case MLX4_QP_STATE_RTS:      return IB_QPS_RTS;
2986 	case MLX4_QP_STATE_SQ_DRAINING:
2987 	case MLX4_QP_STATE_SQD:      return IB_QPS_SQD;
2988 	case MLX4_QP_STATE_SQER:     return IB_QPS_SQE;
2989 	case MLX4_QP_STATE_ERR:      return IB_QPS_ERR;
2990 	default:		     return -1;
2991 	}
2992 }
2993 
2994 static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
2995 {
2996 	switch (mlx4_mig_state) {
2997 	case MLX4_QP_PM_ARMED:		return IB_MIG_ARMED;
2998 	case MLX4_QP_PM_REARM:		return IB_MIG_REARM;
2999 	case MLX4_QP_PM_MIGRATED:	return IB_MIG_MIGRATED;
3000 	default: return -1;
3001 	}
3002 }
3003 
3004 static int to_ib_qp_access_flags(int mlx4_flags)
3005 {
3006 	int ib_flags = 0;
3007 
3008 	if (mlx4_flags & MLX4_QP_BIT_RRE)
3009 		ib_flags |= IB_ACCESS_REMOTE_READ;
3010 	if (mlx4_flags & MLX4_QP_BIT_RWE)
3011 		ib_flags |= IB_ACCESS_REMOTE_WRITE;
3012 	if (mlx4_flags & MLX4_QP_BIT_RAE)
3013 		ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
3014 
3015 	return ib_flags;
3016 }
3017 
3018 static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr,
3019 				struct mlx4_qp_path *path)
3020 {
3021 	struct mlx4_dev *dev = ibdev->dev;
3022 	int is_eth;
3023 
3024 	memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
3025 	ib_ah_attr->port_num	  = path->sched_queue & 0x40 ? 2 : 1;
3026 
3027 	if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)
3028 		return;
3029 
3030 	is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) ==
3031 		IB_LINK_LAYER_ETHERNET;
3032 	if (is_eth)
3033 		ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) |
3034 		((path->sched_queue & 4) << 1);
3035 	else
3036 		ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;
3037 
3038 	ib_ah_attr->dlid	  = be16_to_cpu(path->rlid);
3039 	ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;
3040 	ib_ah_attr->static_rate   = path->static_rate ? path->static_rate - 5 : 0;
3041 	ib_ah_attr->ah_flags      = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
3042 	if (ib_ah_attr->ah_flags) {
3043 		ib_ah_attr->grh.sgid_index = path->mgid_index;
3044 		ib_ah_attr->grh.hop_limit  = path->hop_limit;
3045 		ib_ah_attr->grh.traffic_class =
3046 			(be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff;
3047 		ib_ah_attr->grh.flow_label =
3048 			be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
3049 		memcpy(ib_ah_attr->grh.dgid.raw,
3050 			path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
3051 	}
3052 }
3053 
3054 int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
3055 		     struct ib_qp_init_attr *qp_init_attr)
3056 {
3057 	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
3058 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
3059 	struct mlx4_qp_context context;
3060 	int mlx4_state;
3061 	int err = 0;
3062 
3063 	mutex_lock(&qp->mutex);
3064 
3065 	if (qp->state == IB_QPS_RESET) {
3066 		qp_attr->qp_state = IB_QPS_RESET;
3067 		goto done;
3068 	}
3069 
3070 	err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
3071 	if (err) {
3072 		err = -EINVAL;
3073 		goto out;
3074 	}
3075 
3076 	mlx4_state = be32_to_cpu(context.flags) >> 28;
3077 
3078 	qp->state		     = to_ib_qp_state(mlx4_state);
3079 	qp_attr->qp_state	     = qp->state;
3080 	qp_attr->path_mtu	     = context.mtu_msgmax >> 5;
3081 	qp_attr->path_mig_state	     =
3082 		to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
3083 	qp_attr->qkey		     = be32_to_cpu(context.qkey);
3084 	qp_attr->rq_psn		     = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
3085 	qp_attr->sq_psn		     = be32_to_cpu(context.next_send_psn) & 0xffffff;
3086 	qp_attr->dest_qp_num	     = be32_to_cpu(context.remote_qpn) & 0xffffff;
3087 	qp_attr->qp_access_flags     =
3088 		to_ib_qp_access_flags(be32_to_cpu(context.params2));
3089 
3090 	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
3091 		to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
3092 		to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
3093 		qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
3094 		qp_attr->alt_port_num	= qp_attr->alt_ah_attr.port_num;
3095 	}
3096 
3097 	qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
3098 	if (qp_attr->qp_state == IB_QPS_INIT)
3099 		qp_attr->port_num = qp->port;
3100 	else
3101 		qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
3102 
3103 	/* qp_attr->en_sqd_async_notify is only applicable in modify qp */
3104 	qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
3105 
3106 	qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
3107 
3108 	qp_attr->max_dest_rd_atomic =
3109 		1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
3110 	qp_attr->min_rnr_timer	    =
3111 		(be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
3112 	qp_attr->timeout	    = context.pri_path.ackto >> 3;
3113 	qp_attr->retry_cnt	    = (be32_to_cpu(context.params1) >> 16) & 0x7;
3114 	qp_attr->rnr_retry	    = (be32_to_cpu(context.params1) >> 13) & 0x7;
3115 	qp_attr->alt_timeout	    = context.alt_path.ackto >> 3;
3116 
3117 done:
3118 	qp_attr->cur_qp_state	     = qp_attr->qp_state;
3119 	qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
3120 	qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
3121 
3122 	if (!ibqp->uobject) {
3123 		qp_attr->cap.max_send_wr  = qp->sq.wqe_cnt;
3124 		qp_attr->cap.max_send_sge = qp->sq.max_gs;
3125 	} else {
3126 		qp_attr->cap.max_send_wr  = 0;
3127 		qp_attr->cap.max_send_sge = 0;
3128 	}
3129 
3130 	/*
3131 	 * We don't support inline sends for kernel QPs (yet), and we
3132 	 * don't know what userspace's value should be.
3133 	 */
3134 	qp_attr->cap.max_inline_data = 0;
3135 
3136 	qp_init_attr->cap	     = qp_attr->cap;
3137 
3138 	qp_init_attr->create_flags = 0;
3139 	if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
3140 		qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
3141 
3142 	if (qp->flags & MLX4_IB_QP_LSO)
3143 		qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
3144 
3145 	if (qp->flags & MLX4_IB_QP_NETIF)
3146 		qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP;
3147 
3148 	qp_init_attr->sq_sig_type =
3149 		qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ?
3150 		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
3151 
3152 out:
3153 	mutex_unlock(&qp->mutex);
3154 	return err;
3155 }
3156 
3157